From 21222ab6faab3d379a76a38b30b0a79febc77cc9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:26:12 +0000 Subject: [PATCH 01/65] fix: queryguard refusal data race; CI race/gofmt/govulncheck/staticcheck; pin Nucleus; hardening docs - queryguard: Limiter.Acquire read l.global / l.sites[siteID] after unlocking l.mu to format refusal messages (concurrent map access). Capture under the lock; add multi-site regression test. - CI: full go test -race job, gofmt check, govulncheck and staticcheck jobs, single NUCLEUS_VERSION (v1.1.1) for all jobs, loopback-only published Nucleus port. - compose: pin Nucleus to ${NUCLEUS_VERSION:-v1.1.1}; Caddy profile now uses packaging/caddy/Caddyfile with HSTS, nosniff, Referrer-Policy and frame-ancestors. systemd nucleus binds 127.0.0.1. - docs/operations/hardening.md. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- .github/workflows/ci.yml | 78 ++++++++++++++++++---- docker-compose.yml | 22 ++++--- docs/operations/hardening.md | 105 ++++++++++++++++++++++++++++++ internal/queryguard/queryguard.go | 10 ++- packaging/caddy/Caddyfile | 28 ++++++++ packaging/systemd/nucleus.service | 4 +- 6 files changed, 223 insertions(+), 24 deletions(-) create mode 100644 docs/operations/hardening.md create mode 100644 packaging/caddy/Caddyfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e6c9531..b5abb6d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,11 @@ on: # and a superseded push to the same branch cancels its own stale run. permissions: contents: read +# One Nucleus image version for every job (was v1.1.0 in e2e, v1.1.1 +# elsewhere). Also the default in docker-compose.yml (NUCLEUS_VERSION). +env: + NUCLEUS_VERSION: v1.1.1 + STATICCHECK_VERSION: 2025.1.1 concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -34,14 +39,61 @@ jobs: run: go vet ./... - name: Validate installer run: sh -n scripts/install.sh + - name: gofmt + # Non-vendor, non-Neutron Go files only; any output fails the job. + run: | + out=$(git ls-files '*.go' | grep -v -e '^vendor/' -e '^Neutron/' | xargs gofmt -l) + if [ -n "$out" ]; then + echo "unformatted files (run gofmt -w):" >&2 + echo "$out" >&2 + exit 1 + fi - name: Test run: go test ./... - # AUD-056 (round 2): the concurrency-heavy packages get the race - # detector — the batch-admission, WAL, replay-lock, and audit-head - # regressions from this audit series are exactly the class of bug a - # plain `go test` passes. - - name: Race regression tests - run: go test -race ./internal/ingest/... ./internal/audit/... ./internal/replays/... ./internal/auth/... ./internal/backup/... + + # Full race detector over the main module (supersedes the old partial + # list in the go job). Separate job so it runs in parallel with the plain + # suite; DB-backed tests skip here (no Nucleus), which keeps it bounded. + go-race: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Race + run: go test -race ./... + + # Known-vulnerability scan over the module graph. Separate job so a new + # advisory is its own red check rather than hiding in `go`. Intended to + # be BLOCKING once the first run is triaged (stdlib findings clear by + # bumping the Go toolchain, dependency findings by go.mod). + govulncheck: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: govulncheck + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + + staticcheck: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: staticcheck + run: | + go install honnef.co/go/tools/cmd/staticcheck@${{ env.STATICCHECK_VERSION }} + staticcheck ./... # AUD-056 (round 2): the classic replay tracker is a separately shipped # surface; these contract tests execute the real served script in a vm @@ -227,13 +279,13 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '1.25' - - name: Start pinned Nucleus v1.1.1 + - name: Start pinned Nucleus run: | - docker run -d --name nucleus -p 55432:5432 \ + docker run -d --name nucleus -p 127.0.0.1:55432:5432 \ -e NUCLEUS_ALLOW_NO_AUTH=1 \ -e NUCLEUS_ALLOW_INSECURE_CLUSTER=1 \ -e NUCLEUS_ALLOW_INSECURE_REPLICATION=1 \ - ghcr.io/neutron-build/nucleus:v1.1.1 \ + ghcr.io/neutron-build/nucleus:${{ env.NUCLEUS_VERSION }} \ start --host 0.0.0.0 --port 5432 --cluster-port 5433 --data /data --max-memory 512 for i in $(seq 1 60); do if (exec 3<>/dev/tcp/127.0.0.1/55432) 2>/dev/null; then exec 3>&-; exit 0; fi @@ -277,11 +329,11 @@ jobs: go-version: '1.25' - name: Start pinned Nucleus run: | - docker run -d --name nucleus -p 55432:5432 \ + docker run -d --name nucleus -p 127.0.0.1:55432:5432 \ -e NUCLEUS_ALLOW_NO_AUTH=1 \ -e NUCLEUS_ALLOW_INSECURE_CLUSTER=1 \ -e NUCLEUS_ALLOW_INSECURE_REPLICATION=1 \ - ghcr.io/neutron-build/nucleus:v1.1.1 \ + ghcr.io/neutron-build/nucleus:${{ env.NUCLEUS_VERSION }} \ start --host 0.0.0.0 --port 5432 --cluster-port 5433 --data /data --max-memory 512 for i in $(seq 1 60); do if (exec 3<>/dev/tcp/127.0.0.1/55432) 2>/dev/null; then exec 3>&-; exit 0; fi @@ -334,11 +386,11 @@ jobs: node-version: '20' - name: Start pinned Nucleus run: | - docker run -d --name nucleus -p 55432:5432 \ + docker run -d --name nucleus -p 127.0.0.1:55432:5432 \ -e NUCLEUS_ALLOW_NO_AUTH=1 \ -e NUCLEUS_ALLOW_INSECURE_CLUSTER=1 \ -e NUCLEUS_ALLOW_INSECURE_REPLICATION=1 \ - ghcr.io/neutron-build/nucleus:v1.1.0 \ + ghcr.io/neutron-build/nucleus:${{ env.NUCLEUS_VERSION }} \ start --host 0.0.0.0 --port 5432 --cluster-port 5433 --data /data --max-memory 512 for i in $(seq 1 60); do if (exec 3<>/dev/tcp/127.0.0.1/55432) 2>/dev/null; then exec 3>&-; exit 0; fi diff --git a/docker-compose.yml b/docker-compose.yml index 9401b8e3..334187d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,12 +39,18 @@ services: nucleus: - image: ghcr.io/neutron-build/nucleus:latest + # Pinned (was :latest). Keep in sync with NUCLEUS_VERSION in + # .github/workflows/ci.yml; override with NUCLEUS_VERSION=... to upgrade. + image: ghcr.io/neutron-build/nucleus:${NUCLEUS_VERSION:-v1.1.1} restart: unless-stopped networks: # R43 (round 4): database on the internal backend network only — the # TLS proxy (and anything else on frontend) cannot reach it directly. backend: {} + # No `ports:` on purpose. NUCLEUS_ALLOW_NO_AUTH=1 below means anyone who + # can reach 5432 has full database access; the only network this service + # joins is the `internal: true` backend network. See + # docs/operations/hardening.md before changing this. volumes: - nucleus_data:/data command: ["start", "--port", "5432", "--cluster-port", "5433", "--data", "/data", "--max-memory", "512"] @@ -130,13 +136,13 @@ services: volumes: - caddy_data:/data - caddy_config:/config - command: - - caddy - - reverse-proxy - - --from - - "${OBSERVE_DOMAIN:-observe.localhost}" - - --to - - "observe:3000" + - ./packaging/caddy/Caddyfile:/etc/caddy/Caddyfile:ro + environment: + OBSERVE_DOMAIN: "${OBSERVE_DOMAIN:-observe.localhost}" + # Security headers (HSTS, nosniff, Referrer-Policy, frame-ancestors) live + # in packaging/caddy/Caddyfile; the old `caddy reverse-proxy` one-liner + # set none of them. + command: ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] depends_on: observe: condition: service_healthy diff --git a/docs/operations/hardening.md b/docs/operations/hardening.md new file mode 100644 index 00000000..de309eb2 --- /dev/null +++ b/docs/operations/hardening.md @@ -0,0 +1,105 @@ +# Hardening a self-hosted Observe + +This describes what the repository ships and what the operator must still do. +It does not claim properties the code does not enforce. + +## Nucleus runs with NUCLEUS_ALLOW_NO_AUTH=1 + +`docker-compose.yml`, `packaging/systemd/nucleus.service` and every CI job +start Nucleus with `NUCLEUS_ALLOW_NO_AUTH=1`. Observe connects with +`postgres://nucleus:5432/observe` (no password), so the flag is how the +shipped topology works; it has not been verified here whether the engine can +run Observe with authentication enabled, so the flag was not removed. +`NUCLEUS_ALLOW_INSECURE_CLUSTER=1` and `NUCLEUS_ALLOW_INSECURE_REPLICATION=1` +are set for the same reason (single node, no peers). + +The consequence is that **network isolation is the only access control on the +database**. + +## Network isolation requirement + +- Never publish port 5432 (or the cluster port 5433) on a public or shared + interface. +- Compose: the `nucleus` service has no `ports:` and joins only the + `backend` network, which is `internal: true` (no route to or from the + outside). Only `observe` shares it. Caddy is on `frontend` only and cannot + reach the database. Do not add a `ports:` entry to `nucleus`. +- systemd: `nucleus.service` starts with `--host 127.0.0.1`. `observe.service` + uses `postgres://localhost:5432/observe`, so both stay on one host. Do not + change the host to `0.0.0.0`. +- Image version: compose defaults to `ghcr.io/neutron-build/nucleus:v1.1.1` + (override with `NUCLEUS_VERSION`); CI uses the same value. + +## What an attacker with network access to Nucleus gets + +Full read/write/DDL over everything Observe stores, with no credential: +events, errors, logs, traces, replays, users and password hashes, API keys, +the audit log, and the SQL surface to rewrite or drop any of it (including +forging audit history). Nucleus authentication state is not a second line of +defense. Treat a reachable Nucleus port as total compromise of the instance. + +## JWT secret and session salt persistence + +- `OBSERVE_JWT_SECRET`: unset means a random per-process secret is generated + (logged as "generated random JWT secret"), so all sessions are invalidated + on every restart. If set it must be at least 32 characters or startup + validation rejects it. Set it from a secret store, not the repo; rotating it + logs everyone out. +- `OBSERVE_SESSION_SALT`: unset means a random salt (a warning is logged), so + visitor/session IDs change across restarts. Set it if you want them stable. + Changing it re-keys visitor IDs; do it deliberately. +- Neither has a public default. Compose passes both through as empty unless + you export them. + +## TLS and response headers (Caddy profile) + +`docker compose --profile tls up -d` runs Caddy with +`packaging/caddy/Caddyfile` (mounted read-only). It obtains a certificate for +`OBSERVE_DOMAIN` and adds, as defaults: `Strict-Transport-Security: +max-age=31536000`, `X-Content-Type-Options: nosniff`, `Referrer-Policy: +strict-origin-when-cross-origin`, `Content-Security-Policy: frame-ancestors +'none'`, and strips `Server`. The last three use Caddy's `?` default-only +form so they never override stricter values the application sets itself. + +Not set: a full `script-src`/`style-src` CSP. It has not been validated against +the embedded UI. The direct `127.0.0.1:3000` binding bypasses Caddy and gets +none of these headers. + +What the Go server sets itself (no global middleware does this): the shared +dashboard page (`Referrer-Policy: no-referrer`, `X-Content-Type-Options: +nosniff`, `Cache-Control: private, no-store`, in `cmd/observe/main.go`), +replay asset responses (`X-Content-Type-Options: nosniff`, `Content-Security-Policy: +default-src 'none'; sandbox`, in `internal/replays/assets.go`), and +`Referrer-Policy: no-referrer` for stream-ticket requests and the OIDC flow. +All other responses carry no security headers from the application, so a +deployment without the Caddy profile (or an equivalent proxy) has none. + +## Trusted proxies + +`OBSERVE_TRUSTED_PROXIES` lists the CIDRs whose forwarded-client-IP headers +are honored. Compose defaults it to `172.30.10.2/32`, Caddy's fixed address on +the project-private `frontend` subnet, so no other container is trusted. +Behind a different proxy, set it to that proxy's address only; an overly broad +range lets any client spoof its IP and evade rate limits and IP-based audit +data. `OBSERVE_PUBLIC_URL` sets the canonical external URL used for generated +links (share links, SSO redirects). + +## Ingest listener separation + +To collect telemetry from the public internet, do not expose the dashboard +port. Set `OBSERVE_INGEST_ADDR` (for example `:3001`) and publish only that +port. The second listener serves a default-deny allowlist of telemetry-write +routes (`ingestRoutes` in `cmd/observe/ingest_listener.go`); everything else +returns 404 there. Keep `OBSERVE_ADDR` on loopback or a private network. With +the listener unset, ingest shares the dashboard port, which is the right +default only for private networks. + +## Checklist + +1. Nucleus unreachable from outside the host/compose network. +2. `OBSERVE_JWT_SECRET` (32+ chars) and `OBSERVE_SESSION_SALT` set and stored + outside the repo. +3. TLS terminated by the Caddy profile or an equivalent proxy that sets the + headers above. +4. `OBSERVE_TRUSTED_PROXIES` limited to that proxy. +5. Public telemetry served via `OBSERVE_INGEST_ADDR` only. diff --git a/internal/queryguard/queryguard.go b/internal/queryguard/queryguard.go index c51d4b03..4d9dcccc 100644 --- a/internal/queryguard/queryguard.go +++ b/internal/queryguard/queryguard.go @@ -205,17 +205,23 @@ func (l *Limiter) Acquire(ctx context.Context, siteID string) (func(), error) { l.mu.Lock() if l.global >= l.globalSlots { l.refused[CodeConcurrencyGlobal]++ + // Capture under the lock: l.global is written by concurrent + // Acquire/release, so it must not be read after Unlock. + inUse, limit := l.global, l.globalSlots l.mu.Unlock() return nil, refusal(CodeConcurrencyGlobal, - fmt.Sprintf("%d of %d global query slots in use", l.global, l.globalSlots), + fmt.Sprintf("%d of %d global query slots in use", inUse, limit), "retry when in-flight queries finish, or raise OBSERVE_QUERY_GLOBAL_CONCURRENCY", 429) } if l.sites[siteID] >= l.siteSlots { l.refused[CodeConcurrencySite]++ + // Capture under the lock: l.sites is a map mutated by concurrent + // Acquire/release; reading it after Unlock is a fatal map race. + inUse, limit := l.sites[siteID], l.siteSlots l.mu.Unlock() return nil, refusal(CodeConcurrencySite, - fmt.Sprintf("site has %d of %d query slots in use", l.sites[siteID], l.siteSlots), + fmt.Sprintf("site has %d of %d query slots in use", inUse, limit), "retry when this site's in-flight queries finish, or raise OBSERVE_QUERY_SITE_CONCURRENCY", 429) } diff --git a/packaging/caddy/Caddyfile b/packaging/caddy/Caddyfile new file mode 100644 index 00000000..2b68d5c6 --- /dev/null +++ b/packaging/caddy/Caddyfile @@ -0,0 +1,28 @@ +# Caddy front for the compose "tls" profile (docker-compose.yml mounts this +# at /etc/caddy/Caddyfile). OBSERVE_DOMAIN comes from the compose +# environment; Caddy obtains and renews the certificate automatically. +{$OBSERVE_DOMAIN:observe.localhost} { + encode zstd gzip + + header { + # "?" = default only: the app sets stricter values on share pages + # (no-referrer) and replay assets (CSP default-src none; sandbox), + # which must not be overridden. + # Browsers refuse plain HTTP for this host for a year. No + # includeSubDomains/preload: those depend on the operator's other + # subdomains and are hard to undo. + Strict-Transport-Security "max-age=31536000" + ?X-Content-Type-Options "nosniff" + ?Referrer-Policy "strict-origin-when-cross-origin" + # Clickjacking defense for the dashboard. Deliberately only + # frame-ancestors: a full script-src/style-src policy has not been + # validated against the embedded UI. + ?Content-Security-Policy "frame-ancestors 'none'" + -Server + } + + # observe is the only upstream; the database is not reachable from this + # network. Caddy sets X-Forwarded-For/-Proto/-Host itself, and observe + # trusts only this container via OBSERVE_TRUSTED_PROXIES. + reverse_proxy observe:3000 +} diff --git a/packaging/systemd/nucleus.service b/packaging/systemd/nucleus.service index 2725b704..fe1519b4 100644 --- a/packaging/systemd/nucleus.service +++ b/packaging/systemd/nucleus.service @@ -10,9 +10,11 @@ User=nucleus Group=nucleus WorkingDirectory=/var/lib/nucleus +# NO_AUTH is on, so the listener MUST stay on loopback (--host 127.0.0.1 +# below). Do not widen it; see docs/operations/hardening.md. Environment=NUCLEUS_ALLOW_NO_AUTH=1 -ExecStart=/usr/local/bin/nucleus start --port 5432 --data /var/lib/nucleus/data --max-memory 512 +ExecStart=/usr/local/bin/nucleus start --host 127.0.0.1 --port 5432 --data /var/lib/nucleus/data --max-memory 512 KillSignal=SIGTERM TimeoutStopSec=30s From 86eef482c2f6b5361d0ec17c343802195bae95d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:26:48 +0000 Subject: [PATCH 02/65] feat: retain metric_points, host_metrics, uptime_results, performance_issues, service_dependencies These tables grew unbounded. RetentionPolicy gains a Unit (ms default, ns for metric_points.ts_ns) so the cutoff is scaled to the column; a ms cutoff against ns would delete nothing. New validated env knobs OBSERVE_METRICS_RETENTION_DAYS (30), OBSERVE_INFRA_RETENTION_DAYS (30), OBSERVE_UPTIME_RETENTION_DAYS (90), all >= 1. performance_issues (90d by last_seen) and service_dependencies (30d by ts_bucket, matching service_stats) use fixed windows. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- README.md | 3 + cmd/observe/main.go | 2 + internal/config/config.go | 17 ++++ internal/config/config_test.go | 22 +++++ internal/jobs/retention.go | 60 +++++++++++- internal/jobs/retention_telemetry_test.go | 107 ++++++++++++++++++++++ 6 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 internal/jobs/retention_telemetry_test.go diff --git a/README.md b/README.md index 08fdd37b..562ce8f2 100644 --- a/README.md +++ b/README.md @@ -385,6 +385,9 @@ observeErrors.addBreadcrumb({ type: "user", category: "click", message: "Button" | `OBSERVE_RAW_RETENTION_DAYS` | `30` | Raw event retention. Also the window over which visitor counts are exact from raw events; past it they are counted from the `sessions` table (90 days), and past both the dashboard says which window the figure covers. | | `OBSERVE_HOURLY_RETENTION_DAYS` | `365` | Hourly rollup retention. | | `OBSERVE_LLM_RETENTION_DAYS` | `30` | LLM trace retention, including stored prompts/completions. Must be at least 1. Cleanup runs daily; historical model-price catalog entries are retained. Logical expiry does not guarantee database files immediately shrink. | +| `OBSERVE_METRICS_RETENTION_DAYS` | `30` | OTLP metric point (`metric_points`) retention. Must be at least 1. | +| `OBSERVE_INFRA_RETENTION_DAYS` | `30` | Host metric (`host_metrics`) retention. Must be at least 1. | +| `OBSERVE_UPTIME_RETENTION_DAYS` | `90` | Uptime check result (`uptime_results`) retention. Must be at least 1. Fixed windows on the same daily cleanup: `performance_issues` 90 days (by last detection), `service_dependencies` 30 days. | | `OBSERVE_LOG_ROUTES` | `0` | Set to `1` to print route table at boot. | | `OBSERVE_SMTP_HOST` | | SMTP server for email reports. | | `OBSERVE_SMTP_PORT` | `587` | SMTP port. | diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..d0fe8998 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -374,6 +374,8 @@ func main() { // they dedupe, processed outbox intents prune, dead letters never do. retentionPolicies := append(jobs.DefaultPolicies(cfg.RawRetentionDays, cfg.HourlyRetentionDays, cfg.LLMRetentionDays), jobs.DefaultLedgerPolicies(cfg.ErrorInboxRetentionDays, cfg.ReplayBatchesRetentionDays, cfg.DerivedOutboxRetentionDays)...) + retentionPolicies = append(retentionPolicies, + jobs.DefaultTelemetryPolicies(cfg.MetricsRetentionDays, cfg.InfraRetentionDays, cfg.UptimeRetentionDays)...) // Stats service statsSvc := query.NewStatsService(db).WithRetention(query.RetentionWindows{ diff --git a/internal/config/config.go b/internal/config/config.go index af11cbe7..3f909539 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,6 +34,11 @@ type Config struct { RawRetentionDays int HourlyRetentionDays int LLMRetentionDays int + // Telemetry retention: metric_points, host_metrics and uptime_results + // were previously unbounded. + MetricsRetentionDays int + InfraRetentionDays int + UptimeRetentionDays int // Ledger retention (O01 ADR 5.8, decided 2026-09-23). Expiry means a // retried record is processed as new — no silent infinite exactly-once // promise beyond the window. @@ -97,6 +102,9 @@ func Load() Config { set("OBSERVE_RAW_RETENTION_DAYS", 30, &c.RawRetentionDays) set("OBSERVE_HOURLY_RETENTION_DAYS", 365, &c.HourlyRetentionDays) set("OBSERVE_LLM_RETENTION_DAYS", 30, &c.LLMRetentionDays) + set("OBSERVE_METRICS_RETENTION_DAYS", 30, &c.MetricsRetentionDays) + set("OBSERVE_INFRA_RETENTION_DAYS", 30, &c.InfraRetentionDays) + set("OBSERVE_UPTIME_RETENTION_DAYS", 90, &c.UptimeRetentionDays) // O01 slice 5 ledger retention (decided defaults 2026-09-23): the // error_inbox and replay_batches ledgers must not outlive the data they // dedupe (error_events 180d, replay_sessions 14d); derived_outbox @@ -173,6 +181,15 @@ func (c Config) Validate() error { if c.LLMRetentionDays < 1 { return fmt.Errorf("OBSERVE_LLM_RETENTION_DAYS must be >= 1, got %d", c.LLMRetentionDays) } + if c.MetricsRetentionDays < 1 { + return fmt.Errorf("OBSERVE_METRICS_RETENTION_DAYS must be >= 1, got %d", c.MetricsRetentionDays) + } + if c.InfraRetentionDays < 1 { + return fmt.Errorf("OBSERVE_INFRA_RETENTION_DAYS must be >= 1, got %d", c.InfraRetentionDays) + } + if c.UptimeRetentionDays < 1 { + return fmt.Errorf("OBSERVE_UPTIME_RETENTION_DAYS must be >= 1, got %d", c.UptimeRetentionDays) + } if c.ErrorInboxRetentionDays < 1 { return fmt.Errorf("OBSERVE_ERROR_INBOX_RETENTION_DAYS must be >= 1, got %d", c.ErrorInboxRetentionDays) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 51b23a6e..6c8568f9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -25,6 +25,9 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) { {"huge interval", func(c *Config) { c.FlushInterval = 10 * time.Minute }, "OBSERVE_FLUSH_INTERVAL_MS"}, {"zero rate limit", func(c *Config) { c.RateLimit = 0 }, "OBSERVE_RATE_LIMIT"}, {"zero LLM retention", func(c *Config) { c.LLMRetentionDays = 0 }, "OBSERVE_LLM_RETENTION_DAYS"}, + {"zero metrics retention", func(c *Config) { c.MetricsRetentionDays = 0 }, "OBSERVE_METRICS_RETENTION_DAYS"}, + {"zero infra retention", func(c *Config) { c.InfraRetentionDays = 0 }, "OBSERVE_INFRA_RETENTION_DAYS"}, + {"negative uptime retention", func(c *Config) { c.UptimeRetentionDays = -1 }, "OBSERVE_UPTIME_RETENTION_DAYS"}, {"zero retention", func(c *Config) { c.RawRetentionDays = 0 }, "OBSERVE_RAW_RETENTION_DAYS"}, {"malformed integer", func(c *Config) { c.parseErr = fmt.Errorf("OBSERVE_BUFFER_SIZE must be an integer, got %q", "big") @@ -34,6 +37,7 @@ func TestValidateRejectsBrokenNumericConfig(t *testing.T) { c := Config{ BufferSize: 1000, FlushSize: 100, FlushInterval: time.Second, RateLimit: 100, RawRetentionDays: 30, HourlyRetentionDays: 365, LLMRetentionDays: 30, + MetricsRetentionDays: 30, InfraRetentionDays: 30, UptimeRetentionDays: 90, ErrorInboxRetentionDays: 14, ReplayBatchesRetentionDays: 14, DerivedOutboxRetentionDays: 7, } tc.mut(&c) @@ -48,6 +52,7 @@ func TestValidateAcceptsDefaults(t *testing.T) { c := Config{ BufferSize: 100_000, FlushSize: 500, FlushInterval: 2 * time.Second, RateLimit: 1000, RawRetentionDays: 30, HourlyRetentionDays: 365, LLMRetentionDays: 30, + MetricsRetentionDays: 30, InfraRetentionDays: 30, UptimeRetentionDays: 90, ErrorInboxRetentionDays: 14, ReplayBatchesRetentionDays: 14, DerivedOutboxRetentionDays: 7, } if err := c.Validate(); err != nil { @@ -67,3 +72,20 @@ func TestLoadLLMRetentionWindow(t *testing.T) { t.Fatalf("invalid LLM duration must fail validation, got %v", err) } } + +func TestLoadTelemetryRetentionDefaultsAndOverrides(t *testing.T) { + c := Load() + if c.MetricsRetentionDays != 30 || c.InfraRetentionDays != 30 || c.UptimeRetentionDays != 90 { + t.Fatalf("defaults metrics/infra/uptime = %d/%d/%d, want 30/30/90", + c.MetricsRetentionDays, c.InfraRetentionDays, c.UptimeRetentionDays) + } + t.Setenv("OBSERVE_METRICS_RETENTION_DAYS", "3") + t.Setenv("OBSERVE_UPTIME_RETENTION_DAYS", "bogus") + c = Load() + if c.MetricsRetentionDays != 3 { + t.Fatalf("metrics retention=%d, want 3", c.MetricsRetentionDays) + } + if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "OBSERVE_UPTIME_RETENTION_DAYS") { + t.Fatalf("invalid uptime duration must fail validation, got %v", err) + } +} diff --git a/internal/jobs/retention.go b/internal/jobs/retention.go index 0c040e06..9c571429 100644 --- a/internal/jobs/retention.go +++ b/internal/jobs/retention.go @@ -19,11 +19,37 @@ import ( // internal/metrics/metrics.go) exists to avoid. const retentionChunkSize = 5000 +// TimeUnit is the unit of a policy's BIGINT time column. The zero value is +// epoch milliseconds, which is what nearly every table stores, so existing +// policy literals keep their meaning. +type TimeUnit int + +const ( + // UnitMillis is epoch milliseconds (the default). + UnitMillis TimeUnit = iota + // UnitNanos is epoch nanoseconds (metric_points.ts_ns, OTLP TimeUnixNano). + // A millisecond cutoff compared against such a column is ~1e6 times too + // small, so the TTL would silently match nothing. + UnitNanos +) + +// CutoffAt returns the cutoff for a retention window of days ending at now, +// expressed in the unit of the column it will be compared against. +func (u TimeUnit) CutoffAt(now time.Time, days int) int64 { + t := now.Add(-time.Duration(days) * 24 * time.Hour) + if u == UnitNanos { + return t.UnixNano() + } + return t.UnixMilli() +} + // RetentionPolicy defines the TTL (in days) for a table + the SQL column to compare against. type RetentionPolicy struct { Table string - Column string // BIGINT epoch-ms column, typically "timestamp", "start_time", or "ts_bucket" + Column string // BIGINT epoch column (see Unit), typically "timestamp", "start_time", or "ts_bucket" Days int + // Unit is the time unit of Column; the zero value is epoch milliseconds. + Unit TimeUnit // ExtraWhere is an optional additional predicate ANDed into the // chunk-boundary SELECT and the DELETE (e.g. "processed_at > 0" so the // derived_outbox policy prunes processed intents but never dead @@ -68,6 +94,34 @@ func DefaultPolicies(rawDays, hourlyDays, llmDays int) []RetentionPolicy { } } +// DefaultTelemetryPolicies returns retention for the telemetry tables that +// DefaultPolicies historically did not cover and that therefore grew without +// bound. Units and columns were verified against the writers: +// +// - metric_points.ts_ns is epoch NANOSECONDS (OTLP TimeUnixNano), hence +// UnitNanos. +// - host_metrics.timestamp and uptime_results.timestamp are epoch ms +// (time.Now().UnixMilli() in internal/infra and internal/monitoring). +// - performance_issues is a replacing_mergetree keyed on (tenant, site, +// fingerprint) versioned by last_seen (span-time ms). Deleting +// last_seen < cutoff removes superseded versions and issues not +// re-detected within the window; a live issue keeps writing a newer +// last_seen, so it survives. Fixed 90 days, well beyond the 14-day span +// retention its trace_id drill-down depends on. +// - service_dependencies is a replacing_mergetree whose ORDER BY includes +// ts_bucket (ms), so each time bucket is its own key and a bucket is only +// ever rewritten with the same key. Deleting old buckets is the same shape +// as the existing service_stats policy and is aligned to it (30 days). +func DefaultTelemetryPolicies(metricsDays, infraDays, uptimeDays int) []RetentionPolicy { + return []RetentionPolicy{ + {Table: "metric_points", Column: "ts_ns", Days: metricsDays, Unit: UnitNanos}, + {Table: "host_metrics", Column: "timestamp", Days: infraDays}, + {Table: "uptime_results", Column: "timestamp", Days: uptimeDays}, + {Table: "performance_issues", Column: "last_seen", Days: 90}, + {Table: "service_dependencies", Column: "ts_bucket", Days: 30}, + } +} + // DefaultLedgerPolicies returns the O01 ADR section 5.8 retention for the // durability ledgers. Each window is the decided 2026-09-23 default // (DELEGATED_DECISIONS section 5) and is env-tunable by the caller. @@ -138,7 +192,7 @@ func (r *RetentionService) RunCleanup(ctx context.Context) error { if p.Days <= 0 { continue } - cutoff := now.Add(-time.Duration(p.Days) * 24 * time.Hour).UnixMilli() + cutoff := p.Unit.CutoffAt(now, p.Days) // Bind the cutoff as an int64 and compare against the BIGINT column // directly. The old form (quoted text literal vs CAST(col AS BIGINT)) // matched nothing — Nucleus compared the column's numeric value against @@ -166,7 +220,7 @@ func (r *RetentionService) RunCleanup(ctx context.Context) error { // rollup facts keyed by a composite), so chunking can't key off "id IN // (SELECT id ... LIMIT N)" generically. Instead each iteration finds the // value of the chunkSize-th oldest surviving row in the policy's own cutoff -// column (already BIGINT epoch-ms, already used for the WHERE bound) and +// column (already BIGINT in the policy's Unit, already used for the WHERE bound) and // deletes everything up to and including it; once fewer than a full chunk // remain, one final small DELETE clears the rest. func (r *RetentionService) cleanupTable(ctx context.Context, sql *nucleus.SQLModel, p RetentionPolicy, cutoff int64) (int64, error) { diff --git a/internal/jobs/retention_telemetry_test.go b/internal/jobs/retention_telemetry_test.go new file mode 100644 index 00000000..3874d514 --- /dev/null +++ b/internal/jobs/retention_telemetry_test.go @@ -0,0 +1,107 @@ +package jobs + +import ( + "io" + "log/slog" + "strconv" + "testing" + "time" + + "github.com/neutron-build/neutron/go/nucleus" +) + +// TestDefaultTelemetryPolicies pins the tables that used to grow unbounded and +// the column/unit each is compared on. metric_points MUST be nanoseconds. +func TestDefaultTelemetryPolicies(t *testing.T) { + p := DefaultTelemetryPolicies(30, 30, 90) + want := map[string]RetentionPolicy{ + "metric_points": {Table: "metric_points", Column: "ts_ns", Days: 30, Unit: UnitNanos}, + "host_metrics": {Table: "host_metrics", Column: "timestamp", Days: 30}, + "uptime_results": {Table: "uptime_results", Column: "timestamp", Days: 90}, + "performance_issues": {Table: "performance_issues", Column: "last_seen", Days: 90}, + "service_dependencies": {Table: "service_dependencies", Column: "ts_bucket", Days: 30}, + } + if len(p) != len(want) { + t.Fatalf("want %d telemetry policies, got %d", len(want), len(p)) + } + for _, x := range p { + if w, ok := want[x.Table]; !ok || x != w { + t.Errorf("policy %s = %+v, want %+v", x.Table, x, w) + } + } + // Knobs flow through. + q := DefaultTelemetryPolicies(3, 4, 5) + if PolicyDays(q, "metric_points") != 3 || PolicyDays(q, "host_metrics") != 4 || PolicyDays(q, "uptime_results") != 5 { + t.Errorf("configured days not applied: %+v", q) + } +} + +// TestTimeUnitCutoffScaling: a ns column compared against a ms cutoff would +// delete nothing, so the cutoff must be scaled to the column's unit. +func TestTimeUnitCutoffScaling(t *testing.T) { + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + const dayMs = int64(24 * 60 * 60 * 1000) + + ms := UnitMillis.CutoffAt(now, 30) + if want := now.UnixMilli() - 30*dayMs; ms != want { + t.Errorf("ms cutoff = %d, want %d", ms, want) + } + ns := UnitNanos.CutoffAt(now, 30) + if want := (now.UnixMilli() - 30*dayMs) * 1_000_000; ns != want { + t.Errorf("ns cutoff = %d, want %d", ns, want) + } + if ns/1_000_000 != ms { + t.Errorf("ns and ms cutoffs disagree: %d vs %d", ns, ms) + } + // A 40-day-old point is below the ns cutoff and a 1-day-old one above it. + old := now.Add(-40 * 24 * time.Hour).UnixNano() + recent := now.Add(-24 * time.Hour).UnixNano() + if !(old < ns && recent > ns) { + t.Errorf("ns cutoff misclassifies rows: old=%d recent=%d cutoff=%d", old, recent, ns) + } + // The unscaled ms cutoff is smaller than every ns row, so it would match + // nothing: this is the bug the Unit field exists to prevent. + if old < ms { + t.Errorf("sanity: unscaled ms cutoff should (wrongly) match nothing for ns rows") + } + // Zero value stays milliseconds so legacy policy literals are unchanged. + if (RetentionPolicy{}).Unit != UnitMillis { + t.Errorf("zero Unit must be milliseconds") + } +} + +// UNVERIFIED against live Nucleus (skips without a database): metric_points +// uses a nanosecond column, so the ns-scaled cutoff must delete the old point +// and keep the recent one. +func TestRetentionMetricPointsNanosecondColumn(t *testing.T) { + ctx, db, done := connect(t) + defer done() + + site := "retmp_" + strconv.FormatInt(time.Now().UnixNano(), 36) + now := time.Now() + ins := func(name string, ts int64) { + if _, err := db.SQL().Exec(ctx, + `INSERT INTO metric_points (site_id, metric_name, metric_kind, ts_ns, value) VALUES ($1,$2,'gauge',$3,1)`, + site, name, ts); err != nil { + t.Fatalf("insert metric point: %v", err) + } + } + ins("old", now.Add(-40*24*time.Hour).UnixNano()) + ins("recent", now.Add(-24*time.Hour).UnixNano()) + + svc := NewRetentionServiceWithPolicies(db, slog.New(slog.NewTextHandler(io.Discard, nil)), + []RetentionPolicy{{Table: "metric_points", Column: "ts_ns", Days: 30, Unit: UnitNanos}}) + if err := svc.RunCleanup(ctx); err != nil { + t.Fatalf("RunCleanup: %v", err) + } + for name, want := range map[string]int64{"old": 0, "recent": 1} { + rows, err := nucleus.Query[countRow](ctx, db.SQL(), + `SELECT COUNT(*) AS n FROM metric_points WHERE site_id = $1 AND metric_name = $2`, site, name) + if err != nil { + t.Fatalf("count: %v", err) + } + if rows[0].N != want { + t.Errorf("%s metric point count = %d, want %d", name, rows[0].N, want) + } + } +} From 1dc45b81b007b33a2a88b51342ef55179d03a98d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:27:34 +0000 Subject: [PATCH 03/65] fix: cohort filter fails closed; "!=" property rule means no matching event; cap cohort size resolveFilters used to log and drop the cohort filter when the resolver failed, so charts labeled with a cohort showed all traffic. It now returns an error mapped to HTTP: 503 for resolver/store failure (raw error logged, not echoed), 422 for a cohort over MaxFilterMembers (30000; the filter expands to one SQL parameter per member, wire limit 65535), and the refusal's own status for queryguard refusals (new internal/guardmap). Property rules with operator "!=" matched any user with at least one non-matching event. They now mean "no event has this value", computed as (all identified users) minus (users matching "="), with two plain DISTINCT queries and a Go-side subtraction. SQL builders are unit tested. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- internal/cohorts/cohorts.go | 101 ++++++++++++++--- internal/cohorts/cohorts_sql_test.go | 69 ++++++++++++ internal/guardmap/guardmap.go | 47 ++++++++ internal/guardmap/guardmap_test.go | 44 ++++++++ internal/query/api.go | 137 +++++++++++++++++++---- internal/query/cohort_failclosed_test.go | 77 +++++++++++++ 6 files changed, 436 insertions(+), 39 deletions(-) create mode 100644 internal/cohorts/cohorts_sql_test.go create mode 100644 internal/guardmap/guardmap.go create mode 100644 internal/guardmap/guardmap_test.go create mode 100644 internal/query/cohort_failclosed_test.go diff --git a/internal/cohorts/cohorts.go b/internal/cohorts/cohorts.go index 34f61715..ba984226 100644 --- a/internal/cohorts/cohorts.go +++ b/internal/cohorts/cohorts.go @@ -27,6 +27,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "sort" "strconv" @@ -38,6 +39,19 @@ import ( "github.com/useteploy/teploy-observe/internal/dbutil" ) +// MaxFilterMembers is the largest cohort MembersForFilter will hand to the +// chart pipeline. The pipeline expands a cohort into one SQL parameter per +// member (`distinct_id IN ($4, $5, ...)`), and the wire protocol caps a +// statement at 65535 parameters, so a larger cohort would fail at the +// engine with an opaque error (or, worse, be truncated). Refusing up front +// with ErrTooLarge keeps that failure labeled. The ceiling leaves headroom +// for the fixed site/time and dimension-filter parameters. +const MaxFilterMembers = 30000 + +// ErrTooLarge is returned (wrapped) by MembersForFilter when a cohort has +// more than MaxFilterMembers members. +var ErrTooLarge = errors.New("cohort too large for chart filtering") + // Service exposes cohort CRUD and evaluation. type Service struct { db *nucleus.Client @@ -56,7 +70,8 @@ func NewService(db *nucleus.Client) *Service { // // Property rules apply on the events table (one row per event); the // distinct_id set returned is the set of users who emitted *any* event -// matching the property. That matches the PostHog "person property" +// matching the property ("=") or that have NO event with that value +// ("!="). That matches the PostHog "person property" // semantic in the common case where users carry session-level // attributes (country, browser, etc.) on each event. type Rule struct { @@ -246,23 +261,72 @@ func (s *Service) evalPropertyRule(ctx context.Context, siteID string, r Rule) ( type idRow struct { DistinctID string `db:"distinct_id"` } - // Direct column comparison — safe because isAllowedPropertyKey - // constrains the column name to a hard-coded allow-list, so the - // fmt.Sprintf can't be SQL-injected. - q := fmt.Sprintf(`SELECT DISTINCT distinct_id - FROM events - WHERE site_id = $1 - AND %s %s $2 - AND distinct_id != ''`, r.Key, op) - rows, err := nucleus.Query[idRow](ctx, s.db.SQL(), q, siteID, r.Value) + matched, err := nucleus.Query[idRow](ctx, s.db.SQL(), propertyMatchSQL(r.Key), siteID, r.Value) if err != nil { return nil, fmt.Errorf("property rule query: %w", err) } - out := make([]string, 0, len(rows)) - for _, row := range rows { - out = append(out, row.DistinctID) + matchedIDs := make([]string, 0, len(matched)) + for _, row := range matched { + matchedIDs = append(matchedIDs, row.DistinctID) } - return out, nil + if op == "=" { + return matchedIDs, nil + } + + // "!=" means NO event of the user has this value. The old query + // (`key != value` row filter) returned every user with at least one + // non-matching event, so a user with events in both US and DE matched + // both "country = US" and "country != US". Compute it as the set + // difference (all users) minus (users with a matching event). Two + // plain DISTINCT scans and a Go-side subtraction keep the SQL to forms + // the engine is known to handle (no NOT IN subquery). Events whose + // column is NULL/empty never match "=", so their users are kept. + all, err := nucleus.Query[idRow](ctx, s.db.SQL(), propertyUniverseSQL(), siteID) + if err != nil { + return nil, fmt.Errorf("property rule query: %w", err) + } + allIDs := make([]string, 0, len(all)) + for _, row := range all { + allIDs = append(allIDs, row.DistinctID) + } + return subtractIDs(allIDs, matchedIDs), nil +} + +// propertyMatchSQL is the query for users with at least one event whose +// column equals $2. The column name is interpolated, so the caller must +// have validated key with isAllowedPropertyKey (hard-coded allow-list); +// the value is always a bind parameter. +func propertyMatchSQL(key string) string { + return fmt.Sprintf(`SELECT DISTINCT distinct_id + FROM events + WHERE site_id = $1 + AND %s = $2 + AND distinct_id != ''`, key) +} + +// propertyUniverseSQL is the query for every identified user of the site; +// the "!=" operator subtracts the "=" matches from it. +func propertyUniverseSQL() string { + return `SELECT DISTINCT distinct_id + FROM events + WHERE site_id = $1 + AND distinct_id != ''` +} + +// subtractIDs returns the members of all that are not in remove, +// preserving all's order. +func subtractIDs(all, remove []string) []string { + drop := make(map[string]struct{}, len(remove)) + for _, id := range remove { + drop[id] = struct{}{} + } + out := make([]string, 0, len(all)) + for _, id := range all { + if _, ok := drop[id]; !ok { + out = append(out, id) + } + } + return out } // allowedPropertyKeys returns the columns a property rule may filter @@ -554,7 +618,14 @@ func (s *Service) MembersForFilter(ctx context.Context, siteID, cohortID string) if err != nil { return nil, err } - return s.EvaluateCohort(ctx, siteID, def) + ids, err := s.EvaluateCohort(ctx, siteID, def) + if err != nil { + return nil, err + } + if len(ids) > MaxFilterMembers { + return nil, fmt.Errorf("%w: %d members, limit %d", ErrTooLarge, len(ids), MaxFilterMembers) + } + return ids, nil } func genID() string { diff --git a/internal/cohorts/cohorts_sql_test.go b/internal/cohorts/cohorts_sql_test.go new file mode 100644 index 00000000..35055d72 --- /dev/null +++ b/internal/cohorts/cohorts_sql_test.go @@ -0,0 +1,69 @@ +package cohorts + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// These tests need no database: they pin the SQL the property rule builds +// and the Go-side set arithmetic used for the "!=" operator. + +func TestPropertyMatchSQL(t *testing.T) { + q := propertyMatchSQL("country") + for _, want := range []string{"site_id = $1", "country = $2", "distinct_id != ''", "SELECT DISTINCT distinct_id"} { + if !strings.Contains(q, want) { + t.Errorf("query missing %q:\n%s", want, q) + } + } + // The "!=" operator must never reach SQL as a row filter: that was the + // bug (any non-matching event qualified the user). + if strings.Contains(q, "!= $2") || strings.Contains(q, "<> $2") { + t.Errorf("match query must be equality only:\n%s", q) + } + // The value is always a bind parameter. + if strings.Contains(q, "'US'") { + t.Errorf("value interpolated into SQL:\n%s", q) + } +} + +func TestPropertyUniverseSQL(t *testing.T) { + q := propertyUniverseSQL() + if !strings.Contains(q, "site_id = $1") || !strings.Contains(q, "distinct_id != ''") { + t.Errorf("universe query must be site-scoped and exclude anonymous:\n%s", q) + } + if strings.Contains(q, "$2") { + t.Errorf("universe query takes only the site parameter:\n%s", q) + } +} + +func TestSubtractIDs(t *testing.T) { + // userBoth has a US and a DE event: "country = US" matches, so + // "country != US" must exclude them (old behaviour included them). + all := []string{"userBoth", "userDE", "userUS"} + matchedUS := []string{"userBoth", "userUS"} + got := subtractIDs(all, matchedUS) + if want := []string{"userDE"}; !reflect.DeepEqual(got, want) { + t.Errorf("subtractIDs = %v, want %v", got, want) + } + if got := subtractIDs(nil, matchedUS); len(got) != 0 { + t.Errorf("empty universe: %v", got) + } + if got := subtractIDs(all, nil); !reflect.DeepEqual(got, all) { + t.Errorf("nothing to remove: %v", got) + } +} + +func TestErrTooLargeWraps(t *testing.T) { + err := fmt.Errorf("%w: %d members, limit %d", ErrTooLarge, MaxFilterMembers+1, MaxFilterMembers) + if !errors.Is(err, ErrTooLarge) { + t.Fatal("ErrTooLarge must survive wrapping") + } + // Leave headroom under the 65535-parameter wire limit for the fixed + // and dimension-filter parameters. + if MaxFilterMembers > 60000 { + t.Fatalf("MaxFilterMembers %d leaves no parameter headroom", MaxFilterMembers) + } +} diff --git a/internal/guardmap/guardmap.go b/internal/guardmap/guardmap.go new file mode 100644 index 00000000..c7767c9a --- /dev/null +++ b/internal/guardmap/guardmap.go @@ -0,0 +1,47 @@ +// Package guardmap maps query-admission refusals (internal/queryguard) to +// HTTP problem responses so every read-path handler reports them the same +// way. A bare *queryguard.Refusal returned from a Neutron handler would be +// flattened to an unlabeled 500; mapping it keeps the 429 (load shedding, +// row budget) and 504 (time budget) statuses and the remedy text. +package guardmap + +import ( + "errors" + "net/http" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +// HTTPError returns err unchanged unless it wraps a *queryguard.Refusal, in +// which case it returns a *neutron.AppError carrying the refusal's status, +// code and remedy. A nil err stays nil. +func HTTPError(err error) error { + if err == nil { + return nil + } + var ref *queryguard.Refusal + if !errors.As(err, &ref) { + return err + } + status := ref.Status + if status == 0 { + status = http.StatusTooManyRequests + } + title := "Query Refused" + if status == http.StatusGatewayTimeout { + title = "Query Timed Out" + } + detail := ref.Message + if ref.Remedy != "" { + detail += " (remedy: " + ref.Remedy + ")" + } + return &neutron.AppError{ + Status: status, + Code: "https://neutron.dev/errors/" + ref.Code, + Title: title, + Detail: detail, + Meta: map[string]any{"refusal_code": ref.Code}, + } +} diff --git a/internal/guardmap/guardmap_test.go b/internal/guardmap/guardmap_test.go new file mode 100644 index 00000000..144e2270 --- /dev/null +++ b/internal/guardmap/guardmap_test.go @@ -0,0 +1,44 @@ +package guardmap + +import ( + "errors" + "fmt" + "net/http" + "testing" + "time" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +func TestHTTPError(t *testing.T) { + if HTTPError(nil) != nil { + t.Fatal("nil must stay nil") + } + plain := errors.New("boom") + if HTTPError(plain) != plain { + t.Fatal("non-refusal errors must pass through") + } + cases := []struct { + name string + err error + status int + }{ + {"rows", queryguard.RowBudgetRefusal(nil, 10), http.StatusTooManyRequests}, + {"time", queryguard.TimeBudgetRefusal(nil, time.Second), http.StatusGatewayTimeout}, + {"wrapped", fmt.Errorf("ctx: %w", queryguard.RowBudgetRefusal(nil, 10)), http.StatusTooManyRequests}, + } + for _, c := range cases { + var app *neutron.AppError + if !errors.As(HTTPError(c.err), &app) { + t.Fatalf("%s: not an AppError", c.name) + } + if app.Status != c.status { + t.Fatalf("%s: status %d want %d", c.name, app.Status, c.status) + } + if app.Meta["refusal_code"] == "" { + t.Fatalf("%s: missing refusal code", c.name) + } + } +} diff --git a/internal/query/api.go b/internal/query/api.go index 8315983e..94227a35 100644 --- a/internal/query/api.go +++ b/internal/query/api.go @@ -2,10 +2,14 @@ package query import ( "context" + "errors" "log/slog" "time" "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/cohorts" + "github.com/useteploy/teploy-observe/internal/guardmap" ) // StatsInput is the common query input for dashboard API endpoints. @@ -71,18 +75,21 @@ func (i StatsInput) Filters() *FilterBuilder { // A cohort with zero members short-circuits to a "1 = 0" clause via // FilterBuilder.AddIn, so the chart returns empty rather than the // (mistaken) full unfiltered result. -func (i StatsInput) resolveFilters(ctx context.Context, svc *StatsService) *FilterBuilder { +// +// Fail closed: when the resolver errors, the request fails with a mapped +// HTTP error instead of silently dropping the cohort filter. Falling back +// to "unfiltered" painted a chart of ALL traffic under a cohort label, +// which is a wrong answer, not a degraded one. See cohortResolveError for +// the status mapping. +func (i StatsInput) resolveFilters(ctx context.Context, svc *StatsService) (*FilterBuilder, error) { fb := i.Filters() if i.CohortID == "" || svc == nil { - return fb + return fb, nil } ids, err := svc.ResolveCohort(ctx, i.SiteID, i.CohortID) if err != nil { - // Resolver failure is non-fatal: log and fall back to no - // cohort filter so the chart degrades gracefully (matches the - // "channels query failed" pattern below). - slog.Warn("cohort resolve failed; falling back to unfiltered", "err", err, "site", i.SiteID, "cohort", i.CohortID) - return fb + slog.Error("cohort resolve failed", "err", err, "site", i.SiteID, "cohort", i.CohortID) + return nil, cohortResolveError(err) } if ids == nil { // Resolver wired but cohort_id wasn't found / returned nil. @@ -91,7 +98,26 @@ func (i StatsInput) resolveFilters(ctx context.Context, svc *StatsService) *Filt ids = []string{} } fb.AddIn("distinct_id", ids) - return fb + return fb, nil +} + +// cohortResolveError maps a cohort resolver failure to an HTTP error: +// - cohort larger than the filter can express (one SQL parameter per +// member): 422, the caller must narrow the cohort; +// - query-admission refusal (429 / 504): the refusal's own status; +// - anything else (store failure, unreadable stored rule): 503, a +// retryable server-side condition. The raw error is logged by the +// caller, never echoed to the client. +func cohortResolveError(err error) error { + if errors.Is(err, cohorts.ErrTooLarge) { + return neutron.ErrValidation( + "cohort has more members than a chart filter supports; narrow the cohort definition", + []neutron.ValidationError{{Field: "cohort_id", Message: "cohort too large for chart filtering"}}) + } + if mapped := guardmap.HTTPError(err); mapped != err { + return mapped + } + return neutron.ErrServiceUnavailable("cohort filter could not be resolved; retry later") } // UTMInput extends StatsInput with a UTM type selector. @@ -277,13 +303,20 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa // a shorter window than the one the user picked. neutron.Get(api, "/unique-coverage", func(ctx context.Context, input StatsInput) (UniqueCoverage, error) { from, to := input.TimeRange() - return svc.UniqueCoverageFor(ctx, input.SiteID, from, to, input.resolveFilters(ctx, svc)), nil + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return UniqueCoverage{}, err + } + return svc.UniqueCoverageFor(ctx, input.SiteID, from, to, filters), nil }, neutron.WithTags("stats"), neutron.WithSummary("Which table answers this range's unique counts, and how much of it they cover")) neutron.Get(api, "/overview", func(ctx context.Context, input StatsInput) (any, error) { from, to := input.TimeRange() - filters := input.resolveFilters(ctx, svc) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } if input.Compare != "" { result, err := svc.OverviewWithComparison(ctx, input.SiteID, from, to, input.Compare, filters) if err != nil { @@ -300,7 +333,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/timeseries", func(ctx context.Context, input StatsInput) ([]TimeSeriesPoint, error) { from, to := input.TimeRange() - result, err := svc.PageviewTimeSeries(ctx, input.SiteID, from, to, input.Interval, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.PageviewTimeSeries(ctx, input.SiteID, from, to, input.Interval, filters) if err != nil { slog.Error("timeseries query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -309,7 +346,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/pages", func(ctx context.Context, input StatsInput) ([]TopPage, error) { from, to := input.TimeRange() - result, err := svc.TopPages(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopPages(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("pages query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -318,7 +359,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/referrers", func(ctx context.Context, input StatsInput) ([]TopReferrer, error) { from, to := input.TimeRange() - result, err := svc.TopReferrers(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopReferrers(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("referrers query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -327,7 +372,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/browsers", func(ctx context.Context, input StatsInput) ([]BrowserStat, error) { from, to := input.TimeRange() - result, err := svc.TopBrowsers(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopBrowsers(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("browsers query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -336,7 +385,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/countries", func(ctx context.Context, input StatsInput) ([]CountryStat, error) { from, to := input.TimeRange() - result, err := svc.TopCountries(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopCountries(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("countries query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -345,7 +398,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/os", func(ctx context.Context, input StatsInput) ([]OSStat, error) { from, to := input.TimeRange() - result, err := svc.TopOS(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopOS(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("os query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -354,7 +411,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/devices", func(ctx context.Context, input StatsInput) ([]DeviceStat, error) { from, to := input.TimeRange() - result, err := svc.TopDevices(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopDevices(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("devices query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -363,7 +424,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/channels", func(ctx context.Context, input StatsInput) ([]ChannelStat, error) { from, to := input.TimeRange() - result, err := svc.TopChannels(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopChannels(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("channels query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -372,7 +437,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/languages", func(ctx context.Context, input StatsInput) ([]LanguageStat, error) { from, to := input.TimeRange() - result, err := svc.TopLanguages(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopLanguages(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("languages query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -381,7 +450,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/screens", func(ctx context.Context, input StatsInput) ([]ScreenStat, error) { from, to := input.TimeRange() - result, err := svc.TopScreens(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopScreens(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("screens query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -394,7 +467,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa if utmType == "" { utmType = "source" } - result, err := svc.TopUTM(ctx, input.SiteID, from, to, utmType, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopUTM(ctx, input.SiteID, from, to, utmType, input.Limit, filters) if err != nil { slog.Error("utm query failed", "err", err, "site", input.SiteID, "type", utmType, "from", from, "to", to) } @@ -403,7 +480,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/entry-pages", func(ctx context.Context, input StatsInput) ([]EntryPageStat, error) { from, to := input.TimeRange() - result, err := svc.TopEntryPages(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopEntryPages(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("entry-pages query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -412,7 +493,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/exit-pages", func(ctx context.Context, input StatsInput) ([]ExitPageStat, error) { from, to := input.TimeRange() - result, err := svc.TopExitPages(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.TopExitPages(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("exit-pages query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } @@ -421,7 +506,11 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa neutron.Get(api, "/events", func(ctx context.Context, input StatsInput) ([]CustomEventStat, error) { from, to := input.TimeRange() - result, err := svc.CustomEvents(ctx, input.SiteID, from, to, input.Limit, input.resolveFilters(ctx, svc)) + filters, err := input.resolveFilters(ctx, svc) + if err != nil { + return nil, err + } + result, err := svc.CustomEvents(ctx, input.SiteID, from, to, input.Limit, filters) if err != nil { slog.Error("custom events query failed", "err", err, "site", input.SiteID, "from", from, "to", to) } diff --git a/internal/query/cohort_failclosed_test.go b/internal/query/cohort_failclosed_test.go new file mode 100644 index 00000000..fa9d3afd --- /dev/null +++ b/internal/query/cohort_failclosed_test.go @@ -0,0 +1,77 @@ +package query + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/cohorts" +) + +func statsWithResolver(r CohortResolver) *StatsService { + return (&StatsService{}).WithCohortResolver(r) +} + +func appErr(t *testing.T, err error) *neutron.AppError { + t.Helper() + var app *neutron.AppError + if !errors.As(err, &app) { + t.Fatalf("not an AppError: %v", err) + } + return app +} + +// A failing resolver must fail the request, never drop the cohort filter. +func TestResolveFiltersFailsClosed(t *testing.T) { + svc := statsWithResolver(func(ctx context.Context, siteID, cohortID string) ([]string, error) { + return nil, errors.New("store down: secret-dsn") + }) + in := StatsInput{SiteID: "s", CohortID: "c1"} + fb, err := in.resolveFilters(context.Background(), svc) + if err == nil || fb != nil { + t.Fatalf("want error and nil builder, got fb=%v err=%v", fb, err) + } + app := appErr(t, err) + if app.Status != http.StatusServiceUnavailable { + t.Errorf("status %d, want 503", app.Status) + } + if strings.Contains(app.Detail, "secret-dsn") { + t.Errorf("raw resolver error leaked to client: %q", app.Detail) + } +} + +func TestResolveFiltersTooLargeIs422(t *testing.T) { + svc := statsWithResolver(func(ctx context.Context, siteID, cohortID string) ([]string, error) { + return nil, fmt.Errorf("%w: 99999 members", cohorts.ErrTooLarge) + }) + _, err := StatsInput{SiteID: "s", CohortID: "c1"}.resolveFilters(context.Background(), svc) + if got := appErr(t, err).Status; got != http.StatusUnprocessableEntity { + t.Errorf("status %d, want 422", got) + } +} + +func TestResolveFiltersOK(t *testing.T) { + svc := statsWithResolver(func(ctx context.Context, siteID, cohortID string) ([]string, error) { + return []string{"a", "b"}, nil + }) + fb, err := StatsInput{SiteID: "s", CohortID: "c1"}.resolveFilters(context.Background(), svc) + if err != nil || fb.SQL() == "" { + t.Fatalf("fb=%q err=%v", fb.SQL(), err) + } + // No cohort: no resolver call, no error. + fb, err = StatsInput{SiteID: "s"}.resolveFilters(context.Background(), svc) + if err != nil || fb.SQL() != "" { + t.Fatalf("no-cohort fb=%q err=%v", fb.SQL(), err) + } + // Missing cohort (nil ids) stays match-nothing, not unfiltered. + svc = statsWithResolver(func(ctx context.Context, siteID, cohortID string) ([]string, error) { return nil, nil }) + fb, _ = StatsInput{SiteID: "s", CohortID: "gone"}.resolveFilters(context.Background(), svc) + if fb.SQL() != " AND 1 = 0" { + t.Errorf("missing cohort SQL = %q", fb.SQL()) + } +} From 01fe902a7af82eaa5546fc560050dd5351608c54 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:27:44 +0000 Subject: [PATCH 04/65] fix: resolve source maps across URL/path/basename forms and count outcomes Upload now stores maps under a canonical name (no query, fragment, host or leading slash) and lookup tries the exact name, the stripped URL, the canonical path, shorter path suffixes and the basename in a fixed, bounded order (existing maps stay reachable via the exact-name candidate). Resolved frames keep the minified position in orig_filename/orig_line/ orig_col with resolved:true; lineno/colno still carry the original source position. Resolved frames are skipped on re-resolution (idempotent). Resolution outcomes (hit, miss_no_map, miss_no_mapping, error) are counted and surfaced at /healthz under "sourcemaps"; errors are logged at debug, rate limited, instead of being swallowed. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/main.go | 3 + internal/sourcemaps/resolve.go | 157 ++++++++++++++++ internal/sourcemaps/resolve_test.go | 270 ++++++++++++++++++++++++++++ internal/sourcemaps/sourcemaps.go | 80 ++++++--- 4 files changed, 488 insertions(+), 22 deletions(-) create mode 100644 internal/sourcemaps/resolve.go create mode 100644 internal/sourcemaps/resolve_test.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..e0c7d8e8 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -1705,6 +1705,9 @@ func main() { // conflicting-id / pending (plus the legacy queued/bytes backlog // fields and replayed-on-restart). health["errors"] = errorBuf.Stats() + // Source-map resolution outcomes per frame (hit / no map for the + // release+file / map without a covering mapping / read error). + health["sourcemaps"] = sourcemaps.Stats() // O08: flag-evaluation condition counters — how many evaluations // could not read their config (unavailable) and how many flags // were quarantined from evaluation by invalid stored config. diff --git a/internal/sourcemaps/resolve.go b/internal/sourcemaps/resolve.go new file mode 100644 index 00000000..c335c668 --- /dev/null +++ b/internal/sourcemaps/resolve.go @@ -0,0 +1,157 @@ +package sourcemaps + +import ( + "context" + "log/slog" + "net/url" + "strings" + "sync/atomic" + "time" +) + +// kvGetter is the one KV method resolution needs; *nucleus.KVModel satisfies +// it and tests substitute an in-memory store. +type kvGetter interface { + Get(ctx context.Context, key string) ([]byte, error) +} + +// Resolution outcome counters. Process-wide (the service is a singleton in +// practice and /healthz is process-wide), restart-reset like the other +// in-memory counters. +var ( + resolveHit atomic.Int64 + resolveNoMap atomic.Int64 + resolveNoMapping atomic.Int64 + resolveError atomic.Int64 + lastErrLogNanos atomic.Int64 +) + +// ResolveStats is the source-map resolution counter block surfaced at +// /healthz. Counts are per frame. A rising miss_no_map with maps uploaded +// means filename or release do not line up with what the SDK reports; a +// rising error means the store or a stored map is unreadable. +type ResolveStats struct { + Hit int64 `json:"hit"` + MissNoMap int64 `json:"miss_no_map"` + MissNoMapping int64 `json:"miss_no_mapping"` + Error int64 `json:"error"` +} + +// Stats returns a snapshot of the resolution counters. +func Stats() ResolveStats { + return ResolveStats{ + Hit: resolveHit.Load(), + MissNoMap: resolveNoMap.Load(), + MissNoMapping: resolveNoMapping.Load(), + Error: resolveError.Load(), + } +} + +// errLogInterval rate-limits the debug log for resolution errors so a +// failing store cannot turn every ingested frame into a log line. +const errLogInterval = 30 * time.Second + +func noteResolveError(site, release, file string, err error) { + resolveError.Add(1) + now := time.Now().UnixNano() + last := lastErrLogNanos.Load() + if now-last < int64(errLogInterval) || !lastErrLogNanos.CompareAndSwap(last, now) { + return + } + slog.Debug("sourcemaps: resolution failed (rate limited; see /healthz sourcemaps.error)", + "site", site, "release", release, "file", file, "err", err) +} + +const ( + maxFilenameLen = 2048 + maxLookupCands = 8 + maxSuffixSegments = 5 +) + +// canonicalName reduces a filename or URL to the form maps are stored under: +// no query, no fragment, no scheme/host, no leading "/", "./" or "~/". +// "https://cdn.x/static/app.js?v=3#a" and "/static/app.js" both become +// "static/app.js", so an upload keyed by path matches a frame carrying the +// full URL and vice versa. +func canonicalName(name string) string { + return trimLead(pathOf(stripQF(name))) +} + +func stripQF(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexAny(s, "#?"); i >= 0 { + s = s[:i] + } + return s +} + +// pathOf returns the path of an absolute URL, or s unchanged. +func pathOf(s string) string { + if strings.Contains(s, "://") { + if u, err := url.Parse(s); err == nil && u.Host != "" { + return u.Path + } + } + return s +} + +func trimLead(s string) string { + for { + switch { + case strings.HasPrefix(s, "~/"): + s = s[2:] + case strings.HasPrefix(s, "./"): + s = s[2:] + case strings.HasPrefix(s, "/"): + s = s[1:] + default: + return s + } + } +} + +// lookupCandidates lists, in deterministic most-specific-first order, the +// stored names a frame filename may have been uploaded under: the exact +// string (pre-normalisation behaviour, so existing maps still resolve), the +// query/fragment-stripped full URL, the canonical path, shorter path +// suffixes, and finally the basename. Bounded so an attacker-supplied +// filename cannot fan out into many KV reads. +func lookupCandidates(filename string) []string { + if filename == "" || len(filename) > maxFilenameLen { + return nil + } + var out []string + seen := map[string]struct{}{} + add := func(c string) { + if c == "" || len(out) >= maxLookupCands { + return + } + if _, ok := seen[c]; ok { + return + } + seen[c] = struct{}{} + out = append(out, c) + } + add(filename) + add(stripQF(filename)) + canon := canonicalName(filename) + add(canon) + segs := strings.Split(canon, "/") + for i := 1; i < len(segs) && i <= maxSuffixSegments; i++ { + add(strings.Join(segs[i:], "/")) + } + add(segs[len(segs)-1]) + return out +} + +// findSourceMap tries each candidate name for (site, release). Returns the +// first map found; a nil meta with nil error means no candidate has a map. +func findSourceMap(ctx context.Context, kv kvGetter, siteID, release, filename string) (*SourceMapMeta, error) { + for _, cand := range lookupCandidates(filename) { + meta, err := loadSourceMap(ctx, kv, siteID, release, cand) + if err != nil || meta != nil { + return meta, err + } + } + return nil, nil +} diff --git a/internal/sourcemaps/resolve_test.go b/internal/sourcemaps/resolve_test.go new file mode 100644 index 00000000..eac11b04 --- /dev/null +++ b/internal/sourcemaps/resolve_test.go @@ -0,0 +1,270 @@ +package sourcemaps + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +const vlqAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +// encodeVLQ is a reference base64-VLQ encoder so the fixture's mappings are +// produced by the real format rather than hand-typed. +func encodeVLQ(vals ...int) string { + var b strings.Builder + for _, v := range vals { + u := v << 1 + if v < 0 { + u = (-v << 1) | 1 + } + for { + digit := u & 0x1f + u >>= 5 + if u > 0 { + digit |= 0x20 + } + b.WriteByte(vlqAlphabet[digit]) + if u == 0 { + break + } + } + } + return b.String() +} + +// fixtureMap: generated line 1 col 1 -> src/app.ts 1:1; generated line 2, +// genCol 10 -> src/app.ts line 7 col 3, name handleClick. +func fixtureMap(t *testing.T) []byte { + t.Helper() + mappings := encodeVLQ(0, 0, 0, 0) + ";" + encodeVLQ(10, 0, 6, 2, 0) + raw, err := json.Marshal(SourceMapMeta{ + Version: 3, File: "app.min.js", + Sources: []string{"src/app.ts"}, Names: []string{"handleClick"}, + Mappings: mappings, + }) + if err != nil { + t.Fatal(err) + } + return raw +} + +type fakeKV struct { + data map[string][]byte + err error + gets int +} + +func (f *fakeKV) Get(_ context.Context, key string) ([]byte, error) { + f.gets++ + if f.err != nil { + return nil, f.err + } + return f.data[key], nil +} + +// store mimics Upload's keying. +func (f *fakeKV) store(site, release, filename string, m []byte) { + if f.data == nil { + f.data = map[string][]byte{} + } + f.data[kvKeyV2(site, release, canonicalName(filename))] = m +} + +func resetCounters() { + resolveHit.Store(0) + resolveNoMap.Store(0) + resolveNoMapping.Store(0) + resolveError.Store(0) + lastErrLogNanos.Store(0) +} + +func stackOf(filename string, line, col int) string { + b, _ := json.Marshal([]stackFrame{{Filename: filename, Function: "a", Lineno: line, Colno: col, InApp: true}}) + return string(b) +} + +func resolveOne(t *testing.T, kv kvGetter, release, stack string) stackFrame { + t.Helper() + out, err := resolveStack(context.Background(), kv, "s1", release, stack) + if err != nil { + t.Fatal(err) + } + var fr []stackFrame + if err := json.Unmarshal([]byte(out), &fr); err != nil || len(fr) != 1 { + t.Fatalf("bad output %q: %v", out, err) + } + return fr[0] +} + +func TestFixtureDecodes(t *testing.T) { + m := decodeMappings(encodeVLQ(0, 0, 0, 0)+";"+encodeVLQ(10, 0, 6, 2, 0), []string{"src/app.ts"}, []string{"handleClick"}, 2, 11) + if m == nil || m.OriginalLine != 7 || m.OriginalColumn != 3 || m.OriginalName != "handleClick" { + t.Fatalf("fixture mapping wrong: %+v", m) + } +} + +func TestResolve_FilenameForms(t *testing.T) { + cases := []struct { + name string + uploaded string + frame string + hit bool + }{ + {"exact", "app.min.js", "app.min.js", true}, + {"url frame, basename upload", "app.min.js", "https://cdn.example.com/static/js/app.min.js", true}, + {"url frame, path upload", "/static/js/app.min.js", "https://cdn.example.com/static/js/app.min.js", true}, + {"url upload, other host frame", "https://origin.example.com/static/js/app.min.js", "https://cdn.example.com/static/js/app.min.js", true}, + {"query on frame", "static/js/app.min.js", "https://cdn.example.com/static/js/app.min.js?v=abc123", true}, + {"fragment on frame", "app.min.js", "https://cdn.example.com/app.min.js#frag", true}, + {"query and fragment on upload name", "app.min.js?v=1#x", "https://cdn.example.com/app.min.js", true}, + {"tilde upload", "~/static/app.min.js", "https://h/static/app.min.js", true}, + {"different basename", "other.js", "https://cdn.example.com/app.min.js", false}, + {"different directory, full path upload", "other/app.min.js", "https://cdn.example.com/static/app.min.js", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + resetCounters() + kv := &fakeKV{} + kv.store("s1", "r1", c.uploaded, fixtureMap(t)) + fr := resolveOne(t, kv, "r1", stackOf(c.frame, 2, 11)) + if c.hit { + if !fr.Resolved || fr.Filename != "src/app.ts" || fr.Lineno != 7 || fr.Colno != 3 || fr.Function != "handleClick" { + t.Fatalf("expected resolution, got %+v", fr) + } + if Stats().Hit != 1 { + t.Fatalf("hit counter: %+v", Stats()) + } + } else { + if fr.Resolved || fr.Filename != c.frame { + t.Fatalf("expected raw frame, got %+v", fr) + } + if Stats().MissNoMap != 1 { + t.Fatalf("miss_no_map counter: %+v", Stats()) + } + } + }) + } +} + +func TestResolve_PreservesOriginalPosition(t *testing.T) { + resetCounters() + kv := &fakeKV{} + kv.store("s1", "r1", "app.min.js", fixtureMap(t)) + fr := resolveOne(t, kv, "r1", stackOf("https://x/app.min.js?v=1", 2, 11)) + if !fr.Resolved || fr.OrigFilename != "https://x/app.min.js?v=1" || fr.OrigLine != 2 || fr.OrigCol != 11 { + t.Fatalf("minified position lost: %+v", fr) + } + if fr.Lineno != 7 || fr.Colno != 3 { + t.Fatalf("original position not in lineno/colno: %+v", fr) + } +} + +func TestResolve_ExistingConsumersKeepShape(t *testing.T) { + kv := &fakeKV{} + kv.store("s1", "r1", "app.min.js", fixtureMap(t)) + out, _ := resolveStack(context.Background(), kv, "s1", "r1", stackOf("app.min.js", 2, 11)) + var generic []map[string]any + if err := json.Unmarshal([]byte(out), &generic); err != nil { + t.Fatal(err) + } + for _, k := range []string{"filename", "function", "lineno", "colno", "in_app"} { + if _, ok := generic[0][k]; !ok { + t.Fatalf("legacy key %q missing: %v", k, generic[0]) + } + } + // Unresolved frames do not grow the new keys. + out, _ = resolveStack(context.Background(), &fakeKV{}, "s1", "r1", stackOf("app.min.js", 2, 11)) + if strings.Contains(out, "orig_") || strings.Contains(out, "resolved") { + t.Fatalf("unresolved frame gained new keys: %s", out) + } +} + +func TestResolve_WrongRelease(t *testing.T) { + resetCounters() + kv := &fakeKV{} + kv.store("s1", "r1", "app.min.js", fixtureMap(t)) + fr := resolveOne(t, kv, "r2", stackOf("app.min.js", 2, 11)) + if fr.Resolved || fr.Lineno != 2 { + t.Fatalf("wrong release must not resolve: %+v", fr) + } + if s := Stats(); s.MissNoMap != 1 || s.Hit != 0 { + t.Fatalf("counters: %+v", s) + } +} + +func TestResolve_WrongSite(t *testing.T) { + kv := &fakeKV{} + kv.store("other-site", "r1", "app.min.js", fixtureMap(t)) + fr := resolveOne(t, kv, "r1", stackOf("app.min.js", 2, 11)) + if fr.Resolved { + t.Fatal("a map uploaded for another site must never resolve") + } +} + +func TestResolve_NoMappingAtPosition(t *testing.T) { + resetCounters() + kv := &fakeKV{} + kv.store("s1", "r1", "app.min.js", fixtureMap(t)) + fr := resolveOne(t, kv, "r1", stackOf("app.min.js", 99, 1)) + if fr.Resolved { + t.Fatalf("unexpected resolution: %+v", fr) + } + if s := Stats(); s.MissNoMapping != 1 || s.MissNoMap != 0 { + t.Fatalf("counters: %+v", s) + } +} + +func TestResolve_ErrorsAreCountedNotSwallowed(t *testing.T) { + resetCounters() + kv := &fakeKV{err: errors.New("store down")} + out, err := resolveStack(context.Background(), kv, "s1", "r1", stackOf("app.min.js", 2, 11)) + if err != nil || out != stackOf("app.min.js", 2, 11) { + t.Fatalf("raw stack must be kept on error: %q %v", out, err) + } + if Stats().Error != 1 { + t.Fatalf("error not counted: %+v", Stats()) + } + + resetCounters() + kv = &fakeKV{} + kv.store("s1", "r1", "app.min.js", []byte("{not json")) + resolveStack(context.Background(), kv, "s1", "r1", stackOf("app.min.js", 2, 11)) + if Stats().Error != 1 { + t.Fatalf("corrupt map not counted: %+v", Stats()) + } + + resetCounters() + resolveStack(context.Background(), &fakeKV{}, "s1", "r1", `{"not":"an array"}`) + if Stats().Error != 1 { + t.Fatalf("unparseable stack not counted: %+v", Stats()) + } +} + +func TestResolve_Idempotent(t *testing.T) { + kv := &fakeKV{} + kv.store("s1", "r1", "app.min.js", fixtureMap(t)) + once, _ := resolveStack(context.Background(), kv, "s1", "r1", stackOf("app.min.js", 2, 11)) + // A second pass (read path re-resolving a stored stack) must not treat the + // original source position as a minified one. + kv.store("s1", "r1", "src/app.ts", fixtureMap(t)) + twice, _ := resolveStack(context.Background(), kv, "s1", "r1", once) + if once != twice { + t.Fatalf("not idempotent:\n%s\n%s", once, twice) + } +} + +func TestLookupCandidates_BoundedAndDeterministic(t *testing.T) { + a := lookupCandidates("https://h/a/b/c/d/e/f/g/h/app.js?x=1#y") + b := lookupCandidates("https://h/a/b/c/d/e/f/g/h/app.js?x=1#y") + if len(a) == 0 || len(a) > maxLookupCands || strings.Join(a, "|") != strings.Join(b, "|") { + t.Fatalf("candidates: %v", a) + } + if a[0] != "https://h/a/b/c/d/e/f/g/h/app.js?x=1#y" { + t.Fatalf("exact name must come first: %v", a) + } + if lookupCandidates(strings.Repeat("a", maxFilenameLen+1)) != nil { + t.Fatal("oversized filename must not be looked up") + } +} diff --git a/internal/sourcemaps/sourcemaps.go b/internal/sourcemaps/sourcemaps.go index f342fe32..993539d3 100644 --- a/internal/sourcemaps/sourcemaps.go +++ b/internal/sourcemaps/sourcemaps.go @@ -65,6 +65,11 @@ const DefaultKeepReleases = 10 // legacy key so pre-upgrade maps stay resolvable until pruned. func (s *SourceMapService) Upload(ctx context.Context, siteID, release, filename string, mapData []byte) error { kv := s.db.KV() + // Store under the canonical name (no query/fragment/host/leading slash) + // so lookups can match a frame's full URL, path or basename to it. + if c := canonicalName(filename); c != "" { + filename = c + } key := kvKeyV2(siteID, release, filename) if err := kv.Set(ctx, key, mapData); err != nil { return err @@ -265,8 +270,7 @@ func releasesSetKey(siteID string) string { // unavailable store used to look identical to an unsymbolicated stack, so // outages produced silently wrong output instead of a diagnosable one. func (s *SourceMapService) ResolveFrame(ctx context.Context, siteID, release, filename string, line, col int) (*SourceMapping, error) { - kv := s.db.KV() - meta, err := loadSourceMap(ctx, kv, siteID, release, filename) + meta, err := findSourceMap(ctx, s.db.KV(), siteID, release, filename) if err != nil || meta == nil { return nil, err } @@ -281,7 +285,7 @@ const maxSourceMapBytes = 8 << 20 // loadSourceMap fetches and parses the map for one (site, release, file), // trying the v2 key first and falling back to the legacy raw key. -func loadSourceMap(ctx context.Context, kv *nucleus.KVModel, siteID, release, filename string) (*SourceMapMeta, error) { +func loadSourceMap(ctx context.Context, kv kvGetter, siteID, release, filename string) (*SourceMapMeta, error) { data, err := kv.Get(ctx, kvKeyV2(siteID, release, filename)) if err != nil { return nil, fmt.Errorf("read source map: %w", err) @@ -312,21 +316,39 @@ func loadSourceMap(ctx context.Context, kv *nucleus.KVModel, siteID, release, fi // AUD-048 (round 2): maps are loaded and parsed at most ONCE per distinct // file within one stack request (a 50-frame stack used to fetch and decode // the same map 50 times), with failures memoized for the request. +// +// A resolved frame keeps its minified position in orig_filename/orig_line/ +// orig_col and is marked resolved:true; filename/lineno/colno carry the +// original source position as before, so existing consumers are unchanged. +// Already-resolved frames are skipped, which makes the operation idempotent +// (the read path re-resolves stored stacks). Outcomes are counted per frame; +// failures leave the raw frame in place and are counted and (rate-limited) +// logged rather than dropped silently. func (s *SourceMapService) ResolveStackTrace(ctx context.Context, siteID, release, stackJSON string) (string, error) { + return resolveStack(ctx, s.db.KV(), siteID, release, stackJSON) +} + +type stackFrame struct { + Filename string `json:"filename"` + Function string `json:"function"` + Lineno int `json:"lineno"` + Colno int `json:"colno"` + InApp bool `json:"in_app"` + // Set only on frames resolved through a source map. + OrigFilename string `json:"orig_filename,omitempty"` + OrigLine int `json:"orig_line,omitempty"` + OrigCol int `json:"orig_col,omitempty"` + Resolved bool `json:"resolved,omitempty"` +} + +func resolveStack(ctx context.Context, kv kvGetter, siteID, release, stackJSON string) (string, error) { if stackJSON == "" || release == "" { return stackJSON, nil } - type frame struct { - Filename string `json:"filename"` - Function string `json:"function"` - Lineno int `json:"lineno"` - Colno int `json:"colno"` - InApp bool `json:"in_app"` - } - - var frames []frame + var frames []stackFrame if err := json.Unmarshal([]byte(stackJSON), &frames); err != nil { + noteResolveError(siteID, release, "", fmt.Errorf("stack is not a frame array: %w", err)) return stackJSON, nil } @@ -335,28 +357,39 @@ func (s *SourceMapService) ResolveStackTrace(ctx context.Context, siteID, releas err error } cache := make(map[string]mapResult) - kv := s.db.KV() + changed := false for i, f := range frames { + if f.Resolved || f.Filename == "" { + continue + } hit, cached := cache[f.Filename] if !cached { - meta, err := loadSourceMap(ctx, kv, siteID, release, f.Filename) - if err != nil { - // A store/parse failure is observable but must not discard - // the raw frame — record it and keep the stack readable. - hit = mapResult{err: err} - } else { - hit = mapResult{meta: meta} - } + meta, err := findSourceMap(ctx, kv, siteID, release, f.Filename) + hit = mapResult{meta: meta, err: err} cache[f.Filename] = hit } - if hit.err != nil || hit.meta == nil { + if hit.err != nil { + // A store/parse failure is observable but must not discard + // the raw frame — record it and keep the stack readable. + noteResolveError(siteID, release, f.Filename, hit.err) + continue + } + if hit.meta == nil { + resolveNoMap.Add(1) continue } mapping := decodeMappings(hit.meta.Mappings, hit.meta.Sources, hit.meta.Names, f.Lineno, f.Colno) if mapping == nil { + resolveNoMapping.Add(1) continue } + resolveHit.Add(1) + changed = true + frames[i].OrigFilename = f.Filename + frames[i].OrigLine = f.Lineno + frames[i].OrigCol = f.Colno + frames[i].Resolved = true frames[i].Filename = mapping.OriginalFile frames[i].Lineno = mapping.OriginalLine frames[i].Colno = mapping.OriginalColumn @@ -365,6 +398,9 @@ func (s *SourceMapService) ResolveStackTrace(ctx context.Context, siteID, releas } } + if !changed { + return stackJSON, nil + } result, _ := json.Marshal(frames) return string(result), nil } From 80774537e48f3c3606dc7ff392583d1a339ca31c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:28:52 +0000 Subject: [PATCH 05/65] feat: dedupe flag_evaluations writes per (site, flag, variant, user) window Every enabled /flags/evaluate wrote a row. Nothing reads flag_evaluations (experiment exposure and conversion counts come from experiment_exposures / experiment_conversions, written only by the explicit RecordExposure and RecordConversion calls), so the table is an audit/volume trail and one row per key per window is enough. Bounded in-memory LRU+TTL (hashed 32-byte keys, default 5 min / 50k entries; OBSERVE_FLAG_EVAL_DEDUP_SECONDS, 0 disables; OBSERVE_FLAG_EVAL_DEDUP_MAX). A variant change is always recorded; a failed write is forgotten and retried; written/deduped/failed/evicted counters surface under /healthz "flags". The INSERT itself is unchanged, moved behind a seam. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/main.go | 1 + internal/flags/evaldedup.go | 169 +++++++++++++++++++++++++++++++ internal/flags/evaldedup_test.go | 169 +++++++++++++++++++++++++++++++ internal/flags/flags.go | 75 +++++++++++--- 4 files changed, 401 insertions(+), 13 deletions(-) create mode 100644 internal/flags/evaldedup.go create mode 100644 internal/flags/evaldedup_test.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..e0ccec30 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -443,6 +443,7 @@ func main() { groupSvc := groups.NewGroupService(db) ssoSvc := sso.NewSSOService(db) flagSvc := flags.NewFlagService(db) + flagSvc.WithEvalDedup(flags.LoadEvalDedupFromEnv(os.Getenv)) experimentSvc := experiments.NewExperimentService(db) surveySvc := surveys.NewSurveyService(db, cfg.SessionSalt, siteSvc) logSvc := logs.NewLogService(db) diff --git a/internal/flags/evaldedup.go b/internal/flags/evaldedup.go new file mode 100644 index 00000000..52879bda --- /dev/null +++ b/internal/flags/evaldedup.go @@ -0,0 +1,169 @@ +package flags + +import ( + "container/list" + "crypto/sha256" + "strconv" + "strings" + "sync" + "time" +) + +// Evaluation-write dedupe. +// +// POST /flags/evaluate is a public, per-pageview SDK call. Every enabled +// decision used to append a flag_evaluations row, so one busy user (or a +// polling client) wrote a row per call. Nothing reads flag_evaluations: +// experiment exposure and conversion counts come from the separate +// experiment_exposures / experiment_conversions tables, which only the +// explicit RecordExposure / RecordConversion endpoints write. The table is +// an audit/volume trail, so it is safe to write at most one row per +// (site, flag, variant, user) per window. +// +// The window is measured from the last row actually written (a hit does not +// extend it), so a continuously-evaluating user still leaves one row per +// window rather than one forever. A different variant is a different key: a +// variant change is always recorded. The cache is a bounded LRU; evicting a +// key can only cause one extra row, never a lost one. A failed write +// forgets its key so the next evaluation retries. Dedup state is +// per-process, so N replicas can write up to N rows per window - acceptable +// for an unread trail. + +const ( + // DefaultEvalDedupWindow is the safe default: short enough that the + // trail still shows ongoing activity, long enough to collapse a + // polling client or a page that evaluates a flag on every render. + DefaultEvalDedupWindow = 5 * time.Minute + // DefaultEvalDedupMax bounds the dedupe cache (one 32-byte hash key plus + // list bookkeeping per entry, so roughly 10 MB at the default). + DefaultEvalDedupMax = 50000 + + envEvalDedupWindowSeconds = "OBSERVE_FLAG_EVAL_DEDUP_SECONDS" + envEvalDedupMax = "OBSERVE_FLAG_EVAL_DEDUP_MAX" +) + +// LoadEvalDedupFromEnv reads the dedupe knobs. OBSERVE_FLAG_EVAL_DEDUP_SECONDS +// is the window in seconds (0 disables dedupe: every evaluation writes a row, +// the pre-existing behaviour); OBSERVE_FLAG_EVAL_DEDUP_MAX bounds the cache. +// Unset or malformed values fall back to the defaults. +func LoadEvalDedupFromEnv(getenv func(string) string) (window time.Duration, max int) { + window, max = DefaultEvalDedupWindow, DefaultEvalDedupMax + if raw := strings.TrimSpace(getenv(envEvalDedupWindowSeconds)); raw != "" { + if n, err := strconv.ParseInt(raw, 10, 64); err == nil && n >= 0 && n <= 86400 { + window = time.Duration(n) * time.Second + } + } + if raw := strings.TrimSpace(getenv(envEvalDedupMax)); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n > 0 && n <= 5_000_000 { + max = n + } + } + return window, max +} + +// evalDedup is a bounded LRU with a fixed TTL. Safe for concurrent use. +type evalDedup struct { + mu sync.Mutex + window time.Duration + max int + order *list.List // front = most recently touched + entries map[[32]byte]*list.Element + now func() time.Time + + evicted int64 +} + +type evalDedupEntry struct { + key [32]byte + recordedAt time.Time +} + +func newEvalDedup(window time.Duration, max int) *evalDedup { + if max <= 0 { + max = DefaultEvalDedupMax + } + return &evalDedup{ + window: window, + max: max, + order: list.New(), + entries: make(map[[32]byte]*list.Element), + now: time.Now, + } +} + +// evalKey hashes the tuple so memory per entry is fixed regardless of how +// long an attacker-chosen user_id is. NUL separators prevent field-boundary +// ambiguity ("a","bc" vs "ab","c"). +func evalKey(siteID, flagKey, variant, userID string) [32]byte { + h := sha256.New() + for _, p := range [...]string{siteID, flagKey, variant, userID} { + h.Write([]byte(p)) + h.Write([]byte{0}) + } + var k [32]byte + copy(k[:], h.Sum(nil)) + return k +} + +// admit reports whether a row should be written for key now. True means the +// caller must write (and call forget on failure); false means a row for this +// key was already written within the window. +func (d *evalDedup) admit(key [32]byte) bool { + if d == nil || d.window <= 0 { + return true + } + d.mu.Lock() + defer d.mu.Unlock() + now := d.now() + if el, ok := d.entries[key]; ok { + e := el.Value.(*evalDedupEntry) + if now.Sub(e.recordedAt) < d.window { + d.order.MoveToFront(el) + return false + } + // Window elapsed: record a fresh row and restart the window. + e.recordedAt = now + d.order.MoveToFront(el) + return true + } + d.entries[key] = d.order.PushFront(&evalDedupEntry{key: key, recordedAt: now}) + for d.order.Len() > d.max { + back := d.order.Back() + d.order.Remove(back) + delete(d.entries, back.Value.(*evalDedupEntry).key) + d.evicted++ + } + return true +} + +// forget drops key so the next evaluation writes again (used when the write +// failed). +func (d *evalDedup) forget(key [32]byte) { + if d == nil || d.window <= 0 { + return + } + d.mu.Lock() + defer d.mu.Unlock() + if el, ok := d.entries[key]; ok { + d.order.Remove(el) + delete(d.entries, key) + } +} + +func (d *evalDedup) size() int { + if d == nil { + return 0 + } + d.mu.Lock() + defer d.mu.Unlock() + return d.order.Len() +} + +func (d *evalDedup) evictedTotal() int64 { + if d == nil { + return 0 + } + d.mu.Lock() + defer d.mu.Unlock() + return d.evicted +} diff --git a/internal/flags/evaldedup_test.go b/internal/flags/evaldedup_test.go new file mode 100644 index 00000000..79bb4f2a --- /dev/null +++ b/internal/flags/evaldedup_test.go @@ -0,0 +1,169 @@ +package flags + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +func enabledSvc(rows *[]string, mu *sync.Mutex) *FlagService { + svc := NewFlagService(nil) + f := FeatureFlag{FlagID: "id", SiteID: "s", FlagKey: "f", FlagType: "boolean", Enabled: true, RolloutPct: 100} + svc.fetchFlag = func(context.Context, string, string) ([]FeatureFlag, error) { return []FeatureFlag{f}, nil } + svc.persistEval = func(_ context.Context, siteID, flagKey, userID, variant string) error { + mu.Lock() + defer mu.Unlock() + *rows = append(*rows, strings.Join([]string{siteID, flagKey, userID, variant}, "|")) + return nil + } + return svc +} + +func TestEvalDedupCollapsesRepeatsWithinWindow(t *testing.T) { + var rows []string + var mu sync.Mutex + svc := enabledSvc(&rows, &mu) + clock := time.Unix(1000, 0) + svc.dedup.now = func() time.Time { return clock } + ctx := context.Background() + + for i := 0; i < 50; i++ { + res, err := svc.Evaluate(ctx, "s", "f", "u1", nil) + if err != nil || !res.Enabled { + t.Fatalf("evaluate: %v %+v", err, res) + } + } + if len(rows) != 1 { + t.Fatalf("50 repeats wrote %d rows, want 1", len(rows)) + } + // A different user is a different key. + _, _ = svc.Evaluate(ctx, "s", "f", "u2", nil) + if len(rows) != 2 { + t.Fatalf("second user wrote %d rows total, want 2", len(rows)) + } + // A hit does not extend the window: after it elapses, one more row. + clock = clock.Add(DefaultEvalDedupWindow + time.Second) + _, _ = svc.Evaluate(ctx, "s", "f", "u1", nil) + _, _ = svc.Evaluate(ctx, "s", "f", "u1", nil) + if len(rows) != 3 { + t.Fatalf("after window got %d rows, want 3", len(rows)) + } + st := svc.Stats() + if st["eval_rows_written_total"] != 3 || st["eval_rows_deduped_total"] != 49+1 { + t.Fatalf("stats = %v", st) + } +} + +func TestEvalDedupVariantChangeIsRecorded(t *testing.T) { + d := newEvalDedup(time.Minute, 10) + a := evalKey("s", "f", "control", "u") + b := evalKey("s", "f", "treatment", "u") + if !d.admit(a) || d.admit(a) { + t.Fatal("same variant must dedupe") + } + if !d.admit(b) { + t.Fatal("a variant change must be recorded") + } + // Field-boundary ambiguity must not collide. + if evalKey("s", "ab", "", "c") == evalKey("s", "a", "", "bc") { + t.Fatal("keys collide across field boundaries") + } +} + +func TestEvalDedupDisabledWritesEveryRow(t *testing.T) { + var rows []string + var mu sync.Mutex + svc := enabledSvc(&rows, &mu).WithEvalDedup(0, 10) + for i := 0; i < 5; i++ { + _, _ = svc.Evaluate(context.Background(), "s", "f", "u", nil) + } + if len(rows) != 5 { + t.Fatalf("window 0 wrote %d rows, want 5", len(rows)) + } +} + +func TestEvalDedupLRUBoundedAndCounted(t *testing.T) { + d := newEvalDedup(time.Hour, 100) + for i := 0; i < 1000; i++ { + d.admit(evalKey("s", "f", "", fmt.Sprintf("u%d", i))) + } + if d.size() != 100 { + t.Fatalf("size %d, want bound 100", d.size()) + } + if d.evictedTotal() != 900 { + t.Fatalf("evicted %d, want 900", d.evictedTotal()) + } + // Recently touched keys survive eviction (LRU, not FIFO). + hot := evalKey("s", "f", "", "hot") + d.admit(hot) + for i := 0; i < 90; i++ { + d.admit(evalKey("s", "f", "", fmt.Sprintf("x%d", i))) + d.admit(hot) + } + if d.admit(hot) { + t.Fatal("hot key was evicted despite constant use") + } +} + +func TestEvalDedupFailedWriteRetries(t *testing.T) { + svc := NewFlagService(nil) + calls := 0 + svc.persistEval = func(context.Context, string, string, string, string) error { + calls++ + if calls == 1 { + return errors.New("store down") + } + return nil + } + svc.recordEvaluation(context.Background(), "s", "f", "u", "") + svc.recordEvaluation(context.Background(), "s", "f", "u", "") + svc.recordEvaluation(context.Background(), "s", "f", "u", "") + if calls != 2 { + t.Fatalf("persist calls = %d, want 2 (failure retried once, then deduped)", calls) + } + if svc.Stats()["eval_rows_failed_total"] != 1 { + t.Fatalf("stats = %v", svc.Stats()) + } +} + +func TestEvalDedupConcurrent(t *testing.T) { + var rows []string + var mu sync.Mutex + svc := enabledSvc(&rows, &mu) + var wg sync.WaitGroup + for g := 0; g < 16; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + _, _ = svc.Evaluate(context.Background(), "s", "f", "shared", nil) + } + }() + } + wg.Wait() + if len(rows) != 1 { + t.Fatalf("concurrent identical evaluations wrote %d rows, want 1", len(rows)) + } +} + +func TestLoadEvalDedupFromEnv(t *testing.T) { + env := map[string]string{} + get := func(k string) string { return env[k] } + if w, m := LoadEvalDedupFromEnv(get); w != DefaultEvalDedupWindow || m != DefaultEvalDedupMax { + t.Fatalf("defaults = %v %d", w, m) + } + env["OBSERVE_FLAG_EVAL_DEDUP_SECONDS"] = "0" + env["OBSERVE_FLAG_EVAL_DEDUP_MAX"] = "123" + if w, m := LoadEvalDedupFromEnv(get); w != 0 || m != 123 { + t.Fatalf("explicit = %v %d", w, m) + } + env["OBSERVE_FLAG_EVAL_DEDUP_SECONDS"] = "-5" + env["OBSERVE_FLAG_EVAL_DEDUP_MAX"] = "junk" + if w, m := LoadEvalDedupFromEnv(get); w != DefaultEvalDedupWindow || m != DefaultEvalDedupMax { + t.Fatalf("malformed must fall back, got %v %d", w, m) + } +} diff --git a/internal/flags/flags.go b/internal/flags/flags.go index afba149a..f8edb07c 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -29,10 +29,29 @@ type FlagService struct { unavailableTotal atomic.Int64 invalidTotal atomic.Int64 quarantineSeen sync.Map // flagID "|" part -> struct{}{} — log-once gate + + // persistEval writes one flag_evaluations row. A seam so the dedupe + // path is testable without a store; nil-db services (hermetic unit + // tests) record nothing. + persistEval func(ctx context.Context, siteID, flagKey, userID, variant string) error + dedup *evalDedup + evalWritten atomic.Int64 + evalDeduped atomic.Int64 + evalFailed atomic.Int64 } func NewFlagService(db *nucleus.Client) *FlagService { - s := &FlagService{db: db} + s := &FlagService{db: db, dedup: newEvalDedup(DefaultEvalDedupWindow, DefaultEvalDedupMax)} + s.persistEval = func(ctx context.Context, siteID, flagKey, userID, variant string) error { + if s.db == nil { + return nil + } + _, err := s.db.SQL().Exec(ctx, + `INSERT INTO flag_evaluations (eval_id, tenant_id, site_id, flag_key, user_id, variant, timestamp) + VALUES ($1, 'default', $2, $3, $4, $5, $6)`, + genID(), siteID, flagKey, userID, variant, time.Now().UTC().UnixMilli()) + return err + } s.fetchFlag = func(ctx context.Context, siteID, flagKey string) ([]FeatureFlag, error) { return nucleus.Query[FeatureFlag](ctx, s.db.SQL(), `SELECT flag_id, tenant_id, site_id, flag_key, name, description, flag_type, enabled, rollout_pct, @@ -42,6 +61,14 @@ func NewFlagService(db *nucleus.Client) *FlagService { return s } +// WithEvalDedup sets the flag_evaluations write-dedupe window and cache +// bound (see evaldedup.go). A window <= 0 disables dedupe. Returns the +// receiver for fluent boot-time setup. +func (s *FlagService) WithEvalDedup(window time.Duration, max int) *FlagService { + s.dedup = newEvalDedup(window, max) + return s +} + // Stats exposes the O08 condition counters: how many evaluations could not // read their config (unavailable) and how many were quarantined from // evaluation by invalid stored config (total plus distinct flag|part @@ -53,6 +80,14 @@ func (s *FlagService) Stats() map[string]int64 { "config_unavailable_total": s.unavailableTotal.Load(), "invalid_config_total": s.invalidTotal.Load(), "invalid_config_distinct": distinct, + // flag_evaluations write accounting: rows written, evaluations + // whose row was suppressed by the dedupe window, failed writes, + // and the dedupe cache's current size / LRU evictions. + "eval_rows_written_total": s.evalWritten.Load(), + "eval_rows_deduped_total": s.evalDeduped.Load(), + "eval_rows_failed_total": s.evalFailed.Load(), + "eval_dedup_entries": int64(s.dedup.size()), + "eval_dedup_evicted": s.dedup.evictedTotal(), } } @@ -461,22 +496,36 @@ func (s *FlagService) Evaluate(ctx context.Context, siteID, flagKey, userID stri } } - // Fire-and-forget: evaluation tracking is best-effort, must not block - // response. Reached only for real decisions (unavailable and invalid - // return above) — exposure recording semantics unchanged (O08 scope). - // The nil-db guard keeps the seam-injected unit path hermetic; the - // recording is already best-effort. - if s.db != nil { - evalID := genID() - _, _ = s.db.SQL().Exec(ctx, - `INSERT INTO flag_evaluations (eval_id, tenant_id, site_id, flag_key, user_id, variant, timestamp) - VALUES ($1, 'default', $2, $3, $4, $5, $6)`, - evalID, siteID, flagKey, userID, result.Variant, time.Now().UTC().UnixMilli()) - } + // Evaluation tracking is best-effort and never changes the answer. + // Reached only for real decisions (unavailable and invalid return + // above). At most one row per (site, flag, variant, user) per dedupe + // window; nothing reads flag_evaluations for experiment exposure + // counts (those come from experiment_exposures), see evaldedup.go. + s.recordEvaluation(ctx, siteID, flagKey, userID, result.Variant) return result, nil } +// recordEvaluation applies the dedupe window and writes at most one +// flag_evaluations row. Failures are counted, forgotten (so the next +// evaluation retries) and never surfaced to the caller. +func (s *FlagService) recordEvaluation(ctx context.Context, siteID, flagKey, userID, variant string) { + if s.persistEval == nil { + return + } + key := evalKey(siteID, flagKey, variant, userID) + if !s.dedup.admit(key) { + s.evalDeduped.Add(1) + return + } + if err := s.persistEval(ctx, siteID, flagKey, userID, variant); err != nil { + s.dedup.forget(key) + s.evalFailed.Add(1) + return + } + s.evalWritten.Add(1) +} + // quarantine refuses evaluation of a flag whose stored config failed // validation: counts the condition, logs the first occurrence per // flag+part (the public evaluate endpoint would otherwise flood the log), From 8786518d7d62b1e7ab8083bd7b36d6cb00e25763 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:29:14 +0000 Subject: [PATCH 06/65] docs: correct migration guides and README claims, add operations runbooks and design notes Fix Sentry source-map curl field, shim/breadcrumb claims, state no Sentry protocol ingest; PostHog cohorts/flags/surveys truth; README integrations, SAML, alert metrics, surveys. Add install/upgrade/backup-restore runbooks and NOT-BUILT design notes (per-site RBAC, SAML, HA, PromQL). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- README.md | 23 ++- docs/design/ha-and-clustering.md | 71 ++++++++++ .../design/multi-tenancy-and-per-site-rbac.md | 70 ++++++++++ docs/design/promql-decision.md | 67 +++++++++ docs/design/saml.md | 68 +++++++++ docs/migrations/from-posthog.md | 68 +++++++-- docs/migrations/from-sentry.md | 84 +++++++---- docs/operations/backup-restore.md | 126 +++++++++++++++++ docs/operations/install.md | 132 ++++++++++++++++++ docs/operations/upgrade.md | 106 ++++++++++++++ 10 files changed, 776 insertions(+), 39 deletions(-) create mode 100644 docs/design/ha-and-clustering.md create mode 100644 docs/design/multi-tenancy-and-per-site-rbac.md create mode 100644 docs/design/promql-decision.md create mode 100644 docs/design/saml.md create mode 100644 docs/operations/backup-restore.md create mode 100644 docs/operations/install.md create mode 100644 docs/operations/upgrade.md diff --git a/README.md b/README.md index 08fdd37b..9ccb0f33 100644 --- a/README.md +++ b/README.md @@ -253,9 +253,13 @@ Declared limits, fixture-tested scale points, and where to read them live: a row. ### Product tools -- Feature flags (boolean + multivariate, rollout %, user targeting). +- Feature flags (boolean + multivariate, rollout %, user targeting), + evaluated over a public HTTP endpoint (`POST /api/v1/flags/evaluate`); + there is no flag client in the browser SDK. - A/B experiments (frequentist p-value + Bayesian probability-to-beat). -- Surveys, custom dashboards with panels. +- Surveys (create / activate / collect responses via the API and UI; there + is no on-page survey widget, so you render them yourself), custom + dashboards with panels, and dynamic AND-only cohorts. ### Platform - **RBAC** enforced — JWT carries a role claim (`admin` / `editor` / @@ -295,10 +299,17 @@ Declared limits, fixture-tested scale points, and where to read them live: - **Per-site rate limiting** — each site has its own token bucket. One noisy site can't starve a quiet one. Admin-editable via `PUT /api/v1/sites/{id}/ratelimit`. -- Alerting (threshold per metric, cooldown, silence). Alert-fire - auto-opens an incident marker. -- Integrations (Jira, GitHub, PagerDuty, Slack, email) + webhooks. -- SSO / SAML, email digests, data export (CSV/JSON). +- Alerting (threshold rules on `pageviews`, `visitors`, `error_count` and + `error_rate`, with cooldown and silence). Alert-fire auto-opens an + incident marker and delivers to webhooks. +- Integrations (Jira, GitHub, PagerDuty, Slack, email) can be configured, + tested and replayed from `/api/v1/integrations`, but alerts and issue + events do not trigger them today: the delivery path + (`IntegrationService.Fire`) has no production caller. Only webhooks + receive alert notifications. +- SSO via OIDC (see below). SAML is not available: the SAML callback is + disabled and returns 501 (`internal/sso/sso.go`). Also email digests and + data export (CSV/JSON). - SQL query explorer with lexer-guarded read-only enforcement (rejects `/* comment */ INSERT ...` and stacked statements). - `POST /api/v1/query/explain` returns the Nucleus plan. diff --git a/docs/design/ha-and-clustering.md b/docs/design/ha-and-clustering.md new file mode 100644 index 00000000..a841cab1 --- /dev/null +++ b/docs/design/ha-and-clustering.md @@ -0,0 +1,71 @@ +# High availability and clustering (NOT BUILT) + +Status: design note. Observe runs as one instance on one Nucleus. There is no +failover, no multi-node storage and no supported way to run two Observe +processes against the same database. + +## Current state + +- README, "Operational limits": "There is no clustering, no multi-node + storage; scale is a single instance plus the budgets above." +- Ingest durability is a local-disk WAL under `$OBSERVE_QUEUE_DIR` + ([O01 durable ingest ADR](../O01_DURABLE_INGEST_ADR.md)). A 200 means fsynced + on that machine. The ADR's outbox section states the single-process claim: + "Single-process claim (AUD-018 posture) - multi-replica needs a lease/CAS + design before externally-visible kinds ride it." Its dedupe horizons (24 h + flush filter, 10 min admission cache) are in-process state. +- Other in-process state that assumes one instance: the metrics series + registry (bounded, in-memory), rate-limiter buckets, background jobs + (rollups, retention, alerts, cron/uptime monitors, scheduled exports), and + the upgrade tool (stop, swap, start on one host). +- Nucleus: `docker-compose.yml` starts it with `--cluster-port 5433` and + `NUCLEUS_ALLOW_INSECURE_CLUSTER` / `NUCLEUS_ALLOW_INSECURE_REPLICATION` set + to `1`, but Observe does not use or test replication. Nothing in this repo + configures a replica or failover, or reads from one. +- Recovery today is restore from backup + ([backup-restore.md](../operations/backup-restore.md)) plus WAL replay on + restart. For database loss the recovery point is the last backup (the WAL + covers only the unapplied tail); recovery time is operator time. + +## Requirements if built + +1. Database HA depends entirely on Nucleus replication and failover; Observe + cannot provide it above pgwire. Upstream capabilities, consistency + guarantees (async vs sync), promotion procedure and client reconnect + behaviour must be documented by Nucleus first. This repo cannot verify them + (no live Nucleus here). +2. Multiple Observe replicas need a leader lease/CAS for every singleton job + (rollups, retention, alert evaluation, monitors, scheduled exports, outbox + drain, audit checkpoints), a story for each node's WAL (acknowledged events + are replayable only by the node that holds them), and externalized dedupe + state. `internal/backup/lease.go` is a Nucleus snapshot lease, not a leader + election. +3. Shared rate limiting and WAL budgets, or an explicit per-node contract. +4. A load-balancer story for the ingest and dashboard listeners + (`OBSERVE_INGEST_ADDR` already separates them). +5. Rolling upgrades: migrations are forward-only and run at boot, so old and + new binaries running together need compatibility rules that do not exist + (see [upgrade.md](../operations/upgrade.md), AUDIT_OPEN L9). + +## Risks + +- A naive "run two replicas" setup double-runs singleton jobs, double-fires + alerts, and loses acknowledged events if a node's disk dies. +- Cannot be validated here; every claim needs a live multi-node Nucleus. +- Each in-memory structure above is a separate change. + +## Recommended approach + +Stay single-node and make recovery boring: tested backups, a documented +restore drill, and a warm standby that is a restored copy rather than a live +replica. Revisit HA once Nucleus ships and documents replication with failover +semantics; do Nucleus HA first (Observe is a client and may only need +reconnect handling) and leave multi-replica Observe for last. + +## Open owner decisions + +- What recovery point and time objectives are promised to users? That decides + whether backup-and-restore is enough. +- Is Nucleus replication on the upstream roadmap, and who owns documenting it + (an upstream report, not a fix from this repo)? +- Is multi-replica Observe wanted at all, or only database HA? diff --git a/docs/design/multi-tenancy-and-per-site-rbac.md b/docs/design/multi-tenancy-and-per-site-rbac.md new file mode 100644 index 00000000..2e6a802d --- /dev/null +++ b/docs/design/multi-tenancy-and-per-site-rbac.md @@ -0,0 +1,70 @@ +# Multi-tenancy and per-site RBAC (NOT BUILT) + +Status: design note. Nothing here is implemented. Observe is single-tenant per +deployment, and roles are global to the instance. + +## Current state + +- **One tenant.** Rows carry a `tenant_id` column (for example `sites`, + `logs`, the rollup tables; migration 001 calls it a "placeholder for future + multi-tenant"), but the only value ever written is `"default"` + (`internal/sites/sites.go` `EnsureDefault` inserts `tenant_id = "default"`). + No code resolves a tenant from a request, and nothing enforces isolation by + tenant. `docs/migrations/from-sentry.md` states "each Observe deployment is + one tenant". +- **Roles are instance-wide.** The JWT carries one `role` claim + (`admin` / `editor` / `viewer`, `internal/auth/auth.go`, + `internal/auth/middleware.go`); local users store a single `role` + (migration 010 on `admin_users`, unified principals in 040); OIDC maps IdP + claims or groups to that single role (`internal/auth/oidc.go`). No table + links a user to a set of sites (searched the migrations and Go code for a + user-to-site or site-ACL structure; none exists). Any authenticated user can + name any `site_id` in a request, subject only to the role gate. +- **API keys are site-scoped** (`internal/auth/apikeys.go`): a key writes only + to its own site. That is the one per-site boundary that exists, and it + covers ingest and publish-scope uploads only. + +## Requirements if built + +1. A membership model: user (or IdP group) -> site -> role, with a rule for + the default (instance admin sees all sites). +2. Every site-addressed read and write path (dashboard API, SQL explorer, MCP + tools, exports, share links, replays, source maps) checks membership, not + only role. This is the large part: `cmd/observe/main.go` registers on the + order of 180 routes, most taking `site_id` from the query or body. +3. The SQL explorer (`POST /api/v1/query`) lets an editor run arbitrary + read-only SQL; a role check alone cannot scope it per site. It needs + rewriting through site-filtered views, or to stay admin-only. +4. Real multi-tenancy additionally needs `tenant_id` enforced in every query + and rollup, separate salts and secrets per tenant, per-tenant quotas + (`OBSERVE_RATE_LIMIT` and the WAL budget are global today), and + tenant-aware backup/restore (`internal/backup` dumps all tables as one + unit). +5. OIDC role mapping extended to per-site group mappings. + +## Risks + +- **IDOR across sites** is the main hazard; the project already treats it as a + mandatory adversarial test class. A missed handler is a data leak. +- Retrofitting site filtering onto Nucleus queries cannot be verified by CI + here (integration tests need a live Nucleus and are skipped). +- ALTER-ADD migrations are banned on populated tables (AUDIT_OPEN L9), so a + membership table must be a new `CREATE TABLE IF NOT EXISTS`, and adding a + column to existing tables means rename-aside. + +## Recommended approach + +Do per-site RBAC (not multi-tenancy) first, as deny-by-default middleware: a +`user_site_roles` table, a `RequireSiteAccess(role)` wrapper applied centrally +where handlers resolve `site_id`, and a test that enumerates the route table +and fails on any `site_id`-taking route without the wrapper. Defer true +tenants; if customer isolation is needed, run one Observe per tenant (the model +that works today). + +## Open owner decisions + +- Is the target "teams inside one company" (per-site RBAC) or "separate + customers" (tenancy)? They are different projects. +- Should the SQL explorer remain admin-only once site-restricted users exist? +- Do site-restricted users get share links and exports? +- Where do memberships come from: the local admin UI only, or OIDC groups too? diff --git a/docs/design/promql-decision.md b/docs/design/promql-decision.md new file mode 100644 index 00000000..609019b1 --- /dev/null +++ b/docs/design/promql-decision.md @@ -0,0 +1,67 @@ +# PromQL support (NOT BUILT; decision pending) + +Status: Observe does not implement PromQL. The README calls PromQL "an +explicit future decision, not an implied compatibility". This note frames that +decision; it does not make it. + +## Current state + +- Metrics arrive via OTLP (`POST /v1/metrics`, protobuf or JSON) and are stored + per series. They are queried through a structured API, + `GET /api/v1/metrics/query` and `GET /api/v1/metrics/series` + (`cmd/observe/metrics_handlers.go`): metric name, exact-match label filters, + group-by, step and the reducers `last|avg|sum|min|max|rate|p50|p95|p99`. +- Reducer semantics are pinned by `internal/metrics/o07_reference_test.go`: + per-series `rate` before aggregation, counter-reset handling by the + restart-at-zero convention, cumulative-histogram differencing, bucket + interpolation, and an `estimate`/`method` label on every point. +- There is no PromQL parser, no Prometheus-compatible `query_range` endpoint, + and no remote-write or scrape ingest. The only PromQL mentions in the code + are the "deliberately does not implement" comment in + `internal/metrics/metrics.go` and README text. +- Consequences: Grafana's Prometheus data source, PromQL alert rules and + existing PromQL dashboards do not work against Observe. Observe's own alert + rules cover only `pageviews`, `visitors`, `error_count` and `error_rate`; + there are no metric-series alerts. + +## Requirements if built + +1. A scope decision: (a) a read-only PromQL query API for Grafana, (b) plus + Prometheus remote-write ingest, (c) plus PromQL alerting. +2. Semantic parity for the advertised subset. PromQL has staleness handling, + lookback delta, `rate`/`increase` extrapolation and `histogram_quantile` + rules that differ from the pinned O07 semantics (Prometheus extrapolates at + window edges; Observe time-weights buckets). The two would disagree on the + same data unless one is declared authoritative. +3. Query admission: PromQL range queries easily exceed the row and wall-time + budgets in `docs/operations/capacity.md`; they must run under the same + admission control. +4. The existing cardinality limits (20k series per site) bound what is + queryable. + +## Risks + +- A hand-rolled subset invites silent wrong answers. The README already + commits that "if it ever lands it will wrap the upstream Prometheus engine, + never a hand-rolled subset". +- Embedding the Prometheus engine pulls a large dependency tree. New + dependencies must already be vendored (no network here), and binary size and + the idle footprint in BENCHMARKS.md would change. +- The engine needs a storage interface implemented over Observe's series + store, backed by Nucleus queries that CI here cannot exercise. + +## Recommended approach + +Defer until there is concrete demand. If Grafana compatibility is needed, +build only (a): a read-only adapter implementing the Prometheus storage +interface over the existing series reader, wrapping the upstream engine, under +the same admission control, with a published compatibility table in the style +of `docs/sdk/COMPATIBILITY.md` that states the differences from O07. Leave +PromQL alerting and remote-write out of the first pass. + +## Open owner decisions + +- Is Grafana compatibility a goal, and for which panels? +- Accept the dependency and size cost of the upstream Prometheus engine? +- Which semantics are authoritative when the engine and O07 disagree? +- Should metric-threshold alerts exist at all before any PromQL work? diff --git a/docs/design/saml.md b/docs/design/saml.md new file mode 100644 index 00000000..aa274d63 --- /dev/null +++ b/docs/design/saml.md @@ -0,0 +1,68 @@ +# SAML SSO (NOT BUILT; hard-disabled) + +Status: SAML login does not work and is deliberately disabled. OIDC is the +supported SSO path (README, "Single sign-on (OIDC)"). + +## Current state + +- `internal/sso/sso.go` `SAMLCallbackHandler` always answers + `501 Not Implemented` ("SAML SSO login is not enabled on this instance"). + It is registered at `POST /api/v1/sso/callback` (`cmd/observe/main.go`). +- The earlier implementation took the email out of an unsigned SAML assertion + and could have been used to assert any identity once JWT minting was wired + in. It was disabled rather than patched: no XML signature verification, no + audience/conditions checking, no replay protection, no defence against XML + signature wrapping (XSW). The unsigned assertion is no longer parsed at all. +- Still present: the `sso_configs` table (migration 008), admin CRUD at + `/api/v1/sso/configs` (admin role), and `GET /api/v1/sso/metadata`, which + renders SP metadata (`GetSAMLMetadata`). An admin can store a SAML + configuration that never takes effect, and nothing says so. +- README does not claim SAML; it describes OIDC only. + +## What a safe implementation needs + +1. A maintained SAML library doing the cryptography; the code comment names + `crewjam/saml`. It is not in `vendor/` (checked `vendor/modules.txt`), and + the repo rule is that new dependencies must already be vendored (no + network), so adding one is a dependency decision in itself. +2. Verify the XML signature (XML-DSig) against the certificate stored in + `sso_configs`, over the element that is then actually consumed (XSW + defence), with canonicalization handled by the library. +3. Enforce `Audience`, `Recipient`/destination, `NotBefore`/`NotOnOrAfter` + with bounded clock skew, and `InResponseTo` bound to a server-side + `AuthnRequest` id (SP-initiated flow), so responses cannot be replayed or + injected. +4. Replay cache for assertion ids over their validity window; in-memory is + acceptable only while Observe is single-instance. +5. Reject unsigned responses and unsigned assertions; no DTD/entity + processing in the XML parser. +6. Map attributes to a role through `AttributeMap`, defaulting to `viewer`, + mirroring OIDC's resolution order; never trust a role attribute outside the + signed element. +7. Mint the normal Observe JWT only after all of the above; audit-log + `auth.sso_login` as OIDC does. `OBSERVE_PUBLIC_URL` must be set so entity + and ACS URLs are not derived from a spoofable Host header. +8. Adversarial tests: XSW variants, signature stripping, expired and + not-yet-valid assertions, wrong audience, replay, comment injection in + NameID, oversized body. + +## Risks + +- This is authentication code; a mistake is account takeover. +- IdP quirks (signed assertion vs signed response, encrypted assertions) need + real IdP fixtures to test, and none are in the repo. + +## Recommended approach + +Do not build SAML until a customer needs it. Point such customers at OIDC: +most SAML IdPs (Okta, Azure AD, Keycloak, Google Workspace) also speak OIDC. +If it is built, use the library route above with a standalone design review. +Meanwhile consider making the admin API refuse to store SAML configs (or mark +them inactive) so the configuration surface stops implying support. + +## Open owner decisions + +- Is there a concrete customer requirement, or is OIDC sufficient? +- Approve the new vendored dependency (`crewjam/saml` or an alternative)? +- Remove the dead `sso_configs` CRUD and metadata endpoint, or keep them for a + future implementation? diff --git a/docs/migrations/from-posthog.md b/docs/migrations/from-posthog.md index 9daa421c..699fea86 100644 --- a/docs/migrations/from-posthog.md +++ b/docs/migrations/from-posthog.md @@ -6,7 +6,8 @@ swap. > **TL;DR** — Observe's event model and flag evaluation are simpler than > PostHog's (fewer concepts, no Kafka/Clickhouse) but cover the 80% path. -> You lose cohort-as-query and retention-by-SQL; you gain a single binary. +> Cohorts exist but are simpler than PostHog's (AND-only rules, see +> [Cohorts](#cohorts)); you lose retention-by-SQL and gain a single binary. ## Concept mapping @@ -23,8 +24,8 @@ swap. | Feature flag | Feature flag (`/flags`) | | Experiment | Experiment (`/experiments`) | | Survey | Survey (`/surveys`) | -| Cohort | (no direct equivalent — use funnel breakdown) | -| Plugin / app | Integration (`/integrations`) | +| Cohort | Cohort (`/api/v1/cohorts`; dynamic, AND-only, see [Cohorts](#cohorts)) | +| Plugin / app | Webhooks (alert delivery); `/integrations` are test/replay-only today | ## JavaScript / TypeScript @@ -44,15 +45,23 @@ init({ endpoint: "https://observe.example.com", siteId: "default" }); track("signup", { plan: "pro" }); identify(user.id, { email: user.email }); -// Flag evaluation is a server call: -const r = await fetch("/api/v1/flags/evaluate", { +// Flag evaluation is a plain HTTP call. @teploy/observe-browser has no +// flag client, so you write this fetch yourself. The endpoint is public +// (no API key or JWT) and rate limited per IP and per site/IP: +const r = await fetch("https://observe.example.com/api/v1/flags/evaluate", { method: "POST", - headers: { "Content-Type": "application/json", "X-API-Key": OBSERVE_API_KEY }, + headers: { "Content-Type": "application/json" }, body: JSON.stringify({ site_id: "default", flag_key: "new-checkout", user_id: user.id }), }); const { enabled, variant } = await r.json(); ``` +The handler (`flagEvaluateHandler` in `cmd/observe/main.go`) reads `site_id`, +`flag_key`, `user_id` and an optional string map `context` used for targeting, +and returns `{enabled, variant?}`. There is no bootstrap/`/decide`-style call +that returns all flags at once, so the `isFeatureEnabled` shim below only works +for flags you have already evaluated by key. + ### Drop-in shim ```ts @@ -143,10 +152,50 @@ done Reserve `/api/v1/events/batch` (up to 100 events per call) for larger imports. +## Cohorts + +Observe has cohorts (`internal/cohorts`, routes under `/api/v1/cohorts`: +list, create, preview, get, members). What they are, and are not: + +- **Dynamic.** A cohort is a saved rule definition, re-evaluated on every + request that uses it. There is no stored member table or refresh job + (the package comment calls that a phase-2 item). +- **AND-only.** `Definition.Op` must be `"and"`; any other value, including + `"or"`, is rejected, and rules cannot be nested. +- **Two rule types.** `event` (an event name seen at least `min_count` times + within a window such as `7d`, `30d`, `24h`) and `property` (`=` or `!=` + against an allow-listed event column (`isAllowedPropertyKey`), matching users who + emitted *any* event with that value). +- **Identified users only.** Anonymous events (empty `distinct_id`) are never + members. +- **Used as an analytics filter.** Pass `cohort_id` to the stats routes to + filter a chart to the cohort. If the cohort cannot be resolved, the query + logs a warning and falls back to **unfiltered** results + (`internal/query/api.go`), so check a filtered chart really is filtered. +- **Not supported:** behavioral cohorts built from funnels or paths, static + (uploaded) cohorts, cohort-to-cohort references, and cohorts as a flag + targeting source. + +## Flags, experiments, surveys: what is and is not client-side + +- **Feature flags**: management UI and the public evaluate endpoint exist; + there is no flag client in `sdk/browser` and no flag bootstrap payload. +- **Experiments**: assignment rides on a flag. Exposures and conversions are + posted to `POST /api/v1/experiments/expose` (API key); analysis shows a + frequentist p-value and a Bayesian probability-to-beat. +- **Surveys**: you can create, activate, close and read responses via the + API/UI, and the public endpoints `GET /api/v1/surveys/active`, + `POST /api/v1/surveys/expose` and `POST /api/v1/surveys/respond` exist. No + on-page survey widget ships: none of the tracker scripts nor `sdk/browser` + renders a survey. You must fetch active surveys and render and submit them + yourself. (`observe-feedback.js` is a separate feedback widget that posts to + `/api/v1/feedback`; it is not the survey runner.) + ## What doesn't port cleanly -- **Cohort-as-query** — Observe doesn't have a separate cohort concept; - use funnel breakdowns or journey filters instead. +- **Behavioral / OR cohorts** — Observe cohorts are AND-only (see + [Cohorts](#cohorts)); recreate OR logic as separate cohorts or use funnel + breakdowns / journey filters. - **PostHog Apps/Plugins** — no equivalent. Consider webhooks + integrations. - **Data pipelines** — PostHog's destinations (BigQuery, S3) aren't built in. Use the `/api/v1/export` endpoint or database backups. @@ -157,6 +206,7 @@ Reserve `/api/v1/events/batch` (up to 100 events per call) for larger imports. - [ ] Swap init + capture / identify / flag calls. - [ ] Re-create key dashboards on `/dashboards`. -- [ ] Migrate cohorts to equivalent funnels with breakdowns. +- [ ] Recreate simple cohorts as AND-only cohort rules; use funnel breakdowns for anything else. +- [ ] If you used PostHog surveys, plan to render them yourself (no on-page widget ships). - [ ] If you used session replays, add `observe-replay.js` to your tag setup. - [ ] Delete the PostHog project and API key from env + secrets. diff --git a/docs/migrations/from-sentry.md b/docs/migrations/from-sentry.md index a2e82694..e7379766 100644 --- a/docs/migrations/from-sentry.md +++ b/docs/migrations/from-sentry.md @@ -1,12 +1,21 @@ # Migrating from Sentry Observe's error tracking covers the same core surface as Sentry: grouped issues, -stack traces with source maps, breadcrumbs, releases, and webhook/Slack alerts. +stack traces with source maps, breadcrumbs, releases, and webhook alerts. This guide walks through the SDK-level changes and the concept mapping. -> **TL;DR** — Replace your Sentry DSN + SDK with one of Observe's SDKs. The -> payload shapes map almost 1-to-1; the only concept Sentry has that Observe -> doesn't is "organization" (Observe uses sites instead). +> **TL;DR** — Replace your Sentry SDK with one of Observe's SDKs (or the +> `@teploy/observe-sentry-shim` package). The payload shapes map almost +> 1-to-1; the only concept Sentry has that Observe doesn't is "organization" +> (Observe uses sites instead). + +> **Stock Sentry SDKs cannot be pointed at Observe by changing the DSN.** +> Observe has no Sentry-protocol ingest: there is no `/envelope` or `/store` +> route (nor `/api//envelope/`). Its error endpoint is +> `POST /api/v1/errors` with Observe's own JSON body (`internal/errors/handler.go`). +> A stock `@sentry/*`, `sentry-sdk` or `sentry-go` client given an Observe URL +> will get 404s and drop events. You must swap the SDK (or use the shim, which +> speaks Observe's API, not Sentry's). ## Concept mapping @@ -18,11 +27,11 @@ This guide walks through the SDK-level changes and the concept mapping. | Event | Error event (`error_events` table) | | Release | Release tag (string field) | | Environment | Environment tag (string field) | -| Breadcrumb | Breadcrumb (JSON array on each event) | +| Breadcrumb | Breadcrumb (JSON array on each event; see [Breadcrumbs](#breadcrumbs)) | | Source map | Source map (upload via `/api/v1/sourcemaps/upload`) | | Organization | (none — each Observe deployment is one tenant) | | Alert rule | Alert rule (`/alerts`) | -| Webhook integration | Integration (`/integrations`) | +| Webhook integration | Webhook (alert delivery); `/integrations` entries are not fired by alerts today | ## JavaScript / TypeScript @@ -45,25 +54,45 @@ captureException(err, { release: "v1.4.2" }); ### Drop-in shim -If you want to swap Sentry for Observe without touching call sites, register a -shim on `window.Sentry`: +To keep Sentry call sites, use the `@teploy/observe-sentry-shim` package +(`sdk/sentry-shim`) instead of writing your own `window.Sentry` object. It +exports the Sentry-named functions and posts to Observe's API +(`captureException` to `/api/v1/errors`, `captureMessage` to `/api/v1/logs`): ```ts -import { init, captureException, track, identify } from "@teploy/observe-browser"; - -init({ endpoint: "https://observe.example.com" }); - -(globalThis as any).Sentry = { - init: () => {}, // already initialized - captureException, // same signature - captureMessage: (msg: string) => track("message", { message: msg }), - setUser: (u: { id?: string }) => u?.id && identify(u.id, u), - setTag: () => {}, // tags roll into events via props - addBreadcrumb: () => {}, // Observe captures breadcrumbs automatically -}; +import * as Sentry from "@teploy/observe-sentry-shim"; + +Sentry.init({ + endpoint: "https://observe.example.com", + siteId: "default", + apiKey: process.env.OBSERVE_API_KEY, // required: keyless ingest gets 401 + release: "v1.4.2", +}); +Sentry.captureException(err); ``` -Existing code calling `Sentry.captureException(err)` continues to work unchanged. +`init({ dsn })` is accepted, but only the shim parses it (an Observe-style +`https://host/__observe__/` or a classic `https://key@host/` +whose last path segment is used as the site id). That is the shim reading a +string, not Observe accepting Sentry traffic; the shim still sends Observe's +own payloads and still needs `apiKey`. + +`setTag`, `setUser`, `setContext`, `setExtra` and `addBreadcrumb` are +**implemented**, not no-ops: tags ride in `contexts.tags`, breadcrumbs are kept +in a bounded per-scope ring (`maxBreadcrumbs`, default 100) and sent with the +event. Some APIs are no-ops or unsupported (`tracesSampleRate`, `startSession`, +`captureEvent`, `addEventProcessor`, ...). The per-API status table is +[docs/sdk/COMPATIBILITY.md](../sdk/COMPATIBILITY.md); treat it as the authority. + +#### Breadcrumbs + +Automatic breadcrumb capture (clicks, navigation, console, fetch/XHR) exists +only in the browser tracker script `observe-errors.js` +(`cmd/observe/tracker/observe-errors.js`, bounded by `data-max-breadcrumbs`, +default 30). The Python and Go SDKs and the `@teploy/observe-browser` package +do **not** collect breadcrumbs automatically, and the Python and Go SDKs have +no breadcrumb API at all. The shim and the tracker's `addBreadcrumb` let you +add them by hand. ## Go @@ -119,10 +148,17 @@ symbolicated on read. curl -X POST https://observe.example.com/api/v1/sourcemaps/upload \ -H "X-API-Key: $OBSERVE_PUBLISH_KEY" \ -F release=v1.4.2 \ - -F file=@dist/app.js.map \ - -F filename=app.js + -F filename=app.js \ + -F sourcemap=@dist/app.js.map ``` +The multipart form fields are read by `srcmapUploadHandler` in +`cmd/observe/main.go`: `release` and `filename` are required text fields, and +the map itself must be in the file field named `sourcemap` (a field named +`file` is rejected with `sourcemap file required`). With an API key the site +comes from the key; `site_id` is only read when authenticating with an +editor/admin JWT. The whole body is capped at 12 MiB and the map at 10 MiB. + After upload, events with `release_tag=v1.4.2` get their stack frames resolved to original file/line/col on the `/errors` page. @@ -153,7 +189,7 @@ Issues are grouped by `group_hash` on ingest, so duplicates merge automatically. ## Checklist -- [ ] Swap SDK init + `captureException` calls (or use the shim above). +- [ ] Swap SDK init + `captureException` calls (or use `@teploy/observe-sentry-shim`). - [ ] Upload source maps for your current release. - [ ] Confirm errors appear at `/errors` with resolved stack frames. - [ ] Set up alert rules at `/alerts` for your critical thresholds. diff --git a/docs/operations/backup-restore.md b/docs/operations/backup-restore.md new file mode 100644 index 00000000..6d1117ba --- /dev/null +++ b/docs/operations/backup-restore.md @@ -0,0 +1,126 @@ +# Backup and restore + +Source: `cmd/observe/main.go` (`runBackup`, `runRestore`) and +`internal/backup/` (`backup.go`, `restore.go`, `crypto.go`, `lease.go`, +`kvsrcmap.go`). Both commands connect to Nucleus using `OBSERVE_NUCLEUS_URL` +(same variable as the server). Diagnostics go to stderr; the archive goes to +stdout (backup) or is read from stdin (restore). + +## Take a backup + +```bash +observe backup | zstd > observe-$(date +%F).tar.zst +``` + +(`observe help` prints the same example with the legacy `teploy-observe` +name.) Exit status is 0 on success, 1 for a bad encryption configuration or +connection failure, 2 if the dump finished with per-table errors. Always check +the exit status of the pipeline (`set -o pipefail`); a status 2 archive is +incomplete and its trailing results record marks the failed tables, which +restore refuses. + +What is in it: + +- Every table listed in `backup.Tables` (a test, `TestTablesMatchSchema`, + keeps that list equal to the tables the migrations create). Excluded: + `_neutron_migrations` (rebuilt by migrations) and `_preNNN` + rename-aside recovery copies. +- The KV source-map section (uploaded source maps), when present. +- A manifest recording the table list, whether the dump ran inside a Nucleus + snapshot lease (one point-in-time moment across SQL tables; writers wait at + the engine while it is held, so expect a brief ingest stall on a large + database), and the non-secret audit-key id. + +What is **not** in it, and must be backed up separately: + +- `$OBSERVE_DATA_DIR/audit.key` (default `./data`, `/var/lib/observe` in + Compose and systemd) or the `OBSERVE_AUDIT_KEY` / `OBSERVE_AUDIT_KEYRING` + values. Without it, keyed audit history stops verifying after a restore onto + a new host. The manifest only says which key id was in use. +- `OBSERVE_SECRET_KEY` (decrypts the stored LLM key and S3/R2 credentials), + `OBSERVE_JWT_SECRET`, `OBSERVE_SESSION_SALT` and the rest of + `/etc/observe/observe.env`. Keep them with the backup key, but in a + separate secret store. +- The ingest WAL (`$OBSERVE_QUEUE_DIR`). Events acknowledged but not yet + applied to Nucleus live there; a database-level backup does not include + them. Stop the server gracefully (it flushes) before a final backup you + plan to restore from. + +## Encryption + +Archives contain password hashes, API keys, integration and webhook secrets +and share tokens as plaintext JSON. Set `OBSERVE_BACKUP_ENCRYPTION_KEY` to +wrap the whole archive in AES-256-GCM (chunked, 64 KiB per chunk, magic header +`OBSBKv1`): + +```bash +export OBSERVE_BACKUP_ENCRYPTION_KEY="$(openssl rand -base64 32)" # 32 raw bytes, standard base64 +observe backup > observe-$(date +%F).obsbk +``` + +Encryption happens inside `observe backup`, so its output is already +ciphertext and will not compress; compress nothing (or use a filesystem that +does). + +With the variable unset, `observe backup` writes a plaintext archive and logs +a warning. A malformed key is a hard error (exit 1). Restore needs the **same** +variable set to decrypt; losing the key loses the backup. An encrypted archive cannot be read without it. + +## Restore + +```bash +zstdcat observe-2026-04-17.tar.zst | observe restore +# encrypted: +OBSERVE_BACKUP_ENCRYPTION_KEY=... observe restore < observe-2026-04-17.obsbk +``` + +Behaviour (`internal/backup/restore.go`): + +1. The stream is spooled to a local temp file (bounded at 200 GiB), so make + sure the temp directory has room. +2. A full validation pass runs before anything is written: manifest present + and compatible, table names on the allowlist, every row structurally valid, + exactly one trailing results record with no failed tables, row counts + matching. A bad, truncated or partial archive changes nothing. +3. **The target must be empty.** Restore appends; if any backed-up table + (or the KV source-map namespace, when the archive has KV data) already + holds rows it refuses with `restore target is not empty - nothing was + written`. There is no merge mode. +4. Rows are applied in batches per table. It is not one transaction: an + infrastructure failure mid-apply can leave earlier batches and tables + committed, and the command exits non-zero naming the table. Recover by + discarding that database and starting the restore over on a fresh one. + +Restore does not create tables. The schema must already exist, which today +means starting the `observe` server once against the new database so it runs +the migrations. **Caveat found by reading the code, not by running it:** server +startup also inserts the `default` site (`SiteService.EnsureDefault`), and +the bootstrap admin when `OBSERVE_ADMIN_PASSWORD` is set. Both `sites` and +`admin_users` are restore tables, so a restore into a database the server has +already started against would be refused by the emptiness check. Before +relying on this procedure, rehearse it (below) and be ready to empty those +rows by hand; the repository has no supported command for it yet. + +## Restore drill (do this before you need it) + +Run on a scratch host or a second Compose project, never the live database. + +1. Create a fresh, empty Nucleus (new data directory or volume). +2. Point `OBSERVE_NUCLEUS_URL` at it, set the same `OBSERVE_SECRET_KEY`, + `OBSERVE_JWT_SECRET` and `OBSERVE_SESSION_SALT` as production, leave + `OBSERVE_ADMIN_PASSWORD` unset, and copy `audit.key` into the new + `OBSERVE_DATA_DIR`. +3. Start `observe` once so migrations run, then stop it. Note whether the + emptiness check then rejects `sites` (see the caveat) and record the exact + manual step you needed. +4. Run `observe restore` with the encryption key; confirm exit status 0 and + the log line `restore complete`. +5. Start `observe`. Check `/healthz` is `ok`. +6. Log in with a production account; compare site list, a known issue, a + known dashboard and a recent day of stats against production. +7. Open a source-mapped stack trace to confirm the KV source maps came back. +8. Check the audit log still verifies (the audit chain is keyed by + `audit.key`; the manifest records the expected key id). +9. Record: archive size, backup duration, restore duration, and the gaps you + hit. Repeat on a schedule and after every upgrade that adds migrations. +10. Destroy the scratch instance; it holds production data. diff --git a/docs/operations/install.md b/docs/operations/install.md new file mode 100644 index 00000000..cc001bb3 --- /dev/null +++ b/docs/operations/install.md @@ -0,0 +1,132 @@ +# Installing Observe + +Observe is two processes: the `observe` binary and a Nucleus database. Every +install path below starts both (Compose) or expects you to run Nucleus yourself +(systemd, script, source). Configuration reference: the environment-variable +tables in the top-level [README](../../README.md). + +Pick the path, then read "After install". + +## Docker Compose (recommended for a single host) + +`docker-compose.yml` in the repo root runs the `nucleus` and `observe` +services (and an optional Caddy TLS proxy). + +```bash +git clone https://github.com/useteploy/teploy-observe.git +cd teploy-observe +docker compose up -d +``` + +- The dashboard binds **loopback only** (`127.0.0.1:3000`). Reach it over SSH + or change the `ports:` binding to a tailnet/LAN address (see the header + comment in `docker-compose.yml`). +- Nucleus is on an internal `backend` network and is not published. +- Public TLS via Caddy: `OBSERVE_DOMAIN=observe.example.com docker compose --profile tls up -d` + (needs ports 80/443 and DNS pointing at the host). +- To collect telemetry from the public internet without exposing the dashboard, + set `OBSERVE_INGEST_ADDR: ":3001"` on the `observe` service and publish only + that port. +- Pin a release with `OBSERVE_VERSION=1.2.3` (image + `ghcr.io/useteploy/teploy-observe:${OBSERVE_VERSION:-latest}`). +- Data lives in the named volumes `nucleus_data` (database) and `observe_data` + (`/var/lib/observe`: WAL queue, `audit.key`). Back up both; see + [backup-restore.md](backup-restore.md). +- No default password: the first browser visit lands on the `/setup` wizard. + Setting `OBSERVE_ADMIN_PASSWORD` in the environment skips the wizard and + creates that account at boot. +- Warning: the Compose file starts Nucleus with `NUCLEUS_ALLOW_NO_AUTH=1`, + relying on the internal network for protection. Do not publish the Nucleus + port. + +## Install script (Linux with systemd; also macOS binary only) + +`scripts/install.sh` (published as `install.sh` on each release). Verify the +script against the release `checksums.txt` before running it, exactly as the +README shows: + +```bash +( + set -e + curl -fsSLO https://github.com/useteploy/teploy-observe/releases/latest/download/install.sh + curl -fsSLO https://github.com/useteploy/teploy-observe/releases/latest/download/checksums.txt + grep " install.sh\$" checksums.txt > checksum.txt + if command -v sha256sum >/dev/null 2>&1; then sha256sum -c checksum.txt || exit 1; else shasum -a 256 -c checksum.txt || exit 1; fi + sh install.sh +) +``` + +Environment variables read by the script (`scripts/install.sh`): + +| Variable | Default | Meaning | +|---|---|---| +| `OBSERVE_VERSION` | `latest` | Release tag to install (`v` prefix optional). | +| `OBSERVE_PREFIX` | `/usr/local/bin` | Install directory. A non-default prefix requires `OBSERVE_NO_SERVICE=1`. | +| `OBSERVE_NO_SERVICE` | (unset) | Set to skip systemd user/unit/env-file creation. | +| `OBSERVE_HEALTH_URL` | `http://127.0.0.1:3000/healthz` | Readiness URL polled after restarting an already-running service. | + +What it does on Linux with systemd: verifies the Ed25519-signed +`checksums.txt` and the archive hash (fails closed), installs the binary, +creates the `observe` system user and `/var/lib/observe`, writes +`/etc/observe/observe.env` (mode 0600) **only if it does not exist** with a +random `OBSERVE_JWT_SECRET`, `OBSERVE_SESSION_SALT` and a random +`OBSERVE_ADMIN_PASSWORD` (printed once at the end), and installs and enables +`observe.service`. If the service was already running it restarts it and, on +failed readiness, restores the previous binary (`observe.prev`). + +The script does **not** install Nucleus. Install it separately (next section) +and point `OBSERVE_NUCLEUS_URL` at it. The default unit uses +`postgres://localhost:5432/observe`. + +## systemd (manual) + +Unit files and the full step list are in `packaging/systemd/` (`README.md`, +`observe.service`, `nucleus.service`). Summary of that README: + +```bash +sudo useradd --system --home /var/lib/observe --shell /usr/sbin/nologin observe +sudo useradd --system --home /var/lib/nucleus --shell /usr/sbin/nologin nucleus +sudo mkdir -p /var/lib/observe /var/lib/nucleus/data /var/log/observe /etc/observe +sudo chown observe:observe /var/lib/observe /var/log/observe /etc/observe +sudo chown nucleus:nucleus /var/lib/nucleus /var/lib/nucleus/data + +sudo install -m 0755 observe /usr/local/bin/observe +sudo install -m 0755 nucleus /usr/local/bin/nucleus + +# /etc/observe/observe.env (mode 0600, owner observe): set at least +# OBSERVE_JWT_SECRET, OBSERVE_SESSION_SALT (openssl rand -hex 32) +# OBSERVE_ADMIN_PASSWORD (or leave unset to use /setup) + +sudo install -m 0644 nucleus.service /etc/systemd/system/nucleus.service +sudo install -m 0644 observe.service /etc/systemd/system/observe.service +sudo systemctl daemon-reload +sudo systemctl enable --now nucleus observe +sudo journalctl -u observe -f +``` + +Notes taken from the units: `nucleus.service` runs +`nucleus start --port 5432 --data /var/lib/nucleus/data --max-memory 512` with +`NUCLEUS_ALLOW_NO_AUTH=1`, so keep port 5432 on loopback or a firewalled +interface. `observe.service` sets `TimeoutStopSec=90s` (graceful flush on +SIGTERM) and `ReadWritePaths=/var/lib/observe`; add a reverse proxy for TLS. + +## Homebrew and from source + +```bash +brew install useteploy/tap/observe +go build ./cmd/observe # from a clone; dependencies are vendored +``` + +You still need a running Nucleus; see `docker-compose.yml` for the exact image +and flags. + +## After install + +1. `curl -fsS http://127.0.0.1:3000/healthz` returns status `ok` and the + running `version`. +2. Create the admin account (`/setup`, or the bootstrap env vars). +3. Create a site-scoped API key (Settings > API keys, or + `POST /api/v1/sites/default/keys`) and send a first event; the README + "First success" section has the tested path. +4. Back up before you rely on it: [backup-restore.md](backup-restore.md). +5. Understand how to upgrade: [upgrade.md](upgrade.md). diff --git a/docs/operations/upgrade.md b/docs/operations/upgrade.md new file mode 100644 index 00000000..07a7cce7 --- /dev/null +++ b/docs/operations/upgrade.md @@ -0,0 +1,106 @@ +# Upgrading Observe + +Use the tool that installed Observe. `observe upgrade` refuses to touch a +Homebrew or container install and tells you which command to use instead +(`runUpgrade` in `cmd/observe/main.go`). + +## Before every upgrade + +1. Take a backup and check it (see [backup-restore.md](backup-restore.md)). + This is the only way back from a bad schema change; see "What rollback + covers" below. +2. Read `CHANGELOG.md` for the target version, especially migration notes. +3. Check `/healthz` is `ok` on the current version. + +## Commands + +```bash +# Direct Linux/systemd install (must be root; binary installed by install.sh) +sudo observe upgrade +sudo observe upgrade --version v1.2.3 +# --version release to install (default: latest) +# --service systemd unit name (default: observe.service) +# --health-url readiness URL (default derived from OBSERVE_ADDR) + +# Homebrew +brew upgrade useteploy/tap/observe + +# Docker Compose +docker compose pull && docker compose up -d +# pin with OBSERVE_VERSION=1.2.3 + +# Re-running the install script also upgrades a direct install +OBSERVE_VERSION=v1.2.3 sh install.sh +``` + +`observe upgrade` requires Linux, root, systemd, and a release build (a +development build cannot be compared). It takes an upgrade lock, refuses to +downgrade (`refusing downgrade X -> Y`) or reinstall the same version, and +requires the service to be active. + +## What `observe upgrade` does (internal/upgrade/upgrade.go) + +1. Downloads `checksums.txt` and `checksums.txt.sig` for the target release, + verifies the Ed25519 signature against the public key compiled into the + binary, then verifies the archive SHA-256. Fails closed; the running server + stays online during this staging. +2. Asks systemd to stop the unit (graceful: Observe flushes buffers on + SIGTERM; `TimeoutStopSec=90s`). +3. Atomically replaces the binary, keeping the previous one for restore. +4. Starts the unit and polls `/healthz` until it has returned three + consecutive healthy responses reporting the exact new version. +5. If start or the readiness check fails, it stops the unit, restores the + previous binary, restarts it and waits for it to be healthy again. + +There is a short restart window with no ingest. SDKs retry; the WAL +(`OBSERVE_QUEUE_DIR`) replays acknowledged-but-unapplied events on the next +start. + +## What rollback covers: the binary only + +The automatic rollback restores the **previous executable**. It does not +restore data. + +- **Migrations run at startup, in the new binary, and are forward-only.** + `internal/schema/migrations/` contains only `NNN_name.up.sql` files; there + are no down migrations and no tooling to reverse one. +- If the new binary applied migrations and then failed its readiness check, + the old binary is restored **against the already-migrated database**. A + migration that failed part-way may also have left tables half-changed. The + old binary may or may not run against the new schema; nothing in the repo + verifies that combination. +- Downgrading deliberately is refused by `observe upgrade`. To go back to an + older version after a migration you must restore a pre-upgrade backup into + an empty database with the older binary (see the caveats in + [backup-restore.md](backup-restore.md)), which loses data written since the + backup. + +### Known risk: ALTER-ADD migrations on populated tables (AUDIT_OPEN L9) + +Open P1 item L9 in `AUDIT_OPEN.md`: migrations 048, 049, 051, 052, 054 and 056 +use `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` on existing tables. On a +populated Nucleus store, migration 054 failed twice with `corrupt tuple` +and left `llm_traces` unreadable (an upstream engine defect; the table could +not be read afterwards). The test suite cannot catch this because it runs +migrations against empty tables. As of this writing the item is open: there is +no populated-store migration test, and no repair has been decided. Practical +consequences for upgrades: + +- Always take and verify a backup first. +- After upgrading across those migrations on a populated install, check the + LLM views and `/healthz`; a failure there is this issue, not something your + rollback fixes. +- New migrations are required by project rules to avoid ALTER-ADD on existing + tables (rename-aside + create + copy instead), but earlier ones are already + shipped. + +## Verify after upgrading + +```bash +observe version +curl -fsS http://127.0.0.1:3000/healthz # status ok, expected version +sudo journalctl -u observe -n 100 # look for "migrations complete" +``` + +Then spot-check a dashboard page that reads recently written data and confirm +new events arrive. From 43d56747f612c4a9cd9ab08f1709dc8077161dc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:29:46 +0000 Subject: [PATCH 07/65] fix: attribute experiment conversions to the first exposure; add EventDistinctID helper RecordConversion stored the variant of the user's LATEST exposure, so a user exposed to several arms was attributed to whichever exposed last and the answer moved as exposures arrived. It now reads the FIRST exposure (documented on the method). Results does not read the stored variant, so reported counts are unchanged. Exposure/conversion rows carry the caller's raw user_id while events carry the hashed distinct_id. ExperimentService.EventDistinctID derives the event-pipeline id (per-site salt, raw opt-out, global-salt fallback, via internal/identity) so callers can join; stored values are not rewritten. Wired with WithPrivacy(siteSvc.PrivacyConfig, salt) in main.go. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/main.go | 2 +- internal/experiments/experiments.go | 68 +++++++++++++++++++++++++-- internal/experiments/identity_test.go | 64 +++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 internal/experiments/identity_test.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index e0ccec30..3b1f78fd 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -444,7 +444,7 @@ func main() { ssoSvc := sso.NewSSOService(db) flagSvc := flags.NewFlagService(db) flagSvc.WithEvalDedup(flags.LoadEvalDedupFromEnv(os.Getenv)) - experimentSvc := experiments.NewExperimentService(db) + experimentSvc := experiments.NewExperimentService(db).WithPrivacy(siteSvc.PrivacyConfig, cfg.SessionSalt) surveySvc := surveys.NewSurveyService(db, cfg.SessionSalt, siteSvc) logSvc := logs.NewLogService(db) logSvc.SetPipelines(pipelineSvc) diff --git a/internal/experiments/experiments.go b/internal/experiments/experiments.go index 81fa0b72..7afbc11a 100644 --- a/internal/experiments/experiments.go +++ b/internal/experiments/experiments.go @@ -10,10 +10,58 @@ import ( "time" "github.com/neutron-build/neutron/go/nucleus" + + "github.com/useteploy/teploy-observe/internal/identity" ) type ExperimentService struct { db *nucleus.Client + + // Optional per-site privacy lookup for EventDistinctID; nil means the + // helper falls back to the global salt (matches the ingest path for + // unknown sites). + privacy PrivacyLookup + salt string +} + +// PrivacyLookup resolves a site's distinct_id hashing config: the per-site +// salt and whether the site stores raw distinct_ids. ok=false means the +// site is unknown. Satisfied by (*sites.SiteService).PrivacyConfig. +type PrivacyLookup func(ctx context.Context, siteID string) (salt string, rawOptIn bool, ok bool) + +// WithPrivacy installs the per-site lookup and the global fallback salt used +// by EventDistinctID. Returns the receiver for fluent boot-time setup. +func (s *ExperimentService) WithPrivacy(lookup PrivacyLookup, fallbackSalt string) *ExperimentService { + s.privacy = lookup + s.salt = fallbackSalt + return s +} + +// EventDistinctID derives, from a raw experiment user id, the exact +// distinct_id the event pipeline stores on events for the same user: the +// HMAC under the site's salt (or the raw value for a raw_distinct_id site), +// resolved the same way internal/ingest does (per-site config, global salt +// fallback for unknown sites). +// +// Identity contract (see docs/IDENTITY_MODEL_ADR.md): exposure and +// conversion rows keep the user_id the caller sent, verbatim, and are joined +// to each other on it. Events carry the hashed distinct_id. Anything that +// must join experiment rows to the events table (a goal measured from +// events, a per-user drill-down) has to hash the experiment user_id with +// this helper first, or send the already-hashed id to the expose/convert +// endpoints from the start. Stored values are deliberately not rewritten +// (D5: no silent history rewrites). +func (s *ExperimentService) EventDistinctID(ctx context.Context, siteID, rawUserID string) string { + if rawUserID == "" { + return "" + } + salt, rawOptIn := s.salt, false + if s.privacy != nil { + if ps, raw, ok := s.privacy(ctx, siteID); ok { + salt, rawOptIn = ps, raw + } + } + return identity.MaybeHashDistinctID(rawUserID, salt, rawOptIn) } func NewExperimentService(db *nucleus.Client) *ExperimentService { @@ -136,19 +184,31 @@ func (s *ExperimentService) RecordExposure(ctx context.Context, experimentID, si return err } +// firstExposureSQL selects the user's earliest exposure (attribution +// choice documented on RecordConversion). +const firstExposureSQL = `SELECT variant FROM experiment_exposures + WHERE experiment_id = $1 AND site_id = $2 AND user_id = $3 + ORDER BY timestamp ASC LIMIT 1` + // RecordConversion records that an exposed user converted. The conversion is // stored in its own append-only table (not a row-copy back into exposures, // which used to duplicate rows and corrupt counts). The user's variant is // resolved from their exposure so Results can attribute the conversion without // a join. A conversion with no prior exposure is ignored. +// +// Attribution is FIRST-exposure: the stored variant is the one the user was +// first assigned to, the same choice every first-touch experiment readout +// makes. It used to read the LATEST exposure, so a user exposed to several +// variants (an allocation change mid-run, a client that re-sends expose with +// a different arm) was attributed to whichever arm happened to expose last, +// and the answer moved as more exposures arrived. Note Results does not read +// this stored variant: it counts a converting user in every arm whose +// in-interval exposure window contains the conversion. func (s *ExperimentService) RecordConversion(ctx context.Context, experimentID, siteID, userID string) error { type vrow struct { Variant string `db:"variant"` } - rows, err := nucleus.Query[vrow](ctx, s.db.SQL(), - `SELECT variant FROM experiment_exposures - WHERE experiment_id = $1 AND site_id = $2 AND user_id = $3 - ORDER BY timestamp DESC LIMIT 1`, + rows, err := nucleus.Query[vrow](ctx, s.db.SQL(), firstExposureSQL, experimentID, siteID, userID) if err != nil { return err diff --git a/internal/experiments/identity_test.go b/internal/experiments/identity_test.go new file mode 100644 index 00000000..01711932 --- /dev/null +++ b/internal/experiments/identity_test.go @@ -0,0 +1,64 @@ +package experiments + +import ( + "context" + "strings" + "testing" + + "github.com/useteploy/teploy-observe/internal/identity" +) + +func TestEventDistinctIDMatchesIngestDerivation(t *testing.T) { + ctx := context.Background() + svc := NewExperimentService(nil).WithPrivacy( + func(_ context.Context, siteID string) (string, bool, bool) { + switch siteID { + case "site-salted": + return "site-salt", false, true + case "site-raw": + return "ignored", true, true + } + return "", false, false + }, "global-salt") + + // Known site: HMAC under the per-site salt, exactly what ingest stores. + if got, want := svc.EventDistinctID(ctx, "site-salted", "alice@example.com"), + identity.HashDistinctID("alice@example.com", "site-salt"); got != want || len(got) != identity.HashedIDLength { + t.Errorf("salted: got %q want %q", got, want) + } + // Raw opt-out site: unchanged. + if got := svc.EventDistinctID(ctx, "site-raw", "alice@example.com"); got != "alice@example.com" { + t.Errorf("raw site: got %q", got) + } + // Unknown site: global-salt fallback, same as ingest. + if got, want := svc.EventDistinctID(ctx, "nope", "u1"), identity.HashDistinctID("u1", "global-salt"); got != want { + t.Errorf("fallback: got %q want %q", got, want) + } + // Sites do not share identities. + if svc.EventDistinctID(ctx, "site-salted", "u1") == svc.EventDistinctID(ctx, "nope", "u1") { + t.Error("different salts must give different ids") + } + // No identify value, no identity. + if got := svc.EventDistinctID(ctx, "site-salted", ""); got != "" { + t.Errorf("empty raw id hashed to %q", got) + } + // Without a lookup wired the global salt is used. + if got, want := NewExperimentService(nil).WithPrivacy(nil, "g").EventDistinctID(ctx, "s", "u"), identity.HashDistinctID("u", "g"); got != want { + t.Errorf("no lookup: got %q want %q", got, want) + } +} + +// Conversions attribute to the user's FIRST exposure, never the latest. +func TestFirstExposureAttributionQuery(t *testing.T) { + if !strings.Contains(firstExposureSQL, "ORDER BY timestamp ASC LIMIT 1") { + t.Fatalf("attribution must read the earliest exposure:\n%s", firstExposureSQL) + } + if strings.Contains(firstExposureSQL, "DESC") { + t.Fatalf("attribution must not read the latest exposure:\n%s", firstExposureSQL) + } + for _, scope := range []string{"experiment_id = $1", "site_id = $2", "user_id = $3"} { + if !strings.Contains(firstExposureSQL, scope) { + t.Errorf("query not scoped by %s", scope) + } + } +} From 8b0e5a071b9d8a54739f16c9d1c82006e583b230 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:30:29 +0000 Subject: [PATCH 08/65] fix: keep false/0/0.0/"" OTLP attributes and persist span links (migration 057) AttrsToMap (tracing, metrics) and otlpAny.text (logs) tested the scalar fields against their zero values, so attributes set to false, 0, 0.0 or "" were dropped. AnyValue now carries an unexported kind tag set by the JSON and protobuf decoders; kind-less literals keep the old fallback. Tracing also folds array/kvlist/bytes values to deterministic text (sorted compact JSON / hex, identical across wire formats, depth-bounded) and marshals AnyValue by kind so span-event JSON keeps zero values. Span links were dropped on ingest. Both OTLP trace paths now parse links (trace_id, span_id, trace_state, attributes), bounded to 128 links per span and 32 attributes per link, with truncations counted (IngestResponse links_truncated + warn log). Stored in new span_links table (migration 057, CREATE TABLE IF NOT EXISTS) inserted via chunked INSERT in the same ingest transaction as spans. QueryService.GetTraceLinks/GetSpanLinks added; trace detail spans gain an optional `links` field (omitted when empty). span_links retention is not wired (internal/jobs not touched here). UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- internal/logs/otlp.go | 65 +++- internal/logs/otlp_presence_test.go | 29 ++ internal/metrics/attrs.go | 11 +- internal/metrics/attrs_presence_test.go | 37 ++ internal/metrics/otlp_proto.go | 10 +- internal/metrics/types.go | 85 ++++- .../schema/migrations/057_span_links.up.sql | 32 ++ internal/tracing/attrs_presence_test.go | 176 ++++++++++ internal/tracing/ingest.go | 23 +- internal/tracing/links.go | 201 +++++++++++ internal/tracing/links_test.go | 153 ++++++++ internal/tracing/otlp_proto.go | 61 +++- internal/tracing/query.go | 11 + internal/tracing/types.go | 326 ++++++++++++++++-- 14 files changed, 1155 insertions(+), 65 deletions(-) create mode 100644 internal/logs/otlp_presence_test.go create mode 100644 internal/metrics/attrs_presence_test.go create mode 100644 internal/schema/migrations/057_span_links.up.sql create mode 100644 internal/tracing/attrs_presence_test.go create mode 100644 internal/tracing/links.go create mode 100644 internal/tracing/links_test.go diff --git a/internal/logs/otlp.go b/internal/logs/otlp.go index 0a5a900d..0fc04c87 100644 --- a/internal/logs/otlp.go +++ b/internal/logs/otlp.go @@ -10,6 +10,7 @@ import ( "io" "mime" "net/http" + "strconv" "strings" "google.golang.org/protobuf/encoding/protojson" @@ -60,21 +61,69 @@ type otlpAny struct { IntValue string `json:"intValue,omitempty"` BoolValue bool `json:"boolValue,omitempty"` DoubleValue float64 `json:"doubleValue,omitempty"` + + // kind records which field the producer set, so false, 0, 0.0 and "" are + // kept instead of being indistinguishable from "not set". + kind byte } -func (v otlpAny) text() string { +const ( + otlpUnset byte = iota + otlpString + otlpInt + otlpBool + otlpDouble +) + +func (v *otlpAny) UnmarshalJSON(b []byte) error { + *v = otlpAny{} + var w struct { + StringValue *string `json:"stringValue"` + IntValue *string `json:"intValue"` + BoolValue *bool `json:"boolValue"` + DoubleValue *float64 `json:"doubleValue"` + } + if err := json.Unmarshal(b, &w); err != nil { + return err + } switch { - case v.StringValue != "": + case w.StringValue != nil: + *v = otlpAny{StringValue: *w.StringValue, kind: otlpString} + case w.IntValue != nil: + *v = otlpAny{IntValue: *w.IntValue, kind: otlpInt} + case w.BoolValue != nil: + *v = otlpAny{BoolValue: *w.BoolValue, kind: otlpBool} + case w.DoubleValue != nil: + *v = otlpAny{DoubleValue: *w.DoubleValue, kind: otlpDouble} + } + return nil +} + +func (v otlpAny) text() string { + k := v.kind + if k == otlpUnset { + switch { + case v.StringValue != "": + k = otlpString + case v.IntValue != "": + k = otlpInt + case v.DoubleValue != 0: + k = otlpDouble + case v.BoolValue: + k = otlpBool + } + } + switch k { + case otlpString: return v.StringValue - case v.IntValue != "": + case otlpInt: return v.IntValue - case v.DoubleValue != 0: + case otlpDouble: return fmt.Sprintf("%v", v.DoubleValue) - case v.BoolValue: - return "true" - default: - return "" + case otlpBool: + return strconv.FormatBool(v.BoolValue) } + return "" } // severityToLevel maps an OTLP severity number onto the level vocabulary the diff --git a/internal/logs/otlp_presence_test.go b/internal/logs/otlp_presence_test.go new file mode 100644 index 00000000..bfdef54a --- /dev/null +++ b/internal/logs/otlp_presence_test.go @@ -0,0 +1,29 @@ +package logs + +import ( + "encoding/json" + "testing" +) + +// false, 0 and 0.0 set by the producer must render, not collapse to "". +func TestOTLPAny_JSONPreservesZeroValues(t *testing.T) { + cases := map[string]string{ + `{"boolValue":false}`: "false", + `{"boolValue":true}`: "true", + `{"intValue":"0"}`: "0", + `{"doubleValue":0}`: "0", + `{"doubleValue":1.5}`: "1.5", + `{"stringValue":""}`: "", + `{"stringValue":"ok"}`: "ok", + `{}`: "", + } + for in, want := range cases { + var v otlpAny + if err := json.Unmarshal([]byte(in), &v); err != nil { + t.Fatalf("%s: %v", in, err) + } + if got := v.text(); got != want { + t.Errorf("%s: got %q want %q", in, got, want) + } + } +} diff --git a/internal/metrics/attrs.go b/internal/metrics/attrs.go index 4d1ceac0..cf6a048a 100644 --- a/internal/metrics/attrs.go +++ b/internal/metrics/attrs.go @@ -15,15 +15,8 @@ func AttrsToMap(attrs []KeyValue) map[string]string { } m := make(map[string]string, len(attrs)) for _, kv := range attrs { - switch { - case kv.Value.StringValue != "": - m[kv.Key] = kv.Value.StringValue - case kv.Value.IntValue != "": - m[kv.Key] = string(kv.Value.IntValue) - case kv.Value.BoolValue: - m[kv.Key] = "true" - case kv.Value.DoubleValue != 0: - m[kv.Key] = fmt.Sprintf("%g", kv.Value.DoubleValue) + if v, ok := kv.Value.text(); ok { + m[kv.Key] = v } } return m diff --git a/internal/metrics/attrs_presence_test.go b/internal/metrics/attrs_presence_test.go new file mode 100644 index 00000000..7f1f34d0 --- /dev/null +++ b/internal/metrics/attrs_presence_test.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "encoding/json" + "reflect" + "testing" + + commonpb "go.opentelemetry.io/proto/otlp/common/v1" +) + +// false, 0, 0.0 and "" are values a producer set; they must not vanish. +func TestAttrsToMap_JSONPreservesZeroValues(t *testing.T) { + body := `[{"key":"s","value":{"stringValue":""}},{"key":"i","value":{"intValue":"0"}}, + {"key":"b","value":{"boolValue":false}},{"key":"d","value":{"doubleValue":0}}, + {"key":"t","value":{"boolValue":true}},{"key":"none","value":{}}]` + var kvs []KeyValue + if err := json.Unmarshal([]byte(body), &kvs); err != nil { + t.Fatal(err) + } + want := map[string]string{"s": "", "i": "0", "b": "false", "d": "0", "t": "true"} + if got := AttrsToMap(kvs); !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v want %#v", got, want) + } +} + +func TestAttrsToMap_ProtoPreservesZeroValues(t *testing.T) { + pb := []*commonpb.KeyValue{ + {Key: "s", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: ""}}}, + {Key: "i", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: 0}}}, + {Key: "b", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_BoolValue{BoolValue: false}}}, + {Key: "d", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_DoubleValue{DoubleValue: 0}}}, + } + want := map[string]string{"s": "", "i": "0", "b": "false", "d": "0"} + if got := AttrsToMap(protoAttrs(pb)); !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v want %#v", got, want) + } +} diff --git a/internal/metrics/otlp_proto.go b/internal/metrics/otlp_proto.go index 09863a0f..762fccca 100644 --- a/internal/metrics/otlp_proto.go +++ b/internal/metrics/otlp_proto.go @@ -120,15 +120,15 @@ func protoAttrs(kvs []*commonpb.KeyValue) []KeyValue { func protoAnyValue(v *commonpb.AnyValue) AnyValue { switch val := v.GetValue().(type) { case *commonpb.AnyValue_StringValue: - return AnyValue{StringValue: val.StringValue} + return stringAny(val.StringValue) case *commonpb.AnyValue_BoolValue: - return AnyValue{BoolValue: val.BoolValue} + return boolAny(val.BoolValue) case *commonpb.AnyValue_IntValue: - return AnyValue{IntValue: jsonInt(fmt.Sprintf("%d", val.IntValue))} + return intAny(fmt.Sprintf("%d", val.IntValue)) case *commonpb.AnyValue_DoubleValue: - return AnyValue{DoubleValue: val.DoubleValue} + return doubleAny(val.DoubleValue) case *commonpb.AnyValue_BytesValue: - return AnyValue{StringValue: hex.EncodeToString(val.BytesValue)} + return stringAny(hex.EncodeToString(val.BytesValue)) case nil: return AnyValue{} default: diff --git a/internal/metrics/types.go b/internal/metrics/types.go index cebef32c..438b3fc4 100644 --- a/internal/metrics/types.go +++ b/internal/metrics/types.go @@ -1,6 +1,11 @@ package metrics -import "strings" +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) // OTLP JSON types for ExportMetricsServiceRequest. // These mirror the OpenTelemetry protobuf-to-JSON mapping for the metrics @@ -107,10 +112,86 @@ type AnyValue struct { IntValue jsonInt `json:"intValue,omitempty"` BoolValue bool `json:"boolValue,omitempty"` DoubleValue float64 `json:"doubleValue,omitempty"` + + // kind records which field the decoder saw set. The scalar fields cannot + // say that themselves (false, 0, 0.0 and "" are Go zero values), so + // without it AttrsToMap dropped those attributes. A literal with no kind + // falls back to non-zero detection. + kind valueKind +} + +type valueKind uint8 + +const ( + kindUnset valueKind = iota + kindString + kindInt + kindBool + kindDouble +) + +func stringAny(s string) AnyValue { return AnyValue{StringValue: s, kind: kindString} } +func intAny(s string) AnyValue { return AnyValue{IntValue: jsonInt(s), kind: kindInt} } +func boolAny(b bool) AnyValue { return AnyValue{BoolValue: b, kind: kindBool} } +func doubleAny(f float64) AnyValue { return AnyValue{DoubleValue: f, kind: kindDouble} } + +func (a *AnyValue) UnmarshalJSON(b []byte) error { + *a = AnyValue{} + var w struct { + StringValue *string `json:"stringValue"` + IntValue *jsonInt `json:"intValue"` + BoolValue *bool `json:"boolValue"` + DoubleValue *float64 `json:"doubleValue"` + } + if err := json.Unmarshal(b, &w); err != nil { + return err + } + switch { + case w.StringValue != nil: + *a = stringAny(*w.StringValue) + case w.IntValue != nil: + *a = intAny(string(*w.IntValue)) + case w.BoolValue != nil: + *a = boolAny(*w.BoolValue) + case w.DoubleValue != nil: + *a = doubleAny(*w.DoubleValue) + } + return nil +} + +// text renders the value for the string-map label store; ok is false when the +// producer set no value. +func (a AnyValue) text() (string, bool) { + k := a.kind + if k == kindUnset { + switch { + case a.StringValue != "": + k = kindString + case a.IntValue != "": + k = kindInt + case a.BoolValue: + k = kindBool + case a.DoubleValue != 0: + k = kindDouble + } + } + switch k { + case kindString: + return a.StringValue, true + case kindInt: + return string(a.IntValue), true + case kindBool: + return strconv.FormatBool(a.BoolValue), true + case kindDouble: + return fmt.Sprintf("%g", a.DoubleValue), true + } + return "", false } // AggregationTemporality maps the OTLP enum to the column value. -// 1 = delta, 2 = cumulative. +// +// 1 = delta, 2 = cumulative. +// // Anything else collapses to "cumulative" — matches the OTLP default // for SDKs that omit the field. func AggregationTemporality(t int) string { diff --git a/internal/schema/migrations/057_span_links.up.sql b/internal/schema/migrations/057_span_links.up.sql new file mode 100644 index 00000000..29c0d5b3 --- /dev/null +++ b/internal/schema/migrations/057_span_links.up.sql @@ -0,0 +1,32 @@ +-- 057 (2026-10-03): WP1.1 - persist OTLP span links. +-- +-- Links (a span's causal references to spans in the same or another trace: +-- batch consumers, fan-in, retries) were parsed by no ingest path and so were +-- silently dropped. span_links stores them, one row per (owning span, link). +-- +-- trace_id, span_id the OWNING span (joins to spans) +-- link_idx position of the link on the span, so two +-- links to the same target stay distinct +-- linked_trace_id/_span_id the link target, lowercase hex as on spans +-- trace_state W3C tracestate of the target, may be empty +-- attributes JSONB string map, NULL when none +-- start_time the OWNING span's start (epoch ms): the +-- retention column, so links age out with the +-- spans they annotate +-- +-- Ingest caps links per span and attributes per link; see links.go. +-- Mirrors the spans table in 003: mergetree, site_id leading the ordering. + +CREATE TABLE IF NOT EXISTS span_links ( + trace_id TEXT NOT NULL, + span_id TEXT NOT NULL, + link_idx BIGINT NOT NULL DEFAULT 0, + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + linked_trace_id TEXT NOT NULL DEFAULT '', + linked_span_id TEXT NOT NULL DEFAULT '', + trace_state TEXT NOT NULL DEFAULT '', + attributes JSONB, + start_time BIGINT NOT NULL +) WITH (engine = 'mergetree') +ORDER BY (tenant_id, site_id, start_time, trace_id, span_id, link_idx); diff --git a/internal/tracing/attrs_presence_test.go b/internal/tracing/attrs_presence_test.go new file mode 100644 index 00000000..b8c7109d --- /dev/null +++ b/internal/tracing/attrs_presence_test.go @@ -0,0 +1,176 @@ +package tracing + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "google.golang.org/protobuf/proto" + + tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonpb "go.opentelemetry.io/proto/otlp/common/v1" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" +) + +// mixedAttrsJSON is a real OTLP/JSON attribute list covering every scalar kind +// at its zero value plus array, kvlist and bytes. +const mixedAttrsJSON = `[ + {"key":"s.text","value":{"stringValue":"hello"}}, + {"key":"s.empty","value":{"stringValue":""}}, + {"key":"i.num","value":{"intValue":"42"}}, + {"key":"i.bare","value":{"intValue":7}}, + {"key":"i.zero","value":{"intValue":"0"}}, + {"key":"b.true","value":{"boolValue":true}}, + {"key":"b.false","value":{"boolValue":false}}, + {"key":"d.pi","value":{"doubleValue":3.5}}, + {"key":"d.zero","value":{"doubleValue":0}}, + {"key":"a.list","value":{"arrayValue":{"values":[{"stringValue":"x"},{"intValue":"1"},{"boolValue":false},{"doubleValue":0}]}}}, + {"key":"k.map","value":{"kvlistValue":{"values":[{"key":"z","value":{"intValue":"2"}},{"key":"a","value":{"stringValue":"b"}}]}}}, + {"key":"y.bytes","value":{"bytesValue":"AQID"}}, + {"key":"n.none","value":{}} +]` + +// n.none carries no value at all and is skipped. +var mixedAttrsGolden = map[string]string{ + "s.text": "hello", + "s.empty": "", + "i.num": "42", + "i.bare": "7", + "i.zero": "0", + "b.true": "true", + "b.false": "false", + "d.pi": "3.5", + "d.zero": "0", + "a.list": `["x",1,false,0]`, + "k.map": `{"a":"b","z":2}`, + "y.bytes": "010203", +} + +func mixedAttrsProto() []*commonpb.KeyValue { + str := func(s string) *commonpb.AnyValue { + return &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: s}} + } + i64 := func(n int64) *commonpb.AnyValue { + return &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: n}} + } + bl := func(b bool) *commonpb.AnyValue { + return &commonpb.AnyValue{Value: &commonpb.AnyValue_BoolValue{BoolValue: b}} + } + dbl := func(f float64) *commonpb.AnyValue { + return &commonpb.AnyValue{Value: &commonpb.AnyValue_DoubleValue{DoubleValue: f}} + } + return []*commonpb.KeyValue{ + {Key: "s.text", Value: str("hello")}, + {Key: "s.empty", Value: str("")}, + {Key: "i.num", Value: i64(42)}, + {Key: "i.bare", Value: i64(7)}, + {Key: "i.zero", Value: i64(0)}, + {Key: "b.true", Value: bl(true)}, + {Key: "b.false", Value: bl(false)}, + {Key: "d.pi", Value: dbl(3.5)}, + {Key: "d.zero", Value: dbl(0)}, + {Key: "a.list", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_ArrayValue{ArrayValue: &commonpb.ArrayValue{ + Values: []*commonpb.AnyValue{str("x"), i64(1), bl(false), dbl(0)}}}}}, + {Key: "k.map", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_KvlistValue{KvlistValue: &commonpb.KeyValueList{ + Values: []*commonpb.KeyValue{{Key: "z", Value: i64(2)}, {Key: "a", Value: str("b")}}}}}}, + {Key: "y.bytes", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_BytesValue{BytesValue: []byte{1, 2, 3}}}}, + {Key: "n.none", Value: &commonpb.AnyValue{}}, + } +} + +func TestAttrsToMap_JSONPreservesZeroValues(t *testing.T) { + var kvs []KeyValue + if err := json.Unmarshal([]byte(mixedAttrsJSON), &kvs); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got := AttrsToMap(kvs) + if !reflect.DeepEqual(got, mixedAttrsGolden) { + t.Fatalf("json attrs mismatch\n got: %#v\nwant: %#v", got, mixedAttrsGolden) + } +} + +func TestAttrsToMap_ProtobufPreservesZeroValues(t *testing.T) { + req := &tracepb.ExportTraceServiceRequest{ResourceSpans: []*otlptrace.ResourceSpans{{ + ScopeSpans: []*otlptrace.ScopeSpans{{Spans: []*otlptrace.Span{{ + TraceId: []byte{1}, SpanId: []byte{2}, Attributes: mixedAttrsProto(), + }}}}, + }}} + body, err := proto.Marshal(req) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeProtoTraces(body) + if err != nil { + t.Fatal(err) + } + got := AttrsToMap(decoded.ResourceSpans[0].ScopeSpans[0].Spans[0].Attributes) + if !reflect.DeepEqual(got, mixedAttrsGolden) { + t.Fatalf("protobuf attrs mismatch\n got: %#v\nwant: %#v", got, mixedAttrsGolden) + } +} + +// The stored span-event JSON goes through AnyValue.MarshalJSON; false/0 must +// survive that too, and round-trip back through the decoder. +func TestAnyValue_MarshalRoundTripKeepsZeroValues(t *testing.T) { + var kvs []KeyValue + if err := json.Unmarshal([]byte(mixedAttrsJSON), &kvs); err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(kvs) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"boolValue":false`, `"intValue":"0"`, `"doubleValue":0`, `"stringValue":""`} { + if !strings.Contains(string(raw), want) { + t.Errorf("marshalled attrs missing %s: %s", want, raw) + } + } + var back []KeyValue + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatal(err) + } + if got := AttrsToMap(back); !reflect.DeepEqual(got, mixedAttrsGolden) { + t.Fatalf("round trip changed attrs: %#v", got) + } +} + +// A literal built without a kind (the seeder) keeps the legacy behaviour. +func TestAttrsToMap_KindlessLiteralFallback(t *testing.T) { + got := AttrsToMap([]KeyValue{ + {Key: "a", Value: AnyValue{StringValue: "x"}}, + {Key: "b", Value: AnyValue{}}, + }) + if got["a"] != "x" { + t.Fatalf("a = %q", got["a"]) + } + if _, ok := got["b"]; ok { + t.Fatalf("empty literal should be skipped: %#v", got) + } +} + +func TestAnyValue_DoubleStringForms(t *testing.T) { + var kvs []KeyValue + body := `[{"key":"n","value":{"doubleValue":"NaN"}},{"key":"i","value":{"doubleValue":"-Infinity"}},{"key":"s","value":{"doubleValue":"2.5"}}]` + if err := json.Unmarshal([]byte(body), &kvs); err != nil { + t.Fatal(err) + } + got := AttrsToMap(kvs) + if got["n"] != "NaN" || got["i"] != "-Inf" || got["s"] != "2.5" { + t.Fatalf("got %#v", got) + } +} + +func TestAnyValue_DeepNestingBounded(t *testing.T) { + body := `{"stringValue":"leaf"}` + for i := 0; i < 200; i++ { + body = `{"arrayValue":{"values":[` + body + `]}}` + } + var v AnyValue + if err := json.Unmarshal([]byte(body), &v); err != nil { + t.Fatal(err) + } + if !strings.Contains(v.StringValue, `"..."`) { + t.Fatalf("deep nesting not truncated: %.80s", v.StringValue) + } +} diff --git a/internal/tracing/ingest.go b/internal/tracing/ingest.go index ba1dbae9..c6ad374b 100644 --- a/internal/tracing/ingest.go +++ b/internal/tracing/ingest.go @@ -73,6 +73,9 @@ func (s *IngestService) WithLogger(logger *slog.Logger) *IngestService { type IngestResponse struct { OK bool `json:"ok"` Spans int `json:"spans"` + // LinksTruncated counts span links (or link attributes/fields) dropped by + // the ingest caps; omitted when nothing was dropped. + LinksTruncated int `json:"links_truncated,omitempty"` } // Ingest processes an OTLP ExportTraceServiceRequest. Spans AND their @@ -139,6 +142,12 @@ func (s *IngestService) ingest(ctx context.Context, siteID string, req ExportTra return IngestResponse{}, fmt.Errorf("insert span: %w", err) } + // Links ride the same transaction: a rolled-back export leaves no orphan + // links, and a committed one never lacks them. + if err := insertSpanLinks(ctx, sqlc, siteID, flat); err != nil { + return IngestResponse{}, fmt.Errorf("insert span links: %w", err) + } + intentIDs := make([]string, 0, 2) id, err := s.outbox.Enqueue(ctx, sqlc, siteID, outbox.KindTraceRollup, rollupPL) if err != nil { @@ -169,7 +178,14 @@ func (s *IngestService) ingest(ctx context.Context, siteID string, req ExportTra } } - return IngestResponse{OK: true, Spans: total}, nil + truncated := 0 + for i := range flat { + truncated += flat[i].LinksTruncated + } + if truncated > 0 { + s.logger.Warn("trace ingest: span links truncated", "site_id", siteID, "dropped", truncated) + } + return IngestResponse{OK: true, Spans: total, LinksTruncated: truncated}, nil } // handleRollupIntent derives service_stats / service_dependencies rows from @@ -322,6 +338,8 @@ type flatSpan struct { AttributesJSON string ResourceJSON string EventsJSON string + Links []flatLink + LinksTruncated int } // spansCols is the number of bound parameters per row in the batch INSERT @@ -425,6 +443,7 @@ func flattenSpans(req ExportTraceRequest) []flatSpan { svcName = override } + links, linksDropped := flattenLinks(span.Links) out = append(out, flatSpan{ TraceID: span.TraceID, SpanID: span.SpanID, @@ -440,6 +459,8 @@ func flattenSpans(req ExportTraceRequest) []flatSpan { AttributesJSON: jsonOrEmpty(AttrsToMap(span.Attributes)), ResourceJSON: resourceJSON, EventsJSON: jsonOrEmpty(span.Events), + Links: links, + LinksTruncated: linksDropped, }) } } diff --git a/internal/tracing/links.go b/internal/tracing/links.go new file mode 100644 index 00000000..9ef82192 --- /dev/null +++ b/internal/tracing/links.go @@ -0,0 +1,201 @@ +package tracing + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/neutron-build/neutron/go/nucleus" +) + +// Ingest bounds for span links. A hostile or buggy exporter can attach an +// unbounded link list to every span; the caps keep one export from turning +// into an arbitrarily large span_links write. Anything past a cap is dropped +// and counted (IngestResponse.LinksTruncated, plus a warn log). +const ( + maxLinksPerSpan = 128 + maxAttrsPerLink = 32 + maxLinkIDLen = 64 + maxTraceStateLen = 512 + spanLinksCols = 10 + spanLinksBatchLen = 50 + spanLinksColList = `trace_id, span_id, link_idx, tenant_id, site_id, linked_trace_id, linked_span_id, trace_state, attributes, start_time` +) + +// flatLink is one link row, already normalised and bounded. +type flatLink struct { + LinkIdx int + LinkedTraceID string + LinkedSpanID string + TraceState string + AttributesJSON string +} + +// flattenLinks bounds and normalises a span's links. A link without both a +// target trace id and span id is meaningless and is dropped (counted). The +// returned count covers every dropped link, clipped attribute and clipped +// field. +func flattenLinks(links []SpanLink) (out []flatLink, truncated int) { + if len(links) == 0 { + return nil, 0 + } + if len(links) > maxLinksPerSpan { + truncated += len(links) - maxLinksPerSpan + links = links[:maxLinksPerSpan] + } + out = make([]flatLink, 0, len(links)) + for _, l := range links { + if l.TraceID == "" || l.SpanID == "" || len(l.TraceID) > maxLinkIDLen || len(l.SpanID) > maxLinkIDLen { + truncated++ + continue + } + attrs := l.Attributes + if len(attrs) > maxAttrsPerLink { + truncated += len(attrs) - maxAttrsPerLink + attrs = attrs[:maxAttrsPerLink] + } + state := l.TraceState + if len(state) > maxTraceStateLen { + truncated++ + state = state[:maxTraceStateLen] + } + out = append(out, flatLink{ + LinkIdx: len(out), + LinkedTraceID: l.TraceID, + LinkedSpanID: l.SpanID, + TraceState: state, + AttributesJSON: jsonOrEmpty(AttrsToMap(attrs)), + }) + } + return out, truncated +} + +func buildSpanLinkPlaceholders(rows int) string { + var b strings.Builder + n := 1 + for r := 0; r < rows; r++ { + if r > 0 { + b.WriteByte(',') + } + b.WriteByte('(') + for c := 0; c < spanLinksCols; c++ { + if c > 0 { + b.WriteByte(',') + } + b.WriteByte('$') + b.WriteString(strconv.Itoa(n)) + n++ + } + b.WriteByte(')') + } + return b.String() +} + +// spanLinkArgs flattens a batch's links into (args per chunk) rows. Split out +// from the INSERT so the row shape is unit-testable without a database. +type spanLinkRow struct { + sp *flatSpan + l *flatLink +} + +func collectLinkRows(flat []flatSpan) []spanLinkRow { + var rows []spanLinkRow + for i := range flat { + for j := range flat[i].Links { + rows = append(rows, spanLinkRow{&flat[i], &flat[i].Links[j]}) + } + } + return rows +} + +func spanLinkArgs(dst []any, siteID string, r spanLinkRow) []any { + return append(dst, + r.sp.TraceID, r.sp.SpanID, r.l.LinkIdx, "default", siteID, + r.l.LinkedTraceID, r.l.LinkedSpanID, r.l.TraceState, + nullableJSON(r.l.AttributesJSON), r.sp.StartMs, + ) +} + +// insertSpanLinks writes every link of the batch in chunked multi-row INSERTs +// on the caller's connection - the ingest passes its transaction so links +// commit or roll back with the spans they belong to. +func insertSpanLinks(ctx context.Context, sql *nucleus.SQLModel, siteID string, flat []flatSpan) error { + rows := collectLinkRows(flat) + for start := 0; start < len(rows); start += spanLinksBatchLen { + end := start + spanLinksBatchLen + if end > len(rows) { + end = len(rows) + } + chunk := rows[start:end] + args := make([]any, 0, len(chunk)*spanLinksCols) + for _, r := range chunk { + args = spanLinkArgs(args, siteID, r) + } + q := "INSERT INTO span_links (" + spanLinksColList + ") VALUES " + buildSpanLinkPlaceholders(len(chunk)) + if _, err := sql.Exec(ctx, q, args...); err != nil { + return fmt.Errorf("batch insert span_links %d-%d: %w", start+1, end, err) + } + } + return nil +} + +// StoredSpanLink is a persisted span link as returned to API clients. +type StoredSpanLink struct { + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + LinkIdx int64 `json:"link_idx"` + LinkedTraceID string `json:"linked_trace_id"` + LinkedSpanID string `json:"linked_span_id"` + TraceState string `json:"trace_state"` + Attributes string `json:"attributes"` +} + +// GetTraceLinks returns the links of every span in a trace, scoped to the site. +func (q *QueryService) GetTraceLinks(ctx context.Context, traceID, siteID string) ([]StoredSpanLink, error) { + return nucleus.Query[StoredSpanLink](ctx, q.db.SQL(), + `SELECT trace_id, span_id, + CAST(link_idx AS TEXT) AS link_idx, + linked_trace_id, linked_span_id, trace_state, + COALESCE(attributes, '') AS attributes + FROM span_links + WHERE trace_id = $1 AND site_id = $2 + ORDER BY span_id ASC, link_idx ASC`, + traceID, siteID, + ) +} + +// GetSpanLinks returns the links of one span. +func (q *QueryService) GetSpanLinks(ctx context.Context, traceID, spanID, siteID string) ([]StoredSpanLink, error) { + all, err := q.GetTraceLinks(ctx, traceID, siteID) + if err != nil { + return nil, err + } + out := make([]StoredSpanLink, 0) + for _, l := range all { + if l.SpanID == spanID { + out = append(out, l) + } + } + return out, nil +} + +// attachLinks fills Span.Links from one trace-wide links query. Best effort: +// a failure (for example the table not yet migrated) leaves Links empty rather +// than failing the waterfall, which is still correct without them. +func (q *QueryService) attachLinks(ctx context.Context, traceID, siteID string, spans []Span) { + if len(spans) == 0 { + return + } + links, err := q.GetTraceLinks(ctx, traceID, siteID) + if err != nil || len(links) == 0 { + return + } + bySpan := make(map[string][]StoredSpanLink, len(links)) + for _, l := range links { + bySpan[l.SpanID] = append(bySpan[l.SpanID], l) + } + for i := range spans { + spans[i].Links = bySpan[spans[i].SpanID] + } +} diff --git a/internal/tracing/links_test.go b/internal/tracing/links_test.go new file mode 100644 index 00000000..4eb71165 --- /dev/null +++ b/internal/tracing/links_test.go @@ -0,0 +1,153 @@ +package tracing + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "google.golang.org/protobuf/proto" + + tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonpb "go.opentelemetry.io/proto/otlp/common/v1" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" +) + +const linksTraceJSON = `{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"consumer"}}]}, +"scopeSpans":[{"scope":{"name":"t"},"spans":[{ + "traceId":"0102030405060708090a0b0c0d0e0f10","spanId":"1112131415161718","name":"process batch","kind":5, + "startTimeUnixNano":"1700000000000000000","endTimeUnixNano":"1700000000500000000", + "links":[ + {"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"bbbbbbbbbbbbbbbb","traceState":"k=v", + "attributes":[{"key":"retry","value":{"boolValue":false}},{"key":"attempt","value":{"intValue":"0"}}],"droppedAttributesCount":2}, + {"traceId":"cccccccccccccccccccccccccccccccc","spanId":"dddddddddddddddd"} + ]}]}]}]}` + +func TestSpanLinks_JSONFixture(t *testing.T) { + var req ExportTraceRequest + if err := json.Unmarshal([]byte(linksTraceJSON), &req); err != nil { + t.Fatal(err) + } + assertLinksFixture(t, req) +} + +func TestSpanLinks_ProtobufFixture(t *testing.T) { + rep := func(b byte, n int) []byte { return []byte(strings.Repeat(string([]byte{b}), n)) } + pb := &tracepb.ExportTraceServiceRequest{ResourceSpans: []*otlptrace.ResourceSpans{{ + ScopeSpans: []*otlptrace.ScopeSpans{{Spans: []*otlptrace.Span{{ + TraceId: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, + SpanId: []byte{0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}, + Name: "process batch", + StartTimeUnixNano: 1700000000000000000, + EndTimeUnixNano: 1700000000500000000, + Links: []*otlptrace.Span_Link{ + { + TraceId: rep(0xaa, 16), + SpanId: rep(0xbb, 8), + TraceState: "k=v", + Attributes: []*commonpb.KeyValue{ + {Key: "retry", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_BoolValue{BoolValue: false}}}, + {Key: "attempt", Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: 0}}}, + }, + DroppedAttributesCount: 2, + }, + {TraceId: rep(0xcc, 16), SpanId: rep(0xdd, 8)}, + }, + }}}}, + }}} + body, err := proto.Marshal(pb) + if err != nil { + t.Fatal(err) + } + req, err := decodeProtoTraces(body) + if err != nil { + t.Fatal(err) + } + assertLinksFixture(t, req) +} + +// Both wire formats must flatten to identical link rows. +func assertLinksFixture(t *testing.T, req ExportTraceRequest) { + t.Helper() + flat := flattenSpans(req) + if len(flat) != 1 { + t.Fatalf("spans = %d", len(flat)) + } + want := []flatLink{ + {LinkIdx: 0, LinkedTraceID: strings.Repeat("aa", 16), LinkedSpanID: strings.Repeat("bb", 8), + TraceState: "k=v", AttributesJSON: `{"attempt":"0","retry":"false"}`}, + {LinkIdx: 1, LinkedTraceID: strings.Repeat("cc", 16), LinkedSpanID: strings.Repeat("dd", 8)}, + } + if !reflect.DeepEqual(flat[0].Links, want) { + t.Fatalf("links\n got: %#v\nwant: %#v", flat[0].Links, want) + } + if flat[0].LinksTruncated != 0 { + t.Fatalf("truncated = %d", flat[0].LinksTruncated) + } + + rows := collectLinkRows(flat) + if len(rows) != 2 { + t.Fatalf("rows = %d", len(rows)) + } + args := spanLinkArgs(nil, "site-1", rows[0]) + if len(args) != spanLinksCols { + t.Fatalf("args = %d want %d", len(args), spanLinksCols) + } + if args[4] != "site-1" || args[9] != int64(1700000000000) { + t.Fatalf("site/start args wrong: %v", args) + } + if got := spanLinkArgs(nil, "site-1", rows[1])[8]; got != nil { + t.Fatalf("link without attributes must bind NULL, got %v", got) + } +} + +func TestFlattenLinks_Bounds(t *testing.T) { + var links []SpanLink + for i := 0; i < maxLinksPerSpan+10; i++ { + links = append(links, SpanLink{TraceID: "aa", SpanID: "bb"}) + } + out, trunc := flattenLinks(links) + if len(out) != maxLinksPerSpan || trunc != 10 { + t.Fatalf("len=%d trunc=%d", len(out), trunc) + } + + var attrs []KeyValue + for i := 0; i < maxAttrsPerLink+5; i++ { + attrs = append(attrs, KeyValue{Key: "k" + string(rune('a'+i%26)) + string(rune('a'+i/26)), Value: stringAny("v")}) + } + out, trunc = flattenLinks([]SpanLink{ + {TraceID: "aa", SpanID: "bb", Attributes: attrs, TraceState: strings.Repeat("x", maxTraceStateLen+1)}, + {TraceID: "", SpanID: "bb"}, // dropped: no trace id + {TraceID: strings.Repeat("a", maxLinkIDLen+1), SpanID: "bb"}, // dropped: oversized id + }) + if len(out) != 1 || trunc != 5+1+2 { + t.Fatalf("len=%d trunc=%d", len(out), trunc) + } + if len(out[0].TraceState) != maxTraceStateLen { + t.Fatalf("trace state not clipped: %d", len(out[0].TraceState)) + } + var m map[string]string + if err := json.Unmarshal([]byte(out[0].AttributesJSON), &m); err != nil || len(m) != maxAttrsPerLink { + t.Fatalf("attrs kept = %d err=%v", len(m), err) + } +} + +func TestFlattenLinks_NoLinks(t *testing.T) { + if out, trunc := flattenLinks(nil); out != nil || trunc != 0 { + t.Fatalf("got %v %d", out, trunc) + } +} + +func TestBuildSpanLinkPlaceholders(t *testing.T) { + got := buildSpanLinkPlaceholders(2) + if !strings.HasPrefix(got, "($1,$2,") || !strings.HasSuffix(got, "$20)") || strings.Count(got, "(") != 2 { + t.Fatalf("placeholders: %s", got) + } +} + +func TestInsertSpanLinks_NoLinksNoExec(t *testing.T) { + // nil SQL would panic if an INSERT were attempted for a link-free batch. + if err := insertSpanLinks(nil, nil, "s", []flatSpan{{TraceID: "t", SpanID: "s"}}); err != nil { + t.Fatal(err) + } +} diff --git a/internal/tracing/otlp_proto.go b/internal/tracing/otlp_proto.go index 1440c2ba..26a035b6 100644 --- a/internal/tracing/otlp_proto.go +++ b/internal/tracing/otlp_proto.go @@ -6,8 +6,8 @@ import ( "google.golang.org/protobuf/proto" - commonpb "go.opentelemetry.io/proto/otlp/common/v1" tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonpb "go.opentelemetry.io/proto/otlp/common/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" ) @@ -77,12 +77,21 @@ func protoSpan(s *otlptrace.Span) OTLPSpan { Attributes: protoAttrs(e.GetAttributes()), }) } + for _, l := range s.GetLinks() { + span.Links = append(span.Links, SpanLink{ + TraceID: hex.EncodeToString(l.GetTraceId()), + SpanID: hex.EncodeToString(l.GetSpanId()), + TraceState: l.GetTraceState(), + Attributes: protoAttrs(l.GetAttributes()), + DroppedAttributesCount: int(l.GetDroppedAttributesCount()), + }) + } return span } // protoAttrs flattens OTLP attribute values into the same shape the JSON path -// unmarshals into. Array, kvlist and bytes values are rendered as their string -// form rather than dropped — a resource attribute that arrives as a list is +// unmarshals into. Array, kvlist and bytes values are rendered as deterministic text +// (compact sorted JSON / hex) rather than dropped — a resource attribute that arrives as a list is // still worth keeping, and the JSON path has no richer representation either. func protoAttrs(kvs []*commonpb.KeyValue) []KeyValue { if len(kvs) == 0 { @@ -96,22 +105,46 @@ func protoAttrs(kvs []*commonpb.KeyValue) []KeyValue { } func protoAnyValue(v *commonpb.AnyValue) AnyValue { + return protoAnyValueDepth(v, 0) +} + +func protoAnyValueDepth(v *commonpb.AnyValue, depth int) AnyValue { switch val := v.GetValue().(type) { case *commonpb.AnyValue_StringValue: - return AnyValue{StringValue: val.StringValue} + return stringAny(val.StringValue) case *commonpb.AnyValue_BoolValue: - return AnyValue{BoolValue: val.BoolValue} + return boolAny(val.BoolValue) case *commonpb.AnyValue_IntValue: - return AnyValue{IntValue: jsonInt(fmt.Sprintf("%d", val.IntValue))} + return intAny(fmt.Sprintf("%d", val.IntValue)) case *commonpb.AnyValue_DoubleValue: - return AnyValue{DoubleValue: val.DoubleValue} + return doubleAny(val.DoubleValue) case *commonpb.AnyValue_BytesValue: - return AnyValue{StringValue: hex.EncodeToString(val.BytesValue)} - case nil: - return AnyValue{} - default: - // Arrays and kvlists: keep the protobuf text form rather than lose the - // attribute entirely. - return AnyValue{StringValue: v.String()} + return stringAny(hex.EncodeToString(val.BytesValue)) + case *commonpb.AnyValue_ArrayValue, *commonpb.AnyValue_KvlistValue: + return stringAny(renderNative(protoNative(v, depth))) + } + return AnyValue{} +} + +// protoNative mirrors jsonNative so an array or kvlist renders to the same +// text whichever wire format carried it. +func protoNative(v *commonpb.AnyValue, depth int) any { + if depth > maxAnyDepth { + return "..." + } + switch val := v.GetValue().(type) { + case *commonpb.AnyValue_ArrayValue: + list := make([]any, 0, len(val.ArrayValue.GetValues())) + for _, e := range val.ArrayValue.GetValues() { + list = append(list, protoNative(e, depth+1)) + } + return list + case *commonpb.AnyValue_KvlistValue: + m := make(map[string]any, len(val.KvlistValue.GetValues())) + for _, e := range val.KvlistValue.GetValues() { + m[e.GetKey()] = protoNative(e.GetValue(), depth+1) + } + return m } + return protoAnyValueDepth(v, depth).native() } diff --git a/internal/tracing/query.go b/internal/tracing/query.go index f7b99d2a..78f6a3ea 100644 --- a/internal/tracing/query.go +++ b/internal/tracing/query.go @@ -375,10 +375,21 @@ type Span struct { Attributes string `json:"attributes"` Resource string `json:"resource"` Events string `json:"events"` + // Links is additive: omitted when the span has none. + Links []StoredSpanLink `json:"links,omitempty"` } // GetTrace returns all spans for a trace, ordered for waterfall rendering. func (q *QueryService) GetTrace(ctx context.Context, traceID, siteID string) ([]Span, error) { + spans, err := q.getTraceSpans(ctx, traceID, siteID) + if err != nil { + return nil, err + } + q.attachLinks(ctx, traceID, siteID, spans) + return spans, nil +} + +func (q *QueryService) getTraceSpans(ctx context.Context, traceID, siteID string) ([]Span, error) { return nucleus.Query[Span](ctx, q.db.SQL(), `SELECT trace_id, span_id, parent_span_id, service_name, operation_name, span_kind, diff --git a/internal/tracing/types.go b/internal/tracing/types.go index 4efeffef..2fe6bdb9 100644 --- a/internal/tracing/types.go +++ b/internal/tracing/types.go @@ -1,7 +1,12 @@ package tracing import ( + "encoding/base64" + "encoding/hex" + "encoding/json" "fmt" + "math" + "strconv" "strings" ) @@ -50,16 +55,27 @@ type InstrumentationScope struct { } type OTLPSpan struct { - TraceID string `json:"traceId"` - SpanID string `json:"spanId"` - ParentSpanID string `json:"parentSpanId"` - Name string `json:"name"` - Kind int `json:"kind"` - StartTimeUnixNano string `json:"startTimeUnixNano"` - EndTimeUnixNano string `json:"endTimeUnixNano"` - Attributes []KeyValue `json:"attributes"` - Status SpanStatus `json:"status"` - Events []SpanEvent `json:"events"` + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + ParentSpanID string `json:"parentSpanId"` + Name string `json:"name"` + Kind int `json:"kind"` + StartTimeUnixNano string `json:"startTimeUnixNano"` + EndTimeUnixNano string `json:"endTimeUnixNano"` + Attributes []KeyValue `json:"attributes"` + Status SpanStatus `json:"status"` + Events []SpanEvent `json:"events"` + Links []SpanLink `json:"links"` +} + +// SpanLink is an OTLP span link: a causal reference from this span to a span +// in the same or another trace (batch consumers, fan-in, retries). +type SpanLink struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + TraceState string `json:"traceState"` + Attributes []KeyValue `json:"attributes"` + DroppedAttributesCount int `json:"droppedAttributesCount"` } type SpanStatus struct { @@ -68,21 +84,284 @@ type SpanStatus struct { } type SpanEvent struct { - Name string `json:"name"` - TimeUnixNano string `json:"timeUnixNano"` - Attributes []KeyValue `json:"attributes"` + Name string `json:"name"` + TimeUnixNano string `json:"timeUnixNano"` + Attributes []KeyValue `json:"attributes"` } type KeyValue struct { - Key string `json:"key"` - Value AnyValue `json:"value"` + Key string `json:"key"` + Value AnyValue `json:"value"` } +// valueKind records WHICH AnyValue field the producer actually set. The four +// scalar fields cannot say that on their own: false, 0, 0.0 and "" are the Go +// zero values, so a presence check on the value drops exactly the attributes +// (http.error=false, retry.count=0) an operator most wants to filter on. +type valueKind uint8 + +const ( + kindUnset valueKind = iota // zero AnyValue, or built by a literal that predates kind + kindString + kindInt + kindBool + kindDouble +) + +// AnyValue is an OTLP attribute value. Decoders (UnmarshalJSON, the protobuf +// translator) record which field was present in an unexported kind tag, so +// AttrsToMap can keep false/0/0.0/"". A literal built outside the decoders +// (the seeder) has no tag and falls back to non-zero detection. +// +// OTLP array, kvlist and bytes values are folded into StringValue at decode +// time as deterministic text (compact JSON with sorted keys; lowercase hex for +// bytes) - there is no richer representation in the string-map storage. type AnyValue struct { - StringValue string `json:"stringValue,omitempty"` + StringValue string `json:"stringValue,omitempty"` IntValue jsonInt `json:"intValue,omitempty"` - BoolValue bool `json:"boolValue,omitempty"` + BoolValue bool `json:"boolValue,omitempty"` DoubleValue float64 `json:"doubleValue,omitempty"` + + kind valueKind +} + +func stringAny(s string) AnyValue { return AnyValue{StringValue: s, kind: kindString} } +func intAny(s string) AnyValue { return AnyValue{IntValue: jsonInt(s), kind: kindInt} } +func boolAny(b bool) AnyValue { return AnyValue{BoolValue: b, kind: kindBool} } +func doubleAny(f float64) AnyValue { return AnyValue{DoubleValue: f, kind: kindDouble} } + +// maxAnyDepth bounds array/kvlist nesting when folding to text; deeper levels +// are replaced by a marker so a hostile payload cannot drive the recursion. +const maxAnyDepth = 16 + +// anyWire is the OTLP/JSON shape with pointer/raw fields so presence survives. +type anyWire struct { + StringValue *string `json:"stringValue"` + IntValue *jsonInt `json:"intValue"` + BoolValue *bool `json:"boolValue"` + DoubleValue json.RawMessage `json:"doubleValue"` + ArrayValue json.RawMessage `json:"arrayValue"` + KvlistValue json.RawMessage `json:"kvlistValue"` + BytesValue *string `json:"bytesValue"` +} + +func (a *AnyValue) UnmarshalJSON(b []byte) error { + return a.unmarshalDepth(b, 0) +} + +func (a *AnyValue) unmarshalDepth(b []byte, depth int) error { + *a = AnyValue{} + if s := strings.TrimSpace(string(b)); s == "null" || s == "" { + return nil + } + var w anyWire + if err := json.Unmarshal(b, &w); err != nil { + return err + } + switch { + case w.StringValue != nil: + *a = stringAny(*w.StringValue) + case w.IntValue != nil: + *a = intAny(string(*w.IntValue)) + case w.BoolValue != nil: + *a = boolAny(*w.BoolValue) + case isPresent(w.DoubleValue): + f, err := parseJSONDouble(w.DoubleValue) + if err != nil { + return err + } + *a = doubleAny(f) + case isPresent(w.ArrayValue) || isPresent(w.KvlistValue): + n, err := containerNative(w, depth) + if err != nil { + return err + } + *a = stringAny(renderNative(n)) + case w.BytesValue != nil: + // OTLP/JSON bytes are base64; the protobuf path renders hex, so + // normalise to hex to keep both wire formats byte-identical. + raw, err := base64.StdEncoding.DecodeString(*w.BytesValue) + if err != nil { + *a = stringAny(*w.BytesValue) + } else { + *a = stringAny(hex.EncodeToString(raw)) + } + } + return nil +} + +// parseJSONDouble accepts a JSON number or the protobuf-JSON string forms +// ("1.5", "NaN", "Infinity", "-Infinity"). +func parseJSONDouble(raw json.RawMessage) (float64, error) { + s := strings.Trim(strings.TrimSpace(string(raw)), `"`) + switch s { + case "NaN": + return math.NaN(), nil + case "Infinity": + return math.Inf(1), nil + case "-Infinity": + return math.Inf(-1), nil + } + return strconv.ParseFloat(s, 64) +} + +func isPresent(raw json.RawMessage) bool { + return len(raw) > 0 && string(raw) != "null" +} + +// containerNative converts an OTLP/JSON array or kvlist into nested plain Go +// values (slices / maps), so nesting renders as real JSON rather than as +// escaped strings inside strings. +func containerNative(w anyWire, depth int) (any, error) { + if isPresent(w.ArrayValue) { + var arr struct { + Values []json.RawMessage `json:"values"` + } + if err := json.Unmarshal(w.ArrayValue, &arr); err != nil { + return nil, err + } + list := make([]any, 0, len(arr.Values)) + for _, raw := range arr.Values { + list = append(list, jsonNative(raw, depth+1)) + } + return list, nil + } + var kv struct { + Values []struct { + Key string `json:"key"` + Value json.RawMessage `json:"value"` + } `json:"values"` + } + if err := json.Unmarshal(w.KvlistValue, &kv); err != nil { + return nil, err + } + m := make(map[string]any, len(kv.Values)) + for _, e := range kv.Values { + m[e.Key] = jsonNative(e.Value, depth+1) + } + return m, nil +} + +// jsonNative converts a nested OTLP/JSON AnyValue into a plain Go value for +// deterministic rendering. +func jsonNative(raw json.RawMessage, depth int) any { + if depth > maxAnyDepth { + return "..." + } + var w anyWire + if s := strings.TrimSpace(string(raw)); s == "null" || s == "" || json.Unmarshal(raw, &w) != nil { + return nil + } + if isPresent(w.ArrayValue) || isPresent(w.KvlistValue) { + n, err := containerNative(w, depth) + if err != nil { + return nil + } + return n + } + var v AnyValue + if err := v.unmarshalDepth(raw, depth); err != nil { + return nil + } + return v.native() +} + +// native returns the typed Go value of a scalar AnyValue (nested containers +// are already folded to text and therefore come back as strings). +func (a AnyValue) native() any { + switch a.effectiveKind() { + case kindString: + return a.StringValue + case kindInt: + if _, err := strconv.ParseInt(string(a.IntValue), 10, 64); err == nil { + return json.Number(a.IntValue) + } + return string(a.IntValue) + case kindBool: + return a.BoolValue + case kindDouble: + if math.IsNaN(a.DoubleValue) || math.IsInf(a.DoubleValue, 0) { + return formatDouble(a.DoubleValue) + } + return json.Number(formatDouble(a.DoubleValue)) + } + return nil +} + +// renderNative is compact JSON; encoding/json sorts map keys, so equal inputs +// render byte-identical regardless of wire format or producer key order. +func renderNative(v any) string { + out, err := json.Marshal(v) + if err != nil { + return "" + } + return string(out) +} + +func formatDouble(f float64) string { return fmt.Sprintf("%g", f) } + +// effectiveKind is the recorded kind, or - for a literal with no tag - the +// first non-zero field (the pre-tag behaviour). +func (a AnyValue) effectiveKind() valueKind { + if a.kind != kindUnset { + return a.kind + } + switch { + case a.StringValue != "": + return kindString + case a.IntValue != "": + return kindInt + case a.BoolValue: + return kindBool + case a.DoubleValue != 0: + return kindDouble + } + return kindUnset +} + +// text renders a value for the string-map attribute store; ok is false when +// the producer set no value at all. +func (a AnyValue) text() (string, bool) { + switch a.effectiveKind() { + case kindString: + return a.StringValue, true + case kindInt: + return string(a.IntValue), true + case kindBool: + return strconv.FormatBool(a.BoolValue), true + case kindDouble: + return formatDouble(a.DoubleValue), true + } + return "", false +} + +// MarshalJSON emits the field the producer set, including false/0/0.0/"", +// so span events (stored as marshalled JSON) keep those values too. +func (a AnyValue) MarshalJSON() ([]byte, error) { + switch a.effectiveKind() { + case kindString: + return json.Marshal(struct { + V string `json:"stringValue"` + }{a.StringValue}) + case kindInt: + return json.Marshal(struct { + V jsonInt `json:"intValue"` + }{a.IntValue}) + case kindBool: + return json.Marshal(struct { + V bool `json:"boolValue"` + }{a.BoolValue}) + case kindDouble: + if math.IsNaN(a.DoubleValue) || math.IsInf(a.DoubleValue, 0) { + return json.Marshal(struct { + V string `json:"doubleValue"` + }{formatDouble(a.DoubleValue)}) + } + return json.Marshal(struct { + V float64 `json:"doubleValue"` + }{a.DoubleValue}) + } + return []byte("{}"), nil } // SpanKind maps OTLP integer to string. @@ -130,21 +409,16 @@ func ExtractServiceName(attrs []KeyValue) string { } // AttrsToMap converts OTLP KeyValue slice to a string map for JSONB storage. +// A key whose value the producer set is always kept - including false, 0, +// 0.0 and the empty string - and only a value with no field set is skipped. func AttrsToMap(attrs []KeyValue) map[string]string { if len(attrs) == 0 { return nil } m := make(map[string]string, len(attrs)) for _, kv := range attrs { - switch { - case kv.Value.StringValue != "": - m[kv.Key] = kv.Value.StringValue - case kv.Value.IntValue != "": - m[kv.Key] = string(kv.Value.IntValue) - case kv.Value.BoolValue: - m[kv.Key] = "true" - case kv.Value.DoubleValue != 0: - m[kv.Key] = fmt.Sprintf("%g", kv.Value.DoubleValue) + if v, ok := kv.Value.text(); ok { + m[kv.Key] = v } } return m From 0871cdd8f1ef9b78f431273d1595f7a4578b9149 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:31:08 +0000 Subject: [PATCH 09/65] chore: retain span_links alongside spans (14 days) UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- internal/jobs/retention.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/jobs/retention.go b/internal/jobs/retention.go index 9c571429..bce3c07b 100644 --- a/internal/jobs/retention.go +++ b/internal/jobs/retention.go @@ -85,6 +85,7 @@ func DefaultPolicies(rawDays, hourlyDays, llmDays int) []RetentionPolicy { {Table: "logs", Column: "timestamp", Days: 30}, {Table: "llm_traces", Column: "timestamp", Days: llmDays}, {Table: "spans", Column: "start_time", Days: 14}, + {Table: "span_links", Column: "start_time", Days: 14}, {Table: "service_stats", Column: "ts_bucket", Days: 30}, {Table: "replay_sessions", Column: "start_time", Days: 14}, // O10: the evaluation ledger is a log, not a dedupe set - 14 days From 7f4aaf34628da5fd669c770e1355cfbee29e84b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:32:47 +0000 Subject: [PATCH 10/65] feat: deliver new-issue and regression events to site integrations IntegrationService.Fire had no caller outside the test/replay endpoints, so configured integrations never fired for real issues. IssueService now emits IssueEvents (first occurrence; resolved issue reopened, detected from the real status on a cache miss and from a resolved flag on the grouphash cache entry on a hit) through an IssueNotifier callback. internal/errors does not import internal/integrations; cmd/observe/integrations_wiring.go connects them. Delivery runs on a bounded in-memory queue (256, drop-oldest with counter and loud log), 4 workers, a per-delivery timeout, and a per-(integration, issue) cooldown of 15 minutes so a flapping issue cannot spam. Counters are surfaced at /healthz under issue_notifications. Each attempt is still recorded in integration_deliveries. Ingest never waits: Notify only enqueues and a notifier panic is recovered. Hardening on the shared fire path: transport errors are redacted (Slack webhook URL is the credential) before being logged or stored, Slack/Jira URLs are scheme-validated, and the email channel now resolves smtp_host once, refuses non-public addresses and dials the pinned IP (SSRF and DNS rebinding). Alert title text is stripped of control characters. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/integrations_wiring.go | 40 ++ cmd/observe/main.go | 13 + internal/errors/issues.go | 51 ++ internal/errors/notify.go | 64 +++ internal/errors/notify_test.go | 53 ++ internal/integrations/integrations.go | 81 ++- internal/integrations/issue_dispatch.go | 362 +++++++++++++ internal/integrations/issue_dispatch_test.go | 505 +++++++++++++++++++ 8 files changed, 1166 insertions(+), 3 deletions(-) create mode 100644 cmd/observe/integrations_wiring.go create mode 100644 internal/errors/notify.go create mode 100644 internal/errors/notify_test.go create mode 100644 internal/integrations/issue_dispatch.go create mode 100644 internal/integrations/issue_dispatch_test.go diff --git a/cmd/observe/integrations_wiring.go b/cmd/observe/integrations_wiring.go new file mode 100644 index 00000000..8a0675bf --- /dev/null +++ b/cmd/observe/integrations_wiring.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "time" + + obserrors "github.com/useteploy/teploy-observe/internal/errors" + "github.com/useteploy/teploy-observe/internal/integrations" +) + +// wireIssueNotifications connects issue lifecycle events (a new issue's first +// occurrence, a resolved issue regressing) to the site's configured +// integrations. internal/errors only knows the IssueNotifier callback type; +// the translation to the integrations package happens here so neither package +// imports the other. The callback only enqueues (bounded, drop-oldest), so +// error ingest never waits on a notification. +func wireIssueNotifications(issueSvc *obserrors.IssueService, integrationSvc *integrations.IntegrationService, publicURL string) *integrations.IssueDispatcher { + d := integrationSvc.NewIssueDispatcher(publicURL) + issueSvc.SetNotifier(func(_ context.Context, ev obserrors.IssueEvent) { + d.Notify(integrations.IssueEvent{ + Kind: ev.Kind, + SiteID: ev.SiteID, + IssueID: ev.IssueID, + Title: ev.Title, + Culprit: ev.Culprit, + Level: ev.Level, + Release: ev.Release, + EventCount: ev.EventCount, + }) + }) + return d +} + +// shutdownIssueNotifications drains queued notifications on shutdown, bounded +// so a hung receiver cannot hold the process open. +func shutdownIssueNotifications(d *integrations.IssueDispatcher) error { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + return d.Shutdown(ctx) +} diff --git a/cmd/observe/main.go b/cmd/observe/main.go index e0c7d8e8..befbd547 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -434,6 +434,9 @@ func main() { // Feature expansion services reportSvc := reports.NewReportService(db, logger) integrationSvc := integrations.NewIntegrationService(db, logger) + // New-issue and regression events fire the site's integrations. Wired + // after the demo seed above so seeded issues do not page anyone. + issueNotifier := wireIssueNotifications(issueSvc, integrationSvc, cfg.PublicURL) feedbackSvc := feedback.NewFeedbackService(db) viewSvc := views.NewViewService(db) explorerSvc := explorer.NewExplorerService(db) @@ -654,6 +657,12 @@ func main() { }), // R22 (round 4): webhook delivery drains on shutdown instead of // dying with the process mid-alert. + neutron.WithLifecycle(neutron.LifecycleHook{ + Name: "issue-notifications", + OnStop: func(ctx context.Context) error { + return shutdownIssueNotifications(issueNotifier) + }, + }), neutron.WithLifecycle(neutron.LifecycleHook{ Name: "webhook-delivery", OnStop: func(ctx context.Context) error { @@ -1708,6 +1717,10 @@ func main() { // Source-map resolution outcomes per frame (hit / no map for the // release+file / map without a covering mapping / read error). health["sourcemaps"] = sourcemaps.Stats() + // New-issue / regression integration notifications: queue depth, + // delivered / failed / timed-out, and the overflow and cooldown + // suppression counters. + health["issue_notifications"] = issueNotifier.Stats() // O08: flag-evaluation condition counters — how many evaluations // could not read their config (unavailable) and how many flags // were quarantined from evaluation by invalid stored config. diff --git a/internal/errors/issues.go b/internal/errors/issues.go index 44d6af06..df6a6929 100644 --- a/internal/errors/issues.go +++ b/internal/errors/issues.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "strconv" + "sync/atomic" "time" "github.com/neutron-build/neutron/go/nucleus" @@ -48,6 +49,8 @@ const issueSelectCols = `issue_id, tenant_id, site_id, group_hash, title, culpri // IssueService manages error grouping, issue creation, and the grouphash-to-issue KV cache. type IssueService struct { db *nucleus.Client + // notifier receives new-issue and regression events (see notify.go). + notifier atomic.Pointer[IssueNotifier] } func NewIssueService(db *nucleus.Client) *IssueService { @@ -172,6 +175,10 @@ func kvCacheKey(siteID, groupHash string) string { type cachedIssue struct { IssueID string `json:"id"` EventCount int64 `json:"ec"` + // Resolved marks an issue UpdateStatus resolved, so the cache-hit path + // can recognise the next event as a regression without a SQL read. The + // cache-miss path reads the real status and does not depend on it. + Resolved bool `json:"rs,omitempty"` } // ResolveIssue looks up or creates an issue for the given grouphash. @@ -188,6 +195,12 @@ func (s *IssueService) ResolveIssue(ctx context.Context, siteID, groupHash, titl newCount := ci.EventCount + 1 _ = s.bumpIssue(ctx, ci.IssueID, siteID, ts, newCount) ci.EventCount = newCount + if ci.Resolved { + // bumpIssue reopened it (resolved -> open). + ci.Resolved = false + s.notify(ctx, IssueEvent{Kind: IssueEventRegression, SiteID: siteID, IssueID: ci.IssueID, + Title: title, Culprit: culprit, Level: level, Release: release, EventCount: newCount}) + } if raw, err := json.Marshal(ci); err == nil { _ = kv.Set(ctx, cacheKey, raw) } @@ -204,6 +217,10 @@ func (s *IssueService) ResolveIssue(ctx context.Context, siteID, groupHash, titl if raw, err := json.Marshal(ci); err == nil { _ = kv.Set(ctx, cacheKey, raw) } + if existing.Status == "resolved" { + s.notify(ctx, IssueEvent{Kind: IssueEventRegression, SiteID: siteID, IssueID: existing.IssueID, + Title: existing.Title, Culprit: existing.Culprit, Level: existing.Level, Release: release, EventCount: newCount}) + } return existing.IssueID, nil } @@ -229,9 +246,40 @@ func (s *IssueService) ResolveIssue(ctx context.Context, siteID, groupHash, titl _ = kv.Set(ctx, cacheKey, raw) } + s.notify(ctx, IssueEvent{Kind: IssueEventNew, SiteID: siteID, IssueID: issueID, + Title: title, Culprit: culprit, Level: level, Release: release, EventCount: 1}) return issueID, nil } +// markCachedResolved records on the grouphash cache entry (when one exists) +// whether the issue is now resolved. Best-effort: a miss only costs the +// cache-hit path a regression notification until the entry is rebuilt from +// the database, which reads the real status. +func (s *IssueService) markCachedResolved(ctx context.Context, issueID, siteID string, resolved bool) { + rows, err := nucleus.Query[issueScan](ctx, s.db.SQL(), + `SELECT `+issueSelectCols+` + FROM `+issuesLatest("issue_id = $1 AND site_id = $2"), + issueID, siteID, + ) + if err != nil || len(rows) == 0 { + return + } + kv := s.db.KV() + key := kvCacheKey(siteID, rows[0].GroupHash) + data, err := kv.Get(ctx, key) + if err != nil || data == nil { + return + } + var ci cachedIssue + if json.Unmarshal(data, &ci) != nil || ci.IssueID != issueID || ci.Resolved == resolved { + return + } + ci.Resolved = resolved + if raw, err := json.Marshal(ci); err == nil { + _ = kv.Set(ctx, key, raw) + } +} + func (s *IssueService) findIssueByHash(ctx context.Context, siteID, groupHash string) (*Issue, error) { rows, err := nucleus.Query[issueScan](ctx, s.db.SQL(), `SELECT `+issueSelectCols+` @@ -317,6 +365,9 @@ func (s *IssueService) UpdateStatus(ctx context.Context, issueID, siteID, status FROM `+issuesLatest("issue_id = $1 AND site_id = $2"), issueID, siteID, status, now, untilStr, ) + if err == nil { + s.markCachedResolved(ctx, issueID, siteID, status == "resolved") + } return err } diff --git a/internal/errors/notify.go b/internal/errors/notify.go new file mode 100644 index 00000000..c9ad11d5 --- /dev/null +++ b/internal/errors/notify.go @@ -0,0 +1,64 @@ +package errors + +import ( + "context" + "log/slog" + "time" +) + +// Issue lifecycle events a notifier is told about. +const ( + // IssueEventNew is the first occurrence of a grouped error. + IssueEventNew = "new_issue" + // IssueEventRegression is a resolved (or snoozed) issue receiving a new + // event and reopening. + IssueEventRegression = "regression" +) + +// IssueEvent describes one issue lifecycle transition worth telling a human +// about. +type IssueEvent struct { + Kind string + SiteID string + IssueID string + Title string + Culprit string + Level string + Release string + EventCount int64 + At time.Time +} + +// IssueNotifier receives issue lifecycle events. It is called inline on the +// ingest path, so an implementation MUST NOT block or do I/O: enqueue and +// return. This package deliberately knows nothing about who consumes the +// events (cmd/observe wires the integrations dispatcher in), and a panic in a +// notifier is recovered here so it can never fail ingest. +type IssueNotifier func(ctx context.Context, ev IssueEvent) + +// SetNotifier installs (or, with nil, removes) the issue event notifier. +// Safe to call while ingest is running. +func (s *IssueService) SetNotifier(n IssueNotifier) { + if n == nil { + s.notifier.Store(nil) + return + } + s.notifier.Store(&n) +} + +func (s *IssueService) notify(ctx context.Context, ev IssueEvent) { + p := s.notifier.Load() + if p == nil { + return + } + defer func() { + if r := recover(); r != nil { + slog.Error("errors: issue notifier panicked; event dropped, ingest unaffected", + "kind", ev.Kind, "site", ev.SiteID, "issue", ev.IssueID, "panic", r) + } + }() + if ev.At.IsZero() { + ev.At = time.Now().UTC() + } + (*p)(ctx, ev) +} diff --git a/internal/errors/notify_test.go b/internal/errors/notify_test.go new file mode 100644 index 00000000..20cd14e6 --- /dev/null +++ b/internal/errors/notify_test.go @@ -0,0 +1,53 @@ +package errors + +import ( + "context" + "encoding/json" + "testing" +) + +func TestNotify_NoNotifierIsNoop(t *testing.T) { + s := &IssueService{} + s.notify(context.Background(), IssueEvent{Kind: IssueEventNew}) // must not panic +} + +func TestNotify_DeliversEventWithTimestamp(t *testing.T) { + s := &IssueService{} + var got IssueEvent + s.SetNotifier(func(_ context.Context, ev IssueEvent) { got = ev }) + s.notify(context.Background(), IssueEvent{Kind: IssueEventRegression, SiteID: "s", IssueID: "i"}) + if got.Kind != IssueEventRegression || got.IssueID != "i" || got.At.IsZero() { + t.Fatalf("event: %+v", got) + } + s.SetNotifier(nil) + got = IssueEvent{} + s.notify(context.Background(), IssueEvent{Kind: IssueEventNew}) + if got.Kind != "" { + t.Fatal("removed notifier still called") + } +} + +// A misbehaving notifier must never fail ingest. +func TestNotify_PanicIsContained(t *testing.T) { + s := &IssueService{} + s.SetNotifier(func(context.Context, IssueEvent) { panic("boom") }) + s.notify(context.Background(), IssueEvent{Kind: IssueEventNew}) +} + +// The cache entry stays backward compatible: entries written before the +// resolved flag existed decode as not-resolved, and an unset flag is not +// serialised. +func TestCachedIssue_ResolvedFlagCompat(t *testing.T) { + var old cachedIssue + if err := json.Unmarshal([]byte(`{"id":"i1","ec":4}`), &old); err != nil || old.Resolved { + t.Fatalf("old entry: %+v %v", old, err) + } + raw, _ := json.Marshal(cachedIssue{IssueID: "i1", EventCount: 4}) + if string(raw) != `{"id":"i1","ec":4}` { + t.Fatalf("unset flag serialised: %s", raw) + } + raw, _ = json.Marshal(cachedIssue{IssueID: "i1", EventCount: 4, Resolved: true}) + if string(raw) != `{"id":"i1","ec":4,"rs":true}` { + t.Fatalf("flag: %s", raw) + } +} diff --git a/internal/integrations/integrations.go b/internal/integrations/integrations.go index cc9807d9..aed1ffd9 100644 --- a/internal/integrations/integrations.go +++ b/internal/integrations/integrations.go @@ -6,12 +6,15 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "log/slog" + "net" "net/http" "net/mail" "net/smtp" + "net/url" "strconv" "strings" "time" @@ -145,8 +148,10 @@ func (s *IntegrationService) fireAndRecord(ctx context.Context, i Integration, p errMsg := "" if err != nil { status = "failed" - errMsg = err.Error() - s.logger.Error("integration fire failed", "type", i.IntType, "name", i.Name, "err", err) + // Transport errors embed the request URL, and for Slack the URL is + // the credential. Redact before the message is logged or stored. + errMsg = redactDeliveryError(err, i.Config) + s.logger.Error("integration fire failed", "type", i.IntType, "name", i.Name, "err", errMsg) } body, _ := json.Marshal(payload) _, recErr := s.db.SQL().Exec(ctx, @@ -163,6 +168,34 @@ func (s *IntegrationService) fireAndRecord(ctx context.Context, i Integration, p return err } +// secretConfigKeys are the config fields that carry credentials; their values +// are scrubbed from any error text. +var secretConfigKeys = []string{"webhook_url", "api_token", "token", "routing_key", "password"} + +// redactDeliveryError renders err without credentials: a *url.Error is +// reduced to scheme://host (path and query, where webhook tokens live, are +// dropped), and any secret config value that still appears is masked. +func redactDeliveryError(err error, configJSON string) string { + msg := err.Error() + var ue *url.Error + if errors.As(err, &ue) { + dest := "[url]" + if u, perr := url.Parse(ue.URL); perr == nil && u.Host != "" { + dest = u.Scheme + "://" + u.Host + "/[redacted]" + } + msg = fmt.Sprintf("%s %s: %v", ue.Op, dest, ue.Err) + } + var cfg map[string]any + if json.Unmarshal([]byte(configJSON), &cfg) == nil { + for _, k := range secretConfigKeys { + if v, ok := cfg[k].(string); ok && len(v) >= 4 { + msg = strings.ReplaceAll(msg, v, "[redacted]") + } + } + } + return msg +} + func boolStr(b bool) string { if b { return "true" @@ -298,6 +331,9 @@ func (s *IntegrationService) fireJira(configJSON string, p AlertPayload) error { if cfg.BaseURL == "" || cfg.Project == "" { return fmt.Errorf("jira: missing base_url or project") } + if netsafe.ValidateURL(cfg.BaseURL) != nil { + return fmt.Errorf("jira: base_url must be an http(s) URL without userinfo") + } body, _ := json.Marshal(map[string]any{ "fields": map[string]any{ "project": map[string]string{"key": cfg.Project}, @@ -385,9 +421,43 @@ func (s *IntegrationService) fireEmail(configJSON string, p AlertPayload) error } msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: [Observe Alert] %s\r\n\r\n%s\r\n\r\nMetric: %s = %s (threshold: %s)", cfg.From, strings.Join(recipients, ", "), p.Title, p.Message, p.Metric, p.Value, p.Threshold) + // SSRF guard: resolve once, refuse non-public addresses, and dial the + // resolved IP so the name cannot re-resolve to a private address between + // the check and the connect (DNS rebinding). The host name is still what + // TLS and AUTH are bound to. + dialIP, err := publicSMTPAddr(cfg.SMTPHost, lookupIP) + if err != nil { + return err + } auth := smtp.PlainAuth("", cfg.Username, cfg.Password, cfg.SMTPHost) // Timeout-bounded send so a hung relay can't block the alert dispatch path. - return mailx.SendMail(cfg.SMTPHost+":"+port, cfg.SMTPHost, auth, cfg.From, recipients, []byte(msg), 0) + return mailx.SendMail(net.JoinHostPort(dialIP, port), cfg.SMTPHost, auth, cfg.From, recipients, []byte(msg), 0) +} + +// lookupIP is the resolver used by the SMTP guard (replaceable in tests). +var lookupIP = func(ctx context.Context, host string) ([]net.IPAddr, error) { + return net.DefaultResolver.LookupIPAddr(ctx, host) +} + +// publicSMTPAddr resolves host and returns one address to dial. Every +// resolved address must be public: a name that resolves to ANY blocked +// address is refused, so a mixed answer cannot be used to steer the dial. +func publicSMTPAddr(host string, lookup func(context.Context, string) ([]net.IPAddr, error)) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + addrs, err := lookup(ctx, host) + if err != nil { + return "", fmt.Errorf("email: resolving smtp_host: %w", err) + } + if len(addrs) == 0 { + return "", fmt.Errorf("email: smtp_host did not resolve") + } + for _, a := range addrs { + if netsafe.IsBlockedIP(a.IP) { + return "", fmt.Errorf("email: smtp_host resolves to a non-public address; refusing to connect") + } + } + return addrs[0].IP.String(), nil } // parseRecipients splits and validates a comma-separated recipient list, @@ -422,6 +492,11 @@ func (s *IntegrationService) fireSlack(configJSON string, p AlertPayload) error if cfg.WebhookURL == "" { return fmt.Errorf("slack: missing webhook_url") } + if netsafe.ValidateURL(cfg.WebhookURL) != nil { + // Do not echo the URL or the parse error (which quotes it): the URL + // is the credential. + return fmt.Errorf("slack: webhook_url must be an http(s) URL without userinfo") + } text := fmt.Sprintf("*%s*\n%s\nMetric: %s = %s (threshold: %s)", p.Title, p.Message, p.Metric, p.Value, p.Threshold) body, _ := json.Marshal(map[string]string{"text": text}) resp, err := s.client.Post(cfg.WebhookURL, "application/json", bytes.NewReader(body)) diff --git a/internal/integrations/issue_dispatch.go b/internal/integrations/issue_dispatch.go new file mode 100644 index 00000000..f8b6d37f --- /dev/null +++ b/internal/integrations/issue_dispatch.go @@ -0,0 +1,362 @@ +package integrations + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Issue event kinds the dispatcher accepts (mirrors internal/errors; kept as +// plain strings so this package does not import the errors package). +const ( + IssueKindNew = "new_issue" + IssueKindRegression = "regression" +) + +// Dispatch bounds. Delivery of issue events is advisory and must never push +// back on error ingest, so everything here is bounded and lossy-with-a-counter +// rather than blocking. +const ( + // IssueNotifyQueueSize bounds events waiting for a worker. A full queue + // drops the OLDEST event (loudly, counted) so the newest issue is the one + // a human hears about. + IssueNotifyQueueSize = 256 + // IssueNotifyWorkers is the number of concurrent delivery workers. + IssueNotifyWorkers = 4 + // IssueNotifyCooldown is the minimum gap between notifications for the + // SAME issue to the SAME integration, whatever the event kind. It is what + // stops a flapping issue (resolve, regress, resolve, regress) from + // spamming a channel. + IssueNotifyCooldown = 15 * time.Minute + // IssueNotifyDeliveryTimeout bounds one integration delivery end to end + // (the HTTP client and SMTP dialer carry their own shorter timeouts; this + // is the backstop that frees the worker regardless). + IssueNotifyDeliveryTimeout = 30 * time.Second + + issueNotifyListTimeout = 10 * time.Second + issueNotifyCooldownCap = 10000 + issueNotifyMaxTitleChars = 200 + issueNotifyMaxBodyChars = 1000 +) + +// IssueEvent is an issue lifecycle event to deliver to a site's integrations. +type IssueEvent struct { + Kind string + SiteID string + IssueID string + Title string + Culprit string + Level string + Release string + EventCount int64 +} + +// IssueDispatcherStats is the counter block for /healthz. +type IssueDispatcherStats struct { + Enqueued int64 `json:"enqueued"` + Pending int `json:"pending"` + Delivered int64 `json:"delivered"` + Failed int64 `json:"failed"` + TimedOut int64 `json:"timed_out"` + DroppedOverflow int64 `json:"dropped_overflow"` + Suppressed int64 `json:"suppressed_cooldown"` + ListErrors int64 `json:"list_errors"` +} + +// IssueDispatcher delivers new-issue and regression events to a site's +// integrations on a bounded in-memory queue with a bounded worker set. +// +// Why in memory rather than the notification outbox: that outbox is a durable +// table keyed on alert rules and webhook targets, so reusing it would need a +// new intent kind, a schema change and edits to internal/platform. Issue +// events are advisory, the issue itself is already durable, and each attempt +// is still recorded in integration_deliveries. The trade-off is that events +// queued at a crash are lost; the counters make overflow visible. +type IssueDispatcher struct { + list func(ctx context.Context, siteID string) ([]Integration, error) + deliver func(ctx context.Context, i Integration, p AlertPayload) error + logger *slog.Logger + baseURL string + now func() time.Time + // deliverTimeout bounds one delivery; tests shorten it. + deliverTimeout time.Duration + + mu sync.Mutex + q []IssueEvent + stopped bool + cool map[string]time.Time + sem chan struct{} + wg sync.WaitGroup + + enqueued, delivered, failed, timedOut atomic.Int64 + dropped, suppressed, listErrors atomic.Int64 +} + +// NewIssueDispatcher builds a dispatcher backed by this service. publicURL +// (may be empty) is the instance base URL used for the link in a payload. +func (s *IntegrationService) NewIssueDispatcher(publicURL string) *IssueDispatcher { + return newIssueDispatcher(s.List, func(ctx context.Context, i Integration, p AlertPayload) error { + return s.fireAndRecord(ctx, i, p, false, false) + }, s.logger, publicURL) +} + +func newIssueDispatcher( + list func(ctx context.Context, siteID string) ([]Integration, error), + deliver func(ctx context.Context, i Integration, p AlertPayload) error, + logger *slog.Logger, publicURL string, +) *IssueDispatcher { + if logger == nil { + logger = slog.Default() + } + return &IssueDispatcher{ + list: list, deliver: deliver, logger: logger, + baseURL: strings.TrimRight(publicURL, "/"), + now: time.Now, + deliverTimeout: IssueNotifyDeliveryTimeout, + cool: make(map[string]time.Time), + sem: make(chan struct{}, IssueNotifyWorkers), + } +} + +// Notify enqueues an event. It never blocks and never fails the caller. +func (d *IssueDispatcher) Notify(ev IssueEvent) { + if ev.SiteID == "" || ev.IssueID == "" { + return + } + d.mu.Lock() + if d.stopped { + d.mu.Unlock() + return + } + if len(d.q) >= IssueNotifyQueueSize { + old := d.q[0] + d.q = d.q[1:] + d.dropped.Add(1) + d.logger.Error("issue notification queue overflow - dropped oldest queued event", + "kind", old.Kind, "site", old.SiteID, "issue", old.IssueID) + } + d.q = append(d.q, ev) + d.enqueued.Add(1) + // The worker slot is taken (and wg incremented) under the lock so + // Shutdown cannot miss a worker, and released under the lock by the + // worker that finds the queue empty so an event is never stranded + // between "queue empty" and "slot released". + select { + case d.sem <- struct{}{}: + d.wg.Add(1) + go d.worker() + default: + // Enough workers are already draining; one will pick this up. + } + d.mu.Unlock() +} + +func (d *IssueDispatcher) worker() { + defer d.wg.Done() + for { + d.mu.Lock() + if len(d.q) == 0 { + <-d.sem + d.mu.Unlock() + return + } + ev := d.q[0] + d.q = d.q[1:] + d.mu.Unlock() + d.process(ev) + } +} + +func (d *IssueDispatcher) process(ev IssueEvent) { + lctx, cancel := context.WithTimeout(context.Background(), issueNotifyListTimeout) + intgs, err := d.list(lctx, ev.SiteID) + cancel() + if err != nil { + d.listErrors.Add(1) + d.logger.Warn("issue notification: listing integrations failed", + "site", ev.SiteID, "issue", ev.IssueID, "err", err) + return + } + payload := d.payload(ev) + for _, intg := range intgs { + if !d.claim(intg.IntegrationID, ev.IssueID) { + d.suppressed.Add(1) + continue + } + d.deliverOne(intg, payload) + } +} + +// claim applies the per-(integration, issue) cooldown. It claims at decision +// time, so a failing integration is also rate limited rather than retried on +// every flap. +func (d *IssueDispatcher) claim(integrationID, issueID string) bool { + key := integrationID + "|" + issueID + now := d.now() + d.mu.Lock() + defer d.mu.Unlock() + if last, ok := d.cool[key]; ok && now.Sub(last) < IssueNotifyCooldown { + return false + } + if len(d.cool) >= issueNotifyCooldownCap { + for k, t := range d.cool { + if now.Sub(t) >= IssueNotifyCooldown { + delete(d.cool, k) + } + } + if len(d.cool) >= issueNotifyCooldownCap { + // Everything is still hot: forget the entries closest to expiry + // rather than grow without bound. Worst case an old issue may + // notify again early. + for k := range d.cool { + delete(d.cool, k) + if len(d.cool) < issueNotifyCooldownCap/2 { + break + } + } + } + } + d.cool[key] = now + return true +} + +func (d *IssueDispatcher) deliverOne(i Integration, p AlertPayload) { + ctx, cancel := context.WithTimeout(context.Background(), d.deliverTimeout) + defer cancel() + done := make(chan error, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic during delivery: %v", r) + } + }() + done <- d.deliver(ctx, i, p) + }() + select { + case err := <-done: + if err != nil { + // The error text is already redacted and recorded by the + // delivery path; keep this line free of config and URLs. + d.failed.Add(1) + d.logger.Warn("issue notification delivery failed", + "integration", i.IntegrationID, "type", i.IntType, "issue_event", p.RuleName) + return + } + d.delivered.Add(1) + case <-ctx.Done(): + d.timedOut.Add(1) + d.logger.Warn("issue notification delivery timed out", + "integration", i.IntegrationID, "type", i.IntType, "issue_event", p.RuleName) + } +} + +// payload renders an event as the AlertPayload every integration type +// already understands. +func (d *IssueDispatcher) payload(ev IssueEvent) AlertPayload { + label := "New issue" + if ev.Kind == IssueKindRegression { + label = "Regression" + } + title := clean(ev.Title, issueNotifyMaxTitleChars) + if title == "" { + title = "Error" + } + var msg strings.Builder + if ev.Kind == IssueKindRegression { + msg.WriteString("A resolved issue received a new event and was reopened.") + } else { + msg.WriteString("First occurrence of a new error.") + } + if c := clean(ev.Culprit, issueNotifyMaxBodyChars); c != "" { + msg.WriteString("\nCulprit: " + c) + } + if ev.Release != "" { + msg.WriteString("\nRelease: " + clean(ev.Release, 100)) + } + if ev.Level != "" { + msg.WriteString("\nLevel: " + clean(ev.Level, 30)) + } + link := "" + if d.baseURL != "" { + link = d.baseURL + "/errors" + } + return AlertPayload{ + Title: label + ": " + title, + Message: msg.String(), + Severity: severityFor(ev.Level), + SiteID: ev.SiteID, + RuleName: ev.Kind, + Metric: "error_events", + Value: strconv.FormatInt(ev.EventCount, 10), + URL: link, + } +} + +func severityFor(level string) string { + switch strings.ToLower(level) { + case "fatal": + return "critical" + case "warning", "warn": + return "warning" + case "info", "debug": + return "info" + default: + return "error" + } +} + +// clean makes attacker-influenced error text safe for headers and chat bodies: +// control characters (CR/LF would otherwise reach an SMTP Subject) become +// spaces, and the length is capped. +func clean(s string, max int) string { + s = strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, s) + s = strings.TrimSpace(s) + if r := []rune(s); len(r) > max { + s = string(r[:max]) + "..." + } + return s +} + +// Stats returns the dispatcher counters. +func (d *IssueDispatcher) Stats() IssueDispatcherStats { + d.mu.Lock() + pending := len(d.q) + d.mu.Unlock() + return IssueDispatcherStats{ + Enqueued: d.enqueued.Load(), + Pending: pending, + Delivered: d.delivered.Load(), + Failed: d.failed.Load(), + TimedOut: d.timedOut.Load(), + DroppedOverflow: d.dropped.Load(), + Suppressed: d.suppressed.Load(), + ListErrors: d.listErrors.Load(), + } +} + +// Shutdown stops admission and waits for in-flight and queued deliveries +// (each bounded by IssueNotifyDeliveryTimeout), or until ctx ends. +func (d *IssueDispatcher) Shutdown(ctx context.Context) error { + d.mu.Lock() + d.stopped = true + d.mu.Unlock() + done := make(chan struct{}) + go func() { d.wg.Wait(); close(done) }() + select { + case <-done: + return nil + case <-ctx.Done(): + return errors.New("issue notification shutdown: " + ctx.Err().Error()) + } +} diff --git a/internal/integrations/issue_dispatch_test.go b/internal/integrations/issue_dispatch_test.go new file mode 100644 index 00000000..7f6ba72b --- /dev/null +++ b/internal/integrations/issue_dispatch_test.go @@ -0,0 +1,505 @@ +package integrations + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/netsafe" +) + +// ---- helpers --------------------------------------------------------------- + +func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +type logBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (l *logBuf) Write(p []byte) (int, error) { l.mu.Lock(); defer l.mu.Unlock(); return l.b.Write(p) } +func (l *logBuf) String() string { l.mu.Lock(); defer l.mu.Unlock(); return l.b.String() } + +// loopbackAllow lets tests reach httptest servers; the production client uses +// no allowlist. +func loopbackAllow(t *testing.T) netsafe.Allow { + t.Helper() + a, err := netsafe.ParseAllow("127.0.0.0/8") + if err != nil { + t.Fatal(err) + } + return a +} + +func testSvc(t *testing.T, client *http.Client) *IntegrationService { + return &IntegrationService{logger: quietLogger(), client: client} +} + +// rewriteTransport sends every request to the test server regardless of host, +// for providers whose endpoint is hard-coded (GitHub, PagerDuty). +type rewriteTransport struct{ target *url.URL } + +func (r rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.URL.Scheme = r.target.Scheme + req.URL.Host = r.target.Host + return http.DefaultTransport.RoundTrip(req) +} + +type capture struct { + mu sync.Mutex + path string + body string + headers http.Header + user string +} + +func captureServer(t *testing.T, status int) (*httptest.Server, *capture) { + c := &capture{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + c.mu.Lock() + c.path, c.body, c.headers = r.URL.Path, string(b), r.Header.Clone() + c.user, _, _ = r.BasicAuth() + c.mu.Unlock() + w.WriteHeader(status) + })) + t.Cleanup(srv.Close) + return srv, c +} + +func sampleEvent(kind, issue string) IssueEvent { + return IssueEvent{Kind: kind, SiteID: "site-1", IssueID: issue, Title: "TypeError: x is undefined", + Culprit: "app.js in handleClick", Level: "error", Release: "1.2.3", EventCount: 1} +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// ---- per-kind delivery through the real fire path --------------------------- + +func TestIssueEventDelivery_PerIntegrationKind(t *testing.T) { + d := newIssueDispatcher(nil, nil, quietLogger(), "https://obs.example.com/") + payload := d.payload(sampleEvent(IssueKindNew, "i1")) + if !strings.HasPrefix(payload.Title, "New issue: ") || payload.URL != "https://obs.example.com/errors" { + t.Fatalf("payload: %+v", payload) + } + + t.Run("slack", func(t *testing.T) { + srv, c := captureServer(t, 200) + s := testSvc(t, netsafe.ClientWithAllow(5*time.Second, loopbackAllow(t))) + cfg := fmt.Sprintf(`{"webhook_url":%q}`, srv.URL+"/services/T0/B0/secret") + if err := s.fireOne(Integration{IntType: "slack", Config: cfg}, payload); err != nil { + t.Fatal(err) + } + if !strings.Contains(c.body, "New issue: TypeError") || c.path != "/services/T0/B0/secret" { + t.Fatalf("slack request: path=%q body=%s", c.path, c.body) + } + }) + + t.Run("jira", func(t *testing.T) { + srv, c := captureServer(t, 201) + s := testSvc(t, netsafe.ClientWithAllow(5*time.Second, loopbackAllow(t))) + cfg := fmt.Sprintf(`{"base_url":%q,"email":"a@b.c","api_token":"tok-123456","project":"OBS"}`, srv.URL) + if err := s.fireOne(Integration{IntType: "jira", Config: cfg}, payload); err != nil { + t.Fatal(err) + } + if c.path != "/rest/api/2/issue" || c.user != "a@b.c" || !strings.Contains(c.body, "New issue: TypeError") || !strings.Contains(c.body, `"OBS"`) { + t.Fatalf("jira request: path=%q user=%q body=%s", c.path, c.user, c.body) + } + }) + + t.Run("github", func(t *testing.T) { + srv, c := captureServer(t, 201) + u, _ := url.Parse(srv.URL) + s := testSvc(t, &http.Client{Transport: rewriteTransport{u}, Timeout: 5 * time.Second}) + cfg := `{"token":"ghp_secret_value","owner":"o","repo":"r"}` + if err := s.fireOne(Integration{IntType: "github", Config: cfg}, payload); err != nil { + t.Fatal(err) + } + if c.path != "/repos/o/r/issues" || c.headers.Get("Authorization") != "Bearer ghp_secret_value" || !strings.Contains(c.body, "New issue: TypeError") { + t.Fatalf("github request: path=%q body=%s", c.path, c.body) + } + }) + + t.Run("pagerduty", func(t *testing.T) { + srv, c := captureServer(t, 202) + u, _ := url.Parse(srv.URL) + s := testSvc(t, &http.Client{Transport: rewriteTransport{u}, Timeout: 5 * time.Second}) + if err := s.fireOne(Integration{IntType: "pagerduty", Config: `{"routing_key":"rk-abcdef"}`}, payload); err != nil { + t.Fatal(err) + } + if c.path != "/v2/enqueue" || !strings.Contains(c.body, "New issue: TypeError") || !strings.Contains(c.body, `"severity":"error"`) { + t.Fatalf("pagerduty request: path=%q body=%s", c.path, c.body) + } + }) + + t.Run("remote failure is an error", func(t *testing.T) { + srv, _ := captureServer(t, 500) + s := testSvc(t, netsafe.ClientWithAllow(5*time.Second, loopbackAllow(t))) + cfg := fmt.Sprintf(`{"webhook_url":%q}`, srv.URL) + if err := s.fireOne(Integration{IntType: "slack", Config: cfg}, payload); err == nil { + t.Fatal("a 500 must surface as an error") + } + }) +} + +// The email channel exists (it is not faked); its network leg is covered by the +// SSRF guard tests below. A live SMTP exchange is not exercised here. +func TestEmail_RefusesNonPublicSMTPHost(t *testing.T) { + s := testSvc(t, nil) + for _, host := range []string{"127.0.0.1", "localhost", "10.0.0.5", "169.254.169.254", "[::1]"} { + cfg := fmt.Sprintf(`{"smtp_host":%q,"smtp_port":"25","from":"a@b.c","to":"x@y.z"}`, strings.Trim(host, "[]")) + err := s.fireEmail(cfg, AlertPayload{Title: "t"}) + if err == nil || !strings.Contains(err.Error(), "non-public") { + t.Fatalf("host %s: expected non-public refusal, got %v", host, err) + } + } +} + +func TestPublicSMTPAddr_RebindingMixedAnswerRefused(t *testing.T) { + mixed := func(context.Context, string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}, {IP: net.ParseIP("10.1.2.3")}}, nil + } + if _, err := publicSMTPAddr("smtp.evil.example", mixed); err == nil { + t.Fatal("a name with any private answer must be refused") + } + public := func(context.Context, string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil + } + ip, err := publicSMTPAddr("smtp.ok.example", public) + if err != nil || ip != "93.184.216.34" { + t.Fatalf("public host: %q %v", ip, err) + } +} + +// ---- SSRF ------------------------------------------------------------------- + +func TestSSRF_StrictClientBlocksLoopbackAndPrivate(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hits.Add(1) })) + defer srv.Close() + s := testSvc(t, netsafe.Client(3*time.Second)) // exactly what production uses + + _, port, _ := net.SplitHostPort(strings.TrimPrefix(srv.URL, "http://")) + for _, target := range []string{ + srv.URL, // loopback IP literal + "http://localhost:" + port, // name resolving to loopback (dial-time check) + "http://10.0.0.1:" + port, // RFC1918 + "http://169.254.169.254/latest/", // cloud metadata + "http://[::1]:" + port, // v6 loopback + "http://100.64.0.1:" + port, // CGNAT / tailscale + } { + slack := fmt.Sprintf(`{"webhook_url":%q}`, target) + if err := s.fireOne(Integration{IntType: "slack", Config: slack}, AlertPayload{Title: "t"}); err == nil { + t.Fatalf("slack to %s must be blocked", target) + } + jira := fmt.Sprintf(`{"base_url":%q,"project":"P"}`, target) + if err := s.fireOne(Integration{IntType: "jira", Config: jira}, AlertPayload{Title: "t"}); err == nil { + t.Fatalf("jira to %s must be blocked", target) + } + } + if hits.Load() != 0 { + t.Fatalf("blocked destinations must never be contacted, got %d hits", hits.Load()) + } +} + +func TestSSRF_RedirectToMetadataBlocked(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://169.254.169.254/latest/meta-data/", http.StatusFound) + })) + defer srv.Close() + // Even with loopback allowed for the first hop, the redirect hop is + // link-local, which no allowlist can open. + s := testSvc(t, netsafe.ClientWithAllow(3*time.Second, loopbackAllow(t))) + cfg := fmt.Sprintf(`{"webhook_url":%q}`, srv.URL) + if err := s.fireOne(Integration{IntType: "slack", Config: cfg}, AlertPayload{Title: "t"}); err == nil { + t.Fatal("redirect to the metadata address must fail") + } +} + +func TestSSRF_NonHTTPSchemeRejected(t *testing.T) { + s := testSvc(t, netsafe.Client(time.Second)) + for _, u := range []string{"file:///etc/passwd", "gopher://x/", "http://user:pw@example.com/"} { + err := s.fireOne(Integration{IntType: "slack", Config: fmt.Sprintf(`{"webhook_url":%q}`, u)}, AlertPayload{}) + if err == nil { + t.Fatalf("%s must be rejected", u) + } + if strings.Contains(err.Error(), "pw@") || strings.Contains(err.Error(), "passwd") { + t.Fatalf("error echoes the URL: %v", err) + } + } +} + +// ---- redaction --------------------------------------------------------------- + +func TestRedaction_DeliveryErrorNeverCarriesWebhookSecret(t *testing.T) { + const secret = "T000/B000/XXXXsuperSecretTokenXXXX" + s := testSvc(t, netsafe.Client(2*time.Second)) + hook := "http://localhost:1/services/" + secret + cfg := fmt.Sprintf(`{"webhook_url":%q}`, hook) + err := s.fireOne(Integration{IntType: "slack", Config: cfg}, AlertPayload{Title: "t"}) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), secret) { + t.Fatalf("precondition: raw transport error should embed the URL, got %v", err) + } + red := redactDeliveryError(err, cfg) + if strings.Contains(red, secret) || strings.Contains(red, "/services/") { + t.Fatalf("redacted error still leaks the secret: %s", red) + } + if !strings.Contains(red, "localhost") { + t.Fatalf("redacted error lost the host: %s", red) + } +} + +func TestRedaction_ConfigValuesMaskedInPlainErrors(t *testing.T) { + cfg := `{"api_token":"tok-supersecret","password":"pw-hunter22","routing_key":"rk-abcdef","base_url":"https://j.example.com"}` + err := errors.New("remote said: bad credentials tok-supersecret / pw-hunter22 / rk-abcdef") + red := redactDeliveryError(err, cfg) + for _, s := range []string{"tok-supersecret", "pw-hunter22", "rk-abcdef"} { + if strings.Contains(red, s) { + t.Fatalf("%s leaked: %s", s, red) + } + } +} + +func TestRedaction_DispatcherLogsCarryNoSecretsOrConfig(t *testing.T) { + lb := &logBuf{} + logger := slog.New(slog.NewTextHandler(lb, &slog.HandlerOptions{Level: slog.LevelDebug})) + intg := Integration{IntegrationID: "int-1", SiteID: "site-1", IntType: "slack", + Config: `{"webhook_url":"https://hooks.slack.com/services/SECRETSECRET"}`} + d := newIssueDispatcher( + func(context.Context, string) ([]Integration, error) { return []Integration{intg}, nil }, + func(_ context.Context, i Integration, _ AlertPayload) error { + return fmt.Errorf("Post %q: boom", "https://hooks.slack.com/services/SECRETSECRET") + }, logger, "") + d.Notify(sampleEvent(IssueKindNew, "i1")) + waitFor(t, "failure counted", func() bool { return d.Stats().Failed == 1 }) + if strings.Contains(lb.String(), "SECRETSECRET") || strings.Contains(lb.String(), "hooks.slack.com") { + t.Fatalf("dispatcher log leaked destination: %s", lb.String()) + } +} + +// ---- dispatcher behaviour ------------------------------------------------------ + +type recorder struct { + mu sync.Mutex + calls []string +} + +func (r *recorder) add(s string) { r.mu.Lock(); r.calls = append(r.calls, s); r.mu.Unlock() } +func (r *recorder) n() int { r.mu.Lock(); defer r.mu.Unlock(); return len(r.calls) } + +func fixedList(intgs ...Integration) func(context.Context, string) ([]Integration, error) { + return func(_ context.Context, site string) ([]Integration, error) { + var out []Integration + for _, i := range intgs { + if i.SiteID == site { + out = append(out, i) + } + } + return out, nil + } +} + +func TestDispatcher_NewAndRegressionReachEverySiteIntegration(t *testing.T) { + rec := &recorder{} + d := newIssueDispatcher( + fixedList( + Integration{IntegrationID: "a", SiteID: "site-1", IntType: "slack"}, + Integration{IntegrationID: "b", SiteID: "site-1", IntType: "pagerduty"}, + Integration{IntegrationID: "other", SiteID: "site-2", IntType: "slack"}, + ), + func(_ context.Context, i Integration, p AlertPayload) error { + rec.add(i.IntegrationID + ":" + p.RuleName) + return nil + }, quietLogger(), "") + d.Notify(sampleEvent(IssueKindNew, "i1")) + d.Notify(sampleEvent(IssueKindRegression, "i2")) + waitFor(t, "4 deliveries", func() bool { return rec.n() == 4 }) + got := strings.Join(rec.calls, ",") + for _, want := range []string{"a:new_issue", "b:new_issue", "a:regression", "b:regression"} { + if !strings.Contains(got, want) { + t.Fatalf("missing %s in %s", want, got) + } + } + if strings.Contains(got, "other") { + t.Fatalf("another site's integration fired: %s", got) + } + if s := d.Stats(); s.Delivered != 4 { + t.Fatalf("stats: %+v", s) + } +} + +func TestDispatcher_CooldownSuppressesFlapping(t *testing.T) { + rec := &recorder{} + d := newIssueDispatcher( + fixedList(Integration{IntegrationID: "a", SiteID: "site-1", IntType: "slack"}), + func(_ context.Context, i Integration, p AlertPayload) error { rec.add(p.RuleName); return nil }, + quietLogger(), "") + clock := time.Unix(1_700_000_000, 0) + var cm sync.Mutex + d.now = func() time.Time { cm.Lock(); defer cm.Unlock(); return clock } + + for i := 0; i < 5; i++ { // resolve/regress flapping: five events for one issue + d.Notify(sampleEvent(IssueKindRegression, "flappy")) + } + d.Notify(sampleEvent(IssueKindNew, "other-issue")) + waitFor(t, "2 deliveries", func() bool { return rec.n() == 2 }) + time.Sleep(50 * time.Millisecond) + if rec.n() != 2 { + t.Fatalf("flapping issue was not collapsed: %d deliveries", rec.n()) + } + if d.Stats().Suppressed != 4 { + t.Fatalf("suppressed: %+v", d.Stats()) + } + + cm.Lock() + clock = clock.Add(IssueNotifyCooldown + time.Second) + cm.Unlock() + d.Notify(sampleEvent(IssueKindRegression, "flappy")) + waitFor(t, "delivery after cooldown", func() bool { return rec.n() == 3 }) +} + +func TestDispatcher_OverflowDropsOldestAndNeverBlocks(t *testing.T) { + gate := make(chan struct{}) + defer close(gate) + var listed atomic.Int32 + d := newIssueDispatcher( + func(ctx context.Context, site string) ([]Integration, error) { + listed.Add(1) + select { + case <-gate: + case <-ctx.Done(): + } + return nil, nil + }, + func(context.Context, Integration, AlertPayload) error { return nil }, quietLogger(), "") + + // Occupy every worker, then flood well past the queue bound. + for i := 0; i < IssueNotifyWorkers; i++ { + d.Notify(sampleEvent(IssueKindNew, fmt.Sprintf("w%d", i))) + } + waitFor(t, "workers busy", func() bool { return int(listed.Load()) == IssueNotifyWorkers }) + + start := time.Now() + const flood = IssueNotifyQueueSize + 50 + for i := 0; i < flood; i++ { + d.Notify(sampleEvent(IssueKindNew, fmt.Sprintf("f%d", i))) + } + if el := time.Since(start); el > time.Second { + t.Fatalf("Notify blocked the caller for %v with all workers busy", el) + } + s := d.Stats() + if s.Pending != IssueNotifyQueueSize || s.DroppedOverflow != 50 { + t.Fatalf("expected queue at bound with 50 dropped, got %+v", s) + } + d.mu.Lock() + oldest, newest := d.q[0].IssueID, d.q[len(d.q)-1].IssueID + d.mu.Unlock() + if oldest != "f50" || newest != fmt.Sprintf("f%d", flood-1) { + t.Fatalf("drop-oldest violated: oldest=%s newest=%s", oldest, newest) + } +} + +func TestDispatcher_SlowReceiverTimesOutAndFreesWorker(t *testing.T) { + block := make(chan struct{}) + defer close(block) + d := newIssueDispatcher( + fixedList(Integration{IntegrationID: "a", SiteID: "site-1", IntType: "slack"}), + func(context.Context, Integration, AlertPayload) error { <-block; return nil }, + quietLogger(), "") + d.deliverTimeout = 50 * time.Millisecond + d.Notify(sampleEvent(IssueKindNew, "i1")) + waitFor(t, "timeout counted", func() bool { return d.Stats().TimedOut == 1 }) +} + +func TestDispatcher_PanicAndListErrorAreContained(t *testing.T) { + d := newIssueDispatcher( + fixedList(Integration{IntegrationID: "a", SiteID: "site-1", IntType: "slack"}), + func(context.Context, Integration, AlertPayload) error { panic("kaboom") }, + quietLogger(), "") + d.Notify(sampleEvent(IssueKindNew, "i1")) + waitFor(t, "panic counted as failure", func() bool { return d.Stats().Failed == 1 }) + + d2 := newIssueDispatcher( + func(context.Context, string) ([]Integration, error) { return nil, errors.New("db down") }, + nil, quietLogger(), "") + d2.Notify(sampleEvent(IssueKindNew, "i1")) + waitFor(t, "list error counted", func() bool { return d2.Stats().ListErrors == 1 }) +} + +func TestDispatcher_PayloadSanitisesAttackerText(t *testing.T) { + d := newIssueDispatcher(nil, nil, quietLogger(), "") + ev := sampleEvent(IssueKindNew, "i1") + ev.Title = "bad\r\nBcc: evil@example.com\x00" + strings.Repeat("x", 500) + p := d.payload(ev) + if strings.ContainsAny(p.Title, "\r\n\x00") || len([]rune(p.Title)) > 230 { + t.Fatalf("title not sanitised: %q", p.Title) + } + if p.Severity != "error" { + t.Fatalf("severity: %s", p.Severity) + } + ev.Level = "fatal" + if d.payload(ev).Severity != "critical" { + t.Fatal("fatal must map to critical") + } + // The payload must still pass the email header-injection check. + if strings.ContainsAny(p.Title, "\r\n") { + t.Fatal("CR/LF would reach the SMTP Subject") + } +} + +func TestDispatcher_ShutdownDrainsAndRefusesNew(t *testing.T) { + rec := &recorder{} + d := newIssueDispatcher( + fixedList(Integration{IntegrationID: "a", SiteID: "site-1", IntType: "slack"}), + func(context.Context, Integration, AlertPayload) error { + time.Sleep(20 * time.Millisecond) + rec.add("x") + return nil + }, + quietLogger(), "") + d.Notify(sampleEvent(IssueKindNew, "i1")) + if err := d.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + if rec.n() != 1 { + t.Fatalf("queued delivery not drained: %d", rec.n()) + } + d.Notify(sampleEvent(IssueKindNew, "i2")) + if d.Stats().Enqueued != 1 { + t.Fatal("Notify after Shutdown must be refused") + } +} + +func TestDispatcher_IgnoresIncompleteEvents(t *testing.T) { + d := newIssueDispatcher(nil, nil, quietLogger(), "") + d.Notify(IssueEvent{Kind: IssueKindNew}) + if d.Stats().Enqueued != 0 { + t.Fatal("event without site/issue must be ignored") + } +} From 0ad3305ebadee1defdd672cc053d7c2c499c6198 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:33:24 +0000 Subject: [PATCH 11/65] feat: guard log and metric read paths with the O12 limiter and budgets SearchLogs, LogStats, Histogram, ListMetrics and QuerySeries now take a concurrency slot, run under the time budget, and refuse with the labeled queryguard refusal (429 concurrency/rows, 504 time) instead of running unbounded. QuerySeries used to load every raw point in range into Go memory: its SQL now carries LIMIT budget+1 and one row past the ceiling is a refusal, never a truncated series. Histogram and ListMetrics get the same cap; SearchLogs refuses a page (limit+offset) beyond the row budget. internal/guardmap gains Guard (Begin / RowRefusal / DeadlineError, the pattern of internal/query/guard.go packaged for reuse, nil-safe with default budgets) and HTTPError, which maps refusals to problem responses. Log handlers and the metrics list/query/series handlers use it; other errors keep their historical statuses. Shared limiter wired in main.go. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/main.go | 13 ++- cmd/observe/metrics_guard_test.go | 40 ++++++++ cmd/observe/metrics_handlers.go | 18 +++- internal/guardmap/guard.go | 100 +++++++++++++++++++ internal/logs/logs.go | 75 +++++++++++++- internal/logs/queryguard_test.go | 98 +++++++++++++++++++ internal/metrics/metrics.go | 20 ++++ internal/metrics/query.go | 55 +++++++++-- internal/metrics/queryguard_test.go | 147 ++++++++++++++++++++++++++++ 9 files changed, 550 insertions(+), 16 deletions(-) create mode 100644 cmd/observe/metrics_guard_test.go create mode 100644 internal/guardmap/guard.go create mode 100644 internal/logs/queryguard_test.go create mode 100644 internal/metrics/queryguard_test.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index 3b1f78fd..b92f975f 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -39,6 +39,7 @@ import ( "github.com/useteploy/teploy-observe/internal/feedback" "github.com/useteploy/teploy-observe/internal/flags" "github.com/useteploy/teploy-observe/internal/groups" + "github.com/useteploy/teploy-observe/internal/guardmap" "github.com/useteploy/teploy-observe/internal/heatmaps" "github.com/useteploy/teploy-observe/internal/incidents" "github.com/useteploy/teploy-observe/internal/infra" @@ -448,6 +449,9 @@ func main() { surveySvc := surveys.NewSurveyService(db, cfg.SessionSalt, siteSvc) logSvc := logs.NewLogService(db) logSvc.SetPipelines(pipelineSvc) + // The same limiter and budgets guard the log and metric read paths. + logSvc.WithQueryGuard(queryLimiter, queryBudgets) + metricsSvc.WithQueryGuard(queryLimiter, queryBudgets) uptimeSvc := monitoring.NewUptimeService(db, logger) cronSvc := monitoring.NewCronService(db, logger) linkSvc := tracking.NewLinkService(db) @@ -4554,7 +4558,8 @@ func logSearchHandler(svc *logs.LogService) neutron.HandlerFunc[logSearchInput, if err != nil { return nil, neutron.ErrBadRequest(err.Error()) } - return emptyOnNil(svc.SearchLogs(ctx, input.SiteID, from, to, input.Level, input.Service, input.Query, input.Limit, input.Offset)) + out, err := svc.SearchLogs(ctx, input.SiteID, from, to, input.Level, input.Service, input.Query, input.Limit, input.Offset) + return emptyOnNil(out, guardmap.HTTPError(err)) } } @@ -4573,7 +4578,8 @@ func logStatsHandler(svc *logs.LogService) neutron.HandlerFunc[logStatsInput, [] if err != nil { return nil, neutron.ErrBadRequest(err.Error()) } - return emptyOnNil(svc.LogStats(ctx, input.SiteID, from, to)) + out, err := svc.LogStats(ctx, input.SiteID, from, to) + return emptyOnNil(out, guardmap.HTTPError(err)) } } @@ -4593,7 +4599,8 @@ func logHistogramHandler(svc *logs.LogService) neutron.HandlerFunc[logHistogramI if err != nil { return nil, neutron.ErrBadRequest(err.Error()) } - return emptyOnNil(svc.Histogram(ctx, input.SiteID, from, to, input.BucketMs)) + out, err := svc.Histogram(ctx, input.SiteID, from, to, input.BucketMs) + return emptyOnNil(out, guardmap.HTTPError(err)) } } diff --git a/cmd/observe/metrics_guard_test.go b/cmd/observe/metrics_guard_test.go new file mode 100644 index 00000000..f668f90c --- /dev/null +++ b/cmd/observe/metrics_guard_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +// Guard refusals keep their labeled status through the metrics handlers; +// other errors keep the handler's historical status. +func TestWriteMetricsErrorMapsRefusals(t *testing.T) { + cases := []struct { + name string + err error + fallback int + want int + body string + }{ + {"rows", queryguard.RowBudgetRefusal(nil, 5), http.StatusBadRequest, http.StatusTooManyRequests, "query_budget_rows"}, + {"time", queryguard.TimeBudgetRefusal(nil, time.Second), http.StatusBadRequest, http.StatusGatewayTimeout, "query_budget_time"}, + {"plain-400", errors.New("bad agg"), http.StatusBadRequest, http.StatusBadRequest, "bad agg"}, + {"plain-500", errors.New("boom"), http.StatusInternalServerError, http.StatusInternalServerError, "boom"}, + } + for _, c := range cases { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/api/v1/metrics/series", nil) + writeMetricsError(w, r, c.err, c.fallback) + if w.Code != c.want { + t.Errorf("%s: status %d, want %d", c.name, w.Code, c.want) + } + if !strings.Contains(w.Body.String(), c.body) { + t.Errorf("%s: body %q missing %q", c.name, w.Body.String(), c.body) + } + } +} diff --git a/cmd/observe/metrics_handlers.go b/cmd/observe/metrics_handlers.go index ff6efd3f..fd0efbb7 100644 --- a/cmd/observe/metrics_handlers.go +++ b/cmd/observe/metrics_handlers.go @@ -8,9 +8,21 @@ import ( "github.com/neutron-build/neutron/go/neutron" + "github.com/useteploy/teploy-observe/internal/guardmap" "github.com/useteploy/teploy-observe/internal/metrics" ) +// writeMetricsError answers a failed read. O12 admission refusals keep +// their labeled 429/504 problem response; every other error keeps the +// handler's historical status. +func writeMetricsError(w http.ResponseWriter, r *http.Request, err error, fallback int) { + if mapped := guardmap.HTTPError(err); mapped != err { + neutron.WriteError(w, r, mapped) + return + } + http.Error(w, err.Error(), fallback) +} + // RegisterMetricsRoutes wires the metrics API onto the root router. Three // concerns are bundled here: // @@ -44,7 +56,7 @@ func metricsListHandler(svc *metrics.Service) http.HandlerFunc { } list, err := svc.ListMetrics(r.Context(), siteID) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + writeMetricsError(w, r, err, http.StatusInternalServerError) return } if list == nil { @@ -77,7 +89,7 @@ func metricsQueryHandler(svc *metrics.Service) http.HandlerFunc { } points, err := svc.Query(r.Context(), req.siteID, req.name, req.labels, req.fromMs, req.toMs, req.agg) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writeMetricsError(w, r, err, http.StatusBadRequest) return } if points == nil { @@ -108,7 +120,7 @@ func metricsSeriesHandler(svc *metrics.Service) http.HandlerFunc { GroupBy: req.groupBy, }) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writeMetricsError(w, r, err, http.StatusBadRequest) return } if series == nil { diff --git a/internal/guardmap/guard.go b/internal/guardmap/guard.go new file mode 100644 index 00000000..afebeabc --- /dev/null +++ b/internal/guardmap/guard.go @@ -0,0 +1,100 @@ +package guardmap + +import ( + "context" + "errors" + "time" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +// Guard is the O12 admission state for one read service (logs, metrics, +// and any later read path): an optional concurrency limiter plus the +// mandatory budget set. It carries the same pattern internal/query/guard.go +// implements for the stats service, packaged so each service does not +// re-derive it. The nil *Guard is valid and means "no concurrency +// admission, default budgets", so zero-value services built by tests keep +// the budgets in force. +type Guard struct { + limiter *queryguard.Limiter + budgets queryguard.Budgets +} + +// NewGuard returns a Guard. A nil limiter disables concurrency admission; +// zero-valued budget fields fall back to the declared defaults so a +// hand-built Budgets{} cannot disable the row/time ceilings. +func NewGuard(l *queryguard.Limiter, b queryguard.Budgets) *Guard { + d := queryguard.DefaultBudgets() + if b.Timeout <= 0 { + b.Timeout = d.Timeout + } + if b.MaxScanRows <= 0 { + b.MaxScanRows = d.MaxScanRows + } + if b.MaxWindow <= 0 { + b.MaxWindow = d.MaxWindow + } + return &Guard{limiter: l, budgets: b} +} + +func (g *Guard) get() (*queryguard.Limiter, queryguard.Budgets) { + if g == nil { + return nil, queryguard.DefaultBudgets() + } + return g.limiter, g.budgets +} + +// Begin is the admission gate for one expensive query: it takes a +// concurrency slot (labeled refusal when the global or site bound is full) +// and derives the context whose deadline is the time budget. The returned +// release must be called exactly once, on every path. +func (g *Guard) Begin(ctx context.Context, siteID string) (context.Context, func(), error) { + l, b := g.get() + release := func() {} + if l != nil { + rel, err := l.Acquire(ctx, siteID) + if err != nil { + return ctx, release, err + } + release = rel + } + qctx, cancel := context.WithTimeout(ctx, b.Timeout) + return qctx, func() { + cancel() + release() + }, nil +} + +// MaxRows is the declared row ceiling. +func (g *Guard) MaxRows() int64 { + _, b := g.get() + return b.MaxScanRows +} + +// RowRefusal builds (and counts) the labeled row-budget refusal. +func (g *Guard) RowRefusal() *queryguard.Refusal { + l, b := g.get() + return queryguard.RowBudgetRefusal(l, b.MaxScanRows) +} + +// DeadlineError classifies a query failure: a deadline hit on the budget +// context while the caller's context is still live is a labeled +// time-budget refusal; anything else (caller cancellation, engine error) +// propagates as-is. parent is the caller's context (not the one Begin +// returned). +func (g *Guard) DeadlineError(parent context.Context, err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.DeadlineExceeded) && parent.Err() == nil { + l, b := g.get() + return queryguard.TimeBudgetRefusal(l, b.Timeout) + } + return err +} + +// Timeout is the declared wall-time budget (for tests and diagnostics). +func (g *Guard) Timeout() time.Duration { + _, b := g.get() + return b.Timeout +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a140c6b9..7a478755 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -14,6 +14,8 @@ import ( "github.com/neutron-build/neutron/go/nucleus" "github.com/useteploy/teploy-observe/internal/dbutil" + "github.com/useteploy/teploy-observe/internal/guardmap" + "github.com/useteploy/teploy-observe/internal/queryguard" ) // LogService handles log ingestion and querying. @@ -22,6 +24,21 @@ type LogService struct { Bx *Broadcaster pipelines *PipelineService logger *slog.Logger + // guard is the O12 read-path admission (concurrency slot + time and row + // budgets) for SearchLogs / LogStats / Histogram. nil = default budgets, + // no concurrency bound. + guard *guardmap.Guard + + // queryHistogram is a test seam for the histogram read; nil in + // production. + queryHistogram func(ctx context.Context, query string, args ...any) ([]HistogramBucket, error) +} + +// WithQueryGuard installs the O12 admission state on the read paths. A nil +// limiter disables concurrency admission; budgets are always in force. +func (s *LogService) WithQueryGuard(l *queryguard.Limiter, b queryguard.Budgets) *LogService { + s.guard = guardmap.NewGuard(l, b) + return s } func NewLogService(db *nucleus.Client) *LogService { @@ -319,6 +336,9 @@ func (s *LogService) insertLogsBatch(ctx context.Context, batch []*preparedLog) } // SearchLogs queries logs with optional filters. +// +// Guarded (O12): takes a concurrency slot, runs under the time budget, and +// refuses (never truncates) a page that would reach past the row budget. func (s *LogService) SearchLogs(ctx context.Context, siteID string, from, to time.Time, level, service, search string, limit, offset int) ([]Log, error) { fromMs := dbutil.IntParam(from.UnixMilli()) toMs := dbutil.IntParam(to.UnixMilli()) @@ -328,6 +348,16 @@ func (s *LogService) SearchLogs(ctx context.Context, siteID string, from, to tim if offset < 0 { offset = 0 } + // The engine must walk offset+limit rows to serve the page; past the + // declared row budget that is a refusal, not a silently shortened page. + if int64(limit)+int64(offset) > s.guard.MaxRows() { + return nil, s.guard.RowRefusal() + } + qctx, release, err := s.guard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() where := "site_id = $1 AND timestamp >= $2 AND timestamp < $3" params := []any{siteID, fromMs, toMs} @@ -361,7 +391,11 @@ func (s *LogService) SearchLogs(ctx context.Context, siteID string, from, to tim ORDER BY timestamp DESC LIMIT %d OFFSET %d`, where, limit, offset) - return nucleus.Query[Log](ctx, s.db.SQL(), query, params...) + out, err := nucleus.Query[Log](qctx, s.db.SQL(), query, params...) + if err != nil { + return nil, s.guard.DeadlineError(ctx, err) + } + return out, nil } // LevelCount holds the count of logs for a single level. @@ -375,7 +409,13 @@ func (s *LogService) LogStats(ctx context.Context, siteID string, from, to time. fromMs := dbutil.IntParam(from.UnixMilli()) toMs := dbutil.IntParam(to.UnixMilli()) - return nucleus.Query[LevelCount](ctx, s.db.SQL(), + qctx, release, err := s.guard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + + out, err := nucleus.Query[LevelCount](qctx, s.db.SQL(), `SELECT level, COUNT(*) AS count FROM logs WHERE site_id = $1 AND timestamp >= $2 AND timestamp < $3 @@ -383,6 +423,10 @@ func (s *LogService) LogStats(ctx context.Context, siteID string, from, to time. ORDER BY count DESC`, siteID, fromMs, toMs, ) + if err != nil { + return nil, s.guard.DeadlineError(ctx, err) + } + return out, nil } // HistogramBucket represents log counts per level for a single time bucket. @@ -402,7 +446,22 @@ func (s *LogService) Histogram(ctx context.Context, siteID string, from, to time toMs := dbutil.IntParam(to.UnixMilli()) bucketStr := dbutil.IntParam(bucketMs) - return nucleus.Query[HistogramBucket](ctx, s.db.SQL(), + qctx, release, err := s.guard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + + // LIMIT budget+1: the result is one row per (bucket, level), so a tiny + // bucket over a long range could otherwise materialize millions of rows. + // One row past the ceiling is a refusal, never a truncated histogram. + run := s.queryHistogram + if run == nil { + run = func(ctx context.Context, query string, args ...any) ([]HistogramBucket, error) { + return nucleus.Query[HistogramBucket](ctx, s.db.SQL(), query, args...) + } + } + out, err := run(qctx, `SELECT (CAST(timestamp AS BIGINT) / CAST($4 AS BIGINT)) * CAST($4 AS BIGINT) AS bucket, level, COUNT(*) AS count @@ -411,9 +470,17 @@ func (s *LogService) Histogram(ctx context.Context, siteID string, from, to time AND timestamp >= $2 AND timestamp < $3 GROUP BY (CAST(timestamp AS BIGINT) / CAST($4 AS BIGINT)) * CAST($4 AS BIGINT), level - ORDER BY bucket ASC`, + ORDER BY bucket ASC + LIMIT `+strconv.FormatInt(s.guard.MaxRows()+1, 10), siteID, fromMs, toMs, bucketStr, ) + if err != nil { + return nil, s.guard.DeadlineError(ctx, err) + } + if int64(len(out)) > s.guard.MaxRows() { + return nil, s.guard.RowRefusal() + } + return out, nil } func generateID() (string, error) { diff --git a/internal/logs/queryguard_test.go b/internal/logs/queryguard_test.go new file mode 100644 index 00000000..fa00afd5 --- /dev/null +++ b/internal/logs/queryguard_test.go @@ -0,0 +1,98 @@ +package logs + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +func refusalOf(t *testing.T, err error) *queryguard.Refusal { + t.Helper() + var ref *queryguard.Refusal + if !errors.As(err, &ref) { + t.Fatalf("want a *queryguard.Refusal, got %v", err) + } + return ref +} + +func guardedLogs(rows int64, timeout time.Duration, l *queryguard.Limiter) *LogService { + b := queryguard.DefaultBudgets() + b.MaxScanRows = rows + b.Timeout = timeout + return (&LogService{}).WithQueryGuard(l, b) +} + +func TestHistogramRowCapRefuses(t *testing.T) { + s := guardedLogs(2, time.Second, nil) + var gotSQL string + s.queryHistogram = func(_ context.Context, q string, _ ...any) ([]HistogramBucket, error) { + gotSQL = q + return make([]HistogramBucket, 3), nil + } + _, err := s.Histogram(context.Background(), "site", time.Unix(0, 0), time.Unix(100, 0), 1000) + if ref := refusalOf(t, err); ref.Code != queryguard.CodeBudgetRows || ref.Status != 429 { + t.Fatalf("refusal = %+v", ref) + } + if !strings.Contains(gotSQL, "LIMIT 3") { + t.Fatalf("SQL must bound the read at budget+1 rows:\n%s", gotSQL) + } + // At the cap it still answers. + s.queryHistogram = func(context.Context, string, ...any) ([]HistogramBucket, error) { + return make([]HistogramBucket, 2), nil + } + if out, err := s.Histogram(context.Background(), "site", time.Unix(0, 0), time.Unix(100, 0), 1000); err != nil || len(out) != 2 { + t.Fatalf("at cap: %v %d", err, len(out)) + } +} + +func TestHistogramTimeBudgetIs504(t *testing.T) { + s := guardedLogs(100, 20*time.Millisecond, nil) + s.queryHistogram = func(ctx context.Context, _ string, _ ...any) ([]HistogramBucket, error) { + <-ctx.Done() + return nil, ctx.Err() + } + _, err := s.Histogram(context.Background(), "site", time.Unix(0, 0), time.Unix(100, 0), 1000) + if ref := refusalOf(t, err); ref.Code != queryguard.CodeBudgetTime || ref.Status != 504 { + t.Fatalf("refusal = %+v", ref) + } +} + +// Concurrency admission refuses before any engine access (the service has +// no db, so reaching the query would panic). +func TestLogReadsConcurrencyRefusal(t *testing.T) { + l := queryguard.NewLimiter(1, 1) + hold, err := l.Acquire(context.Background(), "site") + if err != nil { + t.Fatal(err) + } + defer hold() + s := guardedLogs(100, time.Second, l) + ctx := context.Background() + from, to := time.Unix(0, 0), time.Unix(100, 0) + + if _, err := s.SearchLogs(ctx, "site", from, to, "", "", "", 10, 0); refusalOf(t, err).Status != 429 { + t.Fatalf("SearchLogs not refused: %v", err) + } + if _, err := s.Histogram(ctx, "site", from, to, 1000); refusalOf(t, err).Status != 429 { + t.Fatalf("Histogram not refused: %v", err) + } + if _, err := s.LogStats(ctx, "site", from, to); refusalOf(t, err).Status != 429 { + t.Fatalf("LogStats not refused: %v", err) + } +} + +// A page that would reach past the row budget is refused, not shortened. +func TestSearchLogsPageBeyondBudgetRefuses(t *testing.T) { + s := guardedLogs(100, time.Second, nil) // nil db: must refuse before querying + from, to := time.Unix(0, 0), time.Unix(100, 0) + for _, c := range []struct{ limit, offset int }{{101, 0}, {50, 51}, {1 << 40, 0}} { + _, err := s.SearchLogs(context.Background(), "site", from, to, "", "", "", c.limit, c.offset) + if ref := refusalOf(t, err); ref.Code != queryguard.CodeBudgetRows { + t.Fatalf("limit=%d offset=%d: %+v", c.limit, c.offset, ref) + } + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index a47c5f6a..d90dd46f 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -20,6 +20,8 @@ import ( "github.com/neutron-build/neutron/go/nucleus" "github.com/useteploy/teploy-observe/internal/dbutil" + "github.com/useteploy/teploy-observe/internal/guardmap" + "github.com/useteploy/teploy-observe/internal/queryguard" ) // Service writes OTLP metric points and answers list / query requests. @@ -29,6 +31,24 @@ type Service struct { db *nucleus.Client logger *slog.Logger guard *seriesGuard + // qguard is the O12 READ-path admission (concurrency slot + time and + // row budgets) for ListMetrics / QuerySeries. Distinct from guard + // above, which bounds INGEST cardinality. nil = default budgets, no + // concurrency bound. + qguard *guardmap.Guard + + // Read-query seams: nil in production (the engine is queried + // directly); tests inject row sets and blocking reads so the budget + // refusals are exercised without a store. + queryPointRows func(ctx context.Context, query string, args ...any) ([]pointRow, error) + queryMetricRows func(ctx context.Context, query string, args ...any) ([]metricRow, error) +} + +// WithQueryGuard installs the O12 read-path admission state. A nil limiter +// disables concurrency admission; budgets are always in force. +func (s *Service) WithQueryGuard(l *queryguard.Limiter, b queryguard.Budgets) *Service { + s.qguard = guardmap.NewGuard(l, b) + return s } func NewService(db *nucleus.Client) *Service { diff --git a/internal/metrics/query.go b/internal/metrics/query.go index 8aa768eb..1c52081d 100644 --- a/internal/metrics/query.go +++ b/internal/metrics/query.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strconv" "strings" "time" @@ -12,6 +13,22 @@ import ( "github.com/useteploy/teploy-observe/internal/dbutil" ) +// fetchPointRows / fetchMetricRows run the read queries through the test +// seams when set, else straight against the engine. +func (s *Service) fetchPointRows(ctx context.Context, query string, args ...any) ([]pointRow, error) { + if s.queryPointRows != nil { + return s.queryPointRows(ctx, query, args...) + } + return nucleus.Query[pointRow](ctx, s.db.SQL(), query, args...) +} + +func (s *Service) fetchMetricRows(ctx context.Context, query string, args ...any) ([]metricRow, error) { + if s.queryMetricRows != nil { + return s.queryMetricRows(ctx, query, args...) + } + return nucleus.Query[metricRow](ctx, s.db.SQL(), query, args...) +} + // MetricInfo names a metric and reports its kind. Used by the /metrics/list // endpoint to populate the left-hand picker in the UI. type MetricInfo struct { @@ -71,13 +88,25 @@ func (s *Service) ListMetrics(ctx context.Context, siteID string) ([]MetricInfo, if siteID == "" { return nil, fmt.Errorf("metrics: site_id required") } - rows, err := nucleus.Query[metricRow](ctx, s.db.SQL(), + // O12: concurrency slot + time budget; the grouped result is bounded + // by LIMIT budget+1 and a result past the ceiling is refused, never + // truncated (a clipped metric picker would silently hide metrics). + qctx, release, err := s.qguard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + rows, err := s.fetchMetricRows(qctx, `SELECT metric_name, metric_kind FROM metric_points WHERE site_id = $1 - GROUP BY metric_name, metric_kind`, + GROUP BY metric_name, metric_kind + LIMIT `+strconv.FormatInt(s.qguard.MaxRows()+1, 10), siteID, ) if err != nil { - return nil, err + return nil, s.qguard.DeadlineError(ctx, err) + } + if int64(len(rows)) > s.qguard.MaxRows() { + return nil, s.qguard.RowRefusal() } // A name can still appear under two kinds if a producer changed kind // mid-stream; last wins, matching the previous behaviour. @@ -229,24 +258,38 @@ func (s *Service) QuerySeries(ctx context.Context, siteID, name string, labels m fromNs := fromMs * 1_000_000 toNs := toMs * 1_000_000 + // O12: concurrency slot + time budget, and a hard row cap. Every raw + // point in range is loaded into Go memory before aggregation, so the + // SQL carries LIMIT budget+1 and one row past the ceiling is a labeled + // refusal - never a truncated series that would draw as a real answer. + qctx, release, err := s.qguard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + // Both sides of the comparison need an explicit BIGINT cast — Nucleus // pgwire advertises ts_ns as TEXT (dogfood finding #6) so a bare // `ts_ns >= literal` returns zero rows even when the row is in range. // The tracing package solved this the same way for `start_time`. - rows, err := nucleus.Query[pointRow](ctx, s.db.SQL(), + rows, err := s.fetchPointRows(qctx, `SELECT ts_ns, value, histogram, metric_kind, attributes, COALESCE(aggregation_temporality, 'cumulative') AS aggregation_temporality FROM metric_points WHERE site_id = $1 AND metric_name = $2 AND ts_ns >= $3 AND ts_ns < $4 - ORDER BY ts_ns ASC`, + ORDER BY ts_ns ASC + LIMIT `+strconv.FormatInt(s.qguard.MaxRows()+1, 10), siteID, name, dbutil.IntParam(fromNs), dbutil.IntParam(toNs), ) if err != nil { - return nil, err + return nil, s.qguard.DeadlineError(ctx, err) + } + if int64(len(rows)) > s.qguard.MaxRows() { + return nil, s.qguard.RowRefusal() } // Group rows by (label-set fingerprint) → list of points. Even when diff --git a/internal/metrics/queryguard_test.go b/internal/metrics/queryguard_test.go new file mode 100644 index 00000000..ff765f36 --- /dev/null +++ b/internal/metrics/queryguard_test.go @@ -0,0 +1,147 @@ +package metrics + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +func refusalOf(t *testing.T, err error) *queryguard.Refusal { + t.Helper() + var ref *queryguard.Refusal + if !errors.As(err, &ref) { + t.Fatalf("want a *queryguard.Refusal, got %v", err) + } + return ref +} + +func guarded(rows int64, timeout time.Duration, l *queryguard.Limiter) *Service { + s := NewService(nil) + b := queryguard.DefaultBudgets() + b.MaxScanRows = rows + b.Timeout = timeout + return s.WithQueryGuard(l, b) +} + +// QuerySeries must refuse, not truncate, once the engine returns more raw +// points than the row budget. +func TestQuerySeriesRowCapRefuses(t *testing.T) { + s := guarded(3, time.Second, nil) + var gotSQL string + s.queryPointRows = func(_ context.Context, q string, _ ...any) ([]pointRow, error) { + gotSQL = q + rows := make([]pointRow, 4) // budget+1 rows: over the ceiling + return rows, nil + } + _, err := s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{}) + ref := refusalOf(t, err) + if ref.Code != queryguard.CodeBudgetRows || ref.Status != 429 { + t.Fatalf("refusal = %+v", ref) + } + if !strings.Contains(gotSQL, "LIMIT 4") { + t.Fatalf("SQL must bound the read at budget+1 rows:\n%s", gotSQL) + } +} + +func TestQuerySeriesAtCapStillAnswers(t *testing.T) { + s := guarded(3, time.Second, nil) + s.queryPointRows = func(context.Context, string, ...any) ([]pointRow, error) { + return []pointRow{{TsNs: 1_000_000, Value: 1, Kind: "gauge"}, {TsNs: 2_000_000, Value: 2, Kind: "gauge"}, {TsNs: 3_000_000, Value: 3, Kind: "gauge"}}, nil + } + series, err := s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{Agg: "sum"}) + if err != nil || len(series) != 1 || len(series[0].Points) == 0 { + t.Fatalf("at-cap query must answer: %v %+v", err, series) + } +} + +func TestQuerySeriesTimeBudgetIs504(t *testing.T) { + s := guarded(100, 20*time.Millisecond, nil) + s.queryPointRows = func(ctx context.Context, _ string, _ ...any) ([]pointRow, error) { + <-ctx.Done() + return nil, ctx.Err() + } + _, err := s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{}) + ref := refusalOf(t, err) + if ref.Code != queryguard.CodeBudgetTime || ref.Status != 504 { + t.Fatalf("refusal = %+v", ref) + } +} + +// A cancelled caller is not a time-budget refusal. +func TestQuerySeriesCallerCancelPassesThrough(t *testing.T) { + s := guarded(100, time.Second, nil) + s.queryPointRows = func(ctx context.Context, _ string, _ ...any) ([]pointRow, error) { + return nil, context.Canceled + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := s.QuerySeries(ctx, "site", "m", nil, 0, 1000, QueryOptions{}) + var ref *queryguard.Refusal + if errors.As(err, &ref) { + t.Fatalf("caller cancellation must not become a refusal: %v", err) + } +} + +func TestQuerySeriesConcurrencyRefusal(t *testing.T) { + l := queryguard.NewLimiter(1, 1) + hold, err := l.Acquire(context.Background(), "site") + if err != nil { + t.Fatal(err) + } + s := guarded(100, time.Second, l) + called := false + s.queryPointRows = func(context.Context, string, ...any) ([]pointRow, error) { called = true; return nil, nil } + _, err = s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{}) + ref := refusalOf(t, err) + if ref.Status != 429 || called { + t.Fatalf("refusal = %+v called=%v", ref, called) + } + if _, err := s.ListMetrics(context.Background(), "site"); refusalOf(t, err).Status != 429 { + t.Fatalf("ListMetrics must share the admission: %v", err) + } + // Releasing the slot readmits; the slot is released after each query. + hold() + s.queryPointRows = func(context.Context, string, ...any) ([]pointRow, error) { return nil, nil } + if _, err := s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{}); err != nil { + t.Fatalf("after release: %v", err) + } + if _, err := s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{}); err != nil { + t.Fatalf("slot leaked after a completed query: %v", err) + } +} + +func TestListMetricsRowCapAndTime(t *testing.T) { + s := guarded(2, 20*time.Millisecond, nil) + s.queryMetricRows = func(context.Context, string, ...any) ([]metricRow, error) { + return make([]metricRow, 3), nil + } + if ref := refusalOf(t, errOnly(s.ListMetrics(context.Background(), "site"))); ref.Code != queryguard.CodeBudgetRows { + t.Fatalf("refusal = %+v", ref) + } + s.queryMetricRows = func(ctx context.Context, _ string, _ ...any) ([]metricRow, error) { + <-ctx.Done() + return nil, ctx.Err() + } + if ref := refusalOf(t, errOnly(s.ListMetrics(context.Background(), "site"))); ref.Code != queryguard.CodeBudgetTime { + t.Fatalf("refusal = %+v", ref) + } +} + +func errOnly[T any](_ T, err error) error { return err } + +// The zero-guard service (tests, embedded use) keeps default budgets. +func TestUnguardedServiceKeepsDefaultBudgets(t *testing.T) { + s := NewService(nil) + var gotSQL string + s.queryPointRows = func(_ context.Context, q string, _ ...any) ([]pointRow, error) { gotSQL = q; return nil, nil } + if _, err := s.QuerySeries(context.Background(), "site", "m", nil, 0, 1000, QueryOptions{}); err != nil { + t.Fatal(err) + } + if !strings.Contains(gotSQL, "LIMIT 1000001") { + t.Fatalf("default row budget missing from SQL:\n%s", gotSQL) + } +} From 9813b7294db2d98e54bcdb51c12838ea3ebe6b36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:34:46 +0000 Subject: [PATCH 12/65] fix: include span_links in backups UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- internal/backup/backup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index f12d723c..6d205f25 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -69,6 +69,7 @@ var Tables = []string{ "logs", "log_pipelines", "spans", + "span_links", "service_stats", "service_dependencies", "derived_outbox", From a6d10fc025a9401d8799d7177fca4735977c6112 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:40:21 +0000 Subject: [PATCH 13/65] test: golden vectors pinning legacy flag evaluation Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- internal/flags/legacy_golden_test.go | 123 + internal/flags/testdata/legacy_fixtures.json | 19 + internal/flags/testdata/legacy_golden.json | 30515 +++++++++++++++++ 3 files changed, 30657 insertions(+) create mode 100644 internal/flags/legacy_golden_test.go create mode 100644 internal/flags/testdata/legacy_fixtures.json create mode 100644 internal/flags/testdata/legacy_golden.json diff --git a/internal/flags/legacy_golden_test.go b/internal/flags/legacy_golden_test.go new file mode 100644 index 00000000..d812e45b --- /dev/null +++ b/internal/flags/legacy_golden_test.go @@ -0,0 +1,123 @@ +package flags + +// Golden vectors for LEGACY stored configs (flat targeting array, variants +// without payloads). They were generated from the evaluator as it stood +// before condition groups, extra operators and payloads were added, and must +// never change: a diff here means existing users were reassigned. +// +// Regenerate ONLY deliberately: OBSERVE_FLAGS_GOLDEN_UPDATE=1 go test ./internal/flags -run LegacyGolden + +import ( + "context" + "encoding/json" + "fmt" + "os" + "testing" +) + +type legacyFixture struct { + Name string `json:"name"` + FlagKey string `json:"flag_key"` + FlagType string `json:"flag_type"` + Enabled bool `json:"enabled"` + RolloutPc int `json:"rollout_pct"` + Variants string `json:"variants"` + Targeting string `json:"targeting"` +} + +type legacyVector struct { + Fixture string `json:"fixture"` + UserID string `json:"user_id"` + Context map[string]string `json:"context,omitempty"` + Enabled bool `json:"enabled"` + Variant string `json:"variant,omitempty"` + Reason string `json:"reason"` + Detail string `json:"detail,omitempty"` +} + +const legacyFixturesPath = "testdata/legacy_fixtures.json" +const legacyVectorsPath = "testdata/legacy_golden.json" + +func legacyUsers() []string { + u := []string{"", "alice", "bob", "carol", "user-with-a-really-long-identifier-0123456789"} + for i := 0; i < 40; i++ { + u = append(u, fmt.Sprintf("u%d", i)) + } + return u +} + +var legacyContexts = []map[string]string{ + nil, + {"plan": "pro", "country": "US"}, + {"plan": "free", "country": "DE"}, + {"plan": "pro", "country": "FR", "email": "a@corp.example"}, +} + +func evalLegacy(t *testing.T, f legacyFixture, user string, c map[string]string) *EvaluationResult { + t.Helper() + svc := NewFlagService(nil) + svc.fetchFlag = func(context.Context, string, string) ([]FeatureFlag, error) { + return []FeatureFlag{{ + FlagID: "id-" + f.Name, SiteID: "s", FlagKey: f.FlagKey, FlagType: f.FlagType, + Enabled: f.Enabled, RolloutPct: f.RolloutPc, Variants: f.Variants, Targeting: f.Targeting, + }}, nil + } + res, err := svc.Evaluate(context.Background(), "s", f.FlagKey, user, c) + if err != nil { + t.Fatalf("evaluate %s: %v", f.Name, err) + } + return res +} + +func TestLegacyGolden(t *testing.T) { + raw, err := os.ReadFile(legacyFixturesPath) + if err != nil { + t.Fatal(err) + } + var fixtures []legacyFixture + if err := json.Unmarshal(raw, &fixtures); err != nil { + t.Fatal(err) + } + if os.Getenv("OBSERVE_FLAGS_GOLDEN_UPDATE") == "1" { + var out []legacyVector + for _, f := range fixtures { + for _, u := range legacyUsers() { + for _, c := range legacyContexts { + r := evalLegacy(t, f, u, c) + out = append(out, legacyVector{f.Name, u, c, r.Enabled, r.Variant, r.Reason, r.Detail}) + } + } + } + b, _ := json.MarshalIndent(out, "", " ") + if err := os.WriteFile(legacyVectorsPath, b, 0o644); err != nil { + t.Fatal(err) + } + return + } + rawV, err := os.ReadFile(legacyVectorsPath) + if err != nil { + t.Fatal(err) + } + var vectors []legacyVector + if err := json.Unmarshal(rawV, &vectors); err != nil { + t.Fatal(err) + } + byName := map[string]legacyFixture{} + for _, f := range fixtures { + byName[f.Name] = f + } + if len(vectors) < 1000 { + t.Fatalf("golden file suspiciously small: %d vectors", len(vectors)) + } + for _, v := range vectors { + f, ok := byName[v.Fixture] + if !ok { + t.Fatalf("vector references unknown fixture %q", v.Fixture) + } + r := evalLegacy(t, f, v.UserID, v.Context) + if r.Enabled != v.Enabled || r.Variant != v.Variant || r.Reason != v.Reason || r.Detail != v.Detail { + t.Fatalf("legacy drift: fixture=%s user=%q ctx=%v got {%v %q %q %q} want {%v %q %q %q}", + v.Fixture, v.UserID, v.Context, r.Enabled, r.Variant, r.Reason, r.Detail, v.Enabled, v.Variant, v.Reason, v.Detail) + } + } +} diff --git a/internal/flags/testdata/legacy_fixtures.json b/internal/flags/testdata/legacy_fixtures.json new file mode 100644 index 00000000..4a188853 --- /dev/null +++ b/internal/flags/testdata/legacy_fixtures.json @@ -0,0 +1,19 @@ +[ + {"name":"bool-100","flag_key":"bool-100","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":""}, + {"name":"bool-disabled","flag_key":"bool-disabled","flag_type":"boolean","enabled":false,"rollout_pct":100,"variants":"","targeting":""}, + {"name":"bool-30","flag_key":"bool-30","flag_type":"boolean","enabled":true,"rollout_pct":30,"variants":"","targeting":"[]"}, + {"name":"bool-50-null","flag_key":"bool-50-null","flag_type":"boolean","enabled":true,"rollout_pct":50,"variants":"","targeting":"null"}, + {"name":"target-eq","flag_key":"target-eq","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"[{\"attribute\":\"plan\",\"operator\":\"eq\",\"value\":\"pro\"}]"}, + {"name":"target-multi","flag_key":"target-multi","flag_type":"boolean","enabled":true,"rollout_pct":75,"variants":"","targeting":"[{\"attribute\":\"plan\",\"operator\":\"neq\",\"value\":\"free\"},{\"attribute\":\"country\",\"operator\":\"in\",\"value\":[\"US\",\"FR\"]}]"}, + {"name":"target-notin","flag_key":"target-notin","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"[{\"attribute\":\"country\",\"operator\":\"not_in\",\"value\":[\"DE\"]}]"}, + {"name":"target-contains","flag_key":"target-contains","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"[{\"attribute\":\"email\",\"operator\":\"contains\",\"value\":\"corp\"}]"}, + {"name":"target-scalar-in","flag_key":"target-scalar-in","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"[{\"attribute\":\"country\",\"operator\":\"in\",\"value\":\"US\"}]"}, + {"name":"target-numeric-eq","flag_key":"target-numeric-eq","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"[{\"attribute\":\"plan\",\"operator\":\"eq\",\"value\":5}]"}, + {"name":"mv-3","flag_key":"mv-3","flag_type":"multivariate","enabled":true,"rollout_pct":100,"variants":"[{\"key\":\"a\",\"name\":\"A\",\"rollout_pct\":50},{\"key\":\"b\",\"name\":\"B\",\"rollout_pct\":30},{\"key\":\"c\",\"name\":\"C\",\"rollout_pct\":20}]","targeting":""}, + {"name":"mv-short-sum","flag_key":"mv-short-sum","flag_type":"multivariate","enabled":true,"rollout_pct":60,"variants":"[{\"key\":\"x\",\"rollout_pct\":10},{\"key\":\"y\",\"rollout_pct\":10}]","targeting":"[{\"attribute\":\"plan\",\"operator\":\"eq\",\"value\":\"pro\"}]"}, + {"name":"mv-legacy-extra-fields","flag_key":"mv-legacy-extra-fields","flag_type":"multivariate","enabled":true,"rollout_pct":100,"variants":"[{\"key\":\"on\",\"name\":\"On\",\"value\":\"1\",\"weight\":9,\"rollout_pct\":50},{\"key\":\"off\",\"rollout_pct\":50}]","targeting":""}, + {"name":"type-string-variants-ignored","flag_key":"type-string-variants-ignored","flag_type":"string","enabled":true,"rollout_pct":100,"variants":"[{\"key\":\"a\",\"rollout_pct\":100}]","targeting":""}, + {"name":"bad-targeting-quarantine","flag_key":"bad-targeting-quarantine","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"{\"attribute\":\"plan\"}"}, + {"name":"bad-operator-quarantine","flag_key":"bad-operator-quarantine","flag_type":"boolean","enabled":true,"rollout_pct":100,"variants":"","targeting":"[{\"attribute\":\"plan\",\"operator\":\"regex\",\"value\":\".*\"}]"}, + {"name":"bad-variants-quarantine","flag_key":"bad-variants-quarantine","flag_type":"multivariate","enabled":true,"rollout_pct":100,"variants":"[{\"key\":\"a\",\"rollout_pct\":150}]","targeting":""} +] diff --git a/internal/flags/testdata/legacy_golden.json b/internal/flags/testdata/legacy_golden.json new file mode 100644 index 00000000..728814f1 --- /dev/null +++ b/internal/flags/testdata/legacy_golden.json @@ -0,0 +1,30515 @@ +[ + { + "fixture": "bool-100", + "user_id": "", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "alice", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "bob", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "carol", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u0", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u1", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u2", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u3", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u4", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u5", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u6", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u7", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u8", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u9", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u10", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u11", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u12", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u13", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u14", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u15", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u16", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u17", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u18", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u19", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u20", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u21", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u22", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u23", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u24", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u25", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u26", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u27", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u28", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u29", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u30", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u31", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u32", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u33", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u34", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u35", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u36", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u37", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u38", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u39", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-100", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-disabled", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-disabled", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "flag disabled" + }, + { + "fixture": "bool-30", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "alice", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "carol", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u0", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u3", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u4", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u6", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u8", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u10", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u11", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u12", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u15", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u19", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u21", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u33", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-30", + "user_id": "u39", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-30", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "alice", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "bob", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u0", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u2", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u3", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u8", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u10", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u12", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u14", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u15", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u18", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u23", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u24", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u25", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u27", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u31", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u34", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u35", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u36", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "bool-50-null", + "user_id": "u38", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u39", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bool-50-null", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-eq", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-eq", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "target-multi", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-multi", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-multi", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-notin", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-notin", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-contains", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-contains", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "target-scalar-in", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-scalar-in", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "target-numeric-eq", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-3", + "user_id": "", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "alice", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "bob", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "carol", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u0", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u1", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u2", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u3", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u4", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u5", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u6", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u7", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u8", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u9", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u10", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u11", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u12", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u13", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u14", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u15", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u16", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u17", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u18", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u19", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u20", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u21", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u22", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u23", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u24", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u25", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u26", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u27", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u28", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u29", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u30", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u31", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u32", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u33", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u34", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u35", + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "c", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u36", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u37", + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "a", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u38", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u39", + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-3", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "b", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "alice", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "bob", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "carol", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u0", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u1", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u2", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u3", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u4", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u5", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u6", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u7", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u8", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u9", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u10", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u11", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u12", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u13", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u14", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u15", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u16", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u17", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u18", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u19", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u20", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u21", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u22", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u23", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u24", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u25", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u26", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u27", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u28", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u29", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u30", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u31", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u32", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u33", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u34", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u35", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u36", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "x", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u37", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u38", + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "evaluated", + "detail": "rollout" + }, + { + "fixture": "mv-short-sum", + "user_id": "u39", + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-short-sum", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "evaluated", + "detail": "targeting" + }, + { + "fixture": "mv-short-sum", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "y", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "alice", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "bob", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "carol", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u0", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u1", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u2", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u3", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u4", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u5", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u6", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u7", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u8", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u9", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u10", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u11", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u12", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u13", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u14", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u15", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u16", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u17", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u18", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u19", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u20", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u21", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u22", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u23", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u24", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u25", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u26", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u27", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u28", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u29", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u30", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u31", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u32", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u33", + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "off", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u34", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u35", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u36", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u37", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u38", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u39", + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "mv-legacy-extra-fields", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "variant": "on", + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "alice", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "bob", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "carol", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u0", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u1", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u2", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u3", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u4", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u5", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u6", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u7", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u8", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u9", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u10", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u11", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u12", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u13", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u14", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u15", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u16", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u17", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u18", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u19", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u20", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u21", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u22", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u23", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u24", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u25", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u26", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u27", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u28", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u29", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u30", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u31", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u32", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u33", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u34", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u35", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u36", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u37", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u38", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u39", + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "type-string-variants-ignored", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": true, + "reason": "evaluated" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "alice", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "bob", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "carol", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u0", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u1", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u2", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u3", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u4", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u5", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u6", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u7", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u8", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u9", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u10", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u11", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u12", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u13", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u14", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u15", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u16", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u17", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u18", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u19", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u20", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u21", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u22", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u23", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u24", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u25", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u26", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u27", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u28", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u29", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u30", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u31", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u32", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u33", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u34", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u35", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u36", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u37", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u38", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u39", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-targeting-quarantine", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: invalid JSON: json: cannot unmarshal object into Go value of type []flags.TargetingRule" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "alice", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "bob", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "carol", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u0", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u1", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u2", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u3", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u4", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u5", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u6", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u7", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u8", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u9", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u10", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u11", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u12", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u13", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u14", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u15", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u16", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u17", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u18", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u19", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u20", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u21", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u22", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u23", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u24", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u25", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u26", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u27", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u28", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u29", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u30", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u31", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u32", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u33", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u34", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u35", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u36", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u37", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u38", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u39", + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-operator-quarantine", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored targeting failed validation: targeting: rule 0: unknown operator \"regex\"" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "alice", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "alice", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "alice", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "alice", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "bob", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "bob", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "bob", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "bob", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "carol", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "carol", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "carol", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "carol", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "user-with-a-really-long-identifier-0123456789", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u0", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u0", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u0", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u0", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u1", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u1", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u1", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u1", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u2", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u2", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u2", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u2", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u3", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u3", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u3", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u3", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u4", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u4", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u4", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u4", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u5", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u5", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u5", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u5", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u6", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u6", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u6", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u6", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u7", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u7", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u7", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u7", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u8", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u8", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u8", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u8", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u9", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u9", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u9", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u9", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u10", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u10", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u10", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u10", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u11", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u11", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u11", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u11", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u12", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u12", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u12", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u12", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u13", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u13", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u13", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u13", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u14", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u14", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u14", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u14", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u15", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u15", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u15", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u15", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u16", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u16", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u16", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u16", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u17", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u17", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u17", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u17", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u18", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u18", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u18", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u18", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u19", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u19", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u19", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u19", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u20", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u20", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u20", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u20", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u21", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u21", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u21", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u21", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u22", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u22", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u22", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u22", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u23", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u23", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u23", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u23", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u24", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u24", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u24", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u24", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u25", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u25", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u25", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u25", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u26", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u26", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u26", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u26", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u27", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u27", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u27", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u27", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u28", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u28", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u28", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u28", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u29", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u29", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u29", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u29", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u30", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u30", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u30", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u30", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u31", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u31", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u31", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u31", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u32", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u32", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u32", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u32", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u33", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u33", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u33", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u33", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u34", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u34", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u34", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u34", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u35", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u35", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u35", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u35", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u36", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u36", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u36", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u36", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u37", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u37", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u37", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u37", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u38", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u38", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u38", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u38", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u39", + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u39", + "context": { + "country": "US", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u39", + "context": { + "country": "DE", + "plan": "free" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + }, + { + "fixture": "bad-variants-quarantine", + "user_id": "u39", + "context": { + "country": "FR", + "email": "a@corp.example", + "plan": "pro" + }, + "enabled": false, + "reason": "invalid", + "detail": "stored variants failed validation: variants: entry 0: rollout_pct 150 out of range 0..100" + } +] \ No newline at end of file From da6103d41bb133e6884ec0bfe48a74bcbcdeedea Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:41:44 +0000 Subject: [PATCH 14/65] feat: add single-instance Helm chart for Kubernetes Chart packaging/helm/observe (Observe + Nucleus StatefulSets, replicas fixed at 1), NetworkPolicy isolating the unauthenticated Nucleus, secret handling that fails templating without JWT secret/salt, values.schema.json, ingest split Service/Ingress, docs/operations/kubernetes.md and a helm.yml workflow (lint + kubeconform). Chart was NOT rendered or linted locally: helm is not available in this environment. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- .github/workflows/helm.yml | 82 ++++++++ docs/operations/kubernetes.md | 198 ++++++++++++++++++ packaging/helm/observe/.helmignore | 2 + packaging/helm/observe/Chart.yaml | 10 + packaging/helm/observe/ci/values-ci.yaml | 13 ++ packaging/helm/observe/templates/NOTES.txt | 30 +++ packaging/helm/observe/templates/_helpers.tpl | 71 +++++++ packaging/helm/observe/templates/ingress.yaml | 65 ++++++ packaging/helm/observe/templates/nucleus.yaml | 151 +++++++++++++ .../templates/observe-statefulset.yaml | 152 ++++++++++++++ packaging/helm/observe/templates/secret.yaml | 38 ++++ .../helm/observe/templates/services.yaml | 39 ++++ packaging/helm/observe/values.schema.json | 174 +++++++++++++++ packaging/helm/observe/values.yaml | 180 ++++++++++++++++ 14 files changed, 1205 insertions(+) create mode 100644 .github/workflows/helm.yml create mode 100644 docs/operations/kubernetes.md create mode 100644 packaging/helm/observe/.helmignore create mode 100644 packaging/helm/observe/Chart.yaml create mode 100644 packaging/helm/observe/ci/values-ci.yaml create mode 100644 packaging/helm/observe/templates/NOTES.txt create mode 100644 packaging/helm/observe/templates/_helpers.tpl create mode 100644 packaging/helm/observe/templates/ingress.yaml create mode 100644 packaging/helm/observe/templates/nucleus.yaml create mode 100644 packaging/helm/observe/templates/observe-statefulset.yaml create mode 100644 packaging/helm/observe/templates/secret.yaml create mode 100644 packaging/helm/observe/templates/services.yaml create mode 100644 packaging/helm/observe/values.schema.json create mode 100644 packaging/helm/observe/values.yaml diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 00000000..d9377fc9 --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,82 @@ +name: Helm chart + +# Lints and renders packaging/helm/observe and validates the rendered manifests +# against the Kubernetes API schemas. Separate from ci.yml on purpose; only runs +# when the chart (or this workflow) changes. + +on: + push: + branches: [main] + paths: + - 'packaging/helm/**' + - '.github/workflows/helm.yml' + pull_request: + paths: + - 'packaging/helm/**' + - '.github/workflows/helm.yml' + +permissions: + contents: read + +env: + CHART: packaging/helm/observe + KUBERNETES_VERSION: 1.30.0 + +jobs: + chart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-helm@v4 + with: + version: v3.16.2 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + - name: Install kubeconform + run: go install github.com/yannh/kubeconform/cmd/kubeconform@v0.6.7 + + - name: helm lint (secrets supplied) + run: helm lint "$CHART" --strict -f "$CHART/ci/values-ci.yaml" + + - name: helm template | kubeconform (all features on) + run: | + helm template obs "$CHART" -f "$CHART/ci/values-ci.yaml" \ + | "$(go env GOPATH)/bin/kubeconform" -strict -summary \ + -kubernetes-version "$KUBERNETES_VERSION" + + - name: helm template | kubeconform (existingSecret, external database) + run: | + helm template obs "$CHART" \ + --set secrets.existingSecret=my-observe-secrets \ + --set nucleus.enabled=false \ + --set nucleus.externalUrl=postgres://db.example:5432/observe \ + | "$(go env GOPATH)/bin/kubeconform" -strict -summary \ + -kubernetes-version "$KUBERNETES_VERSION" + + - name: Render fails without JWT secret and session salt + run: | + if helm template obs "$CHART" >/tmp/out.txt 2>/tmp/err.txt; then + echo "chart rendered without OBSERVE_JWT_SECRET/OBSERVE_SESSION_SALT" >&2 + exit 1 + fi + grep -q OBSERVE_JWT_SECRET /tmp/err.txt + + - name: Schema rejects unpinned Nucleus and unknown keys + run: | + if helm template obs "$CHART" -f "$CHART/ci/values-ci.yaml" --set nucleus.image.tag=latest >/dev/null 2>&1; then + echo "nucleus.image.tag=latest was accepted" >&2; exit 1 + fi + if helm template obs "$CHART" -f "$CHART/ci/values-ci.yaml" --set replicaCount=2 >/dev/null 2>&1; then + echo "unknown key replicaCount was accepted" >&2; exit 1 + fi + + - name: Invariants (single replica, Nucleus isolated) + run: | + helm template obs "$CHART" -f "$CHART/ci/values-ci.yaml" > /tmp/all.yaml + test "$(grep -c '^ replicas: 1$' /tmp/all.yaml)" = 2 + grep -q 'kind: NetworkPolicy' /tmp/all.yaml + # Nucleus is never given an Ingress or non-ClusterIP Service. + if awk '/kind: Ingress/,/^---/' /tmp/all.yaml | grep -q 'obs-observe-nucleus'; then + echo "an Ingress references Nucleus" >&2; exit 1 + fi diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md new file mode 100644 index 00000000..9a61f89a --- /dev/null +++ b/docs/operations/kubernetes.md @@ -0,0 +1,198 @@ +# Running Observe on Kubernetes + +Chart: `packaging/helm/observe`. It deploys one Observe pod and one Nucleus pod +(both StatefulSets), mirroring `docker-compose.yml`. + +Status of this chart: written against the compose file and systemd units. It has +not been installed on a cluster, and `helm lint` / `helm template` had not been +run when it was written (no Helm in the authoring environment). The +`.github/workflows/helm.yml` workflow runs lint, template and kubeconform; treat +a green run of that workflow as the first real validation. + +## Single instance only + +Replicas are fixed at 1 in the templates and there is no `replicaCount` value +(the schema rejects unknown keys, so `--set replicaCount=2` fails). Observe has no +clustering: the ingest WAL and queue under `OBSERVE_DATA_DIR` are owned by one +process, and Nucleus runs as a single node. A second replica, or two releases +pointed at one database, is unsupported and can corrupt WAL state or +double-process events. The HA position is in `docs/design/ha-and-clustering.md` +(that file and `docs/operations/hardening.md` come from a separate change; read +them there if they are not in your checkout). + +Consequences: + +- A PodDisruptionBudget is not provided. With one replica `minAvailable: 1` + would block node drains forever and `maxUnavailable: 1` does nothing. A drain + restarts the pod; acknowledged events are covered by the durable WAL + (default mode) and replayed on start. +- Readiness removes the single pod from the Service while it is degraded, so + senders get connection errors rather than the application's 503. SDKs should + retry. +- Rolling updates restart the pod (the old one stops before the new one starts, + since the volume is ReadWriteOnce). Expect a short ingest gap. + +## Install + +```sh +kubectl create namespace observe +kubectl -n observe create secret generic observe-secrets \ + --from-literal=OBSERVE_JWT_SECRET="$(openssl rand -hex 32)" \ + --from-literal=OBSERVE_SESSION_SALT="$(openssl rand -hex 32)" +helm install observe packaging/helm/observe -n observe \ + --set secrets.existingSecret=observe-secrets \ + --set observe.publicUrl=https://observe.example.com +``` + +The image tag defaults to the chart `appVersion` (release tags have no `v` +prefix). Nucleus is pinned to `v1.1.1` (`nucleus.image.tag`); the schema rejects +`latest`. + +## Secrets + +| Variable | Required | Chart behavior | +|----------|----------|----------------| +| `OBSERVE_JWT_SECRET` | yes | Rendering fails unless provided or autogenerated | +| `OBSERVE_SESSION_SALT` | yes | Same | +| `OBSERVE_SECRET_KEY` | no | Needed to configure LLM/S3 credentials; encrypts them at rest | +| `OBSERVE_ADMIN_PASSWORD` | no | Unset means the `/setup` wizard creates the admin | + +Choose one: + +1. `secrets.existingSecret` (preferred). Key names default to the variable names + and are overridable under `secrets.keys`. The chart cannot check the Secret's + contents, only that you named one. `OBSERVE_SECRET_KEY` and + `OBSERVE_ADMIN_PASSWORD` are optional keys. +2. `secrets.jwtSecret` + `secrets.sessionSalt` as values (they land in the + Helm release record; avoid for real deployments). +3. `secrets.autogenerate=true`: generated once and reused on upgrade through + `lookup`, with `helm.sh/resource-policy: keep`. + +Regenerate-on-restart trap (same failure `teploy.yml` documents): if the JWT +secret or salt is empty, Observe generates fresh random ones on every start, +invalidates every session and changes visitor/session IDs, and logs only a WARN. +On Kubernetes the equivalents are: + +- `lookup` returns nothing under `helm template`, `helm install --dry-run`, + Argo CD and Flux, so `autogenerate` produces new values on every render there. + Use `existingSecret` with those tools. +- Deleting the release removes an inline-value Secret. Re-installing with + different values logs everyone out. +- Changing a Secret does not restart the pod; run + `kubectl rollout restart statefulset/-observe`. + +`OBSERVE_SECRET_KEY` loss makes stored LLM/S3 credentials unreadable. Back it up +with the same care as the data. + +## Nucleus is unauthenticated: network isolation + +Nucleus runs with `NUCLEUS_ALLOW_NO_AUTH=1` (as in compose and systemd). Anyone +who can open a TCP connection to it can read and write all data. The chart: + +- exposes it through a `ClusterIP` Service only, and offers no Ingress, + NodePort or LoadBalancer option for it; +- publishes only the database port (5432) on the Service; the cluster port + (5433, required by the compose flags) is not exposed; +- renders a NetworkPolicy allowing ingress to the Nucleus pod only from the + Observe pod of the same release, on the database port. + +NetworkPolicy is enforced only if your CNI supports it (Calico, Cilium, and +similar). On a cluster without enforcement the policy is silently ignored and +any pod in the cluster can reach the database. Verify with a throwaway pod: + +```sh +kubectl -n observe run np-test --rm -it --image=busybox --restart=Never -- \ + nc -zv -w 3 observe-observe-nucleus 5432 # must time out +``` + +Do not set `nucleus.networkPolicy.enabled=false` on a shared cluster. For +stricter setups add your own egress policy for the Observe pod and, ideally, +dedicate a namespace. With `nucleus.enabled=false` (external database) none of +this applies; securing that database is yours. + +## Probes + +`/healthz` runs `SELECT 1` against Nucleus and returns 503 when the database is +unreachable or ingest durability is degraded (dead flush worker, WAL failure, +memory-only WAL, failing error flush). The chart uses it for: + +- startup probe (up to 180 s) and readiness probe, with 12 s timeouts to cover a + cold Nucleus connect (~7.8 s measured, see the Dockerfile `HEALTHCHECK`). + A 503 marks the pod NotReady. +- Not for liveness. Liveness is a TCP check on the dashboard port, so a database + outage does not restart-loop Observe. Read the `/healthz` body to see why a + pod is NotReady. + +## Storage and backup + +| PVC | Mount | Holds | +|-----|-------|-------| +| `data--observe-0` | `/var/lib/observe` | ingest WAL/queue (`queue/`), `audit.key`, local state | +| `data--nucleus-0` | `/data` | all analytics data | + +Both are ReadWriteOnce (`persistence.*`). `helm uninstall` does not delete +StatefulSet PVCs; do not delete them by hand either. + +Back up: + +1. Both PVCs, with CSI VolumeSnapshots taken as close together as possible. The + WAL is replayed against Nucleus at start, so (by design reading, not tested) + a slightly older Nucleus snapshot with a newer WAL snapshot is the safer + skew; the reverse risks losing the acknowledged tail. Crash-consistency of a Nucleus snapshot has not been + verified here; also take Observe's application-level backup periodically. +2. `audit.key` (in the Observe PVC) and `OBSERVE_AUDIT_KEY` if you set it; losing + the key breaks verification of the audit chain. +3. The Secret holding `OBSERVE_SECRET_KEY`, JWT secret and salt. + +The pod root filesystem is read-only; `/tmp` is an `emptyDir` +(`tmpSizeLimit`, default 1Gi) where backup and restore stage temporary files. +Size it above your largest table export. + +## Exposure + +The dashboard Service is `ClusterIP`. To publish it, use `ingress.*` (with +`ingress.tls` and a pre-created or cert-manager TLS Secret) and set +`observe.publicUrl` and `observe.trustedProxies` to your ingress controller's pod +CIDR so client IPs are not spoofable. + +To collect telemetry from outside without exposing the dashboard, enable the +split listener: + +```yaml +ingest: { enabled: true, port: 3001 } +ingress: + enabled: false # dashboard stays internal + ingest: { enabled: true, host: ingest.example.com, tls: { enabled: true, secretName: ingest-tls } } +``` + +This sets `OBSERVE_INGEST_ADDR=:3001`, adds `-observe-ingest`, and its +Ingress points only at that Service. The first-run `/setup` wizard is on the +dashboard port only; with no admin password set, complete setup (via +port-forward) before exposing the dashboard. + +## Security context and resources + +Observe runs as uid/gid 10001 (the Dockerfile user) with a read-only root +filesystem, all capabilities dropped, no privilege escalation, `RuntimeDefault` +seccomp, and `fsGroup: 10001` so the PVC is writable. The only writable paths +are the data PVC and `/tmp`. The read-only root filesystem was derived from +reading the code (temp files and the data dir are the writes found); it has not +been run. If a feature fails with a read-only error, set +`securityContext.readOnlyRootFilesystem=false` and report the path. + +The Nucleus container drops capabilities and escalation but keeps the image's +user and a writable root filesystem, because its image layout was not verified. + +Resource defaults come from `BENCHMARKS.md` (91 MB idle, 124 MB after the full +benchmark, Nucleus 72 to 100 MB on a 4-core test box) with headroom for the +100000-event buffer and `--max-memory 512`: Observe requests 100m / 128Mi and is +limited to 1Gi; Nucleus requests 100m / 256Mi, limited to 768Mi. No CPU limits +are set, to avoid throttling ingest. Those benchmarks are from small hardware; +size from your own traffic (see `docs/operations/capacity.md`). + +## Values reference + +See `packaging/helm/observe/values.yaml` (commented) and `values.schema.json`. +Any `OBSERVE_*` setting not modeled as a value goes in `observe.env` (plain) or +`observe.extraEnv` (EnvVar form, for secret references such as SMTP or OIDC +client secrets). diff --git a/packaging/helm/observe/.helmignore b/packaging/helm/observe/.helmignore new file mode 100644 index 00000000..531d478b --- /dev/null +++ b/packaging/helm/observe/.helmignore @@ -0,0 +1,2 @@ +*.swp +ci/ diff --git a/packaging/helm/observe/Chart.yaml b/packaging/helm/observe/Chart.yaml new file mode 100644 index 00000000..f1501f96 --- /dev/null +++ b/packaging/helm/observe/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: observe +description: Single-instance Teploy Observe (analytics, errors, logs, traces, replays) with a Nucleus database StatefulSet. +type: application +version: 0.1.0 +appVersion: "0.2.0" +kubeVersion: ">=1.25.0-0" +home: https://github.com/useteploy/teploy-observe +sources: + - https://github.com/useteploy/teploy-observe diff --git a/packaging/helm/observe/ci/values-ci.yaml b/packaging/helm/observe/ci/values-ci.yaml new file mode 100644 index 00000000..89325d38 --- /dev/null +++ b/packaging/helm/observe/ci/values-ci.yaml @@ -0,0 +1,13 @@ +# Used only by CI template validation. Placeholder secrets, not for installs. +secrets: + jwtSecret: ci-only-jwt-secret-0000000000000000 + sessionSalt: ci-only-session-salt-000000000000 +ingest: + enabled: true +ingress: + enabled: true + className: nginx + tls: { enabled: true, secretName: observe-tls } + ingest: + enabled: true + tls: { enabled: true, secretName: ingest-tls } diff --git a/packaging/helm/observe/templates/NOTES.txt b/packaging/helm/observe/templates/NOTES.txt new file mode 100644 index 00000000..251cfbcf --- /dev/null +++ b/packaging/helm/observe/templates/NOTES.txt @@ -0,0 +1,30 @@ +Observe {{ default .Chart.AppVersion .Values.image.tag }} installed as a SINGLE INSTANCE (replicas are fixed at 1; there is no clustering). + +Dashboard (not public by default): + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "observe.fullname" . }} {{ .Values.service.port }}:3000 + then open http://localhost:{{ .Values.service.port }} +{{- if not .Values.secrets.adminPassword }} +{{- if not .Values.secrets.existingSecret }} + +No admin password was set: the first visitor to /setup creates the admin account. +Complete it before exposing the dashboard. +{{- end }} +{{- end }} +{{- if .Values.secrets.autogenerate }} + +JWT secret and session salt were autogenerated and kept in Secret {{ include "observe.fullname" . }}. +Under `helm template`, --dry-run, Argo CD or Flux the lookup is empty and new values are +generated on EVERY render, invalidating all sessions. Use secrets.existingSecret there. +{{- end }} +{{- if .Values.nucleus.enabled }} + +Nucleus runs with NO authentication and is protected only by a NetworkPolicy +({{ if .Values.nucleus.networkPolicy.enabled }}enabled{{ else }}DISABLED - the database is open to the cluster network{{ end }}). +Your CNI must enforce NetworkPolicy. Never expose Service {{ include "observe.nucleusName" . }}. +{{- end }} +{{- if .Values.ingest.enabled }} + +Telemetry ingest listener: Service {{ include "observe.fullname" . }}-ingest port {{ .Values.ingest.port }}. +{{- end }} + +Back up both PVCs (data-{{ include "observe.fullname" . }}-0{{ if .Values.nucleus.enabled }}, data-{{ include "observe.nucleusName" . }}-0{{ end }}); see docs/operations/kubernetes.md. diff --git a/packaging/helm/observe/templates/_helpers.tpl b/packaging/helm/observe/templates/_helpers.tpl new file mode 100644 index 00000000..33453916 --- /dev/null +++ b/packaging/helm/observe/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{- define "observe.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "observe.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "observe.nucleusName" -}} +{{- printf "%s-nucleus" (include "observe.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "observe.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/name: {{ include "observe.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "observe.selectorLabels" -}} +app.kubernetes.io/name: {{ include "observe.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: observe +{{- end -}} + +{{- define "observe.nucleusSelectorLabels" -}} +app.kubernetes.io/name: {{ include "observe.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: nucleus +{{- end -}} + +{{- define "observe.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) -}} +{{- end -}} + +{{- define "observe.secretName" -}} +{{- default (include "observe.fullname" .) .Values.secrets.existingSecret -}} +{{- end -}} + +{{- define "observe.nucleusUrl" -}} +{{- if .Values.nucleus.enabled -}} +{{- printf "postgres://%s:%v/observe" (include "observe.nucleusName" .) .Values.nucleus.port -}} +{{- else -}} +{{- required "nucleus.externalUrl is required when nucleus.enabled=false" .Values.nucleus.externalUrl -}} +{{- end -}} +{{- end -}} + +{{/* +Cross-value validation. Included by the Secret and the Observe StatefulSet so +every render path trips it. +*/}} +{{- define "observe.validate" -}} +{{- if and .Values.ingress.ingest.enabled (not .Values.ingest.enabled) -}} +{{- fail "ingress.ingest.enabled requires ingest.enabled=true" -}} +{{- end -}} +{{- if not .Values.secrets.existingSecret -}} +{{- if not (or .Values.secrets.autogenerate (and .Values.secrets.jwtSecret .Values.secrets.sessionSalt)) -}} +{{- fail "OBSERVE_JWT_SECRET and OBSERVE_SESSION_SALT are required. Set secrets.existingSecret (preferred), set both secrets.jwtSecret and secrets.sessionSalt, or set secrets.autogenerate=true. Without them Observe generates new ones on every restart and every session is invalidated. See docs/operations/kubernetes.md." -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/packaging/helm/observe/templates/ingress.yaml b/packaging/helm/observe/templates/ingress.yaml new file mode 100644 index 00000000..fa39eb53 --- /dev/null +++ b/packaging/helm/observe/templates/ingress.yaml @@ -0,0 +1,65 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "observe.fullname" . }} + labels: + {{- include "observe.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: [{{ .Values.ingress.host | quote }}] + secretName: {{ required "ingress.tls.secretName is required when ingress.tls.enabled" .Values.ingress.tls.secretName }} + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "observe.fullname" . }} + port: + name: http +{{- end }} +{{- if and .Values.ingress.ingest.enabled .Values.ingest.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "observe.fullname" . }}-ingest + labels: + {{- include "observe.labels" . | nindent 4 }} + {{- with .Values.ingress.ingest.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.ingest.tls.enabled }} + tls: + - hosts: [{{ .Values.ingress.ingest.host | quote }}] + secretName: {{ required "ingress.ingest.tls.secretName is required when ingress.ingest.tls.enabled" .Values.ingress.ingest.tls.secretName }} + {{- end }} + rules: + - host: {{ .Values.ingress.ingest.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "observe.fullname" . }}-ingest + port: + name: ingest +{{- end }} diff --git a/packaging/helm/observe/templates/nucleus.yaml b/packaging/helm/observe/templates/nucleus.yaml new file mode 100644 index 00000000..b259ae87 --- /dev/null +++ b/packaging/helm/observe/templates/nucleus.yaml @@ -0,0 +1,151 @@ +{{- if .Values.nucleus.enabled }} +{{- $n := include "observe.nucleusName" . }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $n }} + labels: + {{- include "observe.labels" . | nindent 4 }} + app.kubernetes.io/component: nucleus +spec: + # SINGLE NODE, not configurable. Nucleus runs without authentication and + # without clustering here. + replicas: 1 + serviceName: {{ $n }} + selector: + matchLabels: + {{- include "observe.nucleusSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "observe.labels" . | nindent 8 }} + app.kubernetes.io/component: nucleus + {{- with .Values.nucleus.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 60 + {{- with .Values.nucleus.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: nucleus + image: "{{ .Values.nucleus.image.repository }}:{{ .Values.nucleus.image.tag }}" + imagePullPolicy: {{ .Values.nucleus.image.pullPolicy }} + args: + - start + - --port + - {{ .Values.nucleus.port | quote }} + - --cluster-port + - "5433" + - --data + - /data + - --max-memory + - {{ .Values.nucleus.maxMemoryMB | quote }} + env: + # NO_AUTH: reachable only from the Observe pod (see the + # NetworkPolicy). The cluster/replication flags mirror + # docker-compose.yml; the cluster port is not exposed by the Service. + - { name: NUCLEUS_ALLOW_INSECURE_CLUSTER, value: "1" } + - { name: NUCLEUS_ALLOW_INSECURE_REPLICATION, value: "1" } + - { name: NUCLEUS_ALLOW_NO_AUTH, value: "1" } + ports: + - name: pg + containerPort: {{ .Values.nucleus.port }} + startupProbe: + tcpSocket: { port: pg } + periodSeconds: 5 + failureThreshold: 36 + readinessProbe: + tcpSocket: { port: pg } + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + tcpSocket: { port: pg } + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + securityContext: + {{- toYaml .Values.nucleus.securityContext | nindent 12 }} + resources: + {{- toYaml .Values.nucleus.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + {{- with .Values.nucleus.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nucleus.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nucleus.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "observe.nucleusSelectorLabels" . | nindent 10 }} + spec: + accessModes: + {{- toYaml .Values.nucleus.persistence.accessModes | nindent 10 }} + {{- if .Values.nucleus.persistence.storageClassName }} + storageClassName: {{ .Values.nucleus.persistence.storageClassName | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.nucleus.persistence.size }} +--- +# ClusterIP only. There is intentionally no Ingress, NodePort or LoadBalancer +# option for Nucleus. +apiVersion: v1 +kind: Service +metadata: + name: {{ $n }} + labels: + {{- include "observe.labels" . | nindent 4 }} + app.kubernetes.io/component: nucleus +spec: + type: ClusterIP + selector: + {{- include "observe.nucleusSelectorLabels" . | nindent 4 }} + ports: + - name: pg + port: {{ .Values.nucleus.port }} + targetPort: pg +{{- if .Values.nucleus.networkPolicy.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $n }} + labels: + {{- include "observe.labels" . | nindent 4 }} + app.kubernetes.io/component: nucleus +spec: + # Only the Observe pod of this release may connect, and only to the database + # port. The cluster/replication port accepts nothing. + podSelector: + matchLabels: + {{- include "observe.nucleusSelectorLabels" . | nindent 6 }} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + {{- include "observe.selectorLabels" . | nindent 14 }} + ports: + - protocol: TCP + port: {{ .Values.nucleus.port }} +{{- end }} +{{- end }} diff --git a/packaging/helm/observe/templates/observe-statefulset.yaml b/packaging/helm/observe/templates/observe-statefulset.yaml new file mode 100644 index 00000000..f6dc63f6 --- /dev/null +++ b/packaging/helm/observe/templates/observe-statefulset.yaml @@ -0,0 +1,152 @@ +{{- include "observe.validate" . -}} +{{- $keys := .Values.secrets.keys }} +{{- $secret := include "observe.secretName" . }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "observe.fullname" . }} + labels: + {{- include "observe.labels" . | nindent 4 }} + app.kubernetes.io/component: observe +spec: + # SINGLE INSTANCE, deliberately not configurable: the ingest WAL is + # single-process and Observe has no clustering. A second replica would + # corrupt the WAL/queue state or double-process events. + replicas: 1 + serviceName: {{ include "observe.fullname" . }} + podManagementPolicy: OrderedReady + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "observe.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "observe.labels" . | nindent 8 }} + app.kubernetes.io/component: observe + {{- with .Values.observe.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.observe.terminationGracePeriodSeconds }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: observe + image: {{ include "observe.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - name: http + containerPort: 3000 + {{- if .Values.ingest.enabled }} + - name: ingest + containerPort: {{ .Values.ingest.port }} + {{- end }} + env: + - name: OBSERVE_ADDR + value: ":3000" + - name: OBSERVE_NUCLEUS_URL + value: {{ include "observe.nucleusUrl" . | quote }} + - name: OBSERVE_DATA_DIR + value: /var/lib/observe + - name: OBSERVE_SITE_ID + value: {{ .Values.observe.siteId | quote }} + - name: OBSERVE_ADMIN_USER + value: {{ .Values.observe.adminUser | quote }} + - name: OBSERVE_SEED_DEMO + value: {{ .Values.observe.seedDemo | quote }} + {{- if .Values.observe.publicUrl }} + - name: OBSERVE_PUBLIC_URL + value: {{ .Values.observe.publicUrl | quote }} + {{- end }} + {{- if .Values.observe.trustedProxies }} + - name: OBSERVE_TRUSTED_PROXIES + value: {{ .Values.observe.trustedProxies | quote }} + {{- end }} + {{- if .Values.ingest.enabled }} + - name: OBSERVE_INGEST_ADDR + value: {{ printf ":%v" .Values.ingest.port | quote }} + {{- end }} + {{- range $k, $v := .Values.observe.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + - name: OBSERVE_JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ $secret }} + key: {{ $keys.jwtSecret }} + - name: OBSERVE_SESSION_SALT + valueFrom: + secretKeyRef: + name: {{ $secret }} + key: {{ $keys.sessionSalt }} + - name: OBSERVE_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $secret }} + key: {{ $keys.secretKey }} + optional: true + - name: OBSERVE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $secret }} + key: {{ $keys.adminPassword }} + optional: true + {{- with .Values.observe.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + startupProbe: + httpGet: { path: /healthz, port: http } + {{- toYaml .Values.probes.startup | nindent 12 }} + readinessProbe: + httpGet: { path: /healthz, port: http } + {{- toYaml .Values.probes.readiness | nindent 12 }} + livenessProbe: + tcpSocket: { port: http } + {{- toYaml .Values.probes.liveness | nindent 12 }} + resources: + {{- toYaml .Values.observe.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /var/lib/observe + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + sizeLimit: {{ .Values.tmpSizeLimit }} + {{- with .Values.observe.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.observe.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.observe.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "observe.selectorLabels" . | nindent 10 }} + spec: + accessModes: + {{- toYaml .Values.observe.persistence.accessModes | nindent 10 }} + {{- if .Values.observe.persistence.storageClassName }} + storageClassName: {{ .Values.observe.persistence.storageClassName | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.observe.persistence.size }} diff --git a/packaging/helm/observe/templates/secret.yaml b/packaging/helm/observe/templates/secret.yaml new file mode 100644 index 00000000..9a0a8e92 --- /dev/null +++ b/packaging/helm/observe/templates/secret.yaml @@ -0,0 +1,38 @@ +{{- include "observe.validate" . -}} +{{- if not .Values.secrets.existingSecret }} +{{- $keys := .Values.secrets.keys }} +{{- $name := include "observe.fullname" . }} +{{- /* lookup is empty under helm template / --dry-run / GitOps renderers. */ -}} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name | default dict }} +{{- $jwt := .Values.secrets.jwtSecret }} +{{- $salt := .Values.secrets.sessionSalt }} +{{- if and (not $jwt) .Values.secrets.autogenerate }} +{{- $prev := dig "data" $keys.jwtSecret "" $existing }} +{{- $jwt = ternary ($prev | b64dec) (randAlphaNum 48) (ne $prev "") }} +{{- end }} +{{- if and (not $salt) .Values.secrets.autogenerate }} +{{- $prev := dig "data" $keys.sessionSalt "" $existing }} +{{- $salt = ternary ($prev | b64dec) (randAlphaNum 48) (ne $prev "") }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "observe.labels" . | nindent 4 }} + {{- if .Values.secrets.autogenerate }} + annotations: + # Never delete generated secrets on uninstall: losing them logs everyone out. + helm.sh/resource-policy: keep + {{- end }} +type: Opaque +stringData: + {{ $keys.jwtSecret }}: {{ $jwt | quote }} + {{ $keys.sessionSalt }}: {{ $salt | quote }} + {{- if .Values.secrets.secretKey }} + {{ $keys.secretKey }}: {{ .Values.secrets.secretKey | quote }} + {{- end }} + {{- if .Values.secrets.adminPassword }} + {{ $keys.adminPassword }}: {{ .Values.secrets.adminPassword | quote }} + {{- end }} +{{- end }} diff --git a/packaging/helm/observe/templates/services.yaml b/packaging/helm/observe/templates/services.yaml new file mode 100644 index 00000000..a2ae97d7 --- /dev/null +++ b/packaging/helm/observe/templates/services.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "observe.fullname" . }} + labels: + {{- include "observe.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + selector: + {{- include "observe.selectorLabels" . | nindent 4 }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http +{{- if .Values.ingest.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "observe.fullname" . }}-ingest + labels: + {{- include "observe.labels" . | nindent 4 }} + {{- with .Values.ingest.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.ingest.service.type }} + selector: + {{- include "observe.selectorLabels" . | nindent 4 }} + ports: + - name: ingest + port: {{ .Values.ingest.port }} + targetPort: ingest +{{- end }} diff --git a/packaging/helm/observe/values.schema.json b/packaging/helm/observe/values.schema.json new file mode 100644 index 00000000..d52601cf --- /dev/null +++ b/packaging/helm/observe/values.schema.json @@ -0,0 +1,174 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "nameOverride": {"type": "string"}, + "fullnameOverride": {"type": "string"}, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "repository": {"type": "string", "minLength": 1}, + "tag": {"type": "string"}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "imagePullSecrets": {"type": "array"}, + "observe": { + "type": "object", + "additionalProperties": false, + "properties": { + "siteId": {"type": "string", "minLength": 1}, + "publicUrl": {"type": "string", "pattern": "^(https?://.+)?$"}, + "trustedProxies": {"type": "string"}, + "adminUser": {"type": "string", "minLength": 1}, + "seedDemo": {"type": "boolean"}, + "env": { + "type": "object", + "propertyNames": {"pattern": "^OBSERVE_[A-Z0-9_]+$"}, + "additionalProperties": {"type": "string"} + }, + "extraEnv": {"type": "array"}, + "terminationGracePeriodSeconds": {"type": "integer", "minimum": 30}, + "persistence": {"$ref": "#/definitions/persistence"}, + "resources": {"type": "object"}, + "podAnnotations": {"type": "object"}, + "nodeSelector": {"type": "object"}, + "tolerations": {"type": "array"}, + "affinity": {"type": "object"} + } + }, + "nucleus": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": {"type": "boolean"}, + "externalUrl": {"type": "string", "pattern": "^(postgres(ql)?://.+)?$"}, + "image": { + "type": "object", + "additionalProperties": false, + "properties": { + "repository": {"type": "string", "minLength": 1}, + "tag": {"type": "string", "minLength": 1, "not": {"enum": ["latest"]}}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "maxMemoryMB": {"type": "integer", "minimum": 64}, + "persistence": {"$ref": "#/definitions/persistence"}, + "resources": {"type": "object"}, + "networkPolicy": { + "type": "object", + "additionalProperties": false, + "properties": {"enabled": {"type": "boolean"}} + }, + "securityContext": {"type": "object"}, + "podSecurityContext": {"type": "object"}, + "podAnnotations": {"type": "object"}, + "nodeSelector": {"type": "object"}, + "tolerations": {"type": "array"}, + "affinity": {"type": "object"} + } + }, + "secrets": { + "type": "object", + "additionalProperties": false, + "properties": { + "existingSecret": {"type": "string"}, + "keys": { + "type": "object", + "additionalProperties": false, + "properties": { + "jwtSecret": {"type": "string", "minLength": 1}, + "sessionSalt": {"type": "string", "minLength": 1}, + "secretKey": {"type": "string", "minLength": 1}, + "adminPassword": {"type": "string", "minLength": 1} + } + }, + "jwtSecret": {"type": "string"}, + "sessionSalt": {"type": "string"}, + "secretKey": {"type": "string"}, + "adminPassword": {"type": "string"}, + "autogenerate": {"type": "boolean"} + } + }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"enum": ["ClusterIP", "NodePort", "LoadBalancer"]}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "annotations": {"type": "object"} + } + }, + "ingest": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": {"type": "boolean"}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535, "not": {"const": 3000}}, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"enum": ["ClusterIP", "NodePort", "LoadBalancer"]}, + "annotations": {"type": "object"} + } + } + } + }, + "ingress": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": {"type": "boolean"}, + "className": {"type": "string"}, + "annotations": {"type": "object"}, + "host": {"type": "string"}, + "tls": {"$ref": "#/definitions/tls"}, + "ingest": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": {"type": "boolean"}, + "host": {"type": "string"}, + "annotations": {"type": "object"}, + "tls": {"$ref": "#/definitions/tls"} + } + } + } + }, + "securityContext": {"type": "object"}, + "podSecurityContext": {"type": "object"}, + "tmpSizeLimit": {"type": "string"}, + "probes": { + "type": "object", + "additionalProperties": false, + "properties": { + "startup": {"type": "object"}, + "readiness": {"type": "object"}, + "liveness": {"type": "object"} + } + } + }, + "definitions": { + "persistence": { + "type": "object", + "additionalProperties": false, + "properties": { + "size": {"type": "string", "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti)$"}, + "storageClassName": {"type": "string"}, + "accessModes": {"type": "array", "items": {"type": "string"}, "minItems": 1} + } + }, + "tls": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": {"type": "boolean"}, + "secretName": {"type": "string"} + } + } + } +} diff --git a/packaging/helm/observe/values.yaml b/packaging/helm/observe/values.yaml new file mode 100644 index 00000000..6a423fed --- /dev/null +++ b/packaging/helm/observe/values.yaml @@ -0,0 +1,180 @@ +# Observe is SINGLE-INSTANCE. There is no clustering: the ingest WAL is a +# single-process on-disk queue and Nucleus runs as one node. Both workloads are +# StatefulSets with replicas fixed at 1 in the templates; there is deliberately +# no replicaCount value. Running a second Observe against the same data or +# database is unsupported. See docs/operations/kubernetes.md. + +nameOverride: "" +fullnameOverride: "" + +image: + repository: ghcr.io/useteploy/teploy-observe + # Empty means the chart appVersion (release image tags carry no "v"). + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +observe: + # OBSERVE_SITE_ID, OBSERVE_PUBLIC_URL, OBSERVE_TRUSTED_PROXIES + siteId: default + # External base URL. Set it whenever the instance is reachable by a name. + publicUrl: "" + # Comma-separated CIDRs whose X-Forwarded-For is trusted (your ingress + # controller pods). Empty trusts no one. + trustedProxies: "" + adminUser: admin + seedDemo: false + # Plain (non-secret) OBSERVE_* settings, e.g. OBSERVE_RAW_RETENTION_DAYS. + # Values are rendered as strings. + env: + OBSERVE_BUFFER_SIZE: "100000" + OBSERVE_FLUSH_INTERVAL_MS: "2000" + OBSERVE_FLUSH_SIZE: "500" + OBSERVE_RAW_RETENTION_DAYS: "30" + OBSERVE_HOURLY_RETENTION_DAYS: "365" + # Extra env entries in full Kubernetes EnvVar form (valueFrom etc.). + extraEnv: [] + # Must exceed Observe's own shutdown drain (compose uses 90s). + terminationGracePeriodSeconds: 90 + persistence: + size: 10Gi + storageClassName: "" + accessModes: [ReadWriteOnce] + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { memory: 1Gi } + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +nucleus: + # false = use an external database (set externalUrl); the Nucleus StatefulSet, + # Service and NetworkPolicy are not rendered. + enabled: true + externalUrl: "" + image: + repository: ghcr.io/neutron-build/nucleus + # Pinned. Do not use :latest. Supported release: v1.1.1. + tag: v1.1.1 + pullPolicy: IfNotPresent + port: 5432 + maxMemoryMB: 512 + persistence: + size: 20Gi + storageClassName: "" + accessModes: [ReadWriteOnce] + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { memory: 768Mi } + # Nucleus runs with NUCLEUS_ALLOW_NO_AUTH=1: anyone who can open a TCP + # connection to it owns all data. Keep this NetworkPolicy on. It needs a CNI + # that enforces NetworkPolicy (Calico, Cilium, ...); otherwise it is ignored. + networkPolicy: + enabled: true + # Not verified for the Nucleus image (user/path layout unknown here), so the + # root filesystem is left writable and the user is the image default. + # Dropped capabilities and no privilege escalation still apply. + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: { drop: [ALL] } + podSecurityContext: {} + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# Secrets. OBSERVE_JWT_SECRET and OBSERVE_SESSION_SALT are REQUIRED: the chart +# refuses to render unless one of (a) existingSecret, (b) explicit values, or +# (c) autogenerate=true is chosen. Unset in the app means "random per start", +# which logs everyone out on every restart. +secrets: + # Name of a Secret you manage. Key names are configurable below. + existingSecret: "" + keys: + jwtSecret: OBSERVE_JWT_SECRET + sessionSalt: OBSERVE_SESSION_SALT + secretKey: OBSERVE_SECRET_KEY + adminPassword: OBSERVE_ADMIN_PASSWORD + # Inline values (stored in the release Secret; prefer existingSecret). + jwtSecret: "" + sessionSalt: "" + # Optional. Master key encrypting stored LLM/S3 credentials at rest. + secretKey: "" + # Optional. Empty = first-run /setup wizard creates the admin. + adminPassword: "" + # Generate jwtSecret/sessionSalt once and persist them via lookup. WARNING: + # lookup returns nothing under `helm template`, `--dry-run` and Argo CD/Flux + # rendering, so those regenerate values every render and invalidate all + # sessions. Use existingSecret there. + autogenerate: false + +service: + type: ClusterIP + port: 3000 + annotations: {} + +# Separate telemetry-only listener (OBSERVE_INGEST_ADDR). Expose THIS publicly, +# never the dashboard. +ingest: + enabled: false + port: 3001 + service: + type: ClusterIP + annotations: {} + +ingress: + enabled: false + className: "" + annotations: {} + host: observe.example.com + tls: + enabled: false + secretName: "" + # Ingress for the ingest listener (requires ingest.enabled). + ingest: + enabled: false + host: ingest.example.com + annotations: {} + tls: + enabled: false + secretName: "" + +# Observe runs as 10001 (Dockerfile). The root filesystem is read-only; /tmp +# is an emptyDir because backup/restore stage files there. +securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: [ALL] } + seccompProfile: { type: RuntimeDefault } +podSecurityContext: + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch +tmpSizeLimit: 1Gi + +# Probes target /healthz, which runs SELECT 1 and answers 503 when the +# database is unreachable or ingest durability is degraded. 503 => not Ready +# (removed from Service), which is intended. /healthz is NOT used for liveness: +# a database outage must not restart-loop the pod. Timeouts follow the +# Dockerfile (cold Nucleus connect measured ~7.8s). +probes: + startup: + periodSeconds: 5 + timeoutSeconds: 12 + failureThreshold: 36 + readiness: + periodSeconds: 15 + timeoutSeconds: 12 + failureThreshold: 3 + liveness: + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + +# A PodDisruptionBudget is intentionally not provided: with one replica, +# minAvailable 1 would block node drains forever and maxUnavailable 1 is a +# no-op. A drain restarts the pod; the durable WAL covers acknowledged events. From 757a114075426facecdcdb9a088aaee6fed5ca2e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:42:13 +0000 Subject: [PATCH 15/65] feat(errors): scrub secrets and PII at error admission Case-insensitive recursive key denylist plus value patterns (Luhn cards, JWT, Bearer, query-string keys) applied in ErrorBuffer.Push before the record is frozen for the WAL. Grouping hash is pinned from the raw input (PreGroupHash, server-set) so fingerprints are unchanged; digest is over the scrubbed canonical payload so inbox retries still dedupe. Env: OBSERVE_SCRUB_KEYS, OBSERVE_SCRUB_DISABLE. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- internal/errors/buffer.go | 28 ++++ internal/errors/handler.go | 14 +- internal/errors/scrub.go | 305 ++++++++++++++++++++++++++++++++++ internal/errors/scrub_test.go | 240 ++++++++++++++++++++++++++ 4 files changed, 584 insertions(+), 3 deletions(-) create mode 100644 internal/errors/scrub.go create mode 100644 internal/errors/scrub_test.go diff --git a/internal/errors/buffer.go b/internal/errors/buffer.go index f38ccd87..1e53504f 100644 --- a/internal/errors/buffer.go +++ b/internal/errors/buffer.go @@ -71,6 +71,8 @@ type ErrorBuffer struct { adm *admissionCache // quarantineDir hosts the poison spool when a queue is attached. quarantineDir string + // scrub redacts secrets/PII before the record is frozen (scrub.go). + scrub *Scrubber // O01 §5.10 counters (atomics — the admission path must not take a // second lock for accounting). accepted atomic.Int64 @@ -134,9 +136,13 @@ func NewErrorBuffer(handler *ErrorHandler, maxSize, flushSize int, flushInterval stopCh: make(chan struct{}), wake: make(chan struct{}, 1), adm: newAdmissionCache(admissionCacheTTL, admissionCacheCapacity), + scrub: NewScrubberFromEnv(), } } +// SetScrubber replaces the admission scrubber (tests, explicit wiring). +func (b *ErrorBuffer) SetScrubber(s *Scrubber) { b.scrub = s } + func (b *ErrorBuffer) Start() { b.wg.Add(1) go func() { @@ -324,6 +330,7 @@ func (b *ErrorBuffer) Push(siteID string, input ErrorInput) error { if err := ValidateEventIdentity(input.EventID, input.ProducerID); err != nil { return err } + input = b.prepareRecord(input) raw, err := json.Marshal(input) if err != nil || len(raw) > maxErrorRecordBytes { return ErrErrorBufferFull @@ -402,6 +409,27 @@ func (b *ErrorBuffer) Push(siteID string, input ErrorInput) error { return nil } +// prepareRecord is the pre-freeze step: discard any client-supplied +// PreGroupHash, then (scrubbing on) pin the grouping hash from the RAW +// input and replace the payload with its scrubbed copy. Runs before the +// marshal so the digest and the WAL frame both see scrubbed bytes. +func (b *ErrorBuffer) prepareRecord(input ErrorInput) ErrorInput { + input.PreGroupHash = "" + if b.scrub.Disabled() { + return input + } + raw := input + raw.URL = ingest.CapturedURL(raw.URL) + if raw.ErrorType == "" && raw.ErrorValue == "" { + // Same default insertErrorEvent applies before it hashes. + raw.ErrorType, raw.ErrorValue = "Error", "Unknown error" + } + if h, err := ComputeGroupHash(FingerprintVersion, raw); err == nil { + input.PreGroupHash = h + } + return b.scrub.ScrubInput(input) +} + // release retires a record's reservation on its FINAL disposition (R13): // applied, deduped, conflicting, or quarantined — never a PENDING requeue. func (b *ErrorBuffer) release(ev bufferedError) { diff --git a/internal/errors/handler.go b/internal/errors/handler.go index a259184a..6d2575fd 100644 --- a/internal/errors/handler.go +++ b/internal/errors/handler.go @@ -62,6 +62,11 @@ type ErrorInput struct { EventID string `json:"event_id,omitempty"` // ProducerID optionally namespaces EventID (scoped inbox key). ProducerID string `json:"producer_id,omitempty"` + // PreGroupHash is SERVER-SET: ErrorBuffer.Push overwrites it (a client + // value is discarded) with the grouping hash derived from the raw + // input before scrubbing, so redaction can never move an event to a + // different issue. Empty = derive at insert (seed, scrub disabled). + PreGroupHash string `json:"_pre_group_hash,omitempty"` } // Breadcrumb is a user action that preceded the error. @@ -193,9 +198,12 @@ func (s *Service) insertErrorEvent(ctx context.Context, sqlc *nucleus.SQLModel, // derivation (O05). The issue records the version at create; a // future v2 cutover is a new case in ComputeGroupHash, documented in // grouping.go — existing issues are never rewritten. - groupHash, err := ComputeGroupHash(FingerprintVersion, input) - if err != nil { - return "", "", fmt.Errorf("group fingerprint: %w", err) + groupHash := input.PreGroupHash + if groupHash == "" { + groupHash, err = ComputeGroupHash(FingerprintVersion, input) + if err != nil { + return "", "", fmt.Errorf("group fingerprint: %w", err) + } } title := IssueTitle(input.ErrorType, input.ErrorValue) diff --git a/internal/errors/scrub.go b/internal/errors/scrub.go new file mode 100644 index 00000000..fc684f1e --- /dev/null +++ b/internal/errors/scrub.go @@ -0,0 +1,305 @@ +package errors + +// Server-side PII / secret scrubbing at error admission. +// +// Where it runs: ErrorBuffer.Push, BEFORE the record is marshalled. That +// marshalled body is the frozen record the WAL frame carries and the +// bytes the inbox digest is computed over, so nothing unscrubbed ever +// reaches the WAL, the quarantine spool, or error_events. +// +// Grouping stability: the v1 grouping hash is derived from error_type, +// error_value, the stack frames, an SDK fingerprint and (rage clicks) +// url+selector - several of which scrubbing may rewrite. Push therefore +// computes the group hash from the RAW (URL-canonicalised) input first +// and carries it on the record (ErrorInput.PreGroupHash); the flush-time +// insert uses it instead of re-deriving from scrubbed text. The golden +// test in scrub_test.go pins that the stored hash equals the raw-input +// hash. +// +// Idempotent-inbox consistency: the digest is sha256 over the SCRUBBED +// canonical payload (scrubbing is deterministic, so a retry of the same +// event hashes equal). Consequence, stated plainly: two payloads that +// differ only inside a value that is replaced by "[Filtered]" digest +// equal and dedupe rather than 409 - the server never retained the +// distinguishing bytes, so it cannot honestly call them conflicting. +// +// Configuration: OBSERVE_SCRUB_KEYS (comma list of extra key fragments), +// OBSERVE_SCRUB_DISABLE=true (opt-out; default ON). + +import ( + "os" + "regexp" + "strings" + "unicode" +) + +// Filtered is the replacement for every scrubbed value. +const Filtered = "[Filtered]" + +// truncated replaces a subtree the scrubber refuses to walk (depth or +// node budget exhausted). It is fail-closed: unexamined data is dropped, +// never passed through. +const truncated = "[Truncated]" + +const ( + scrubMaxDepth = 16 + scrubMaxNodes = 20000 +) + +// Substring-matched key fragments (compared against the key lowercased +// with separators removed). +var scrubSubstringKeys = []string{ + "password", "passwd", "passphrase", "secret", "token", "apikey", + "authorization", "cookie", "session", "csrf", "xsrf", "credit", + "privatekey", "accesskey", "bearer", +} + +// Whole-word key fragments: short words that would false-positive as +// substrings ("author", "cardinality", "classname"). +var scrubWordKeys = map[string]bool{ + "auth": true, "ssn": true, "card": true, "pwd": true, "jwt": true, "cvv": true, "cvc": true, +} + +// Scrubber redacts secrets and PII from error payloads. +type Scrubber struct { + disabled bool + extra []string // normalised extra key fragments +} + +// NewScrubber builds a scrubber with extra key fragments. +func NewScrubber(extraKeys ...string) *Scrubber { + s := &Scrubber{} + for _, k := range extraKeys { + if n := normalizeKey(k); n != "" { + s.extra = append(s.extra, n) + } + } + return s +} + +// NewScrubberFromEnv reads OBSERVE_SCRUB_KEYS and OBSERVE_SCRUB_DISABLE. +func NewScrubberFromEnv() *Scrubber { + if v := strings.ToLower(strings.TrimSpace(os.Getenv("OBSERVE_SCRUB_DISABLE"))); v == "true" || v == "1" { + return &Scrubber{disabled: true} + } + return NewScrubber(strings.Split(os.Getenv("OBSERVE_SCRUB_KEYS"), ",")...) +} + +// Disabled reports whether scrubbing is switched off. +func (s *Scrubber) Disabled() bool { return s == nil || s.disabled } + +// normalizeKey lowercases and strips separators so "API-Key", "api_key" +// and "apiKey" compare equal. +func normalizeKey(k string) string { + var b strings.Builder + for _, r := range strings.ToLower(k) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + return b.String() +} + +// keyWords splits a key on non-alphanumerics and camelCase boundaries. +func keyWords(k string) []string { + var words []string + var cur []rune + flush := func() { + if len(cur) > 0 { + words = append(words, strings.ToLower(string(cur))) + cur = cur[:0] + } + } + var prev rune + for _, r := range k { + switch { + case !unicode.IsLetter(r) && !unicode.IsDigit(r): + flush() + case unicode.IsUpper(r) && unicode.IsLower(prev): + flush() + cur = append(cur, r) + default: + cur = append(cur, r) + } + prev = r + } + flush() + return words +} + +// sensitiveKey reports whether a map key names a secret. +func (s *Scrubber) sensitiveKey(k string) bool { + n := normalizeKey(k) + if n == "" { + return false + } + for _, f := range scrubSubstringKeys { + if strings.Contains(n, f) { + return true + } + } + for _, f := range s.extra { + if strings.Contains(n, f) { + return true + } + } + for _, w := range keyWords(k) { + if scrubWordKeys[w] { + return true + } + } + return false +} + +var ( + jwtRe = regexp.MustCompile(`eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}(?:\.[A-Za-z0-9_-]*)?`) + bearerRe = regexp.MustCompile(`(?i)\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}`) + cardRe = regexp.MustCompile(`\b(?:\d[ -]?){13,18}\d\b`) + // Query-string parameters: ?name=value / &name=value / ;name=value. + queryParamRe = regexp.MustCompile(`([?&;])([^=&#;\s"']+)=([^&#;\s"']*)`) + // name=value pairs for obviously secret names in free text. + kvSecretRe = regexp.MustCompile(`(?i)\b(password|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?token|client[_-]?secret)=([^\s&"',;]+)`) +) + +// luhnValid reports whether the digit string passes the Luhn check. +func luhnValid(digits string) bool { + sum, alt := 0, false + for i := len(digits) - 1; i >= 0; i-- { + d := int(digits[i] - '0') + if alt { + d *= 2 + if d > 9 { + d -= 9 + } + } + sum += d + alt = !alt + } + return len(digits) > 0 && sum%10 == 0 +} + +// scrubString applies the value-pattern rules to one string. +func (s *Scrubber) scrubString(v string) string { + if v == "" { + return v + } + v = jwtRe.ReplaceAllString(v, Filtered) + v = bearerRe.ReplaceAllString(v, "$1 "+Filtered) + v = queryParamRe.ReplaceAllStringFunc(v, func(m string) string { + sub := queryParamRe.FindStringSubmatch(m) + name := sub[2] + ln := strings.ToLower(name) + if s.sensitiveKey(name) || ln == "key" || ln == "sig" || ln == "signature" { + return sub[1] + name + "=" + Filtered + } + return m + }) + v = kvSecretRe.ReplaceAllString(v, "$1="+Filtered) + v = cardRe.ReplaceAllStringFunc(v, func(m string) string { + digits := make([]byte, 0, 19) + for i := 0; i < len(m); i++ { + if m[i] >= '0' && m[i] <= '9' { + digits = append(digits, m[i]) + } + } + // 14-19 digits: 13-digit runs are indistinguishable from epoch-ms + // timestamps (10% pass Luhn by chance) and 13-digit cards are + // effectively extinct. + if len(digits) >= 14 && len(digits) <= 19 && luhnValid(string(digits)) { + return Filtered + } + return m + }) + return v +} + +// scrubState carries the per-input node budget. +type scrubState struct { + s *Scrubber + nodes int +} + +// value returns a scrubbed COPY of v (the caller's decoded structure is +// never mutated). depth is the current nesting level. +func (st *scrubState) value(v any, depth int) any { + st.nodes++ + if st.nodes > scrubMaxNodes || depth > scrubMaxDepth { + switch v.(type) { + case nil, bool, float64, int, int64: + return v + } + return truncated + } + switch x := v.(type) { + case string: + return st.s.scrubString(x) + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + if st.s.sensitiveKey(k) { + out[k] = Filtered + continue + } + out[k] = st.value(val, depth+1) + } + return out + case []any: + out := make([]any, len(x)) + for i, val := range x { + out[i] = st.value(val, depth+1) + } + return out + case map[string]string: + out := make(map[string]string, len(x)) + for k, val := range x { + if st.s.sensitiveKey(k) { + out[k] = Filtered + continue + } + out[k] = st.s.scrubString(val) + } + return out + case []string: + out := make([]string, len(x)) + for i, val := range x { + out[i] = st.s.scrubString(val) + } + return out + default: + // Numbers, bools, nil: no pattern applies (a secret-named key + // was already filtered by the map case). + return v + } +} + +// ScrubInput returns a scrubbed copy of the input. The original (and +// every map/slice reachable from it) is left untouched. Disabled or nil +// scrubbers return the input unchanged. +func (s *Scrubber) ScrubInput(in ErrorInput) ErrorInput { + if s.Disabled() { + return in + } + st := &scrubState{s: s} + out := in + out.ErrorType = s.scrubString(in.ErrorType) + out.ErrorValue = s.scrubString(in.ErrorValue) + out.URL = s.scrubString(in.URL) + out.Contexts = st.value(in.Contexts, 0) + out.Extra = st.value(in.Extra, 0) + if in.StackTrace != nil { + out.StackTrace = make([]StackFrame, len(in.StackTrace)) + for i, f := range in.StackTrace { + f.Filename = s.scrubString(f.Filename) + out.StackTrace[i] = f + } + } + if in.Breadcrumbs != nil { + out.Breadcrumbs = make([]Breadcrumb, len(in.Breadcrumbs)) + for i, b := range in.Breadcrumbs { + b.Message = s.scrubString(b.Message) + b.Data = st.value(b.Data, 0) + out.Breadcrumbs[i] = b + } + } + return out +} diff --git a/internal/errors/scrub_test.go b/internal/errors/scrub_test.go new file mode 100644 index 00000000..7f94bf1a --- /dev/null +++ b/internal/errors/scrub_test.go @@ -0,0 +1,240 @@ +package errors + +import ( + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/ingest" +) + +func newScrubBuf() *ErrorBuffer { + return NewErrorBuffer(nil, 50000, 100, time.Hour, slog.New(slog.DiscardHandler)) +} + +func TestScrubKeysCaseInsensitiveRecursive(t *testing.T) { + s := NewScrubber() + in := ErrorInput{ + Extra: map[string]any{ + "Password": "hunter2", + "nested": map[string]any{ + "API-Key": "k", + "list": []any{ + map[string]any{"Set-Cookie": "a=b", "ok": "fine"}, + "plain", + }, + "apiKey": 12345, + }, + "author": "keep me", + "user_id": "u1", + }, + Contexts: map[string]any{"request": map[string]any{"headers": map[string]any{ + "Authorization": "Bearer abcdefghij", "X-CSRF-Token": "t", "Accept": "json"}}}, + Breadcrumbs: []Breadcrumb{{Message: "login", Data: map[string]any{"ssn": "123-45-6789", "page": "/x"}}}, + } + out := s.ScrubInput(in) + b, _ := json.Marshal(out) + js := string(b) + for _, leak := range []string{"hunter2", "a=b", "abcdefghij", "123-45-6789", "12345"} { + if strings.Contains(js, leak) { + t.Fatalf("leaked %q in %s", leak, js) + } + } + for _, keep := range []string{"keep me", "fine", "plain", "json", "/x", "u1"} { + if !strings.Contains(js, keep) { + t.Fatalf("over-scrubbed: lost %q in %s", keep, js) + } + } + // Input must not be mutated. + if in.Extra.(map[string]any)["Password"] != "hunter2" { + t.Fatal("ScrubInput mutated the caller's map") + } +} + +func TestScrubValuePatterns(t *testing.T) { + s := NewScrubber() + jwt := "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTYifQ.SflKxwRJSMeKKF2QT4fwpM" + in := ErrorInput{ + ErrorValue: "bad token " + jwt + " and Authorization: Bearer abc.def-ghi_jkl123 end; card 4111 1111 1111 1111", + URL: "https://x.test/p?ok=1&api_key=SECRET1&key=SECRET2&Sig=zzz", + Breadcrumbs: []Breadcrumb{{Message: "GET /a?token=SECRET3&q=hi", Data: "password=SECRET4 trailing"}}, + } + out := s.ScrubInput(in) + b, _ := json.Marshal(out) + js := string(b) + for _, leak := range []string{jwt, "abc.def-ghi_jkl123", "4111", "SECRET1", "SECRET2", "SECRET3", "SECRET4", "=zzz"} { + if strings.Contains(js, leak) { + t.Fatalf("leaked %q in %s", leak, js) + } + } + if !strings.Contains(out.URL, "ok=1") || !strings.Contains(out.Breadcrumbs[0].Message, "q=hi") { + t.Fatalf("benign query params lost: %q / %q", out.URL, out.Breadcrumbs[0].Message) + } + if !strings.Contains(out.ErrorValue, "Bearer "+Filtered) { + t.Fatalf("bearer not filtered: %q", out.ErrorValue) + } +} + +func TestScrubLuhn(t *testing.T) { + s := NewScrubber() + positives := []string{"4111111111111111", "4111-1111-1111-1111", "5500 0000 0000 0004", "378282246310005", "6011111111111117"} + for _, p := range positives { + if got := s.scrubString("pay " + p + " now"); !strings.Contains(got, Filtered) || strings.Contains(got, p) { + t.Errorf("Luhn positive %q not filtered: %q", p, got) + } + } + negatives := []string{"4111111111111112", "1234567890123456", "1700000000000", "request id 123456789"} + for _, n := range negatives { + if got := s.scrubString(n); got != n { + t.Errorf("negative %q altered: %q", n, got) + } + } +} + +func TestScrubNonStringValuesUnderSecretKeys(t *testing.T) { + s := NewScrubber() + out := s.ScrubInput(ErrorInput{Extra: map[string]any{ + "token": 42, "credit_card": map[string]any{"n": 1}, "ok": 7, "flag": true, "nil": nil, + "cards": []any{1, 2}, "session_id": true, + }}) + m := out.Extra.(map[string]any) + for _, k := range []string{"token", "credit_card", "session_id"} { + if m[k] != Filtered { + t.Errorf("%s = %v", k, m[k]) + } + } + if m["ok"] != float64(7) && m["ok"] != 7 || m["flag"] != true || m["nil"] != nil { + t.Errorf("non-secret scalars changed: %v", m) + } +} + +func TestScrubExtraKeysAndDisable(t *testing.T) { + t.Setenv("OBSERVE_SCRUB_KEYS", "internal_ref, Tenant-Code") + s := NewScrubberFromEnv() + out := s.ScrubInput(ErrorInput{Extra: map[string]any{"InternalRef": "x", "tenantCode": "y", "other": "z"}}) + m := out.Extra.(map[string]any) + if m["InternalRef"] != Filtered || m["tenantCode"] != Filtered || m["other"] != "z" { + t.Fatalf("extra keys: %v", m) + } + t.Setenv("OBSERVE_SCRUB_DISABLE", "true") + d := NewScrubberFromEnv() + if !d.Disabled() { + t.Fatal("disable not honoured") + } + in := ErrorInput{Extra: map[string]any{"password": "p"}} + if got := d.ScrubInput(in).Extra.(map[string]any)["password"]; got != "p" { + t.Fatalf("disabled scrubber altered input: %v", got) + } +} + +func TestScrubDepthAndSizeBounded(t *testing.T) { + s := NewScrubber() + // Deep chain with a secret beyond the depth limit: fail-closed. + var deep any = map[string]any{"leaf": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTYifQ.sig12345"} + for i := 0; i < 100; i++ { + deep = map[string]any{"n": deep} + } + b, _ := json.Marshal(s.ScrubInput(ErrorInput{Extra: deep})) + if strings.Contains(string(b), "eyJ") || !strings.Contains(string(b), truncated) { + t.Fatalf("deep payload not truncated fail-closed: %.200s", b) + } + // Huge breadth: performance guard. + big := make([]any, 200000) + for i := range big { + big[i] = map[string]any{"k": strings.Repeat("a", 50), "password": "p"} + } + start := time.Now() + out := s.ScrubInput(ErrorInput{Extra: big, ErrorValue: strings.Repeat("x ", 100000)}) + if d := time.Since(start); d > 5*time.Second { + t.Fatalf("scrub of huge payload took %v", d) + } + if got := len(out.Extra.([]any)); got != len(big) { + t.Fatalf("len changed %d", got) + } + if tail := out.Extra.([]any)[len(big)-1]; tail != truncated { + t.Fatalf("beyond node budget must be %q, got %v", truncated, tail) + } +} + +// Golden: scrubbing at Push must not move any event to another group. +func TestScrubDoesNotChangeGrouping(t *testing.T) { + frames := []StackFrame{{Filename: "https://a.test/app.js?token=SECRET", Function: "doIt", InApp: true}} + cases := []ErrorInput{ + {ErrorType: "TypeError", ErrorValue: "x is not a function, token=abc12345", StackTrace: frames}, + {ErrorType: "Error", ErrorValue: "Bearer abcdefghijkl rejected for 4111111111111111"}, + {ErrorType: "RageClick", URL: "https://x.test/p?api_key=ZZZ#frag", Selector: "#buy"}, + {ErrorType: "E", ErrorValue: "v", Fingerprint: []string{"custom", "password=hunter2"}}, + {}, + } + b := newScrubBuf() + for i, in := range cases { + in.SiteID = "s" + want := in + want.URL = ingest.CapturedURL(want.URL) + if want.ErrorType == "" && want.ErrorValue == "" { + want.ErrorType, want.ErrorValue = "Error", "Unknown error" + } + wantHash, _ := ComputeGroupHash(FingerprintVersion, want) + if err := b.Push("s", in); err != nil { + t.Fatalf("case %d: %v", i, err) + } + var stored ErrorInput + if err := json.Unmarshal(b.events[len(b.events)-1].Body, &stored); err != nil { + t.Fatal(err) + } + if stored.PreGroupHash != wantHash { + t.Fatalf("case %d: hash %s != raw-derived %s", i, stored.PreGroupHash, wantHash) + } + } + // The derivation itself stays pinned by o05_grouping_test.go goldens. +} + +func TestPushDiscardsClientPreGroupHashAndScrubsBeforeWAL(t *testing.T) { + b := newScrubBuf() + in := ErrorInput{ErrorType: "E", ErrorValue: "v password=hunter2", PreGroupHash: "attacker-chosen", + Extra: map[string]any{"token": "T0KEN"}} + if err := b.Push("s", in); err != nil { + t.Fatal(err) + } + body := string(b.events[0].Body) + if strings.Contains(body, "attacker-chosen") || strings.Contains(body, "hunter2") || strings.Contains(body, "T0KEN") { + t.Fatalf("frozen record leaks: %s", body) + } + // Disabled: still discards the client value. + b2 := newScrubBuf() + b2.SetScrubber(&Scrubber{disabled: true}) + if err := b2.Push("s", in); err != nil { + t.Fatal(err) + } + if strings.Contains(string(b2.events[0].Body), "attacker-chosen") { + t.Fatal("client PreGroupHash survived") + } +} + +// Inbox idempotency under scrubbing: retries hash equal, a real +// difference still conflicts. +func TestScrubKeepsInboxIdempotency(t *testing.T) { + b := newScrubBuf() + mk := func(val string) ErrorInput { + return ErrorInput{EventID: "evt-12345678", ErrorType: "E", ErrorValue: val, + Extra: map[string]any{"password": "p1", "n": 1}} + } + if err := b.Push("s", mk("boom")); err != nil { + t.Fatal(err) + } + if err := b.Push("s", mk("boom")); !errors.Is(err, ErrAdmittedDuplicate) { + t.Fatalf("retry want duplicate, got %v", err) + } + if err := b.Push("s", mk("different")); !errors.Is(err, ErrEventIDConflict) { + t.Fatalf("changed payload want conflict, got %v", err) + } + // Differing only in a filtered secret dedupes (documented in scrub.go). + r := mk("boom") + r.Extra = map[string]any{"password": "OTHER", "n": 1} + if err := b.Push("s", r); !errors.Is(err, ErrAdmittedDuplicate) { + t.Fatalf("secret-only difference want duplicate, got %v", err) + } +} From c401de8aedebdb330d3eac6813a8cc0f35cde70d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:42:46 +0000 Subject: [PATCH 16/65] feat(alerts): trace, log and uptime alert metrics; opt-in fire-only integrations Adds trace_error_rate, trace_p95_ms, log_error_count and uptime_failures rule metrics (site-wide, window-bounded, parameterised, CAST AS BIGINT). Empty windows are no-data, never 0. API whitelist now derives from platform.AlertMetricSet; UI dropdown and docs updated. Alert FIRE edges can route to site integrations behind OBSERVE_ALERTS_TO_INTEGRATIONS=true (default off, detached goroutine, never blocks the engine). UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/alerts_integrations.go | 36 +++ cmd/observe/alerts_integrations_test.go | 37 +++ cmd/observe/main.go | 5 +- internal/platform/alerts.go | 6 + internal/platform/alerts_engine.go | 14 +- internal/platform/alerts_engine_metrics.go | 168 +++++++++++++ .../platform/alerts_engine_metrics_test.go | 220 ++++++++++++++++++ ui/src/routes/alerts.tsx | 4 + ui/src/routes/docs.tsx | 2 +- 9 files changed, 487 insertions(+), 5 deletions(-) create mode 100644 cmd/observe/alerts_integrations.go create mode 100644 cmd/observe/alerts_integrations_test.go create mode 100644 internal/platform/alerts_engine_metrics.go create mode 100644 internal/platform/alerts_engine_metrics_test.go diff --git a/cmd/observe/alerts_integrations.go b/cmd/observe/alerts_integrations.go new file mode 100644 index 00000000..59ced2e0 --- /dev/null +++ b/cmd/observe/alerts_integrations.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "log/slog" + "os" + + "github.com/useteploy/teploy-observe/internal/integrations" + "github.com/useteploy/teploy-observe/internal/platform" +) + +// wireAlertsToIntegrations routes alert FIRE events (the opening edge only) +// to the site's integrations through IntegrationService.Fire, which records +// every attempt in integration_deliveries. Default off: +// OBSERVE_ALERTS_TO_INTEGRATIONS=true opts in globally. RECOVER and repeat +// are deliberately NOT routed - the integration senders are create-only +// (PagerDuty always "trigger", Jira/GitHub always open a new issue), so a +// recovery would page or file a second ticket instead of resolving. +func wireAlertsToIntegrations(alerts *platform.AlertService, svc *integrations.IntegrationService, logger *slog.Logger) { + if os.Getenv("OBSERVE_ALERTS_TO_INTEGRATIONS") != "true" { + return + } + alerts.SetIntegrationsHook(alertFireToIntegrations(svc.Fire)) + logger.Info("alert fire events routed to site integrations (fire-only)") +} + +// alertFireToIntegrations adapts the engine event to the integrations payload. +func alertFireToIntegrations(fire func(ctx context.Context, siteID string, p integrations.AlertPayload)) func(ctx context.Context, ev platform.AlertFireEvent) { + return func(ctx context.Context, ev platform.AlertFireEvent) { + fire(ctx, ev.SiteID, integrations.AlertPayload{ + Title: ev.Title, Message: ev.Message, Severity: ev.Severity, + SiteID: ev.SiteID, RuleName: ev.RuleName, Metric: ev.Metric, + Value: ev.Value, Threshold: ev.Threshold, + }) + } +} diff --git a/cmd/observe/alerts_integrations_test.go b/cmd/observe/alerts_integrations_test.go new file mode 100644 index 00000000..d3272513 --- /dev/null +++ b/cmd/observe/alerts_integrations_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "testing" + + "github.com/useteploy/teploy-observe/internal/integrations" + "github.com/useteploy/teploy-observe/internal/platform" +) + +func TestAlertMetricWhitelistRejectsUnknown(t *testing.T) { + for _, ok := range []string{"error_count", "trace_error_rate", "trace_p95_ms", "log_error_count", "uptime_failures"} { + if _, in := validAlertMetrics[ok]; !in { + t.Errorf("%s should be accepted", ok) + } + } + for _, bad := range []string{"metric_value", "cpu", "error_count'; --", ""} { + if _, in := validAlertMetrics[bad]; in { + t.Errorf("%q should be rejected", bad) + } + } +} + +func TestAlertFireToIntegrationsMapsPayload(t *testing.T) { + var gotSite string + var got integrations.AlertPayload + fn := alertFireToIntegrations(func(_ context.Context, site string, p integrations.AlertPayload) { + gotSite, got = site, p + }) + fn(context.Background(), platform.AlertFireEvent{ + SiteID: "s1", Title: "t", Message: "m", Severity: "critical", + RuleName: "r", Metric: "uptime_failures", Value: "3.00", Threshold: "1", + }) + if gotSite != "s1" || got.Metric != "uptime_failures" || got.Severity != "critical" || got.Value != "3.00" || got.Threshold != "1" || got.SiteID != "s1" { + t.Fatalf("payload = %+v site=%s", got, gotSite) + } +} diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..c44a38f9 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -434,6 +434,7 @@ func main() { // Feature expansion services reportSvc := reports.NewReportService(db, logger) integrationSvc := integrations.NewIntegrationService(db, logger) + wireAlertsToIntegrations(alertSvc, integrationSvc, logger) feedbackSvc := feedback.NewFeedbackService(db) viewSvc := views.NewViewService(db) explorerSvc := explorer.NewExplorerService(db) @@ -5589,9 +5590,7 @@ type createAlertRuleInput struct { Cooldown int `json:"cooldown"` } -var validAlertMetrics = map[string]struct{}{ - "error_count": {}, "error_rate": {}, "pageviews": {}, "visitors": {}, -} +var validAlertMetrics = platform.AlertMetricSet() var validAlertOperators = map[string]struct{}{ "gt": {}, "gte": {}, "lt": {}, "lte": {}, "eq": {}, diff --git a/internal/platform/alerts.go b/internal/platform/alerts.go index 68cc590d..27179a93 100644 --- a/internal/platform/alerts.go +++ b/internal/platform/alerts.go @@ -30,6 +30,12 @@ type AlertService struct { // cooldowns and windows deterministic. evalMu sync.Mutex now func() time.Time + + // scalarHook, when set, replaces the database in scalarMetric (unit + // tests inject a fake query layer). integrationsHook is the optional + // FIRE-only integrations dispatcher (SetIntegrationsHook). + scalarHook func(ctx context.Context, q string, args ...any) (float64, error) + integrationsHook func(ctx context.Context, ev AlertFireEvent) } func NewAlertService(db *nucleus.Client, logger *slog.Logger, webhookSvc *WebhookService, incidentSvc *incidents.Service, notifier *Notifier) *AlertService { diff --git a/internal/platform/alerts_engine.go b/internal/platform/alerts_engine.go index 2622c927..e2995c4c 100644 --- a/internal/platform/alerts_engine.go +++ b/internal/platform/alerts_engine.go @@ -212,6 +212,8 @@ func (s *AlertService) evaluateFiring(ctx context.Context, rule AlertRule, prev if created { s.recordIncidentEvent(ctx, inc.IncidentID, incidents.EventOpened, "alert", fmt.Sprintf("rule %s fired: %s=%.2f samples=%d", rule.RuleID, rule.Metric, value, samples)) + // FIRE only, after the edge committed, never blocking the tick. + s.fireIntegrations(rule, value, fmt.Sprintf("alert rule fired: %s=%.2f (threshold %s %.2f)", rule.Metric, value, rule.Operator, rule.Threshold)) } return nil } @@ -422,6 +424,9 @@ func (r AlertRule) severityOrDefault() string { // no-data; a silent site is no-data) // error_rate value = 100*errors/events; samples = events - an empty // window is NO DATA, never "0%" +// +// trace_error_rate, trace_p95_ms, log_error_count and uptime_failures live +// in alerts_engine_metrics.go with their own sample semantics. func (s *AlertService) queryMetric(ctx context.Context, siteID, metric, fromMs, toMs string) (float64, int64, error) { switch metric { case "pageviews": @@ -478,7 +483,11 @@ func (s *AlertService) queryMetric(ctx context.Context, siteID, metric, fromMs, } return 100.0 * errs / events, int64(events), nil default: - return 0, 0, fmt.Errorf("unknown metric: %s", metric) + v, n, handled, err := s.queryExtendedMetric(ctx, siteID, metric, fromMs, toMs) + if !handled { + return 0, 0, fmt.Errorf("unknown metric: %s", metric) + } + return v, n, err } } @@ -488,6 +497,9 @@ func (s *AlertService) scalarMetric(ctx context.Context, siteID, fromMs, toMs, q type result struct { Value float64 `db:"value"` } + if s.scalarHook != nil { + return s.scalarHook(ctx, q, siteID, fromMs, toMs) + } rows, err := nucleus.Query[result](ctx, s.db.SQL(), q, siteID, fromMs, toMs) if err != nil || len(rows) == 0 { return 0, err diff --git a/internal/platform/alerts_engine_metrics.go b/internal/platform/alerts_engine_metrics.go new file mode 100644 index 00000000..3bb86c7d --- /dev/null +++ b/internal/platform/alerts_engine_metrics.go @@ -0,0 +1,168 @@ +package platform + +// Alert metrics beyond product analytics: traces, logs and uptime. Each is +// computed over the rule's window for the rule's SITE (no per-service filter: +// AlertRule has no field that could carry one, and adding it needs a schema +// change). Every query is window-bounded and parameterised ($1 site, $2/$3 +// the window as CAST($n AS BIGINT) - the Nucleus wire quirk dbutil.IntParam +// documents). +// +// Sample semantics (what "no data" means per metric; queryMetric's contract +// is value + the sample count that backs it, and the engine turns +// samples < min_samples into the labeled no_data state, never a pass): +// +// trace_error_rate value = 100*error spans/spans; samples = spans. No spans +// in the window is NO DATA, never "0%". +// trace_p95_ms value = p95 duration_ms of ROOT spans (parent_span_id=''); +// samples = root spans. No root spans is NO DATA. +// log_error_count value = logs at error/fatal level; samples = all logs +// (a busy log stream with zero errors is HEALTHY, a silent +// pipeline is no-data - the error_count posture). +// uptime_failures value = failed checks (is_up = 'false') across the +// site's monitors in the window; samples = all checks. +// No check results (monitors disabled/absent) is NO DATA. + +import ( + "context" + "fmt" + "sort" +) + +// alertMetricNames is the single source of truth for rule metrics; the API +// whitelist (cmd/observe) derives from it so the layers cannot drift. +var alertMetricNames = []string{ + "pageviews", "visitors", "error_count", "error_rate", + "trace_error_rate", "trace_p95_ms", "log_error_count", "uptime_failures", +} + +// AlertMetricSet returns the supported metric names as a set. +func AlertMetricSet() map[string]struct{} { + m := make(map[string]struct{}, len(alertMetricNames)) + for _, n := range alertMetricNames { + m[n] = struct{}{} + } + return m +} + +// AlertMetricNames returns the supported metric names, sorted. +func AlertMetricNames() []string { + out := append([]string(nil), alertMetricNames...) + sort.Strings(out) + return out +} + +// SQL for the extended metrics. Constants so tests can assert the exact text. +const ( + sqlSpanCount = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM spans WHERE site_id = $1 AND start_time >= CAST($2 AS BIGINT) AND start_time < CAST($3 AS BIGINT)` + + sqlSpanErrorCount = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM spans WHERE site_id = $1 AND start_time >= CAST($2 AS BIGINT) AND start_time < CAST($3 AS BIGINT) AND status_code = 'error'` + + sqlRootSpanCount = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM spans WHERE site_id = $1 AND start_time >= CAST($2 AS BIGINT) AND start_time < CAST($3 AS BIGINT) AND parent_span_id = ''` + + sqlRootSpanP95 = `SELECT CAST(CAST(percentile_cont(duration_ms, 0.95) AS BIGINT) AS TEXT) AS value FROM spans WHERE site_id = $1 AND start_time >= CAST($2 AS BIGINT) AND start_time < CAST($3 AS BIGINT) AND parent_span_id = ''` + + sqlLogCount = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM logs WHERE site_id = $1 AND timestamp >= CAST($2 AS BIGINT) AND timestamp < CAST($3 AS BIGINT)` + + sqlLogErrorCount = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM logs WHERE site_id = $1 AND timestamp >= CAST($2 AS BIGINT) AND timestamp < CAST($3 AS BIGINT) AND level IN ('error', 'fatal')` + + sqlUptimeCount = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM uptime_results WHERE site_id = $1 AND timestamp >= CAST($2 AS BIGINT) AND timestamp < CAST($3 AS BIGINT)` + + sqlUptimeFailures = `SELECT CAST(COUNT(*) AS TEXT) AS value FROM uptime_results WHERE site_id = $1 AND timestamp >= CAST($2 AS BIGINT) AND timestamp < CAST($3 AS BIGINT) AND is_up = 'false'` +) + +// queryExtendedMetric handles the trace/log/uptime metrics. handled=false +// means the name is not one of them (the caller reports unknown metric). +func (s *AlertService) queryExtendedMetric(ctx context.Context, siteID, metric, fromMs, toMs string) (float64, int64, bool, error) { + run := func(q string) (float64, error) { return s.scalarMetric(ctx, siteID, fromMs, toMs, q) } + switch metric { + case "trace_error_rate": + total, err := run(sqlSpanCount) + if err != nil { + return 0, 0, true, err + } + if total == 0 { + return 0, 0, true, nil // no spans: NO DATA, not 0% + } + errs, err := run(sqlSpanErrorCount) + if err != nil { + return 0, 0, true, err + } + return 100.0 * errs / total, int64(total), true, nil + case "trace_p95_ms": + roots, err := run(sqlRootSpanCount) + if err != nil { + return 0, 0, true, err + } + if roots == 0 { + return 0, 0, true, nil // percentile of nothing is not 0 ms + } + p95, err := run(sqlRootSpanP95) + if err != nil { + return 0, 0, true, err + } + return p95, int64(roots), true, nil + case "log_error_count": + v, err := run(sqlLogErrorCount) + if err != nil { + return 0, 0, true, err + } + n, err := run(sqlLogCount) + if err != nil { + return 0, 0, true, err + } + return v, int64(n), true, nil + case "uptime_failures": + n, err := run(sqlUptimeCount) + if err != nil { + return 0, 0, true, err + } + if n == 0 { + return 0, 0, true, nil + } + v, err := run(sqlUptimeFailures) + if err != nil { + return 0, 0, true, err + } + return v, int64(n), true, nil + } + return 0, 0, false, nil +} + +// AlertFireEvent is the opening-edge event handed to the optional +// integrations hook (OBSERVE_ALERTS_TO_INTEGRATIONS). Plain strings so +// platform needs no import of internal/integrations. +type AlertFireEvent struct { + SiteID, Title, Message, Severity string + RuleName, Metric, Value, Threshold string +} + +// SetIntegrationsHook registers the FIRE-only integrations dispatcher. nil +// (the default) disables it. The engine calls it on its own goroutine, after +// the edge has committed. +func (s *AlertService) SetIntegrationsHook(fn func(ctx context.Context, ev AlertFireEvent)) { + s.integrationsHook = fn +} + +// fireIntegrations dispatches an opening edge to the hook, detached from the +// evaluation tick (a slow or failing integration never delays or fails the +// engine; delivery history is the integration service's own). +func (s *AlertService) fireIntegrations(rule AlertRule, value float64, message string) { + fn := s.integrationsHook + if fn == nil { + return + } + ev := AlertFireEvent{ + SiteID: rule.SiteID, Title: rule.Name, Message: message, + Severity: rule.severityOrDefault(), RuleName: rule.Name, Metric: rule.Metric, + Value: fmt.Sprintf("%.2f", value), + Threshold: fmt.Sprintf("%g", rule.Threshold), + } + go func() { + defer func() { + if r := recover(); r != nil { + s.logger.Error("alert integrations hook panicked", "rule", rule.RuleID, "panic", r) + } + }() + fn(context.Background(), ev) + }() +} diff --git a/internal/platform/alerts_engine_metrics_test.go b/internal/platform/alerts_engine_metrics_test.go new file mode 100644 index 00000000..9e42be9d --- /dev/null +++ b/internal/platform/alerts_engine_metrics_test.go @@ -0,0 +1,220 @@ +package platform + +import ( + "context" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" +) + +// fakeScalar answers scalarMetric by matching a distinctive fragment of the +// SQL text, and records every query + args it saw. +type fakeScalar struct { + mu sync.Mutex + answers map[string]float64 // SQL constant -> value + err error + seen []string + args [][]any +} + +func (f *fakeScalar) hook(_ context.Context, q string, args ...any) (float64, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.seen = append(f.seen, q) + f.args = append(f.args, args) + if f.err != nil { + return 0, f.err + } + return f.answers[q], nil +} + +func fakeService(f *fakeScalar) *AlertService { + return &AlertService{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + scalarHook: f.hook, + now: time.Now, + } +} + +func TestQueryMetric_ExtendedMetrics(t *testing.T) { + cases := []struct { + name string + metric string + answers map[string]float64 + wantValue float64 + wantSamples int64 + wantQueries int + }{ + {"trace_error_rate computes percent over spans", "trace_error_rate", + map[string]float64{sqlSpanCount: 200, sqlSpanErrorCount: 10}, 5, 200, 2}, + {"trace_error_rate no spans is no-data not 0%", "trace_error_rate", + map[string]float64{sqlSpanCount: 0, sqlSpanErrorCount: 0}, 0, 0, 1}, + {"trace_error_rate spans with zero errors is healthy 0%", "trace_error_rate", + map[string]float64{sqlSpanCount: 50}, 0, 50, 2}, + {"trace_p95_ms reads p95 of root spans", "trace_p95_ms", + map[string]float64{sqlRootSpanCount: 40, sqlRootSpanP95: 1234}, 1234, 40, 2}, + {"trace_p95_ms no root spans is no-data not 0ms", "trace_p95_ms", + map[string]float64{sqlRootSpanCount: 0, sqlRootSpanP95: 999}, 0, 0, 1}, + {"log_error_count counts error+fatal, samples are all logs", "log_error_count", + map[string]float64{sqlLogErrorCount: 7, sqlLogCount: 900}, 7, 900, 2}, + {"log_error_count silent pipeline is no-data", "log_error_count", + map[string]float64{}, 0, 0, 2}, + {"uptime_failures counts failed checks", "uptime_failures", + map[string]float64{sqlUptimeCount: 30, sqlUptimeFailures: 4}, 4, 30, 2}, + {"uptime_failures no results is no-data", "uptime_failures", + map[string]float64{sqlUptimeCount: 0, sqlUptimeFailures: 3}, 0, 0, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := &fakeScalar{answers: tc.answers} + v, n, err := fakeService(f).queryMetric(context.Background(), "site_a", tc.metric, "1000", "2000") + if err != nil { + t.Fatal(err) + } + if v != tc.wantValue || n != tc.wantSamples { + t.Fatalf("got value=%v samples=%d, want value=%v samples=%d", v, n, tc.wantValue, tc.wantSamples) + } + if len(f.seen) != tc.wantQueries { + t.Fatalf("ran %d queries, want %d", len(f.seen), tc.wantQueries) + } + for i, a := range f.args { + if len(a) != 3 || a[0] != "site_a" || a[1] != "1000" || a[2] != "2000" { + t.Fatalf("query %d args = %v, want [site_a 1000 2000]", i, a) + } + } + }) + } +} + +// The no-data path must reach the engine's min-samples gate: samples below +// the rule minimum is not a decision. This pins the contract the engine +// relies on (samples < max(min_samples,1) => no_data). +func TestQueryMetric_NoDataSamplesBelowMinimum(t *testing.T) { + for _, m := range []string{"trace_error_rate", "trace_p95_ms", "log_error_count", "uptime_failures"} { + f := &fakeScalar{answers: map[string]float64{}} + _, n, err := fakeService(f).queryMetric(context.Background(), "s", m, "1", "2") + if err != nil { + t.Fatalf("%s: %v", m, err) + } + if n >= 1 { + t.Fatalf("%s: empty window reported %d samples, want 0 (no-data)", m, n) + } + } + // min_samples: 10 spans with min 20 is still not enough. + f := &fakeScalar{answers: map[string]float64{sqlSpanCount: 10, sqlSpanErrorCount: 10}} + v, n, _ := fakeService(f).queryMetric(context.Background(), "s", "trace_error_rate", "1", "2") + if v != 100 || n != 10 || n >= 20 { + t.Fatalf("value=%v samples=%d", v, n) + } +} + +func TestQueryMetric_QueryErrorPropagates(t *testing.T) { + for _, m := range []string{"trace_error_rate", "trace_p95_ms", "log_error_count", "uptime_failures"} { + f := &fakeScalar{err: errors.New("boom")} + if _, _, err := fakeService(f).queryMetric(context.Background(), "s", m, "1", "2"); err == nil { + t.Fatalf("%s: query error swallowed (would read as healthy)", m) + } + } +} + +func TestQueryMetric_UnknownMetricRejected(t *testing.T) { + f := &fakeScalar{} + _, _, err := fakeService(f).queryMetric(context.Background(), "s", "bogus; DROP TABLE spans", "1", "2") + if err == nil || !strings.Contains(err.Error(), "unknown metric") { + t.Fatalf("err = %v, want unknown metric", err) + } + if len(f.seen) != 0 { + t.Fatal("unknown metric reached the query layer") + } +} + +func TestAlertMetricWhitelist(t *testing.T) { + set := AlertMetricSet() + for _, ok := range []string{"pageviews", "visitors", "error_count", "error_rate", + "trace_error_rate", "trace_p95_ms", "log_error_count", "uptime_failures"} { + if _, in := set[ok]; !in { + t.Errorf("%s missing from whitelist", ok) + } + } + for _, bad := range []string{"", "metric_value", "TRACE_ERROR_RATE", "trace_error_rate ", "1=1", "cpu"} { + if _, in := set[bad]; in { + t.Errorf("%q must not be whitelisted", bad) + } + } + // Every whitelisted metric must be computable by the engine (no layer drift). + for name := range set { + f := &fakeScalar{answers: map[string]float64{}} + if _, _, err := fakeService(f).queryMetric(context.Background(), "s", name, "1", "2"); err != nil { + t.Errorf("whitelisted metric %s not handled by engine: %v", name, err) + } + } +} + +// SQL text: parameterised, site-scoped, window-bounded, BIGINT-cast; no +// interpolation point for user input. +func TestExtendedMetricSQLText(t *testing.T) { + sqls := map[string]string{ + "span": sqlSpanCount, "spanErr": sqlSpanErrorCount, "root": sqlRootSpanCount, + "p95": sqlRootSpanP95, "log": sqlLogCount, "logErr": sqlLogErrorCount, + "up": sqlUptimeCount, "upFail": sqlUptimeFailures, + } + for name, q := range sqls { + for _, frag := range []string{"site_id = $1", ">= CAST($2 AS BIGINT)", "< CAST($3 AS BIGINT)", "AS value"} { + if !strings.Contains(q, frag) { + t.Errorf("%s: missing %q in %s", name, frag, q) + } + } + } + if !strings.Contains(sqlRootSpanP95, "percentile_cont(duration_ms, 0.95)") || !strings.Contains(sqlRootSpanP95, "parent_span_id = ''") { + t.Error("p95 must be percentile_cont over root spans") + } + if !strings.Contains(sqlSpanErrorCount, "status_code = 'error'") { + t.Error("span error predicate") + } + if !strings.Contains(sqlLogErrorCount, "level IN ('error', 'fatal')") { + t.Error("log error predicate") + } + if !strings.Contains(sqlUptimeFailures, "is_up = 'false'") { + t.Error("uptime failure predicate") + } +} + +func TestFireIntegrations_HookNonBlockingAndFireOnly(t *testing.T) { + f := &fakeScalar{} + s := fakeService(f) + rule := AlertRule{RuleID: "r1", SiteID: "site_a", Name: "High p95", Metric: "trace_p95_ms", Operator: "gt", Threshold: 500} + + s.fireIntegrations(rule, 900, "m") // no hook: no-op, no panic + + got := make(chan AlertFireEvent, 1) + release := make(chan struct{}) + s.SetIntegrationsHook(func(_ context.Context, ev AlertFireEvent) { + <-release // a stuck integration must not block the caller + got <- ev + }) + done := make(chan struct{}) + go func() { s.fireIntegrations(rule, 900, "m"); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("fireIntegrations blocked on a slow hook") + } + close(release) + select { + case ev := <-got: + if ev.SiteID != "site_a" || ev.Metric != "trace_p95_ms" || ev.Value != "900.00" || ev.Threshold != "500" || ev.Severity != "warning" { + t.Fatalf("event = %+v", ev) + } + case <-time.After(time.Second): + t.Fatal("hook never ran") + } + + // A panicking hook is contained. + s.SetIntegrationsHook(func(context.Context, AlertFireEvent) { panic("x") }) + s.fireIntegrations(rule, 1, "m") + time.Sleep(50 * time.Millisecond) +} diff --git a/ui/src/routes/alerts.tsx b/ui/src/routes/alerts.tsx index df257e6e..0fa2cc6b 100644 --- a/ui/src/routes/alerts.tsx +++ b/ui/src/routes/alerts.tsx @@ -17,6 +17,10 @@ const METRICS = [ { value: "error_rate", label: "Error Rate (%)" }, { value: "pageviews", label: "Pageviews" }, { value: "visitors", label: "Visitors" }, + { value: "trace_error_rate", label: "Trace Error Rate (% of spans)" }, + { value: "trace_p95_ms", label: "Trace p95 Latency (ms, root spans)" }, + { value: "log_error_count", label: "Log Errors (error/fatal count)" }, + { value: "uptime_failures", label: "Uptime Failures (failed checks)" }, ]; const OPERATORS = [ diff --git a/ui/src/routes/docs.tsx b/ui/src/routes/docs.tsx index 4ca6a0ca..fe83753c 100644 --- a/ui/src/routes/docs.tsx +++ b/ui/src/routes/docs.tsx @@ -473,7 +473,7 @@ The experiment detail page shows: ## Rules Create rules in **Alerts > Create Rule**: -- **Metric**: error_count, error_rate, pageviews, visitors +- **Metric**: error_count, error_rate, pageviews, visitors, trace_error_rate (% of spans with error status), trace_p95_ms (p95 root-span duration, ms), log_error_count (error/fatal logs), uptime_failures (failed uptime checks). Trace, log and uptime metrics are site-wide; a window with no spans, logs or checks reads as no data, not healthy. - **Operator**: >, >=, <, <=, = - **Threshold**: numeric value - **Window**: minutes over which to evaluate (default 5) From da0c6050fcbe988e23627d725e9051923ba88ce5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:42:55 +0000 Subject: [PATCH 17/65] wip --- cmd/observe/main.go | 15 +- cmd/observe/replay_filter.go | 47 ++++++ internal/replays/capture.go | 249 ++++++++++++++++++++++++++++++++ internal/replays/list_filter.go | 193 +++++++++++++++++++++++++ internal/replays/replays.go | 38 ++--- 5 files changed, 514 insertions(+), 28 deletions(-) create mode 100644 cmd/observe/replay_filter.go create mode 100644 internal/replays/capture.go create mode 100644 internal/replays/list_filter.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..5806294e 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -453,7 +453,8 @@ func main() { dashSvc := dashboards.NewDashboardService(db).WithMetrics(metricsSvc) replaySvc := replays.NewReplayService(db). WithLogger(logger). - WithPrivacy(siteSvc.PrivacyConfig, cfg.SessionSalt) + WithPrivacy(siteSvc.PrivacyConfig, cfg.SessionSalt). + WithQueryGuard(queryLimiter, queryBudgets) heatmapsSvc := heatmaps.NewService(db) aiSvc := aiquery.NewService(db, logger) aiSchema := aiquery.NewSchemaCard(db) @@ -5235,6 +5236,11 @@ type listReplaysInput struct { To string `query:"to"` Limit int `query:"limit"` Offset int `query:"offset"` + // Optional filters (see parseReplayFilter). + HasErrors string `query:"has_errors"` + MinDuration string `query:"min_duration"` + URLContains string `query:"url_contains"` + DistinctID string `query:"distinct_id"` } func listReplaysHandler(svc *replays.ReplayService) neutron.HandlerFunc[listReplaysInput, []replays.ReplaySession] { @@ -5243,7 +5249,12 @@ func listReplaysHandler(svc *replays.ReplayService) neutron.HandlerFunc[listRepl if err != nil { return nil, neutron.ErrBadRequest(err.Error()) } - return emptyOnNil(svc.ListReplays(ctx, input.SiteID, from, to, input.Limit, input.Offset)) + filter, err := parseReplayFilter(input.HasErrors, input.MinDuration, input.URLContains, input.DistinctID) + if err != nil { + return nil, err + } + rows, err := svc.ListReplaysFiltered(ctx, input.SiteID, from, to, input.Limit, input.Offset, filter) + return emptyOnNil(rows, mapGuardRefusal(err)) } } diff --git a/cmd/observe/replay_filter.go b/cmd/observe/replay_filter.go new file mode 100644 index 00000000..f29a06db --- /dev/null +++ b/cmd/observe/replay_filter.go @@ -0,0 +1,47 @@ +package main + +import ( + "errors" + "strconv" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/queryguard" + "github.com/useteploy/teploy-observe/internal/replays" +) + +// parseReplayFilter turns the replay-list query parameters into a +// replays.ReplayFilter. has_errors is strict ("true"/"false"/empty) and +// min_duration is whole milliseconds; anything else is a 400. Length +// bounds on url_contains/distinct_id are enforced by the service. +func parseReplayFilter(hasErrors, minDuration, urlContains, distinctID string) (replays.ReplayFilter, error) { + f := replays.ReplayFilter{URLContains: urlContains, DistinctID: distinctID} + switch hasErrors { + case "", "false": + case "true": + f.HasErrors = true + default: + return f, neutron.ErrBadRequest("has_errors must be true or false") + } + if minDuration != "" { + n, err := strconv.ParseInt(minDuration, 10, 64) + if err != nil || n < 0 { + return f, neutron.ErrBadRequest("min_duration must be a non-negative integer (milliseconds)") + } + f.MinDurationMS = n + } + return f, nil +} + +// mapGuardRefusal converts an O12 admission refusal into the matching +// HTTP error so a shed query reads as 429/504, not a bare 500. +func mapGuardRefusal(err error) error { + var ref *queryguard.Refusal + if errors.As(err, &ref) { + if ref.Status == 429 { + return neutron.ErrRateLimited(ref.Error()) + } + return neutron.ErrServiceUnavailable(ref.Error()) + } + return err +} diff --git a/internal/replays/capture.go b/internal/replays/capture.go new file mode 100644 index 00000000..1a93b800 --- /dev/null +++ b/internal/replays/capture.go @@ -0,0 +1,249 @@ +package replays + +import ( + "fmt" + "math" + "net/url" + "regexp" + "strings" + "unicode/utf8" + + "github.com/neutron-build/neutron/go/neutron" +) + +// Console and network capture events (opt-in on the tracker via data- +// attributes, default OFF). They ride the existing v2 batch transport as two +// extra custom event types. The tracker already truncates and scrubs; the +// server re-validates every bound and re-scrubs, because a producer is not +// trusted (a hand-built batch can carry anything). Violations reject the +// whole batch with a 400, the same rule the rest of the batch validation +// applies. +const ( + // EventConsole and EventNetwork are the stored replay_events.event_type + // values for the captured console and network records. + EventConsole = "console" + EventNetwork = "network" + + // maxConsoleMessageBytes is the per-record cap (1 KiB, UTF-8 bytes). + maxConsoleMessageBytes = 1024 + // maxNetworkURLBytes bounds the stored, query-stripped URL. + maxNetworkURLBytes = 2048 + // maxNetworkDurationMS caps a plausible request duration (1 hour). + maxNetworkDurationMS = 3600 * 1000 + // maxNetworkSizeBytes caps a reported transfer size (1 TiB). + maxNetworkSizeBytes = 1 << 40 + + // Per-batch caps. A batch carries at most 1000 events (tracker + // MAX_FLUSH_EVENTS), so these only bound a hostile producer; the + // tracker's own per-session caps (200 console, 500 network) are far + // below them. The per-session cap is enforced client-side only - a + // server-wide count would need a storage read on the ingest hot path. + maxConsoleEventsPerBatch = 200 + maxNetworkEventsPerBatch = 500 +) + +var ( + consoleLevels = map[string]bool{"log": true, "info": true, "warn": true, "error": true} + + // consoleKeys and networkKeys are the audited key allowlists: anything + // else (headers, bodies, args) is refused rather than stored. + consoleKeys = map[string]bool{"level": true, "message": true, "truncated": true} + networkKeys = map[string]bool{ + "method": true, "url": true, "status": true, "duration_ms": true, + "size": true, "kind": true, + } + + methodRe = regexp.MustCompile(`^[A-Z]{1,16}$`) +) + +// Secret scrubbing for captured console text. Same denylist idea as the +// tracker (observe-replay-capture.js keeps the patterns in lockstep): key +// = value pairs for secret-looking keys, bearer/basic credentials, JWTs, and +// long opaque tokens. +var scrubPatterns = []struct { + re *regexp.Regexp + repl string +}{ + {regexp.MustCompile(`(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}`), "$1 [redacted]"}, + {regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]*`), "[redacted-jwt]"}, + {regexp.MustCompile(`(?i)((?:pass(?:word|wd)?|secret|token|api[_-]?key|authorization|auth|credential|session(?:id)?|cookie|private[_-]?key)["']?\s*[:=]\s*)("[^"]*"|'[^']*'|[^\s,;&}]+)`), "${1}[redacted]"}, + {regexp.MustCompile(`\b[A-Fa-f0-9]{32,}\b`), "[redacted-hex]"}, + {regexp.MustCompile(`\b[A-Za-z0-9_-]{40,}\b`), "[redacted-token]"}, +} + +// ScrubConsoleText removes credential-shaped substrings from captured console +// text. Exported for the tests and so the denylist has one Go home. +func ScrubConsoleText(s string) string { + for _, p := range scrubPatterns { + s = p.re.ReplaceAllString(s, p.repl) + } + return s +} + +// cutUTF8 truncates s to at most max bytes without splitting a rune. +func cutUTF8(s string, max int) string { + if len(s) <= max { + return s + } + s = s[:max] + for len(s) > 0 && !utf8.ValidString(s) { + s = s[:len(s)-1] + } + return s +} + +// numField reads a JSON number (float64 after decode) as an int64, refusing +// fractions, NaN/Inf and out-of-range values. +func numField(v any, min, max int64) (int64, bool) { + f, ok := v.(float64) + if !ok || math.IsNaN(f) || math.IsInf(f, 0) || f != math.Trunc(f) { + return 0, false + } + if f < float64(min) || f > float64(max) { + return 0, false + } + return int64(f), true +} + +// scrubNetworkURL reduces a captured request URL to scheme+host+path and +// masks path segments that look like opaque credentials. +func scrubNetworkURL(raw string) (string, bool) { + if raw == "" || len(raw) > maxNetworkURLBytes { + return "", false + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", false + } + u.User = nil + u.RawQuery = "" + u.ForceQuery = false + u.Fragment = "" + u.RawFragment = "" + segs := strings.Split(u.Path, "/") + for i, seg := range segs { + if looksOpaque(seg) { + segs[i] = ":redacted" + } + } + u.Path = strings.Join(segs, "/") + u.RawPath = "" + out := u.String() + if len(out) > maxNetworkURLBytes { + return "", false + } + return out, true +} + +var opaqueSegmentRe = regexp.MustCompile(`^[A-Za-z0-9_-]{24,}$`) + +// looksOpaque reports a path segment that reads as a credential or random +// id (24+ URL-safe characters including a digit) rather than a slug. +func looksOpaque(seg string) bool { + return opaqueSegmentRe.MatchString(seg) && strings.ContainsAny(seg, "0123456789") +} + +// validateCaptureEvents enforces the bounds on console/network events and +// scrubs them in place. Other event types are not touched. The scrub runs +// before the batch digest is taken, and is deterministic, so retries of one +// batch still digest identically. +func validateCaptureEvents(in *IngestInput) error { + var consoleN, networkN int + for i := range in.Events { + ev := &in.Events[i] + switch ev.Type { + case EventConsole: + consoleN++ + if consoleN > maxConsoleEventsPerBatch { + return neutron.ErrBadRequest(fmt.Sprintf("too many console events in one batch (max %d)", maxConsoleEventsPerBatch)) + } + if err := validateConsoleData(ev.Data); err != nil { + return err + } + m := ev.Data.(map[string]any) + m["message"] = cutUTF8(ScrubConsoleText(m["message"].(string)), maxConsoleMessageBytes) + case EventNetwork: + networkN++ + if networkN > maxNetworkEventsPerBatch { + return neutron.ErrBadRequest(fmt.Sprintf("too many network events in one batch (max %d)", maxNetworkEventsPerBatch)) + } + if err := validateNetworkData(ev.Data); err != nil { + return err + } + m := ev.Data.(map[string]any) + clean, _ := scrubNetworkURL(m["url"].(string)) + m["url"] = clean + } + } + return nil +} + +func validateConsoleData(data any) error { + m, ok := data.(map[string]any) + if !ok { + return neutron.ErrBadRequest("console event data must be an object") + } + for k := range m { + if !consoleKeys[k] { + return neutron.ErrBadRequest(fmt.Sprintf("console event has unknown field %q", k)) + } + } + level, _ := m["level"].(string) + if !consoleLevels[level] { + return neutron.ErrBadRequest("console event level must be log, info, warn or error") + } + msg, ok := m["message"].(string) + if !ok { + return neutron.ErrBadRequest("console event message must be a string") + } + if len(msg) > maxConsoleMessageBytes { + return neutron.ErrBadRequest(fmt.Sprintf("console event message exceeds %d bytes", maxConsoleMessageBytes)) + } + if !utf8.ValidString(msg) { + return neutron.ErrBadRequest("console event message must be valid UTF-8") + } + if t, present := m["truncated"]; present { + if _, isBool := t.(bool); !isBool { + return neutron.ErrBadRequest("console event truncated must be a boolean") + } + } + return nil +} + +func validateNetworkData(data any) error { + m, ok := data.(map[string]any) + if !ok { + return neutron.ErrBadRequest("network event data must be an object") + } + for k := range m { + if !networkKeys[k] { + return neutron.ErrBadRequest(fmt.Sprintf("network event has unknown field %q", k)) + } + } + method, _ := m["method"].(string) + if !methodRe.MatchString(method) { + return neutron.ErrBadRequest("network event method must be 1-16 uppercase letters") + } + raw, _ := m["url"].(string) + if _, ok := scrubNetworkURL(raw); !ok { + return neutron.ErrBadRequest("network event url must be an absolute http(s) URL of at most 2048 bytes") + } + // status 0 = network failure / aborted. + if _, ok := numField(m["status"], 0, 999); !ok { + return neutron.ErrBadRequest("network event status must be an integer 0-999") + } + if _, ok := numField(m["duration_ms"], 0, maxNetworkDurationMS); !ok { + return neutron.ErrBadRequest("network event duration_ms must be an integer 0-3600000") + } + if sz, present := m["size"]; present { + if _, ok := numField(sz, 0, maxNetworkSizeBytes); !ok { + return neutron.ErrBadRequest("network event size must be a non-negative integer") + } + } + if k, present := m["kind"]; present { + if ks, _ := k.(string); ks != "fetch" && ks != "xhr" { + return neutron.ErrBadRequest("network event kind must be fetch or xhr") + } + } + return nil +} diff --git a/internal/replays/list_filter.go b/internal/replays/list_filter.go new file mode 100644 index 00000000..acddd857 --- /dev/null +++ b/internal/replays/list_filter.go @@ -0,0 +1,193 @@ +package replays + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/neutron-build/neutron/go/neutron" + "github.com/neutron-build/neutron/go/nucleus" + + "github.com/useteploy/teploy-observe/internal/dbutil" + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +// Filter bounds for the replay list. +const ( + maxFilterURLContains = 200 + maxFilterDistinctID = 256 + // maxFilterMinDurationMS caps min_duration at 7 days; anything larger + // can never match a recorded session. + maxFilterMinDurationMS = 7 * 24 * 3600 * 1000 + // errorLinkSlackMS extends the error_events lookup window past the + // listing's upper bound: an error is raised after its session starts. + errorLinkSlackMS = 3600 * 1000 +) + +// ReplayFilter narrows ListReplaysFiltered. The zero value filters nothing. +type ReplayFilter struct { + // HasErrors keeps sessions flagged has_error by the tracker or linked + // to an error event by session id or replay id. + HasErrors bool + // MinDurationMS keeps sessions at least this long (milliseconds). + MinDurationMS int64 + // URLContains is a case-insensitive substring match on the stored + // (query-stripped) session URL. Wildcards are matched literally. + URLContains string + // DistinctID is the RAW identifier; it is hashed with the site's ingest + // derivation before comparing, so the raw value never touches SQL. + DistinctID string +} + +func (f ReplayFilter) active() bool { + return f.HasErrors || f.MinDurationMS > 0 || f.URLContains != "" || f.DistinctID != "" +} + +// escapeLike makes s match literally inside a LIKE/ILIKE pattern +// (backslash is the default escape character). +func escapeLike(s string) string { + return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s) +} + +// WithQueryGuard installs O12 query admission for the list: a concurrency +// slot per call plus the wall-time budget and window clamp. A nil limiter +// disables concurrency admission; zero budget fields fall back to defaults. +func (s *ReplayService) WithQueryGuard(l *queryguard.Limiter, b queryguard.Budgets) *ReplayService { + d := queryguard.DefaultBudgets() + if b.Timeout <= 0 { + b.Timeout = d.Timeout + } + if b.MaxWindow <= 0 { + b.MaxWindow = d.MaxWindow + } + if b.MaxScanRows <= 0 { + b.MaxScanRows = d.MaxScanRows + } + s.guard = l + s.budgets = b + return s +} + +// buildListQuery renders the list SQL and its bound arguments. Pure (no +// store access) so the text can be asserted. Every user value is a bound +// parameter; the only interpolated numbers are the clamped LIMIT/OFFSET. +// distinctHash is the already-hashed distinct id ("" when unfiltered). +func buildListQuery(siteID string, from, to time.Time, limit, offset int, f ReplayFilter, distinctHash string) (string, []any) { + if limit <= 0 { + limit = 20 + } + if limit > maxListReplaysLimit { + limit = maxListReplaysLimit + } + if offset < 0 { + offset = 0 + } + args := []any{siteID, dbutil.IntParam(from.UnixMilli()), dbutil.IntParam(to.UnixMilli())} + var outer []string + add := func(v any) string { + args = append(args, v) + return "$" + strconv.Itoa(len(args)) + } + // Non-key columns are compared on the COLLAPSED row (outer WHERE): a + // predicate inside the argMax grouping would match stale versions. + if f.DistinctID != "" { + outer = append(outer, "distinct_id = "+add(distinctHash)) + } + if f.MinDurationMS > 0 { + outer = append(outer, "CAST(duration_ms AS BIGINT) >= CAST("+add(dbutil.IntParam(f.MinDurationMS))+" AS BIGINT)") + } + if f.URLContains != "" { + // Precedent: internal/logs/logs.go message ILIKE '%' || $n || '%'. + outer = append(outer, "url ILIKE '%' || "+add(escapeLike(f.URLContains))+" || '%'") + } + if f.HasErrors { + hi := add(dbutil.IntParam(to.UnixMilli() + errorLinkSlackMS)) + // Precedent for IN (SELECT ...): internal/groups/groups.go. The + // error_events lookups are site- and time-bounded. + outer = append(outer, "(has_error = 'true'"+ + " OR (session_id <> '' AND session_id IN (SELECT session_id FROM error_events WHERE site_id = $1 AND timestamp >= $2 AND timestamp < "+hi+" AND session_id <> ''))"+ + " OR replay_id IN (SELECT replay_id FROM error_events WHERE site_id = $1 AND timestamp >= $2 AND timestamp < "+hi+" AND replay_id <> ''))") + } + where := "" + if len(outer) > 0 { + where = "\n\t\t WHERE " + strings.Join(outer, " AND ") + } + q := `SELECT replay_id, tenant_id, site_id, session_id, + CAST(start_time AS TEXT) AS start_time, + duration_ms, page_count, url, browser, os, device, has_error + FROM ` + replaySessionsLatest("site_id = $1 AND start_time >= $2 AND start_time < $3") + where + + fmt.Sprintf(` + ORDER BY start_time DESC + LIMIT %d OFFSET %d`, limit, offset) + return q, args +} + +// ListReplays returns recent replay sessions for a site, read through the +// version collapse so multi-batch upserts surface as one row per replay. +func (s *ReplayService) ListReplays(ctx context.Context, siteID string, from, to time.Time, limit, offset int) ([]ReplaySession, error) { + return s.ListReplaysFiltered(ctx, siteID, from, to, limit, offset, ReplayFilter{}) +} + +// validateFilter bounds the filter values (400s). +func validateFilter(f ReplayFilter) error { + if len(f.URLContains) > maxFilterURLContains || strings.ContainsRune(f.URLContains, 0) { + return neutron.ErrBadRequest(fmt.Sprintf("url_contains must be at most %d bytes", maxFilterURLContains)) + } + if len(f.DistinctID) > maxFilterDistinctID || strings.ContainsRune(f.DistinctID, 0) { + return neutron.ErrBadRequest(fmt.Sprintf("distinct_id must be at most %d bytes", maxFilterDistinctID)) + } + if f.MinDurationMS < 0 || f.MinDurationMS > maxFilterMinDurationMS { + return neutron.ErrBadRequest("min_duration must be between 0 and 604800000 milliseconds") + } + return nil +} + +// ListReplaysFiltered is ListReplays with optional filters. Invalid filter +// values are 400s; the guard (when installed) admits and bounds the query. +func (s *ReplayService) ListReplaysFiltered(ctx context.Context, siteID string, from, to time.Time, limit, offset int, f ReplayFilter) ([]ReplaySession, error) { + if err := validateFilter(f); err != nil { + return nil, err + } + + distinctHash := "" + if f.DistinctID != "" { + salt, rawOptIn := s.salt, false + if s.privacy != nil { + if siteSalt, raw, ok := s.privacy(ctx, siteID); ok { + salt, rawOptIn = siteSalt, raw + } + } + if salt == "" && !rawOptIn { + // Ingest drops the id in this state, so nothing can match. + return []ReplaySession{}, nil + } + distinctHash = hashDistinctID(f.DistinctID, salt, rawOptIn) + } + + qctx := ctx + if s.guard != nil { + release, err := s.guard.Acquire(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + } + if s.budgets.Timeout > 0 { + if max := s.budgets.MaxWindow; max > 0 && to.Sub(from) > max { + from = to.Add(-max) + } + var cancel context.CancelFunc + qctx, cancel = context.WithTimeout(ctx, s.budgets.Timeout) + defer cancel() + } + + q, args := buildListQuery(siteID, from, to, limit, offset, f, distinctHash) + rows, err := nucleus.Query[ReplaySession](qctx, s.db.SQL(), q, args...) + if err != nil && errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + return nil, queryguard.TimeBudgetRefusal(s.guard, s.budgets.Timeout) + } + return rows, err +} diff --git a/internal/replays/replays.go b/internal/replays/replays.go index 99184e96..a3c30ffa 100644 --- a/internal/replays/replays.go +++ b/internal/replays/replays.go @@ -23,6 +23,7 @@ import ( "github.com/useteploy/teploy-observe/internal/identity" "github.com/useteploy/teploy-observe/internal/ingest" "github.com/useteploy/teploy-observe/internal/query" + "github.com/useteploy/teploy-observe/internal/queryguard" ) // ErrCrossSiteReplay indicates the caller tried to append events to a replay @@ -81,6 +82,11 @@ type ReplayService struct { // process; a multi-replica deployment needs the stable-key + CAS design // deferred with F03-class schema work. replayLocks [64]sync.Mutex + + // guard/budgets are the optional O12 admission state for the list path + // (see WithQueryGuard in list_filter.go). + guard *queryguard.Limiter + budgets queryguard.Budgets } // lockReplay serializes all writes for one replay ID. NOT keyed by site: @@ -522,6 +528,12 @@ func (s *ReplayService) Ingest(ctx context.Context, input IngestInput) (Result, return Result{}, err } + // Console/network capture events: bounds, key allowlist and secret + // scrub (capture.go). Runs before the batch digest is taken. + if err := validateCaptureEvents(&input); err != nil { + return Result{}, err + } + replayID := input.ReplayID if replayID == "" { if input.idempotent() { @@ -784,32 +796,6 @@ func readIntField(data any, key string) (int, bool) { // default but no cap, so an extreme value produced an extreme query). const maxListReplaysLimit = 200 -// ListReplays returns recent replay sessions for a site, read through the -// version collapse so multi-batch upserts surface as one row per replay. -func (s *ReplayService) ListReplays(ctx context.Context, siteID string, from, to time.Time, limit, offset int) ([]ReplaySession, error) { - if limit <= 0 { - limit = 20 - } - if limit > maxListReplaysLimit { - limit = maxListReplaysLimit - } - if offset < 0 { - offset = 0 - } - fromMs := dbutil.IntParam(from.UnixMilli()) - toMs := dbutil.IntParam(to.UnixMilli()) - - return nucleus.Query[ReplaySession](ctx, s.db.SQL(), - fmt.Sprintf(`SELECT replay_id, tenant_id, site_id, session_id, - CAST(start_time AS TEXT) AS start_time, - duration_ms, page_count, url, browser, os, device, has_error - FROM `+replaySessionsLatest("site_id = $1 AND start_time >= $2 AND start_time < $3")+` - ORDER BY start_time DESC - LIMIT %d OFFSET %d`, limit, offset), - siteID, fromMs, toMs, - ) -} - // GetReplayEvents returns the events of one replay, scoped to the site that // owns it (audit F08). Legacy rows written before the site column existed // carry site_id=” and still belong to the owning session's site. From c745c4fef733d1eee53abaee26a92c7b802dc8fc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:43:18 +0000 Subject: [PATCH 18/65] wip --- internal/persons/identity.go | 243 ++++++++++++ internal/persons/persons.go | 173 ++------ internal/persons/service_identity.go | 369 ++++++++++++++++++ internal/persons/store_nucleus.go | 275 +++++++++++++ .../migrations/061_persons_identity.up.sql | 70 ++++ 5 files changed, 998 insertions(+), 132 deletions(-) create mode 100644 internal/persons/identity.go create mode 100644 internal/persons/service_identity.go create mode 100644 internal/persons/store_nucleus.go create mode 100644 internal/schema/migrations/061_persons_identity.up.sql diff --git a/internal/persons/identity.go b/internal/persons/identity.go new file mode 100644 index 00000000..64bd95b6 --- /dev/null +++ b/internal/persons/identity.go @@ -0,0 +1,243 @@ +package persons + +// C3 - person properties, explicit alias/merge and erasure (migration 061). +// +// Everything here follows docs/IDENTITY_MODEL_ADR.md: a person is still an +// aggregate over events.distinct_id; the tables added by 061 are side data +// keyed by that same person_key. Historical events are never rewritten; an +// alias is resolved forward at read time, in Go, over bounded sets. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" +) + +// Bounds. Every limit is enforced before any write. +const ( + MaxPropertyKeys = 50 + MaxPropertyValueLen = 1024 // bytes of the JSON-encoded value + MaxPropertyKeyLen = 64 + MaxPersonKeyLen = 256 + // MaxAliasesPerPerson bounds one merged person's alias fan-out so the + // detail view's per-member reads stay bounded. + MaxAliasesPerPerson = 20 + // MaxAliasesPerSite bounds the alias map loaded for every resolved read. + MaxAliasesPerSite = 10000 + // MaxTombstonesPerSite bounds the exclusion set loaded for list reads. + MaxTombstonesPerSite = 50000 + // resolveScanCap bounds the Go-side fold of a resolved list read. + resolveScanCap = 5000 + // maxResolveHops bounds an alias chain walk. + maxResolveHops = 16 +) + +// Sentinel errors; handlers map them to HTTP statuses. +var ( + ErrInvalid = errors.New("invalid request") + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") + ErrLimit = errors.New("limit exceeded") + ErrErased = errors.New("person erased") +) + +var ( + personKeyRe = regexp.MustCompile(`^[A-Za-z0-9_.:@+=-]{1,256}$`) + propKeyRe = regexp.MustCompile(`^[A-Za-z_$][A-Za-z0-9_.$-]{0,63}$`) +) + +// reservedPropertyKeys are rejected so the raw identify value (which the +// server hashes before storage) cannot be echoed back into a property and +// defeat the hashing. This is identifier hygiene, not a PII filter: any +// other key (including email) is the customer's choice. +var reservedPropertyKeys = map[string]bool{"distinct_id": true, "user_id": true} + +// ValidatePersonKey checks a person_key shape (already-hashed id). +func ValidatePersonKey(k string) error { + if !personKeyRe.MatchString(k) { + return fmt.Errorf("%w: person key must match %s", ErrInvalid, personKeyRe.String()) + } + return nil +} + +// ValidateProperties validates one identify() trait batch. It returns the +// scalar values to set and the keys to remove (null values). Values must be +// strings, numbers or booleans; nested objects and arrays are rejected so a +// property stays bounded and comparable. +func ValidateProperties(in map[string]any) (set map[string]any, remove []string, err error) { + if len(in) > MaxPropertyKeys { + return nil, nil, fmt.Errorf("%w: at most %d properties per request", ErrInvalid, MaxPropertyKeys) + } + set = make(map[string]any, len(in)) + for k, v := range in { + if !propKeyRe.MatchString(k) { + return nil, nil, fmt.Errorf("%w: property key %q must match %s", ErrInvalid, k, propKeyRe.String()) + } + if reservedPropertyKeys[strings.ToLower(k)] { + return nil, nil, fmt.Errorf("%w: property key %q is reserved", ErrInvalid, k) + } + switch v.(type) { + case nil: + remove = append(remove, k) + continue + case string, bool, float64, float32, int, int32, int64, json.Number: + default: + return nil, nil, fmt.Errorf("%w: property %q must be a string, number, boolean or null", ErrInvalid, k) + } + enc, merr := json.Marshal(v) + if merr != nil || len(enc) > MaxPropertyValueLen { + return nil, nil, fmt.Errorf("%w: property %q value exceeds %d bytes", ErrInvalid, k, MaxPropertyValueLen) + } + set[k] = v + } + sort.Strings(remove) + return set, remove, nil +} + +// AliasRow is one active alias edge. +type AliasRow struct { + AliasKey string `json:"alias_key" db:"alias_key"` + CanonicalKey string `json:"canonical_key" db:"canonical_key"` +} + +// Resolver maps alias keys to their canonical person. Built once per read +// from a bounded alias set; chains are followed with a hop cap and cycle +// detection (a corrupt cycle resolves to the key itself - fail safe, no +// merge). +type Resolver struct { + direct map[string]string + members map[string][]string +} + +// NewResolver builds a resolver from active alias edges. +func NewResolver(rows []AliasRow) *Resolver { + r := &Resolver{direct: make(map[string]string, len(rows))} + for _, a := range rows { + if a.AliasKey != "" && a.CanonicalKey != "" && a.AliasKey != a.CanonicalKey { + r.direct[a.AliasKey] = a.CanonicalKey + } + } + r.members = make(map[string][]string) + for alias := range r.direct { + root := r.Canonical(alias) + if root != alias { + r.members[root] = append(r.members[root], alias) + } + } + for _, m := range r.members { + sort.Strings(m) + } + return r +} + +// Empty reports whether no aliases exist. +func (r *Resolver) Empty() bool { return len(r.direct) == 0 } + +// IsAlias reports whether key is itself merged into another person. +func (r *Resolver) IsAlias(key string) bool { _, ok := r.direct[key]; return ok } + +// Canonical returns the root person key for key. +func (r *Resolver) Canonical(key string) string { + cur := key + seen := map[string]bool{key: true} + for hops := 0; hops < maxResolveHops; hops++ { + next, ok := r.direct[cur] + if !ok { + return cur + } + if seen[next] { + return key // cycle: refuse to merge + } + seen[next] = true + cur = next + } + return key // chain too deep: refuse to merge +} + +// Members returns every alias whose root is canonical (sorted, excluding +// canonical itself). +func (r *Resolver) Members(canonical string) []string { return r.members[canonical] } + +// absorb folds o into p: exact event sum, session upper bound, widest +// seen-range, and the most recent row's country/browser. +func (p *Person) absorb(o Person) { + p.EventCount += o.EventCount + p.SessionCount += o.SessionCount + if o.FirstSeenMs < p.FirstSeenMs { + p.FirstSeenMs = o.FirstSeenMs + } + if o.LastSeenMs > p.LastSeenMs { + p.LastSeenMs = o.LastSeenMs + p.TopCountry, p.TopBrowser = o.TopCountry, o.TopBrowser + } +} + +// foldPersons collapses per-distinct_id rows onto their canonical person, +// dropping erased keys. Sums are exact for events; session_count is an +// UPPER BOUND (a monthly session estimate shared by two merged ids is +// counted twice). +func foldPersons(rows []Person, res *Resolver, tomb map[string]bool) []Person { + acc := make(map[string]*Person, len(rows)) + order := make([]string, 0, len(rows)) + for _, p := range rows { + if tomb[p.DistinctID] { + continue + } + c := p.DistinctID + if p.DistinctID != "" { + c = res.Canonical(p.DistinctID) + } + if tomb[c] { + continue + } + cur, ok := acc[c] + if !ok { + cp := p + cp.DistinctID = c + acc[c] = &cp + order = append(order, c) + continue + } + cur.absorb(p) + } + out := make([]Person, 0, len(order)) + for _, k := range order { + out = append(out, *acc[k]) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].LastSeenMs != out[j].LastSeenMs { + return out[i].LastSeenMs > out[j].LastSeenMs + } + return out[i].DistinctID < out[j].DistinctID + }) + return out +} + +// EventSource is the read side over the events table. +type EventSource interface { + // List/Count are the SQL-paginated fast path (no aliases/tombstones). + List(ctx context.Context, siteID string, fromMs, toMs int64, limit, offset int, includeAnonymous bool) ([]Person, error) + Count(ctx context.Context, siteID string, fromMs, toMs int64, includeAnonymous bool) (int64, error) + // ListAll returns up to limit per-distinct_id rows unpaginated. + ListAll(ctx context.Context, siteID string, fromMs, toMs int64, limit int, includeAnonymous bool) ([]Person, error) + // Aggregate returns the lifetime summary of one distinct_id. + Aggregate(ctx context.Context, siteID, key string) (Person, bool, error) + Timeline(ctx context.Context, siteID, key string) ([]PersonEvent, error) + Exists(ctx context.Context, siteID, key string) (bool, error) +} + +// IdentityStore is the side-data store for the 061 tables. Every method is +// scoped by siteID. +type IdentityStore interface { + GetProps(ctx context.Context, siteID, key string) (props string, found bool, err error) + PutProps(ctx context.Context, siteID, key, propsJSON string) error + ListAliases(ctx context.Context, siteID string, limit int) ([]AliasRow, error) + PutAlias(ctx context.Context, siteID, alias, canonical, actor string, active bool) error + ListTombstones(ctx context.Context, siteID string, limit int) ([]string, error) + IsTombstoned(ctx context.Context, siteID, key string) (bool, error) + PutTombstone(ctx context.Context, siteID, key, actor string) error +} diff --git a/internal/persons/persons.go b/internal/persons/persons.go index 987bc1f4..95488c74 100644 --- a/internal/persons/persons.go +++ b/internal/persons/persons.go @@ -5,10 +5,15 @@ // package is the read layer that turns the column into a "Persons" UI. // // Design choices: -// - No table of its own. Persons are an aggregate over events; the -// refresh is implicit on every read. Materialization is a phase-2 -// concern (cohort_members table + periodic refresh per the design -// doc); v1 keeps the contract simple. +// - Persons are an aggregate over events; the refresh is implicit on +// every read. Materialization is a phase-2 concern (cohort_members +// table + periodic refresh per the design doc); v1 keeps the contract +// simple. +// - C3 (migration 061) adds SIDE tables keyed by the same person_key: +// person_properties (identify traits), person_aliases (explicit +// merge, resolved at read time in Go) and person_tombstones (erasure). +// See identity.go and docs/IDENTITY_MODEL_ADR.md. Events are never +// rewritten. // - Anonymous rows (distinct_id = ”) are excluded by default. The // caller can opt them in with a flag — useful for ops who haven't // wired identify() yet and want to see traffic shape. @@ -19,22 +24,29 @@ package persons import ( "context" - "fmt" + "sync" "time" "github.com/neutron-build/neutron/go/nucleus" - - "github.com/useteploy/teploy-observe/internal/dbutil" ) // Service exposes person aggregation queries. type Service struct { db *nucleus.Client + ev EventSource + st IdentityStore + mu sync.Mutex // serializes the read-modify-write paths (single-process boundary) } // NewService constructs a persons.Service backed by the shared nucleus client. func NewService(db *nucleus.Client) *Service { - return &Service{db: db} + return &Service{db: db, ev: &nucleusEvents{db: db}, st: &nucleusIdentity{db: db}} +} + +// NewServiceFromParts builds a Service over explicit stores (tests and the +// in-memory implementation in memory.go). +func NewServiceFromParts(ev EventSource, st IdentityStore) *Service { + return &Service{ev: ev, st: st} } // Person is the aggregate returned by ListPersons. Counts are int64 so @@ -60,17 +72,25 @@ type PersonEvent struct { } // PersonDetail bundles a person's aggregate row with its event timeline. +// +// C3: Properties holds the identify() traits (merged across the person's +// aliases, canonical wins), Aliases lists the person keys merged into this +// one, and CanonicalKey is the key the requested id resolved to. type PersonDetail struct { - Person Person `json:"person"` - Timeline []PersonEvent `json:"timeline"` + Person Person `json:"person"` + Timeline []PersonEvent `json:"timeline"` + Properties map[string]any `json:"properties"` + Aliases []string `json:"aliases"` + CanonicalKey string `json:"canonical_key,omitempty"` } // ListPersons returns one row per distinct_id observed in the time -// window for siteID, ordered by last activity descending. Anonymous -// (empty distinct_id) rows are excluded unless includeAnonymous=true — -// most operators want the identified-users view, but a fresh install -// without identify() wired needs the all-traffic view to verify the -// page is even rendering. +// window for siteID, ordered by last activity descending, WITHOUT alias +// resolution or erasure filtering (use ListResolved for the user-facing +// view). Anonymous (empty distinct_id) rows are excluded unless +// includeAnonymous=true — most operators want the identified-users view, +// but a fresh install without identify() wired needs the all-traffic view +// to verify the page is even rendering. func (s *Service) ListPersons(ctx context.Context, siteID string, fromMs, toMs int64, limit, offset int, includeAnonymous bool) ([]Person, error) { if siteID == "" { return []Person{}, nil @@ -84,42 +104,9 @@ func (s *Service) ListPersons(ctx context.Context, siteID string, fromMs, toMs i if offset < 0 { offset = 0 } - - from := dbutil.IntParam(fromMs) - to := dbutil.IntParam(toMs) - - // Inner aggregate computes the per-distinct_id summary. The outer - // scan is plain so we can sort + paginate in SQL. argMax(country, - // timestamp) picks the most recent country/browser per person — - // nucleus doesn't expose argMax yet, so we approximate with MAX - // over a tuple-style hack: pick by timestamp using a subquery. - // Simpler portable form: use a window-style MAX(country) and - // accept that for ties we get the last grouped value, which is - // good enough for a "where they're connecting from" hint. - anonClause := "AND distinct_id != ''" - if includeAnonymous { - anonClause = "" - } - - q := fmt.Sprintf(`SELECT distinct_id, - MIN(CAST(timestamp AS BIGINT)) AS first_seen_ms, - MAX(CAST(timestamp AS BIGINT)) AS last_seen_ms, - COUNT(*) AS event_count, - COUNT(DISTINCT session_id) AS session_count, - argMax(country, CAST(timestamp AS BIGINT)) AS top_country, - argMax(browser, CAST(timestamp AS BIGINT)) AS top_browser - FROM events - WHERE site_id = $1 - AND timestamp >= $2 - AND timestamp < $3 - %s - GROUP BY distinct_id - ORDER BY last_seen_ms DESC - LIMIT %d OFFSET %d`, anonClause, limit, offset) - - rows, err := nucleus.Query[Person](ctx, s.db.SQL(), q, siteID, from, to) + rows, err := s.ev.List(ctx, siteID, fromMs, toMs, limit, offset, includeAnonymous) if err != nil { - return nil, fmt.Errorf("list persons: %w", err) + return nil, err } if rows == nil { rows = []Person{} @@ -127,92 +114,14 @@ func (s *Service) ListPersons(ctx context.Context, siteID string, fromMs, toMs i return rows, nil } -// PersonDetail returns the aggregate row for distinctID plus the most -// recent 100 events as a vertical timeline. distinctID = ” returns an -// empty result on purpose — anonymous-aggregated detail makes no sense. -func (s *Service) PersonDetail(ctx context.Context, siteID, distinctID string) (PersonDetail, error) { - if siteID == "" || distinctID == "" { - return PersonDetail{}, nil - } - - // Aggregate over the full history of this distinct_id (no time - // window — the detail view always shows the lifetime summary). - aggQ := `SELECT distinct_id, - MIN(CAST(timestamp AS BIGINT)) AS first_seen_ms, - MAX(CAST(timestamp AS BIGINT)) AS last_seen_ms, - COUNT(*) AS event_count, - COUNT(DISTINCT session_id) AS session_count, - argMax(country, CAST(timestamp AS BIGINT)) AS top_country, - argMax(browser, CAST(timestamp AS BIGINT)) AS top_browser - FROM events - WHERE site_id = $1 AND distinct_id = $2 - GROUP BY distinct_id` - - aggRows, err := nucleus.Query[Person](ctx, s.db.SQL(), aggQ, siteID, distinctID) - if err != nil { - return PersonDetail{}, fmt.Errorf("person aggregate: %w", err) - } - - var p Person - if len(aggRows) > 0 { - p = aggRows[0] - } else { - p = Person{DistinctID: distinctID} - } - - tlQ := `SELECT event_id, event_type, - COALESCE(url, '') AS url, - COALESCE(pathname, '') AS pathname, - CAST(timestamp AS BIGINT) AS timestamp - FROM events - WHERE site_id = $1 AND distinct_id = $2 - ORDER BY CAST(timestamp AS BIGINT) DESC - LIMIT 100` - - tl, err := nucleus.Query[PersonEvent](ctx, s.db.SQL(), tlQ, siteID, distinctID) - if err != nil { - // Non-fatal: aggregate is the load-bearing part. Return what we have. - return PersonDetail{Person: p, Timeline: []PersonEvent{}}, nil - } - if tl == nil { - tl = []PersonEvent{} - } - return PersonDetail{Person: p, Timeline: tl}, nil -} - -// CountPersons returns the total distinct_id count in the window. -// Cheap helper used by the UI to render pagination totals without -// pulling a second page. +// CountPersons returns the total distinct_id count in the window (raw: no +// alias resolution or erasure filtering). Cheap helper used by the UI to +// render pagination totals without pulling a second page. func (s *Service) CountPersons(ctx context.Context, siteID string, fromMs, toMs int64, includeAnonymous bool) (int64, error) { if siteID == "" { return 0, nil } - from := dbutil.IntParam(fromMs) - to := dbutil.IntParam(toMs) - - anonClause := "AND distinct_id != ''" - if includeAnonymous { - anonClause = "" - } - - type countRow struct { - Total int64 `db:"total"` - } - q := fmt.Sprintf(`SELECT COUNT(DISTINCT distinct_id) AS total - FROM events - WHERE site_id = $1 - AND timestamp >= $2 - AND timestamp < $3 - %s`, anonClause) - - rows, err := nucleus.Query[countRow](ctx, s.db.SQL(), q, siteID, from, to) - if err != nil { - return 0, fmt.Errorf("count persons: %w", err) - } - if len(rows) == 0 { - return 0, nil - } - return rows[0].Total, nil + return s.ev.Count(ctx, siteID, fromMs, toMs, includeAnonymous) } // DefaultWindow returns the default 30-day query window when the caller diff --git a/internal/persons/service_identity.go b/internal/persons/service_identity.go new file mode 100644 index 00000000..91e91192 --- /dev/null +++ b/internal/persons/service_identity.go @@ -0,0 +1,369 @@ +package persons + +import ( + "context" + "encoding/json" + "fmt" + "sort" +) + +// ListResult is the resolved, erasure-filtered person page. +type ListResult struct { + Persons []Person + Total int64 + // Resolved is true when alias resolution / erasure filtering ran in Go. + Resolved bool + // Truncated is true when the Go-side fold hit its scan cap, so Total + // and ordering cover only the most recently active persons. + Truncated bool +} + +func (s *Service) loadResolver(ctx context.Context, siteID string) (*Resolver, error) { + rows, err := s.st.ListAliases(ctx, siteID, MaxAliasesPerSite+1) + if err != nil { + return nil, err + } + return NewResolver(rows), nil +} + +func (s *Service) loadTombstones(ctx context.Context, siteID string) (map[string]bool, error) { + keys, err := s.st.ListTombstones(ctx, siteID, MaxTombstonesPerSite+1) + if err != nil { + return nil, err + } + if len(keys) > MaxTombstonesPerSite { + // Fail closed: an incomplete exclusion set would leak erased persons. + return nil, fmt.Errorf("%w: tombstone set exceeds %d", ErrLimit, MaxTombstonesPerSite) + } + m := make(map[string]bool, len(keys)) + for _, k := range keys { + m[k] = true + } + return m, nil +} + +// ListResolved is the user-facing person list: aliases folded onto their +// canonical person, erased persons excluded. With no aliases and no +// tombstones for the site it is exactly the SQL-paginated C2 path. +func (s *Service) ListResolved(ctx context.Context, siteID string, fromMs, toMs int64, limit, offset int, includeAnonymous bool) (ListResult, error) { + if siteID == "" { + return ListResult{Persons: []Person{}}, nil + } + if limit <= 0 { + limit = 50 + } + if limit > 500 { + limit = 500 + } + if offset < 0 { + offset = 0 + } + res, err := s.loadResolver(ctx, siteID) + if err != nil { + return ListResult{}, err + } + tomb, err := s.loadTombstones(ctx, siteID) + if err != nil { + return ListResult{}, err + } + if res.Empty() && len(tomb) == 0 { + rows, err := s.ListPersons(ctx, siteID, fromMs, toMs, limit, offset, includeAnonymous) + if err != nil { + return ListResult{}, err + } + total, _ := s.CountPersons(ctx, siteID, fromMs, toMs, includeAnonymous) + return ListResult{Persons: rows, Total: total}, nil + } + + raw, err := s.ev.ListAll(ctx, siteID, fromMs, toMs, resolveScanCap+1, includeAnonymous) + if err != nil { + return ListResult{}, err + } + truncated := len(raw) > resolveScanCap + if truncated { + raw = raw[:resolveScanCap] + } + folded := foldPersons(raw, res, tomb) + total := int64(len(folded)) + if offset >= len(folded) { + folded = nil + } else { + folded = folded[offset:] + if len(folded) > limit { + folded = folded[:limit] + } + } + if folded == nil { + folded = []Person{} + } + return ListResult{Persons: folded, Total: total, Resolved: true, Truncated: truncated}, nil +} + +// PersonDetail returns the lifetime aggregate for distinctID plus the most +// recent 100 events. If distinctID is an alias it resolves to its canonical +// person and the aggregate, timeline and properties cover the whole merged +// group. An erased person returns ErrErased. distinctID = ” returns an +// empty result on purpose — anonymous-aggregated detail makes no sense. +func (s *Service) PersonDetail(ctx context.Context, siteID, distinctID string) (PersonDetail, error) { + if siteID == "" || distinctID == "" { + return PersonDetail{}, nil + } + res, err := s.loadResolver(ctx, siteID) + if err != nil { + return PersonDetail{}, err + } + canon := res.Canonical(distinctID) + tomb, err := s.loadTombstones(ctx, siteID) + if err != nil { + return PersonDetail{}, err + } + if tomb[distinctID] || tomb[canon] { + return PersonDetail{}, ErrErased + } + members := []string{canon} + var aliases []string + for _, m := range res.Members(canon) { + if !tomb[m] { + members = append(members, m) + aliases = append(aliases, m) + } + } + if len(members) > MaxAliasesPerPerson+1 { + members = members[:MaxAliasesPerPerson+1] + } + + var agg Person + have := false + var timeline []PersonEvent + var propParts []map[string]any + for i, m := range members { + p, ok, err := s.ev.Aggregate(ctx, siteID, m) + if err != nil { + return PersonDetail{}, err + } + if ok { + if !have { + agg, have = p, true + } else { + agg.absorb(p) + } + } + if tl, err := s.ev.Timeline(ctx, siteID, m); err == nil { + timeline = append(timeline, tl...) + } + // Canonical (index 0) must win, so apply it last. + raw, found, err := s.st.GetProps(ctx, siteID, m) + if err != nil { + return PersonDetail{}, err + } + if found { + var pm map[string]any + if json.Unmarshal([]byte(raw), &pm) == nil { + if i == 0 { + propParts = append(propParts, pm) + } else { + propParts = append([]map[string]any{pm}, propParts...) + } + } + } + } + agg.DistinctID = canon + if !have { + agg = Person{DistinctID: canon} + } + sort.SliceStable(timeline, func(i, j int) bool { return timeline[i].Timestamp > timeline[j].Timestamp }) + if len(timeline) > 100 { + timeline = timeline[:100] + } + if timeline == nil { + timeline = []PersonEvent{} + } + props := map[string]any{} + for _, part := range propParts { + for k, v := range part { + props[k] = v + } + } + if aliases == nil { + aliases = []string{} + } + return PersonDetail{Person: agg, Timeline: timeline, Properties: props, Aliases: aliases, CanonicalKey: canon}, nil +} + +// SetProperties applies an identify() trait batch to one person. Merge mode +// (default) sets the given keys and removes null-valued ones; replace mode +// stores exactly the given set. The resulting document is bounded to +// MaxPropertyKeys. Writes for an erased person are refused. +func (s *Service) SetProperties(ctx context.Context, siteID, key string, in map[string]any, replace bool) (map[string]any, error) { + if siteID == "" { + return nil, fmt.Errorf("%w: site_id required", ErrInvalid) + } + if err := ValidatePersonKey(key); err != nil { + return nil, err + } + set, remove, err := ValidateProperties(in) + if err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + if gone, err := s.st.IsTombstoned(ctx, siteID, key); err != nil { + return nil, err + } else if gone { + return nil, ErrErased + } + doc := map[string]any{} + if !replace { + raw, found, err := s.st.GetProps(ctx, siteID, key) + if err != nil { + return nil, err + } + if found { + if err := json.Unmarshal([]byte(raw), &doc); err != nil || doc == nil { + doc = map[string]any{} + } + } + for _, k := range remove { + delete(doc, k) + } + } + for k, v := range set { + doc[k] = v + } + if len(doc) > MaxPropertyKeys { + return nil, fmt.Errorf("%w: a person holds at most %d properties", ErrInvalid, MaxPropertyKeys) + } + enc, err := json.Marshal(doc) // map keys marshal sorted: deterministic + if err != nil { + return nil, fmt.Errorf("encode properties: %w", err) + } + if err := s.st.PutProps(ctx, siteID, key, string(enc)); err != nil { + return nil, err + } + return doc, nil +} + +// MergeResult describes a completed merge. +type MergeResult struct { + SiteID string `json:"site_id"` + FromKey string `json:"from_key"` + CanonicalKey string `json:"canonical_key"` +} + +// Merge records fromKey as an alias of intoKey's canonical person. Nothing +// is rewritten: reads resolve the alias forward. Self-merge, merging an +// existing alias, and any merge that would close a cycle are refused; both +// keys must have events in THIS site (so a key from another site is a 404, +// never a cross-site link). +func (s *Service) Merge(ctx context.Context, siteID, fromKey, intoKey, actor string) (MergeResult, error) { + if siteID == "" { + return MergeResult{}, fmt.Errorf("%w: site_id required", ErrInvalid) + } + if err := ValidatePersonKey(fromKey); err != nil { + return MergeResult{}, err + } + if err := ValidatePersonKey(intoKey); err != nil { + return MergeResult{}, err + } + if fromKey == intoKey { + return MergeResult{}, fmt.Errorf("%w: cannot merge a person into itself", ErrInvalid) + } + s.mu.Lock() + defer s.mu.Unlock() + + tomb, err := s.loadTombstones(ctx, siteID) + if err != nil { + return MergeResult{}, err + } + if tomb[fromKey] || tomb[intoKey] { + return MergeResult{}, ErrErased + } + rows, err := s.st.ListAliases(ctx, siteID, MaxAliasesPerSite+1) + if err != nil { + return MergeResult{}, err + } + if len(rows) >= MaxAliasesPerSite { + return MergeResult{}, fmt.Errorf("%w: site already has %d aliases", ErrLimit, MaxAliasesPerSite) + } + res := NewResolver(rows) + if res.IsAlias(fromKey) { + return MergeResult{}, fmt.Errorf("%w: %s is already merged into %s", ErrConflict, fromKey, res.Canonical(fromKey)) + } + root := res.Canonical(intoKey) + if root == fromKey { + return MergeResult{}, fmt.Errorf("%w: %s is already an alias of %s (merge would create a cycle)", ErrConflict, intoKey, fromKey) + } + if tomb[root] { + return MergeResult{}, ErrErased + } + if len(res.Members(root))+1+len(res.Members(fromKey))+1 > MaxAliasesPerPerson { + return MergeResult{}, fmt.Errorf("%w: a person holds at most %d aliases", ErrLimit, MaxAliasesPerPerson) + } + for _, k := range []string{fromKey, intoKey} { + ok, err := s.ev.Exists(ctx, siteID, k) + if err != nil { + return MergeResult{}, err + } + if !ok { + return MergeResult{}, fmt.Errorf("%w: no person %q in this site", ErrNotFound, k) + } + } + if err := s.st.PutAlias(ctx, siteID, fromKey, root, actor, true); err != nil { + return MergeResult{}, err + } + return MergeResult{SiteID: siteID, FromKey: fromKey, CanonicalKey: root}, nil +} + +// EraseResult describes a completed erasure. +type EraseResult struct { + SiteID string `json:"site_id"` + Erased []string `json:"erased_keys"` + // EventsDeleted is always false: see docs/IDENTITY_MODEL_ADR.md. The + // person is tombstoned (excluded from listing/detail, properties and + // aliases cleared) but historical event rows remain until retention. + EventsDeleted bool `json:"events_deleted"` +} + +// Erase is the GDPR erasure path. For the key and, if it is a canonical +// person, every alias merged into it: write a tombstone, deactivate the +// alias edge and blank the stored properties. Idempotent. Events are NOT +// deleted (no proven per-key DELETE on events; see the ADR). +func (s *Service) Erase(ctx context.Context, siteID, key, actor string) (EraseResult, error) { + if siteID == "" { + return EraseResult{}, fmt.Errorf("%w: site_id required", ErrInvalid) + } + if err := ValidatePersonKey(key); err != nil { + return EraseResult{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + + res, err := s.loadResolver(ctx, siteID) + if err != nil { + return EraseResult{}, err + } + keys := []string{key} + keys = append(keys, res.Members(key)...) + // Tombstones first: the exclusion is the load-bearing part, so a later + // failure leaves the person hidden rather than half-cleared and visible. + for _, k := range keys { + if err := s.st.PutTombstone(ctx, siteID, k, actor); err != nil { + return EraseResult{}, err + } + } + for _, k := range keys { + if canon := res.Canonical(k); canon != k { + if err := s.st.PutAlias(ctx, siteID, k, canon, actor, false); err != nil { + return EraseResult{}, err + } + } + if _, found, err := s.st.GetProps(ctx, siteID, k); err != nil { + return EraseResult{}, err + } else if found { + if err := s.st.PutProps(ctx, siteID, k, "{}"); err != nil { + return EraseResult{}, err + } + } + } + return EraseResult{SiteID: siteID, Erased: keys}, nil +} diff --git a/internal/persons/store_nucleus.go b/internal/persons/store_nucleus.go new file mode 100644 index 00000000..f3e2f78c --- /dev/null +++ b/internal/persons/store_nucleus.go @@ -0,0 +1,275 @@ +package persons + +import ( + "context" + "fmt" + "time" + + "github.com/neutron-build/neutron/go/nucleus" + + "github.com/useteploy/teploy-observe/internal/dbutil" +) + +// nucleusEvents reads persons from the events table. The SQL is the C2 +// aggregate, moved verbatim from the original Service methods. +type nucleusEvents struct{ db *nucleus.Client } + +func (n *nucleusEvents) list(ctx context.Context, siteID string, fromMs, toMs int64, limit, offset int, includeAnonymous bool) ([]Person, error) { + from := dbutil.IntParam(fromMs) + to := dbutil.IntParam(toMs) + + // Inner aggregate computes the per-distinct_id summary. argMax picks + // the most recent country/browser per person. + anonClause := "AND distinct_id != ''" + if includeAnonymous { + anonClause = "" + } + + q := fmt.Sprintf(`SELECT distinct_id, + MIN(CAST(timestamp AS BIGINT)) AS first_seen_ms, + MAX(CAST(timestamp AS BIGINT)) AS last_seen_ms, + COUNT(*) AS event_count, + COUNT(DISTINCT session_id) AS session_count, + argMax(country, CAST(timestamp AS BIGINT)) AS top_country, + argMax(browser, CAST(timestamp AS BIGINT)) AS top_browser + FROM events + WHERE site_id = $1 + AND timestamp >= $2 + AND timestamp < $3 + %s + GROUP BY distinct_id + ORDER BY last_seen_ms DESC + LIMIT %d OFFSET %d`, anonClause, limit, offset) + + rows, err := nucleus.Query[Person](ctx, n.db.SQL(), q, siteID, from, to) + if err != nil { + return nil, fmt.Errorf("list persons: %w", err) + } + return rows, nil +} + +func (n *nucleusEvents) List(ctx context.Context, siteID string, fromMs, toMs int64, limit, offset int, includeAnonymous bool) ([]Person, error) { + return n.list(ctx, siteID, fromMs, toMs, limit, offset, includeAnonymous) +} + +func (n *nucleusEvents) ListAll(ctx context.Context, siteID string, fromMs, toMs int64, limit int, includeAnonymous bool) ([]Person, error) { + return n.list(ctx, siteID, fromMs, toMs, limit, 0, includeAnonymous) +} + +func (n *nucleusEvents) Count(ctx context.Context, siteID string, fromMs, toMs int64, includeAnonymous bool) (int64, error) { + from := dbutil.IntParam(fromMs) + to := dbutil.IntParam(toMs) + + anonClause := "AND distinct_id != ''" + if includeAnonymous { + anonClause = "" + } + + type countRow struct { + Total int64 `db:"total"` + } + q := fmt.Sprintf(`SELECT COUNT(DISTINCT distinct_id) AS total + FROM events + WHERE site_id = $1 + AND timestamp >= $2 + AND timestamp < $3 + %s`, anonClause) + + rows, err := nucleus.Query[countRow](ctx, n.db.SQL(), q, siteID, from, to) + if err != nil { + return 0, fmt.Errorf("count persons: %w", err) + } + if len(rows) == 0 { + return 0, nil + } + return rows[0].Total, nil +} + +func (n *nucleusEvents) Aggregate(ctx context.Context, siteID, key string) (Person, bool, error) { + // Aggregate over the full history of this distinct_id (no time + // window — the detail view always shows the lifetime summary). + aggQ := `SELECT distinct_id, + MIN(CAST(timestamp AS BIGINT)) AS first_seen_ms, + MAX(CAST(timestamp AS BIGINT)) AS last_seen_ms, + COUNT(*) AS event_count, + COUNT(DISTINCT session_id) AS session_count, + argMax(country, CAST(timestamp AS BIGINT)) AS top_country, + argMax(browser, CAST(timestamp AS BIGINT)) AS top_browser + FROM events + WHERE site_id = $1 AND distinct_id = $2 + GROUP BY distinct_id` + + rows, err := nucleus.Query[Person](ctx, n.db.SQL(), aggQ, siteID, key) + if err != nil { + return Person{}, false, fmt.Errorf("person aggregate: %w", err) + } + if len(rows) == 0 { + return Person{}, false, nil + } + return rows[0], true, nil +} + +func (n *nucleusEvents) Timeline(ctx context.Context, siteID, key string) ([]PersonEvent, error) { + tlQ := `SELECT event_id, event_type, + COALESCE(url, '') AS url, + COALESCE(pathname, '') AS pathname, + CAST(timestamp AS BIGINT) AS timestamp + FROM events + WHERE site_id = $1 AND distinct_id = $2 + ORDER BY CAST(timestamp AS BIGINT) DESC + LIMIT 100` + return nucleus.Query[PersonEvent](ctx, n.db.SQL(), tlQ, siteID, key) +} + +func (n *nucleusEvents) Exists(ctx context.Context, siteID, key string) (bool, error) { + type idRow struct { + EventID string `db:"event_id"` + } + rows, err := nucleus.Query[idRow](ctx, n.db.SQL(), + `SELECT event_id FROM events WHERE site_id = $1 AND distinct_id = $2 LIMIT 1`, siteID, key) + if err != nil { + return false, fmt.Errorf("person exists: %w", err) + } + return len(rows) > 0, nil +} + +// nucleusIdentity is the 061 side-table store. All three tables are +// ReplacingMergeTree collapsed with argMax over version (precedent: +// internal/platform/maintenance.go); writes are strictly-monotonic new +// versions and nothing is ever DELETEd. +type nucleusIdentity struct{ db *nucleus.Client } + +type propsRow struct { + Props string `db:"props"` + Version int64 `db:"version"` +} + +func (n *nucleusIdentity) latestProps(ctx context.Context, siteID, key string) (propsRow, bool, error) { + rows, err := nucleus.Query[propsRow](ctx, n.db.SQL(), + `SELECT argMax(props, version) AS props, MAX(version) AS version + FROM person_properties + WHERE tenant_id = 'default' AND site_id = $1 AND person_key = $2 + GROUP BY person_key`, siteID, key) + if err != nil { + return propsRow{}, false, fmt.Errorf("get person properties: %w", err) + } + if len(rows) == 0 { + return propsRow{}, false, nil + } + return rows[0], true, nil +} + +func (n *nucleusIdentity) GetProps(ctx context.Context, siteID, key string) (string, bool, error) { + r, ok, err := n.latestProps(ctx, siteID, key) + return r.Props, ok, err +} + +func (n *nucleusIdentity) PutProps(ctx context.Context, siteID, key, propsJSON string) error { + prev, _, err := n.latestProps(ctx, siteID, key) + if err != nil { + return err + } + now := time.Now().UTC().UnixMilli() + v := now + if prev.Version+1 > v { + v = prev.Version + 1 + } + _, err = n.db.SQL().Exec(ctx, + `INSERT INTO person_properties (tenant_id, site_id, person_key, props, updated_at, version) + VALUES ('default', $1, $2, $3, $4, $5)`, + siteID, key, propsJSON, dbutil.IntParam(now), dbutil.IntParam(v)) + if err != nil { + return fmt.Errorf("put person properties: %w", err) + } + return nil +} + +func (n *nucleusIdentity) ListAliases(ctx context.Context, siteID string, limit int) ([]AliasRow, error) { + // active is filtered OUTSIDE the collapse: an un-merge is a newer + // version with active='false' and must win over the superseded row. + rows, err := nucleus.Query[AliasRow](ctx, n.db.SQL(), + fmt.Sprintf(`SELECT alias_key, canonical_key FROM ( + SELECT alias_key, + argMax(canonical_key, version) AS canonical_key, + argMax(active, version) AS active + FROM person_aliases + WHERE tenant_id = 'default' AND site_id = $1 + GROUP BY alias_key) + WHERE active = 'true' + ORDER BY alias_key + LIMIT %d`, limit), siteID) + if err != nil { + return nil, fmt.Errorf("list person aliases: %w", err) + } + return rows, nil +} + +func (n *nucleusIdentity) PutAlias(ctx context.Context, siteID, alias, canonical, actor string, active bool) error { + type vRow struct { + Version int64 `db:"version"` + } + prev, err := nucleus.Query[vRow](ctx, n.db.SQL(), + `SELECT MAX(version) AS version FROM person_aliases + WHERE tenant_id = 'default' AND site_id = $1 AND alias_key = $2 + GROUP BY alias_key`, siteID, alias) + if err != nil { + return fmt.Errorf("read alias version: %w", err) + } + now := time.Now().UTC().UnixMilli() + v := now + if len(prev) > 0 && prev[0].Version+1 > v { + v = prev[0].Version + 1 + } + act := "true" + if !active { + act = "false" + } + _, err = n.db.SQL().Exec(ctx, + `INSERT INTO person_aliases (tenant_id, site_id, alias_key, canonical_key, active, created_by, created_at, version) + VALUES ('default', $1, $2, $3, $4, $5, $6, $7)`, + siteID, alias, canonical, act, actor, dbutil.IntParam(now), dbutil.IntParam(v)) + if err != nil { + return fmt.Errorf("put person alias: %w", err) + } + return nil +} + +type keyRow struct { + PersonKey string `db:"person_key"` +} + +func (n *nucleusIdentity) ListTombstones(ctx context.Context, siteID string, limit int) ([]string, error) { + rows, err := nucleus.Query[keyRow](ctx, n.db.SQL(), + fmt.Sprintf(`SELECT person_key FROM person_tombstones + WHERE tenant_id = 'default' AND site_id = $1 LIMIT %d`, limit), siteID) + if err != nil { + return nil, fmt.Errorf("list person tombstones: %w", err) + } + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.PersonKey) + } + return out, nil +} + +func (n *nucleusIdentity) IsTombstoned(ctx context.Context, siteID, key string) (bool, error) { + rows, err := nucleus.Query[keyRow](ctx, n.db.SQL(), + `SELECT person_key FROM person_tombstones + WHERE tenant_id = 'default' AND site_id = $1 AND person_key = $2 LIMIT 1`, siteID, key) + if err != nil { + return false, fmt.Errorf("check person tombstone: %w", err) + } + return len(rows) > 0, nil +} + +func (n *nucleusIdentity) PutTombstone(ctx context.Context, siteID, key, actor string) error { + now := time.Now().UTC().UnixMilli() + _, err := n.db.SQL().Exec(ctx, + `INSERT INTO person_tombstones (tenant_id, site_id, person_key, erased_by, erased_at, version) + VALUES ('default', $1, $2, $3, $4, $4)`, + siteID, key, actor, dbutil.IntParam(now)) + if err != nil { + return fmt.Errorf("put person tombstone: %w", err) + } + return nil +} diff --git a/internal/schema/migrations/061_persons_identity.up.sql b/internal/schema/migrations/061_persons_identity.up.sql new file mode 100644 index 00000000..1bd1df42 --- /dev/null +++ b/internal/schema/migrations/061_persons_identity.up.sql @@ -0,0 +1,70 @@ +-- 061 (2026-10-03): C3 - person properties, explicit alias/merge, erasure +-- tombstones. Implements the person-graph half of the identity ADR +-- (docs/IDENTITY_MODEL_ADR.md): persons stay an aggregate over +-- events.distinct_id; these tables ADD side data keyed by that same +-- person_key (the HMAC'd distinct_id, or the raw value when the site opted +-- in). Historical event rows are never rewritten (ADR D5). +-- +-- All three tables are ReplacingMergeTree keyed on the entity, collapsed at +-- read time by the argMax-over-version form (internal/query/replacing.go), +-- written as strictly-monotonic new versions (the maintenance_windows +-- pattern). No row is ever DELETEd by the application: an erase or an +-- un-merge is a NEW VERSION (props blanked, active='false'); retention may +-- prune on the time column. +-- +-- person_properties identify(distinct_id, traits) traits. props is a +-- bounded JSON object (<= 50 keys, scalar values +-- <= 1 KiB each; enforced in internal/persons). +-- person_aliases alias_key -> canonical_key, resolved forward at read +-- time in Go. active='false' is an un-merge or an +-- erasure cleanup tombstone. +-- person_tombstones GDPR erasure marker. Insert-once per key; the person +-- listing excludes tombstoned keys and property writes +-- for them are refused. Events are NOT deleted (see the +-- ADR for why). +-- +-- Single-process boundary (the standing AUD-018 posture): the read-modify- +-- write of properties and the merge checks serialize in-process. +-- +-- Comments pure ASCII (038 rule). + +CREATE TABLE IF NOT EXISTS person_properties ( + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + person_key TEXT NOT NULL, + props TEXT NOT NULL DEFAULT '{}', + updated_at BIGINT NOT NULL, + version BIGINT NOT NULL DEFAULT 0 +) WITH ( + engine = 'replacing_mergetree', + version_column = 'version' +) +ORDER BY (tenant_id, site_id, person_key); + +CREATE TABLE IF NOT EXISTS person_aliases ( + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + alias_key TEXT NOT NULL, + canonical_key TEXT NOT NULL, + active TEXT NOT NULL DEFAULT 'true', + created_by TEXT NOT NULL DEFAULT '', + created_at BIGINT NOT NULL, + version BIGINT NOT NULL DEFAULT 0 +) WITH ( + engine = 'replacing_mergetree', + version_column = 'version' +) +ORDER BY (tenant_id, site_id, alias_key); + +CREATE TABLE IF NOT EXISTS person_tombstones ( + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + person_key TEXT NOT NULL, + erased_by TEXT NOT NULL DEFAULT '', + erased_at BIGINT NOT NULL, + version BIGINT NOT NULL DEFAULT 0 +) WITH ( + engine = 'replacing_mergetree', + version_column = 'version' +) +ORDER BY (tenant_id, site_id, person_key); From 301e9899fb6877480f318f3a2bdb6a3059b61ef8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:43:18 +0000 Subject: [PATCH 19/65] wip --- internal/experiments/analysis.go | 179 +++++++++++++++-- internal/experiments/bayesian.go | 32 ++- internal/experiments/bayesian_test.go | 6 +- internal/experiments/compute.go | 279 ++++++++++++++++++++++++++ internal/experiments/config.go | 208 +++++++++++++++++++ internal/experiments/continuous.go | 260 ++++++++++++++++++++++++ 6 files changed, 938 insertions(+), 26 deletions(-) create mode 100644 internal/experiments/compute.go create mode 100644 internal/experiments/config.go create mode 100644 internal/experiments/continuous.go diff --git a/internal/experiments/analysis.go b/internal/experiments/analysis.go index ca2a3d0d..8468fc71 100644 --- a/internal/experiments/analysis.go +++ b/internal/experiments/analysis.go @@ -47,6 +47,10 @@ type PairwiseResult struct { HolmAdjustedP float64 `json:"holm_adjusted_p"` Significant bool `json:"significant"` // survives Holm at alphaOmnibus UsedFisher bool `json:"used_fisher"` + // T and DF are set for continuous metrics (Welch's t statistic and its + // Welch-Satterthwaite degrees of freedom); absent for binary arms. + T float64 `json:"t,omitempty"` + DF float64 `json:"df,omitempty"` } // AnalysisResult is the honest reporting layer: every gate that stands @@ -63,6 +67,50 @@ type AnalysisResult struct { SRM SRMDiagnostic `json:"srm"` Pairwise []PairwiseResult `json:"pairwise_vs_control"` WinnerRule string `json:"winner_rule"` + // PlannedSamplePerArm is the design-time per-arm n from the MDE + // sample-size endpoint, when the experiment declares one. The horizon is + // the larger of it and min_sample unless HorizonOverridden. + PlannedSamplePerArm int `json:"planned_sample_per_arm,omitempty"` + // HorizonOverridden is true when the planned-sample part of the horizon + // was waived (explicit override). The min_sample floor is never waived. + HorizonOverridden bool `json:"horizon_overridden,omitempty"` +} + +// horizonSpec is the fixed-horizon contract for one analysis: the winner gate +// stays shut until every arm reaches Required(). +type horizonSpec struct { + MinSample int // experiment min_sample, per arm; never waived + Planned int // design-time per-arm n (0 = none declared) + Floor int // metric-kind floor (continuous metrics) + Override bool +} + +// Required is the per-arm n at which the horizon is met. +func (h horizonSpec) Required() int { + n := h.MinSample + if h.Floor > n { + n = h.Floor + } + if !h.Override && h.Planned > n { + n = h.Planned + } + return n +} + +// peekingWarning is the fixed-horizon anti-peeking message, empty once the +// horizon is met or when no arm has data yet. +func peekingWarning(minArm int64, anyData bool, h horizonSpec) string { + req := h.Required() + if !anyData || minArm >= int64(req) { + return "" + } + msg := fmt.Sprintf("results requested before the planned sample size (smallest arm %d of %d per arm). "+ + "This analysis is fixed-horizon, not sequential: repeatedly checking p-values and stopping when one dips below 0.05 inflates the false-positive rate. "+ + "Treat these numbers as a progress report and decide at the planned horizon.", minArm, req) + if h.Override { + msg += " The planned-sample gate is overridden; the min_sample floor still applies." + } + return msg } // analyze runs the full decided pipeline over per-arm counts, with the @@ -73,7 +121,14 @@ type AnalysisResult struct { // badge before every arm reaches minSamplePerArm. Estimates and intervals // are always computed and displayed regardless. func analyze(arms []VariantResult, weights []float64, minSamplePerArm int) AnalysisResult { - res := AnalysisResult{MinSamplePerArm: minSamplePerArm, Test: "none"} + return analyzeWithHorizon(arms, weights, horizonSpec{MinSample: minSamplePerArm}) +} + +// analyzeWithHorizon is analyze with the full horizon contract (planned +// sample size and its override). +func analyzeWithHorizon(arms []VariantResult, weights []float64, h horizonSpec) AnalysisResult { + minSamplePerArm := h.Required() + res := AnalysisResult{MinSamplePerArm: minSamplePerArm, Test: "none", PlannedSamplePerArm: h.Planned, HorizonOverridden: h.Override && h.Planned > 0} k := len(arms) if k < 2 { res.WinnerRule = "fewer than two arms have exposures" @@ -86,23 +141,7 @@ func analyze(arms []VariantResult, weights []float64, minSamplePerArm int) Analy exposures[i] = a.Exposures conversions[i] = a.Conversions } - res.MinArmExposures = exposures[0] - for _, n := range exposures { - if n < res.MinArmExposures { - res.MinArmExposures = n - } - } - res.HorizonMet = res.MinArmExposures >= int64(minSamplePerArm) - - // SRM: goodness-of-fit against the declared allocation. - if w := append([]float64(nil), weights...); len(w) == k { - stat, p := srmGoodnessOfFit(exposures, w) - res.SRM = SRMDiagnostic{ChiSquare: stat, PValue: p} - if p < alphaSRM { - res.SRM.Detected = true - res.SRM.Note = "assignment is broken; do not trust these results" - } - } + applyHorizonAndSRM(&res, exposures, weights, minSamplePerArm) // Omnibus test with Fisher fallback for two arms and small cells. omni := chiSquareOmnibus(exposures, conversions) @@ -178,18 +217,118 @@ func analyze(arms []VariantResult, weights []float64, minSamplePerArm int) Analy return res } +// applyHorizonAndSRM fills the per-arm horizon gate and the SRM diagnostic, +// which depend only on the exposure counts (shared by binary and continuous). +func applyHorizonAndSRM(res *AnalysisResult, exposures []int64, weights []float64, required int) { + k := len(exposures) + res.MinArmExposures = exposures[0] + for _, n := range exposures { + if n < res.MinArmExposures { + res.MinArmExposures = n + } + } + res.HorizonMet = res.MinArmExposures >= int64(required) + + // SRM: goodness-of-fit against the declared allocation. + if w := append([]float64(nil), weights...); len(w) == k { + stat, p := srmGoodnessOfFit(exposures, w) + res.SRM = SRMDiagnostic{ChiSquare: stat, PValue: p} + if p < alphaSRM { + res.SRM.Detected = true + res.SRM.Note = "assignment is broken; do not trust these results" + } + } +} + +// analyzeContinuous is the continuous-metric (count / mean) counterpart of +// analyzeWithHorizon: the same horizon and SRM gates, then Welch's t-test of +// every arm against the control (arms[0]) with Holm correction across the +// arms. sums are the per-arm value summaries aligned with arms. There is no +// omnibus test: Holm over the pairwise family already controls the +// family-wise error, and the headline p_value is the smallest Holm-adjusted +// p. Higher is better; the winner is the highest-mean arm whose comparison +// survives Holm. +func analyzeContinuous(arms []VariantResult, sums []contSummary, weights []float64, h horizonSpec) AnalysisResult { + h.Floor = minContinuousN + res := AnalysisResult{MinSamplePerArm: h.Required(), Test: "none", PlannedSamplePerArm: h.Planned, HorizonOverridden: h.Override && h.Planned > 0} + k := len(arms) + if k < 2 { + res.WinnerRule = "fewer than two arms have exposures" + return res + } + exposures := make([]int64, k) + for i, a := range arms { + exposures[i] = a.Exposures + } + applyHorizonAndSRM(&res, exposures, weights, res.MinSamplePerArm) + res.Test = "welch-t" + + rawP := make([]float64, 0, k-1) + welch := make([]WelchResult, 0, k-1) + for i := 1; i < k; i++ { + w := welchTest(sums[0], sums[i], alphaOmnibus) + welch = append(welch, w) + rawP = append(rawP, w.PValue) + } + adj := holmAdjusted(rawP) + minAdj := 1.0 + for i, w := range welch { + pr := PairwiseResult{ + Variant: arms[i+1].Variant, + LiftAbsolute: w.Diff, + CiLow: w.CILow, + CiHigh: w.CIHigh, + PValue: w.PValue, + HolmAdjustedP: adj[i], + Significant: adj[i] <= alphaOmnibus, + T: w.T, + DF: w.DF, + } + if sums[0].Mean != 0 { + pr.LiftRelative = w.Diff / sums[0].Mean + } + if adj[i] < minAdj { + minAdj = adj[i] + } + res.Pairwise = append(res.Pairwise, pr) + } + res.PValue = minAdj + + res.WinnerRule = "winner requires: per-arm horizon (min " + strconv.Itoa(res.MinSamplePerArm) + + " per arm), no SRM, winner's Welch t-test vs control surviving Holm at " + strconv.FormatFloat(alphaOmnibus, 'f', -1, 64) + switch { + case !res.HorizonMet: + res.WinnerRule += fmt.Sprintf(" - waiting for horizon (min arm %d of %d)", res.MinArmExposures, res.MinSamplePerArm) + case res.SRM.Detected: + res.WinnerRule += " - SRM detected: " + res.SRM.Note + case res.PValue > alphaOmnibus: + res.WinnerRule += fmt.Sprintf(" - no arm distinguishable from control (smallest Holm p=%.4g)", res.PValue) + } + return res +} + // winnerFrom returns the winner arm given the analysis and arms (control at // index 0): the best observed conversion-rate arm whose pairwise-vs-control // comparison survived Holm. Empty string when no winner may be claimed. If // the single best-rate arm fails Holm there is no winner: the omnibus fired // but no specific arm is distinguishable from control. func winnerFrom(res AnalysisResult, arms []VariantResult) string { + scores := make([]float64, len(arms)) + for i, a := range arms { + scores[i] = a.ConversionRate + } + return winnerByScore(res, arms, scores) +} + +// winnerByScore is winnerFrom over an arbitrary higher-is-better score per arm +// (conversion rate for binary goals, the arm mean for continuous goals). +func winnerByScore(res AnalysisResult, arms []VariantResult, scores []float64) string { if !res.HorizonMet || res.SRM.Detected || res.Test == "none" || res.PValue >= alphaOmnibus { return "" } bestIdx := -1 - for i, v := range arms[1:] { - if bestIdx == -1 || v.ConversionRate > arms[1+bestIdx].ConversionRate { + for i := range arms[1:] { + if bestIdx == -1 || scores[1+i] > scores[1+bestIdx] { bestIdx = i } } diff --git a/internal/experiments/bayesian.go b/internal/experiments/bayesian.go index c992551e..2400cbe8 100644 --- a/internal/experiments/bayesian.go +++ b/internal/experiments/bayesian.go @@ -1,19 +1,22 @@ package experiments import ( + "crypto/sha256" + "encoding/binary" "math" "math/rand" ) // computeBayesianProbabilities fills in ProbBeatControl for each variant using // a Beta(1+conversions, 1+nonconversions) prior and 4000 Monte Carlo samples. -// The first variant is treated as the control. -func computeBayesianProbabilities(variants []VariantResult) { +// The first variant is treated as the control. seed fixes the Monte Carlo +// stream; see bayesianSeed. +func computeBayesianProbabilities(variants []VariantResult, seed int64) { if len(variants) < 2 { return } const samples = 4000 - rng := rand.New(rand.NewSource(42)) + rng := rand.New(rand.NewSource(seed)) controlSamples := drawBeta(rng, float64(1+variants[0].Conversions), float64(1+variants[0].Exposures-variants[0].Conversions), samples) @@ -32,6 +35,29 @@ func computeBayesianProbabilities(variants []VariantResult) { variants[0].ProbBeatControl = 0 } +// bayesianSeed derives the Monte Carlo seed from the experiment id and a hash +// of the arm counts. Identical calls on identical data return identical +// probabilities (a page refresh never jitters the number), while different +// experiments - and different data of the same experiment - draw independent +// streams, so a shared constant seed cannot correlate the noise of every +// experiment's estimate. The probability stays display-only: it never gates +// the winner. +func bayesianSeed(experimentID string, variants []VariantResult) int64 { + h := sha256.New() + h.Write([]byte(experimentID)) + var buf [8]byte + for _, v := range variants { + h.Write([]byte{0}) + h.Write([]byte(v.Variant)) + binary.BigEndian.PutUint64(buf[:], uint64(v.Exposures)) + h.Write(buf[:]) + binary.BigEndian.PutUint64(buf[:], uint64(v.Conversions)) + h.Write(buf[:]) + } + sum := h.Sum(nil) + return int64(binary.BigEndian.Uint64(sum[:8]) & math.MaxInt64) +} + // drawBeta returns n samples from Beta(alpha, beta) via the ratio of two Gammas. func drawBeta(rng *rand.Rand, alpha, beta float64, n int) []float64 { out := make([]float64, n) diff --git a/internal/experiments/bayesian_test.go b/internal/experiments/bayesian_test.go index 93e39f20..a0fb39ab 100644 --- a/internal/experiments/bayesian_test.go +++ b/internal/experiments/bayesian_test.go @@ -9,7 +9,7 @@ func TestBayesianWinnerClear(t *testing.T) { {Variant: "control", Exposures: 1000, Conversions: 100}, {Variant: "A", Exposures: 1000, Conversions: 300}, } - computeBayesianProbabilities(v) + computeBayesianProbabilities(v, bayesianSeed("test-exp", v)) if v[1].ProbBeatControl < 0.99 { t.Errorf("expected P(A > control) ≈ 1, got %v", v[1].ProbBeatControl) } @@ -24,7 +24,7 @@ func TestBayesianWinnerTight(t *testing.T) { {Variant: "control", Exposures: 1000, Conversions: 100}, {Variant: "A", Exposures: 1000, Conversions: 100}, } - computeBayesianProbabilities(v) + computeBayesianProbabilities(v, bayesianSeed("test-exp", v)) if v[1].ProbBeatControl < 0.35 || v[1].ProbBeatControl > 0.65 { t.Errorf("expected P(A > control) ~0.5, got %v", v[1].ProbBeatControl) } @@ -36,7 +36,7 @@ func TestBayesianLoser(t *testing.T) { {Variant: "control", Exposures: 1000, Conversions: 150}, {Variant: "A", Exposures: 1000, Conversions: 50}, } - computeBayesianProbabilities(v) + computeBayesianProbabilities(v, bayesianSeed("test-exp", v)) if v[1].ProbBeatControl > 0.01 { t.Errorf("expected P(A > control) ~0, got %v", v[1].ProbBeatControl) } diff --git a/internal/experiments/compute.go b/internal/experiments/compute.go new file mode 100644 index 00000000..5c4d35fa --- /dev/null +++ b/internal/experiments/compute.go @@ -0,0 +1,279 @@ +package experiments + +// The pure core of Results: raw per-user rows in, ExperimentResults out. The +// service fetches the rows (SQL, unverified offline); everything that decides +// a number or a gate happens here so it is unit-testable without a database. + +import ( + "sort" +) + +// exposureRow is one user's exposure to one variant inside the running +// interval, with the earliest such exposure time (epoch ms). +type exposureRow struct { + User string + Variant string + First int64 +} + +// userVariant is a distinct (user, variant) conversion attribution row. +type userVariant struct { + User string + Variant string +} + +// metricEvent is one count/mean/secondary observation. +type metricEvent struct { + User string + TS int64 + Value float64 +} + +// ResultsOptions are per-request analysis options. +type ResultsOptions struct { + // AllowEarly waives the planned-sample part of the horizon for this + // request (the winner gate and the peeking warning). The min_sample floor + // is never waived. The experiment's allow_early_winner setting has the + // same effect persistently. + AllowEarly bool +} + +type resultInputs struct { + Exp Experiment + Cfg ExperimentConfig + Opts ResultsOptions + WindowMs int64 + // Exposures: distinct (user, variant) rows in the running interval. + Exposures []exposureRow + // BinaryConversions: distinct (user, variant) rows of the legacy + // experiment_conversions join, used when the primary kind is binary. + BinaryConversions []userVariant + // Events by metric key ("primary" or a secondary key). + Events map[string][]metricEvent + // Truncated marks metric keys whose event read hit the row cap. + Truncated map[string]bool +} + +// MetricResult is one evaluated goal (primary-continuous detail or a +// secondary). Secondary metrics are informational and never gate the winner. +type MetricResult struct { + Key string `json:"key"` + Name string `json:"name,omitempty"` + Kind string `json:"kind"` + // Role is "primary" or "secondary". + Role string `json:"role"` + // GatesWinner is true only for the primary metric. + GatesWinner bool `json:"gates_winner"` + Variants []VariantResult `json:"variants"` + Analysis AnalysisResult `json:"analysis"` + Note string `json:"note,omitempty"` +} + +// cohortOf collapses exposure rows into one entry per user. A user exposed to +// more than one variant is contaminated: excluded from every statistic and +// counted separately. Returns the clean cohort and the contaminated count. +func cohortOf(rows []exposureRow) (map[string]exposureRow, int) { + first := make(map[string]exposureRow, len(rows)) + bad := make(map[string]bool) + for _, r := range rows { + prev, ok := first[r.User] + switch { + case !ok: + first[r.User] = r + case prev.Variant != r.Variant: + bad[r.User] = true + case r.First < prev.First: + first[r.User] = r + } + } + for u := range bad { + delete(first, u) + } + return first, len(bad) +} + +// qualifyingValues maps each cohort user with at least one event inside its +// attribution window [first exposure, first exposure + window] to its metric +// value: 1 for binary, the event count for count, the value sum for mean. +func qualifyingValues(kind string, events []metricEvent, cohort map[string]exposureRow, windowMs int64) map[string]float64 { + out := make(map[string]float64) + for _, e := range events { + c, ok := cohort[e.User] + if !ok || e.TS < c.First || e.TS > c.First+windowMs { + continue + } + switch kind { + case KindBinary: + out[e.User] = 1 + case KindCount: + out[e.User]++ + default: + out[e.User] += e.Value + } + } + return out +} + +// evaluateMetric builds the per-arm results and analysis for one goal over +// the clean cohort. variants is the ordered arm list (control first) with +// Exposures filled; vals maps users to qualifying values (see +// qualifyingValues). Binary goals use the chi-square/Fisher pipeline; +// count/mean goals use Welch with optional winsorizing. +func evaluateMetric(kind string, winsorPct float64, variants []VariantResult, cohort map[string]exposureRow, vals map[string]float64, weights []float64, h horizonSpec) ([]VariantResult, AnalysisResult) { + arms := append([]VariantResult(nil), variants...) + idx := make(map[string]int, len(arms)) + for i, a := range arms { + idx[a.Variant] = i + } + for u := range vals { + arms[idx[cohort[u].Variant]].Conversions++ + } + for i := range arms { + a := &arms[i] + if a.Exposures > 0 { + a.ConversionRate = float64(a.Conversions) / float64(a.Exposures) + } + a.WilsonLow, a.WilsonHigh = wilsonInterval(a.Conversions, a.Exposures, zAlphaTwoSided005) + } + if kind == KindBinary { + return arms, analyzeWithHorizon(arms, weights, h) + } + + // Continuous: every exposed user contributes (zero when no events). + perArm := make([][]float64, len(arms)) + for u, c := range cohort { + perArm[idx[c.Variant]] = append(perArm[idx[c.Variant]], vals[u]) + } + if winsorPct > 0 { + winsorizeArms(perArm, winsorPct) + } + sums := make([]contSummary, len(arms)) + for i := range arms { + sums[i] = summarize(perArm[i]) + m, sd := sums[i].Mean, sums[i].stdDev() + arms[i].Mean, arms[i].StdDev = &m, &sd + } + an := analyzeContinuous(arms, sums, weights, h) + return arms, an +} + +// continuousScores returns each arm's mean for winnerByScore. +func continuousScores(arms []VariantResult) []float64 { + s := make([]float64, len(arms)) + for i, a := range arms { + if a.Mean != nil { + s[i] = *a.Mean + } + } + return s +} + +func computeResults(in resultInputs) *ExperimentResults { + exp, cfg := in.Exp, in.Cfg + cohort, contaminated := cohortOf(in.Exposures) + + // Per-variant clean exposure counts, sorted by variant name (the legacy + // ORDER BY variant) before the declared control is moved to the front. + counts := make(map[string]int64) + for _, c := range cohort { + counts[c.Variant]++ + } + names := make([]string, 0, len(counts)) + for v := range counts { + names = append(names, v) + } + sort.Strings(names) + base := make([]VariantResult, len(names)) + for i, v := range names { + base[i] = VariantResult{Variant: v, Exposures: counts[v]} + } + orderControlFirst(base, controlKey(exp.Variants)) + + minSample := exp.MinSample + if minSample <= 0 { + minSample = 100 + } + h := horizonSpec{ + MinSample: minSample, + Planned: cfg.PlannedSamplePerArm, + Override: in.Opts.AllowEarly || cfg.AllowEarlyWinner, + } + weights := allocationWeights(exp.Variants, len(base)) + + // Primary goal. + var primaryVals map[string]float64 + if cfg.MetricKind == KindBinary { + primaryVals = make(map[string]float64) + for _, c := range in.BinaryConversions { + if cc, ok := cohort[c.User]; ok && cc.Variant == c.Variant { + primaryVals[c.User] = 1 + } + } + } else { + primaryVals = qualifyingValues(cfg.MetricKind, in.Events[PrimaryMetricKey], cohort, in.WindowMs) + } + variants, analysis := evaluateMetric(cfg.MetricKind, cfg.WinsorizePct, base, cohort, primaryVals, weights, h) + + var winner string + var significant bool + if cfg.MetricKind == KindBinary { + // Bayesian: probability each variant beats the control (index 0), + // Beta(1+conv, 1+nonconv) with a Monte Carlo seeded from the experiment + // id and the data (see bayesianSeed). Display-only: it never gates the + // winner (2026-09-23 decision: the fixed-horizon frequentist gates do). + if len(variants) >= 2 { + computeBayesianProbabilities(variants, bayesianSeed(exp.ExperimentID, variants)) + } + winner = winnerFrom(analysis, variants) + significant = analysis.HorizonMet && !analysis.SRM.Detected && + analysis.Test != "none" && analysis.PValue < alphaOmnibus + } else { + winner = winnerByScore(analysis, variants, continuousScores(variants)) + significant = analysis.HorizonMet && !analysis.SRM.Detected && + analysis.Test != "none" && analysis.PValue < alphaOmnibus + } + if in.Truncated[PrimaryMetricKey] { + winner, significant = "", false + analysis.WinnerRule += " - metric events were truncated at the read cap; results are not reliable" + } + + res := &ExperimentResults{ + Experiment: exp, + Variants: variants, + Significant: significant, + Winner: winner, + Analysis: analysis, + MetricKind: cfg.MetricKind, + ContaminatedUsers: contaminated, + PeekingWarning: peekingWarning(analysis.MinArmExposures, len(variants) >= 2, horizonSpec{MinSample: minSample, Planned: h.Planned, Floor: floorFor(cfg.MetricKind), Override: h.Override}), + } + if cfg.MetricKind != KindBinary || cfg.WinsorizePct > 0 || cfg.PlannedSamplePerArm > 0 || cfg.AllowEarlyWinner || len(cfg.Secondary) > 0 { + c := cfg + res.Settings = &c + } + + // Secondary goals: same cohort, same machinery, never gating. + for _, g := range cfg.Secondary { + vals := qualifyingValues(g.Kind, in.Events[g.Key], cohort, in.WindowMs) + arms, an := evaluateMetric(g.Kind, cfg.WinsorizePct, base, cohort, vals, weights, h) + an.WinnerRule = "secondary metric - informational only, does not gate the winner; " + an.WinnerRule + mr := MetricResult{ + Key: g.Key, Name: g.Name, Kind: g.Kind, Role: "secondary", + Variants: arms, Analysis: an, + Note: "secondary: excluded from winner selection; uncorrected across metrics", + } + if in.Truncated[g.Key] { + mr.Note += "; events truncated at the read cap, results are not reliable" + } + res.Secondary = append(res.Secondary, mr) + } + res.MultipleComparisonNote = cfg.multipleComparisonNote() + return res +} + +func floorFor(kind string) int { + if kind == KindBinary { + return 0 + } + return minContinuousN +} diff --git a/internal/experiments/config.go b/internal/experiments/config.go new file mode 100644 index 00000000..84070d5f --- /dev/null +++ b/internal/experiments/config.go @@ -0,0 +1,208 @@ +package experiments + +// Per-experiment analysis configuration (migration 063). The experiments +// table cannot take new columns (ALTER ADD COLUMN corrupts populated tables, +// open P1 L9), so metric kind, secondary goals, winsorizing and the planned +// horizon live in an append-only experiment_settings table holding one JSON +// document per write; the newest write wins. An experiment with no settings +// row behaves exactly as before 063: one binary conversion goal. + +import ( + "context" + "encoding/json" + "fmt" + "math" + "regexp" + "strconv" + "time" + + "github.com/neutron-build/neutron/go/nucleus" +) + +// Metric kinds. Binary is "did the user ever convert"; count is the number of +// events per exposed user; mean is the summed event value per exposed user +// (revenue per exposed user, so non-purchasers contribute zero). +const ( + KindBinary = "binary" + KindCount = "count" + KindMean = "mean" + + // PrimaryMetricKey is the metric_events key of the primary goal when it is + // continuous. A binary primary keeps using experiment_conversions. + PrimaryMetricKey = "primary" + + maxSecondaryGoals = 3 + maxMetricValueAbs = 1e12 + maxPlannedSample = 1_000_000_000 +) + +var metricKeyRE = regexp.MustCompile(`^[a-z0-9_]{1,32}$`) + +// SecondaryGoal is one informational metric evaluated beside the primary. It +// never gates the winner. +type SecondaryGoal struct { + Key string `json:"key"` + Name string `json:"name,omitempty"` + Kind string `json:"kind"` +} + +// ExperimentConfig is the analysis configuration document. +type ExperimentConfig struct { + // MetricKind of the primary goal: binary (default), count or mean. + MetricKind string `json:"metric_kind"` + // WinsorizePct clips continuous values to the pooled [p, 100-p] + // percentiles; 0 (default) is off. Valid range (0, 50). + WinsorizePct float64 `json:"winsorize_pct,omitempty"` + // PlannedSamplePerArm is the design-time n from the sample-size endpoint. + // Until every arm reaches it, results carry a peeking warning and no + // winner is declared (unless AllowEarlyWinner). + PlannedSamplePerArm int `json:"planned_sample_per_arm,omitempty"` + // AllowEarlyWinner is the explicit override of the planned-sample gate. + AllowEarlyWinner bool `json:"allow_early_winner,omitempty"` + // Secondary goals, at most three. + Secondary []SecondaryGoal `json:"secondary_goals,omitempty"` +} + +// DefaultConfig is the pre-063 behaviour. +func DefaultConfig() ExperimentConfig { return ExperimentConfig{MetricKind: KindBinary} } + +func validKind(k string) bool { return k == KindBinary || k == KindCount || k == KindMean } + +// Normalize fills defaults and validates. It returns a copy. +func (c ExperimentConfig) Normalize() (ExperimentConfig, error) { + if c.MetricKind == "" { + c.MetricKind = KindBinary + } + if !validKind(c.MetricKind) { + return c, fmt.Errorf("metric_kind must be binary, count or mean") + } + if math.IsNaN(c.WinsorizePct) || c.WinsorizePct < 0 || c.WinsorizePct >= 50 { + return c, fmt.Errorf("winsorize_pct must be 0 (off) or in (0, 50)") + } + if c.WinsorizePct > 0 && c.MetricKind == KindBinary { + return c, fmt.Errorf("winsorize_pct applies to count and mean metrics only") + } + if c.PlannedSamplePerArm < 0 || c.PlannedSamplePerArm > maxPlannedSample { + return c, fmt.Errorf("planned_sample_per_arm out of range") + } + if len(c.Secondary) > maxSecondaryGoals { + return c, fmt.Errorf("at most %d secondary goals", maxSecondaryGoals) + } + seen := map[string]bool{PrimaryMetricKey: true} + sec := make([]SecondaryGoal, len(c.Secondary)) + for i, g := range c.Secondary { + if !metricKeyRE.MatchString(g.Key) { + return c, fmt.Errorf("secondary goal key must match %s", metricKeyRE.String()) + } + if seen[g.Key] { + return c, fmt.Errorf("duplicate or reserved secondary goal key %q", g.Key) + } + seen[g.Key] = true + if g.Kind == "" { + g.Kind = KindBinary + } + if !validKind(g.Kind) { + return c, fmt.Errorf("secondary goal %q: kind must be binary, count or mean", g.Key) + } + if len(g.Name) > 120 { + return c, fmt.Errorf("secondary goal %q: name too long", g.Key) + } + sec[i] = g + } + c.Secondary = sec + return c, nil +} + +// SecondaryEvaluationsNote is the multiple-comparison caveat attached to any +// result that evaluates more than one metric. +func (c ExperimentConfig) multipleComparisonNote() string { + if len(c.Secondary) == 0 { + return "" + } + return fmt.Sprintf("%d secondary metric(s) are shown beside the primary. Only the primary gates the winner. "+ + "Holm correction applies across the arms of each metric, not across metrics, so a secondary metric crossing p<0.05 "+ + "is exploratory: with several metrics some will cross by chance. Treat secondary results as hypotheses for a follow-up experiment.", + len(c.Secondary)) +} + +// --------------------------------------------------------------------------- +// Storage. + +// SaveConfig validates and appends a settings document for an experiment of +// the site. The experiment must exist for that site. +func (s *ExperimentService) SaveConfig(ctx context.Context, experimentID, siteID string, cfg ExperimentConfig) (ExperimentConfig, error) { + cfg, err := cfg.Normalize() + if err != nil { + return cfg, err + } + if _, err := s.loadExperiment(ctx, experimentID, siteID); err != nil { + return cfg, err + } + doc, err := json.Marshal(cfg) + if err != nil { + return cfg, err + } + _, err = s.db.SQL().Exec(ctx, + `INSERT INTO experiment_settings (setting_id, tenant_id, experiment_id, site_id, config, timestamp) + VALUES ($1, 'default', $2, $3, $4, $5)`, + genID(), experimentID, siteID, string(doc), time.Now().UTC().UnixMilli()) + if err != nil { + return cfg, fmt.Errorf("save experiment settings: %w", err) + } + return cfg, nil +} + +// LoadConfig returns the newest settings document, or DefaultConfig when the +// experiment has none (every pre-063 experiment). +func (s *ExperimentService) LoadConfig(ctx context.Context, experimentID, siteID string) (ExperimentConfig, error) { + type row struct { + Config string `db:"config"` + } + rows, err := nucleus.Query[row](ctx, s.db.SQL(), + `SELECT config FROM experiment_settings + WHERE experiment_id = $1 AND site_id = $2 + ORDER BY timestamp DESC, setting_id DESC LIMIT 1`, + experimentID, siteID) + if err != nil { + return DefaultConfig(), err + } + if len(rows) == 0 { + return DefaultConfig(), nil + } + var cfg ExperimentConfig + if err := json.Unmarshal([]byte(rows[0].Config), &cfg); err != nil { + return DefaultConfig(), fmt.Errorf("experiment settings corrupt: %w", err) + } + return cfg.Normalize() +} + +// RecordMetric appends one metric event (a count/mean/secondary observation) +// for an exposed user. Like RecordConversion, an event from a user with no +// exposure is ignored. value must be finite; binary and count goals ignore it +// at analysis time but it is still validated and stored. +func (s *ExperimentService) RecordMetric(ctx context.Context, experimentID, siteID, userID, metric string, value float64) error { + if metric != PrimaryMetricKey && !metricKeyRE.MatchString(metric) { + return fmt.Errorf("metric must match %s", metricKeyRE.String()) + } + if math.IsNaN(value) || math.IsInf(value, 0) || math.Abs(value) > maxMetricValueAbs { + return fmt.Errorf("value must be a finite number with magnitude <= %g", maxMetricValueAbs) + } + type vrow struct { + Variant string `db:"variant"` + } + rows, err := nucleus.Query[vrow](ctx, s.db.SQL(), + `SELECT variant FROM experiment_exposures + WHERE experiment_id = $1 AND site_id = $2 AND user_id = $3 LIMIT 1`, + experimentID, siteID, userID) + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + _, err = s.db.SQL().Exec(ctx, + `INSERT INTO experiment_metric_events (event_id, tenant_id, experiment_id, site_id, user_id, metric, value, timestamp) + VALUES ($1, 'default', $2, $3, $4, $5, $6, $7)`, + genID(), experimentID, siteID, userID, metric, strconv.FormatFloat(value, 'g', -1, 64), time.Now().UTC().UnixMilli()) + return err +} diff --git a/internal/experiments/continuous.go b/internal/experiments/continuous.go new file mode 100644 index 00000000..ee7f27de --- /dev/null +++ b/internal/experiments/continuous.go @@ -0,0 +1,260 @@ +package experiments + +// Continuous-metric statistics (count and mean/revenue goals): Welch's +// unequal-variance t-test with Welch-Satterthwaite degrees of freedom, the +// mean-difference confidence interval, and percentile winsorizing. Pure +// functions, pinned by scipy goldens in continuous_test.go. + +import ( + "math" + "sort" +) + +// minContinuousN floors the per-arm sample for continuous metrics: a t-test +// on a handful of heavy-tailed revenue values is not trustworthy even when +// the experiment's min_sample is set lower. +const minContinuousN = 30 + +// --------------------------------------------------------------------------- +// Regularized incomplete beta I_x(a, b) and the Student t distribution. + +// betaContinuedFraction is the Lentz evaluation of the incomplete-beta +// continued fraction (Numerical Recipes betacf). +func betaContinuedFraction(a, b, x float64) float64 { + const ( + maxIter = 500 + eps = 1e-15 + tiny = 1e-300 + ) + qab, qap, qam := a+b, a+1, a-1 + c := 1.0 + d := 1 - qab*x/qap + if math.Abs(d) < tiny { + d = tiny + } + d = 1 / d + h := d + for m := 1; m <= maxIter; m++ { + fm := float64(m) + m2 := 2 * fm + aa := fm * (b - fm) * x / ((qam + m2) * (a + m2)) + d = 1 + aa*d + if math.Abs(d) < tiny { + d = tiny + } + c = 1 + aa/c + if math.Abs(c) < tiny { + c = tiny + } + d = 1 / d + h *= d * c + aa = -(a + fm) * (qab + fm) * x / ((a + m2) * (qap + m2)) + d = 1 + aa*d + if math.Abs(d) < tiny { + d = tiny + } + c = 1 + aa/c + if math.Abs(c) < tiny { + c = tiny + } + d = 1 / d + del := d * c + h *= del + if math.Abs(del-1) < eps { + break + } + } + return h +} + +// regIncBeta is the regularized incomplete beta function I_x(a, b). +func regIncBeta(a, b, x float64) float64 { + if x <= 0 { + return 0 + } + if x >= 1 { + return 1 + } + lbeta := lnGamma(a+b) - lnGamma(a) - lnGamma(b) + front := math.Exp(lbeta + a*math.Log(x) + b*math.Log1p(-x)) + if x < (a+1)/(a+b+2) { + return front * betaContinuedFraction(a, b, x) / a + } + return 1 - front*betaContinuedFraction(b, a, 1-x)/b +} + +// studentTTwoSidedP is P(|T| >= |t|) for Student's t with df degrees of +// freedom (df may be fractional, as Welch-Satterthwaite produces). +func studentTTwoSidedP(t, df float64) float64 { + if df <= 0 || math.IsNaN(t) { + return 1 + } + if math.IsInf(t, 0) { + return 0 + } + if t == 0 { + return 1 + } + x := df / (df + t*t) + p := regIncBeta(df/2, 0.5, x) + return clamp01(p) +} + +// studentTQuantileUpper returns the t such that P(T > t) = tail for +// 0 < tail < 0.5 (bisection on the monotone two-sided p; 200 halvings reach +// double precision). +func studentTQuantileUpper(tail, df float64) float64 { + if tail <= 0 || tail >= 0.5 || df <= 0 { + return math.NaN() + } + target := 2 * tail + lo, hi := 0.0, 1.0 + for studentTTwoSidedP(hi, df) > target && hi < 1e12 { + hi *= 2 + } + for i := 0; i < 200; i++ { + mid := (lo + hi) / 2 + if studentTTwoSidedP(mid, df) > target { + lo = mid + } else { + hi = mid + } + } + return (lo + hi) / 2 +} + +// --------------------------------------------------------------------------- +// Per-arm summary and Welch's test. + +// contSummary is the sufficient statistic of one arm's per-user values. +type contSummary struct { + N int64 + Mean float64 + Variance float64 // unbiased sample variance (n-1); 0 when N < 2 +} + +func summarize(values []float64) contSummary { + n := len(values) + if n == 0 { + return contSummary{} + } + var sum float64 + for _, v := range values { + sum += v + } + mean := sum / float64(n) + s := contSummary{N: int64(n), Mean: mean} + if n > 1 { + var ss float64 + for _, v := range values { + d := v - mean + ss += d * d + } + s.Variance = ss / float64(n-1) + } + return s +} + +func (s contSummary) stdDev() float64 { return math.Sqrt(s.Variance) } + +// WelchResult is one two-sample Welch comparison (b minus a). +type WelchResult struct { + Diff float64 + T float64 + DF float64 + PValue float64 + CILow float64 + CIHigh float64 +} + +// welchTest compares arm b against arm a (diff = mean_b - mean_a) with the +// two-sided p-value and the 1-alpha confidence interval of the difference, +// using Welch-Satterthwaite degrees of freedom. Fewer than two users in an +// arm gives no inference (p = 1, unbounded-width CI is reported as the point +// difference with p = 1 so callers never claim significance). +func welchTest(a, b contSummary, alpha float64) WelchResult { + diff := b.Mean - a.Mean + res := WelchResult{Diff: diff, PValue: 1, CILow: diff, CIHigh: diff} + if a.N < 2 || b.N < 2 { + return res + } + va := a.Variance / float64(a.N) + vb := b.Variance / float64(b.N) + se2 := va + vb + if se2 <= 0 { + // Both arms are constant: the difference is exact. Identical + // constants are no evidence; different constants are certain. + res.DF = float64(a.N + b.N - 2) + if diff != 0 { + res.PValue = 0 + } + return res + } + se := math.Sqrt(se2) + res.T = diff / se + res.DF = se2 * se2 / (va*va/float64(a.N-1) + vb*vb/float64(b.N-1)) + res.PValue = studentTTwoSidedP(res.T, res.DF) + q := studentTQuantileUpper(alpha/2, res.DF) + res.CILow = diff - q*se + res.CIHigh = diff + q*se + return res +} + +// --------------------------------------------------------------------------- +// Winsorizing. + +// percentileLinear is the linear-interpolation percentile (numpy.percentile +// default) of an ascending-sorted slice, q in [0, 100]. +func percentileLinear(sorted []float64, q float64) float64 { + n := len(sorted) + if n == 0 { + return 0 + } + if n == 1 { + return sorted[0] + } + pos := q / 100 * float64(n-1) + lo := int(math.Floor(pos)) + if lo < 0 { + lo = 0 + } + if lo >= n-1 { + return sorted[n-1] + } + frac := pos - float64(lo) + return sorted[lo] + frac*(sorted[lo+1]-sorted[lo]) +} + +// winsorizeBounds returns the [pct, 100-pct] percentile clip bounds of the +// pooled values. pct must be in (0, 50); otherwise there is nothing to clip. +func winsorizeBounds(pooled []float64, pct float64) (lo, hi float64, ok bool) { + if !(pct > 0 && pct < 50) || len(pooled) == 0 { + return 0, 0, false + } + sorted := append([]float64(nil), pooled...) + sort.Float64s(sorted) + return percentileLinear(sorted, pct), percentileLinear(sorted, 100-pct), true +} + +// winsorizeArms clips every arm's values to the pooled [pct, 100-pct] +// percentile bounds, in place. Bounds are pooled across arms so the clip is +// identical for every arm (a per-arm clip would bias the comparison). +func winsorizeArms(arms [][]float64, pct float64) { + var pooled []float64 + for _, a := range arms { + pooled = append(pooled, a...) + } + lo, hi, ok := winsorizeBounds(pooled, pct) + if !ok { + return + } + for _, a := range arms { + for i, v := range a { + if v < lo { + a[i] = lo + } else if v > hi { + a[i] = hi + } + } + } +} From b29ca36d63e389415199155b0ecbcfe10b382b72 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:43:31 +0000 Subject: [PATCH 20/65] wip: flags groups, operators, payloads, config endpoint --- cmd/observe/flags_config_handlers.go | 68 ++++ cmd/observe/ingest_listener.go | 1 + cmd/observe/ingest_listener_test.go | 1 + cmd/observe/main.go | 1 + internal/flags/config.go | 59 ++++ internal/flags/eval.go | 480 +++++++++++++++++++++++++++ internal/flags/flags.go | 158 +++------ 7 files changed, 650 insertions(+), 118 deletions(-) create mode 100644 cmd/observe/flags_config_handlers.go create mode 100644 internal/flags/config.go create mode 100644 internal/flags/eval.go diff --git a/cmd/observe/flags_config_handlers.go b/cmd/observe/flags_config_handlers.go new file mode 100644 index 00000000..99b22474 --- /dev/null +++ b/cmd/observe/flags_config_handlers.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "strings" + + "github.com/useteploy/teploy-observe/internal/flags" + "github.com/useteploy/teploy-observe/internal/ingest" +) + +// GET /api/v1/flags/config -- the remote-config bootstrap for SDKs that +// evaluate flags locally. It returns every flag's TARGETING RULES, which +// POST /api/v1/flags/evaluate deliberately never discloses (that endpoint is +// public and answers one decision at a time). So this route is NOT public: +// it requires a site-scoped API key with the telemetry capability +// (X-API-Key), the same credential the browser SDK already holds. The site +// comes from the key, never from the request; a site_id query parameter, if +// sent, must agree with the key or the request is refused (no cross-site +// reads). The key ships to browsers by design, so treat the config as +// readable by anyone who can load your site: keep secrets out of targeting +// values and payloads. + +// flagConfigSource is the slice of FlagService the handler needs. +type flagConfigSource interface { + Config(ctx context.Context, siteID string) (*flags.ConfigBundle, error) +} + +// flagConfigRoute wraps the handler in the ingest chain: API key (resolves +// the site into the context) then the per-site rate limiter. +func flagConfigRoute(apiKeyMW func(http.Handler) http.Handler, rl func(http.Handler) http.Handler, src flagConfigSource) http.Handler { + return apiKeyMW(rl(flagConfigHandler(src))) +} + +func flagConfigHandler(src flagConfigSource) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + siteID := ingest.SiteIDFromContext(r.Context()) + if siteID == "" { + // Unreachable behind the API key middleware; fail closed anyway. + writeJSONError(w, http.StatusUnauthorized, "api key required") + return + } + if q := r.URL.Query().Get("site_id"); q != "" && q != siteID { + writeJSONError(w, http.StatusForbidden, "site_id does not match the api key") + return + } + bundle, err := src.Config(r.Context(), siteID) + if err != nil { + writeJSONError(w, http.StatusServiceUnavailable, "flags config unavailable") + return + } + etag := bundle.ETag() + w.Header().Set("ETag", etag) + // Config carries targeting rules: keep it out of shared caches. + w.Header().Set("Cache-Control", "private, no-cache") + if inm := r.Header.Get("If-None-Match"); inm != "" { + for _, c := range strings.Split(inm, ",") { + if strings.TrimSpace(c) == etag { + w.WriteHeader(http.StatusNotModified) + return + } + } + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _ = json.NewEncoder(w).Encode(bundle) + } +} diff --git a/cmd/observe/ingest_listener.go b/cmd/observe/ingest_listener.go index 7edb1cf9..2975eabc 100644 --- a/cmd/observe/ingest_listener.go +++ b/cmd/observe/ingest_listener.go @@ -49,6 +49,7 @@ var ingestRoutes = map[string]bool{ "POST /api/v1/experiments/expose": true, "POST /api/v1/experiments/convert": true, "POST /api/v1/flags/evaluate": true, + "GET /api/v1/flags/config": true, // API-key authenticated (flag_config handler) "GET /api/v1/surveys/active": true, "POST /api/v1/surveys/respond": true, diff --git a/cmd/observe/ingest_listener_test.go b/cmd/observe/ingest_listener_test.go index 970d667b..d13f093b 100644 --- a/cmd/observe/ingest_listener_test.go +++ b/cmd/observe/ingest_listener_test.go @@ -31,6 +31,7 @@ func TestIngestListenerAllowsTelemetryWrites(t *testing.T) { {"POST", "/api/v1/experiments/expose"}, {"POST", "/api/v1/experiments/convert"}, {"POST", "/api/v1/flags/evaluate"}, + {"GET", "/api/v1/flags/config"}, {"GET", "/api/v1/surveys/active"}, {"POST", "/api/v1/surveys/respond"}, {"POST", "/api/v1/sourcemaps/upload"}, diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..7670bedb 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -1393,6 +1393,7 @@ func main() { // browser evaluation path. flagEvalLimiter := ingest.NewRateLimiter(60, time.Minute, 120) r.HandleFunc("POST /api/v1/flags/evaluate", flagEvaluateHandler(flagSvc, flagEvalLimiter)) + r.Handle("GET /api/v1/flags/config", flagConfigRoute(apiKeyMW, rateLimiter.Middleware, flagSvc)) // --- Experiments (JWT auth; editor+ writes) --- expGroup := r.Group("/api/v1/experiments", jwtMW) diff --git a/internal/flags/config.go b/internal/flags/config.go new file mode 100644 index 00000000..81403b45 --- /dev/null +++ b/internal/flags/config.go @@ -0,0 +1,59 @@ +package flags + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "time" +) + +// ConfigBundle is the body of GET /api/v1/flags/config: every flag of one +// site in evaluable form plus the bucketing spec, so an SDK can evaluate +// locally with EvaluateDefinition-equivalent logic and no per-call round trip. +type ConfigBundle struct { + SiteID string `json:"site_id"` + GeneratedAt int64 `json:"generated_at"` // unix ms + Bucketing map[string]any `json:"bucketing"` + Flags []FlagDefinition `json:"flags"` +} + +// BuildConfig converts stored rows to a bundle. Rows whose stored config +// fails validation are included as quarantined definitions (Invalid set, +// rules omitted) so the SDK answers disabled exactly like the server does, +// rather than silently losing the flag or evaluating it unrestricted. +func BuildConfig(siteID string, rows []FeatureFlag, now time.Time) ConfigBundle { + defs := make([]FlagDefinition, 0, len(rows)) + for _, f := range rows { + def, cerr := ParseDefinition(f) + if cerr != nil { + def = FlagDefinition{ + Key: f.FlagKey, Type: f.FlagType, Enabled: f.Enabled, RolloutPct: f.RolloutPct, + Invalid: clampDetail(fmt.Sprintf("stored %s failed validation: %v", cerr.Part, cerr.Err)), + } + } + defs = append(defs, def) + } + sort.Slice(defs, func(i, j int) bool { return defs[i].Key < defs[j].Key }) + return ConfigBundle{SiteID: siteID, GeneratedAt: now.UnixMilli(), Bucketing: BucketingSpec, Flags: defs} +} + +// ETag identifies the flag content (not GeneratedAt), so unchanged config +// revalidates with 304. +func (b ConfigBundle) ETag() string { + raw, _ := json.Marshal(b.Flags) + sum := sha256.Sum256(raw) + return `"` + hex.EncodeToString(sum[:12]) + `"` +} + +// Config reads every flag of a site and builds the bundle. +func (s *FlagService) Config(ctx context.Context, siteID string) (*ConfigBundle, error) { + rows, err := s.List(ctx, siteID) + if err != nil { + return nil, err + } + b := BuildConfig(siteID, rows, time.Now().UTC()) + return &b, nil +} diff --git a/internal/flags/eval.go b/internal/flags/eval.go new file mode 100644 index 00000000..8db20c12 --- /dev/null +++ b/internal/flags/eval.go @@ -0,0 +1,480 @@ +package flags + +// Pure flag evaluation: no storage, no clock, no randomness. Everything the +// server decides at /api/v1/flags/evaluate is decided here, and the same +// function is what SDKs reimplement for local evaluation against +// GET /api/v1/flags/config (see BucketingSpec and testdata/conformance.json). +// +// Stored-config compatibility. Two targeting formats coexist in the existing +// `targeting` column, distinguished by their first token: +// +// legacy [ {attribute, operator, value}, ... ] one implicit AND group +// groups { "groups": [ {key, conditions[], rollout_pct, variant}, ... ] } +// +// A legacy config evaluates exactly as it always did (legacy_golden_test.go +// pins that); the groups format is only ever produced by new writes. A binary +// rolled back to before groups existed reads a groups config as invalid and +// quarantines it (fails closed to disabled) rather than evaluating it wrongly. + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" +) + +const ( + // MaxPayloadBytes bounds one variant payload (compacted JSON). + MaxPayloadBytes = 4096 + // Bounds on the groups format. The legacy flat format is unbounded for + // compatibility; new writes are not. + maxGroups = 32 + maxGroupConditions = 32 + maxGroupKeyLen = 64 +) + +// ConditionGroup is one release condition: all Conditions must match (AND); +// then the user must fall inside RolloutPct using the group's own bucket. +type ConditionGroup struct { + // Key is the group's stable identity and its bucketing salt. Omitted, it + // defaults to "g" (position). Set it explicitly if groups may be + // reordered without reshuffling users. + Key string `json:"key,omitempty"` + Name string `json:"name,omitempty"` + Conditions []TargetingRule `json:"conditions"` + // RolloutPct is 0..100; omitted means 100. 0 admits nobody. + RolloutPct *int `json:"rollout_pct,omitempty"` + // Variant, when set, overrides the hash-based variant choice for users + // admitted through this group. Multivariate flags only. + Variant string `json:"variant,omitempty"` +} + +type groupsEnvelope struct { + Groups []ConditionGroup `json:"groups"` +} + +// FlagDefinition is the evaluable form of a flag: the parsed, validated +// config. It is also the wire shape of GET /api/v1/flags/config. +type FlagDefinition struct { + Key string `json:"key"` + Type string `json:"type"` + Enabled bool `json:"enabled"` + RolloutPct int `json:"rollout_pct"` + Variants []Variant `json:"variants,omitempty"` + Rules []TargetingRule `json:"rules,omitempty"` // legacy flat AND list + Groups []ConditionGroup `json:"groups,omitempty"` // mutually exclusive with Rules + // Invalid, when non-empty, marks a flag whose stored config failed + // validation. It is quarantined: SDKs must answer disabled, never + // evaluate it. Rules/Groups/Variants are omitted for such flags. + Invalid string `json:"invalid,omitempty"` +} + +// ConfigError is a read-boundary validation failure naming the stored part +// ("targeting" or "variants") so quarantine can attribute it. +type ConfigError struct { + Part string + Err error +} + +func (e *ConfigError) Error() string { return e.Err.Error() } +func (e *ConfigError) Unwrap() error { return e.Err } + +// ParseDefinition validates a stored flag and returns its evaluable form. +// Order matches the original evaluator: targeting first, then variants (only +// for multivariate flags that carry any). +func ParseDefinition(f FeatureFlag) (FlagDefinition, *ConfigError) { + def := FlagDefinition{Key: f.FlagKey, Type: f.FlagType, Enabled: f.Enabled, RolloutPct: f.RolloutPct} + rules, groups, err := parseTargetingConfig(f.Targeting) + if err != nil { + return def, &ConfigError{Part: "targeting", Err: err} + } + def.Rules, def.Groups = rules, groups + if f.FlagType == "multivariate" && f.Variants != "" { + def.Variants, err = ValidateVariants(f.Variants) + if err != nil { + return def, &ConfigError{Part: "variants", Err: err} + } + } + if err := checkGroupVariants(groups, def.Variants, f.FlagType == "multivariate"); err != nil { + return def, &ConfigError{Part: "targeting", Err: err} + } + return def, nil +} + +// ValidateConfig is the write-boundary check: targeting (either format) and +// variants, plus the cross-check that a group's variant override names a +// declared variant of a multivariate flag. +func ValidateConfig(flagType, variants, targeting string) error { + _, groups, err := parseTargetingConfig(targeting) + if err != nil { + return err + } + vs, err := ValidateVariants(variants) + if err != nil { + return err + } + return checkGroupVariants(groups, vs, flagType == "multivariate") +} + +func checkGroupVariants(groups []ConditionGroup, variants []Variant, multivariate bool) error { + for i, g := range groups { + if g.Variant == "" { + continue + } + found := false + if multivariate { + for _, v := range variants { + if v.Key == g.Variant { + found = true + break + } + } + } + if !found { + return fmt.Errorf("targeting: group %d: variant %q is not a declared variant of this multivariate flag", i, g.Variant) + } + } + return nil +} + +// parseTargetingConfig dispatches on format. Anything that is not a JSON +// object carrying a "groups" key goes down the legacy path unchanged, so a +// malformed legacy config keeps its original error text. +func parseTargetingConfig(raw string) ([]TargetingRule, []ConditionGroup, error) { + trimmed := strings.TrimSpace(raw) + if strings.HasPrefix(trimmed, "{") { + var probe map[string]json.RawMessage + if json.Unmarshal([]byte(trimmed), &probe) == nil { + if _, ok := probe["groups"]; ok { + groups, err := parseGroups(trimmed) + return nil, groups, err + } + } + } + rules, err := ValidateTargeting(raw) + return rules, nil, err +} + +func parseGroups(raw string) ([]ConditionGroup, error) { + dec := json.NewDecoder(strings.NewReader(raw)) + dec.DisallowUnknownFields() + var env groupsEnvelope + if err := dec.Decode(&env); err != nil { + return nil, fmt.Errorf("targeting: invalid groups JSON: %w", err) + } + if dec.More() { + return nil, fmt.Errorf("targeting: trailing data after groups object") + } + if len(env.Groups) == 0 { + return nil, fmt.Errorf("targeting: groups must contain at least one group") + } + if len(env.Groups) > maxGroups { + return nil, fmt.Errorf("targeting: %d groups exceeds the limit of %d", len(env.Groups), maxGroups) + } + seen := map[string]bool{} + for i := range env.Groups { + g := &env.Groups[i] + if g.Key == "" { + g.Key = "g" + strconv.Itoa(i) + } + if len(g.Key) > maxGroupKeyLen { + return nil, fmt.Errorf("targeting: group %d: key longer than %d bytes", i, maxGroupKeyLen) + } + if seen[g.Key] { + return nil, fmt.Errorf("targeting: duplicate group key %q", g.Key) + } + seen[g.Key] = true + if g.RolloutPct != nil && (*g.RolloutPct < 0 || *g.RolloutPct > 100) { + return nil, fmt.Errorf("targeting: group %d: rollout_pct %d out of range 0..100", i, *g.RolloutPct) + } + if len(g.Conditions) > maxGroupConditions { + return nil, fmt.Errorf("targeting: group %d: %d conditions exceeds the limit of %d", i, len(g.Conditions), maxGroupConditions) + } + if err := validateRules(g.Conditions, fmt.Sprintf("group %d: condition", i)); err != nil { + return nil, err + } + } + return env.Groups, nil +} + +// --- operators --- + +var targetingOperators = map[string]bool{ + "eq": true, "neq": true, "in": true, "not_in": true, "contains": true, + "gt": true, "gte": true, "lt": true, "lte": true, + "starts_with": true, "ends_with": true, + "is_set": true, "not_set": true, +} + +// validateRules checks a rule list; what names the unit in errors, e.g. +// "rule" or "group 1: condition". Error text for the legacy "rule" unit is +// byte-for-byte what it always was. +func validateRules(rules []TargetingRule, what string) error { + for i, r := range rules { + if r.Attribute == "" { + return fmt.Errorf("targeting: %s %d: attribute is required", what, i) + } + if !targetingOperators[r.Operator] { + return fmt.Errorf("targeting: %s %d: unknown operator %q", what, i, r.Operator) + } + switch r.Operator { + case "is_set", "not_set": + // Value is ignored; none required. + continue + } + if r.Value == nil { + return fmt.Errorf("targeting: %s %d: value is required", what, i) + } + switch r.Operator { + case "gt", "gte", "lt", "lte": + if _, ok := numericValue(r.Value); !ok { + return fmt.Errorf("targeting: %s %d: operator %q needs a finite numeric value", what, i, r.Operator) + } + case "starts_with", "ends_with": + s, ok := r.Value.(string) + if !ok || s == "" { + return fmt.Errorf("targeting: %s %d: operator %q needs a non-empty string value", what, i, r.Operator) + } + } + } + return nil +} + +// numericValue converts a rule value (JSON number or numeric string) to a +// finite float64. +func numericValue(v any) (float64, bool) { + switch x := v.(type) { + case float64: + if math.IsNaN(x) || math.IsInf(x, 0) { + return 0, false + } + return x, true + case string: + return parseNumeric(x) + } + return 0, false +} + +// parseNumeric parses a user-supplied attribute safely: bounded length, +// surrounding whitespace tolerated, and NaN/Inf (which ParseFloat accepts as +// words) rejected so a hostile attribute cannot make every comparison +// quietly false-or-true through IEEE semantics. +func parseNumeric(s string) (float64, bool) { + s = strings.TrimSpace(s) + if s == "" || len(s) > 64 { + return 0, false + } + f, err := strconv.ParseFloat(s, 64) + if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { + return 0, false + } + return f, true +} + +// matchRule evaluates one condition. A missing attribute fails closed for +// every operator except not_set -- including neq and not_in, whose +// "absent != x" reading was deliberately NOT adopted: it would widen +// exposure of every existing flag using them (see docs/flags.md). Use +// not_set / is_set to target absence explicitly. +func matchRule(r TargetingRule, userCtx map[string]string) bool { + actual, present := userCtx[r.Attribute] + switch r.Operator { + case "is_set": + return present && actual != "" + case "not_set": + return !present || actual == "" + } + if !present { + return false + } + switch r.Operator { + case "eq": + return actual == fmt.Sprintf("%v", r.Value) + case "neq": + return actual != fmt.Sprintf("%v", r.Value) + case "in": + for _, v := range toStringSlice(r.Value) { + if actual == v { + return true + } + } + return false + case "not_in": + for _, v := range toStringSlice(r.Value) { + if actual == v { + return false + } + } + return true + case "contains": + return strings.Contains(actual, fmt.Sprintf("%v", r.Value)) + case "starts_with": + s, ok := r.Value.(string) + return ok && s != "" && strings.HasPrefix(actual, s) + case "ends_with": + s, ok := r.Value.(string) + return ok && s != "" && strings.HasSuffix(actual, s) + case "gt", "gte", "lt", "lte": + a, ok := parseNumeric(actual) + if !ok { + return false + } + b, ok := numericValue(r.Value) + if !ok { + return false + } + switch r.Operator { + case "gt": + return a > b + case "gte": + return a >= b + case "lt": + return a < b + default: + return a <= b + } + } + // Fail closed: an unknown operator must not silently match (which + // would expose a flag to everyone), so treat the rule as unmatched. + return false +} + +func matchesTargeting(rules []TargetingRule, userCtx map[string]string) bool { + for _, r := range rules { + if !matchRule(r, userCtx) { + return false + } + } + return true +} + +// --- bucketing --- + +// Bucket returns the user's bucket in 1..100 for a salt: the first two bytes +// of sha256(salt ":" userID) as a big-endian uint16, modulo 100, plus 1. +// Modulo bias: 65536 = 655*100 + 36, so buckets 1..36 each hold 656 of 65536 +// inputs and 37..100 hold 655 (a 0.15% relative skew toward low buckets). +// The mapping is frozen: changing it reassigns every existing user. +func Bucket(salt, userID string) int { + h := sha256.Sum256([]byte(salt + ":" + userID)) + val := int(h[0])<<8 | int(h[1]) + return val%100 + 1 +} + +func groupSalt(flagKey, groupKey string) string { return flagKey + ":grp:" + groupKey } + +// BucketingSpec documents the algorithm for SDK authors; it is embedded in +// the GET /api/v1/flags/config response so the document and the server +// cannot drift apart. +var BucketingSpec = map[string]any{ + "hash": "sha256", + "bucket": "uint16 big-endian from digest bytes 0 and 1, modulo 100, plus 1 (range 1..100)", + "bucket_input": " + \":\" + (UTF-8)", + "flag_rollout": map[string]string{ + "salt": "", + "admit": "rollout_pct >= 100 admits everyone without hashing; otherwise admit when bucket <= rollout_pct", + }, + "group_rollout": map[string]string{ + "salt": " + \":grp:\" + (group key defaults to \"g\" when omitted)", + "admit": "rollout_pct omitted or >= 100 admits everyone without hashing; otherwise admit when bucket <= rollout_pct (0 admits nobody)", + }, + "variant": map[string]string{ + "salt": " + \":variant\"", + "choose": "walk variants in order accumulating rollout_pct; pick the first whose cumulative sum >= bucket; if none, the first variant. A matched group's variant override replaces this choice.", + }, + "evaluation_order": []string{ + "flag disabled -> disabled (detail \"flag disabled\")", + "flag rollout miss -> disabled (detail \"rollout\")", + "invalid flag -> disabled (reason \"invalid\")", + "legacy rules: all must match else disabled (detail \"targeting\")", + "groups: first group whose conditions all match AND whose rollout admits the user wins; none -> disabled (detail \"targeting\" if no group's conditions matched, \"group_rollout\" otherwise)", + "multivariate: pick variant (group override, else hash); attach that variant's payload", + }, + "conditions": "operators eq neq in not_in contains starts_with ends_with gt gte lt lte is_set not_set; a missing attribute fails closed for every operator except not_set; eq/neq/in/not_in/contains compare the attribute string with the %v formatting of the rule value; gt/gte/lt/lte parse both sides as finite float64 and are false when either side does not parse; is_set means present and non-empty", +} + +// --- evaluation --- + +// EvaluateDefinition is the reference evaluator. It is deterministic and +// side-effect free. A definition marked Invalid answers disabled with reason +// "invalid". +func EvaluateDefinition(def FlagDefinition, userID string, userCtx map[string]string) EvaluationResult { + if def.Invalid != "" { + return EvaluationResult{Enabled: false, Reason: ReasonInvalid, Detail: clampDetail(def.Invalid)} + } + if !def.Enabled { + return EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: "flag disabled"} + } + if def.RolloutPct < 100 && Bucket(def.Key, userID) > def.RolloutPct { + return EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: "rollout"} + } + + var group *ConditionGroup + groupKey := "" + if len(def.Groups) > 0 { + conditionMatched := false + for i := range def.Groups { + g := &def.Groups[i] + if !matchesTargeting(g.Conditions, userCtx) { + continue + } + conditionMatched = true + key := g.Key + if key == "" { + key = "g" + strconv.Itoa(i) + } + if g.RolloutPct != nil && *g.RolloutPct < 100 { + if Bucket(groupSalt(def.Key, key), userID) > *g.RolloutPct { + continue + } + } + group, groupKey = g, key + break + } + if group == nil { + detail := "targeting" + if conditionMatched { + detail = "group_rollout" + } + return EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: detail} + } + } else if len(def.Rules) > 0 && !matchesTargeting(def.Rules, userCtx) { + return EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: "targeting"} + } + + res := EvaluationResult{Enabled: true, Reason: ReasonEvaluated} + res.Group = groupKey + if def.Type == "multivariate" && len(def.Variants) > 0 { + chosen := -1 + if group != nil && group.Variant != "" { + for i, v := range def.Variants { + if v.Key == group.Variant { + chosen = i + break + } + } + } + if chosen < 0 { + hash := Bucket(def.Key+":variant", userID) + cumulative := 0 + for i, v := range def.Variants { + cumulative += v.RolloutPct + if hash <= cumulative { + chosen = i + break + } + } + } + if chosen < 0 { + chosen = 0 + } + res.Variant = def.Variants[chosen].Key + if p := def.Variants[chosen].Payload; len(p) > 0 && !bytes.Equal(p, []byte("null")) { + res.Payload = p + } + } + return res +} diff --git a/internal/flags/flags.go b/internal/flags/flags.go index afba149a..a966a399 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -1,17 +1,15 @@ package flags import ( + "bytes" "context" "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log" - "math" "strconv" - "strings" "sync" "sync/atomic" "time" @@ -76,6 +74,9 @@ type Variant struct { Key string `json:"key"` Name string `json:"name"` RolloutPct int `json:"rollout_pct"` + // Payload is optional JSON returned with the variant by evaluate + // (<= MaxPayloadBytes compacted). Absent and null mean no payload. + Payload json.RawMessage `json:"payload,omitempty"` } // Evaluation condition, carried additively on every SDK response (O08). @@ -89,8 +90,13 @@ const ( type EvaluationResult struct { Enabled bool `json:"enabled"` Variant string `json:"variant,omitempty"` - Reason string `json:"reason"` - Detail string `json:"detail,omitempty"` + // Payload is the chosen variant's JSON payload, when it has one. + Payload json.RawMessage `json:"payload,omitempty"` + // Group is the key of the condition group that admitted the user + // (groups-format targeting only). + Group string `json:"group,omitempty"` + Reason string `json:"reason"` + Detail string `json:"detail,omitempty"` } func (s *FlagService) Create(ctx context.Context, siteID, flagKey, name, description, flagType, variants, targeting string, rolloutPct int) (*FeatureFlag, error) { @@ -110,10 +116,7 @@ func (s *FlagService) Create(ctx context.Context, siteID, flagKey, name, descrip // or a semantically invalid ruleset (unknown operator, missing value) // stores fine and used to silently skip the targeting condition at // evaluation. Reject here, naming the error. - if _, err := ValidateTargeting(targeting); err != nil { - return nil, &ValidationError{Err: err} - } - if _, err := ValidateVariants(variants); err != nil { + if err := ValidateConfig(flagType, variants, targeting); err != nil { return nil, &ValidationError{Err: err} } @@ -234,11 +237,8 @@ type ValidationError struct{ Err error } func (e *ValidationError) Error() string { return e.Err.Error() } func (e *ValidationError) Unwrap() error { return e.Err } -var targetingOperators = map[string]bool{ - "eq": true, "neq": true, "in": true, "not_in": true, "contains": true, -} - -// ValidateTargeting parses and sanity-checks a targeting ruleset. Empty, +// ValidateTargeting parses and sanity-checks a LEGACY flat targeting ruleset +// (the groups format is handled by parseTargetingConfig). Empty, // "null" and "[]" mean "no restrictions" and are valid. Anything else must // be a JSON array of rules with a non-empty attribute, a known operator // and a present value — the same operators matchesTargeting enforces. @@ -250,16 +250,8 @@ func ValidateTargeting(raw string) ([]TargetingRule, error) { if err := json.Unmarshal([]byte(raw), &rules); err != nil { return nil, fmt.Errorf("targeting: invalid JSON: %w", err) } - for i, r := range rules { - if r.Attribute == "" { - return nil, fmt.Errorf("targeting: rule %d: attribute is required", i) - } - if !targetingOperators[r.Operator] { - return nil, fmt.Errorf("targeting: rule %d: unknown operator %q", i, r.Operator) - } - if r.Value == nil { - return nil, fmt.Errorf("targeting: rule %d: value is required", i) - } + if err := validateRules(rules, "rule"); err != nil { + return nil, err } return rules, nil } @@ -287,6 +279,16 @@ func ValidateVariants(raw string) ([]Variant, error) { if v.RolloutPct < 0 || v.RolloutPct > 100 { return nil, fmt.Errorf("variants: entry %d: rollout_pct %d out of range 0..100", i, v.RolloutPct) } + if len(v.Payload) > 0 { + var compact bytes.Buffer + if err := json.Compact(&compact, v.Payload); err != nil { + return nil, fmt.Errorf("variants: entry %d: payload is not valid JSON: %w", i, err) + } + if compact.Len() > MaxPayloadBytes { + return nil, fmt.Errorf("variants: entry %d: payload is %d bytes, limit %d", i, compact.Len(), MaxPayloadBytes) + } + variants[i].Payload = json.RawMessage(compact.Bytes()) + } } return variants, nil } @@ -313,53 +315,6 @@ func errorClass(err error) string { } } -func matchesTargeting(rules []TargetingRule, userCtx map[string]string) bool { - for _, r := range rules { - actual, ok := userCtx[r.Attribute] - if !ok { - return false - } - switch r.Operator { - case "eq": - if actual != fmt.Sprintf("%v", r.Value) { - return false - } - case "neq": - if actual == fmt.Sprintf("%v", r.Value) { - return false - } - case "in": - vals := toStringSlice(r.Value) - found := false - for _, v := range vals { - if actual == v { - found = true - break - } - } - if !found { - return false - } - case "not_in": - vals := toStringSlice(r.Value) - for _, v := range vals { - if actual == v { - return false - } - } - case "contains": - if !strings.Contains(actual, fmt.Sprintf("%v", r.Value)) { - return false - } - default: - // Fail closed: an unknown operator must not silently match (which - // would expose a flag to everyone), so treat the rule as unmatched. - return false - } - } - return true -} - func toStringSlice(v any) []string { switch val := v.(type) { case []any: @@ -394,8 +349,11 @@ func toStringSlice(v any) []string { // invalid both answer Enabled=false. Flags gate feature exposure and the // create-path default is disabled, so neither an outage nor a corrupt // config may widen exposure. A per-flag fail-open override (the -// kill-switch pattern) needs a stored default plus an SDK contract — -// recorded as an O08 residual. +// kill-switch pattern) needs a stored default readable WITHOUT the config +// it guards (it cannot live in the targeting JSON: an unavailable or +// invalid config is exactly when it is needed) plus an SDK contract, so it +// stays an O08 residual. SDKs that evaluate locally from +// GET /api/v1/flags/config can keep their last-known config instead. func (s *FlagService) Evaluate(ctx context.Context, siteID, flagKey, userID string, userCtx map[string]string) (*EvaluationResult, error) { rows, err := s.fetchFlag(ctx, siteID, flagKey) if err != nil { @@ -417,48 +375,15 @@ func (s *FlagService) Evaluate(ctx context.Context, siteID, flagKey, userID stri // config surfaces even while the flag is disabled (operators learn // without enabling first), and so invalid rules can never be evaluated // as "no restrictions". - rules, err := ValidateTargeting(flag.Targeting) - if err != nil { - return s.quarantine(flag, "targeting", err), nil - } - var variants []Variant - if flag.FlagType == "multivariate" && flag.Variants != "" { - variants, err = ValidateVariants(flag.Variants) - if err != nil { - return s.quarantine(flag, "variants", err), nil - } - } - - if !flag.Enabled { - return &EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: "flag disabled"}, nil - } - - if flag.RolloutPct < 100 { - hash := hashUser(flagKey, userID) - if hash > flag.RolloutPct { - return &EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: "rollout"}, nil - } + def, cerr := ParseDefinition(flag) + if cerr != nil { + return s.quarantine(flag, cerr.Part, cerr.Err), nil } - if len(rules) > 0 && !matchesTargeting(rules, userCtx) { - return &EvaluationResult{Enabled: false, Reason: ReasonEvaluated, Detail: "targeting"}, nil - } - - result := &EvaluationResult{Enabled: true, Reason: ReasonEvaluated} - - if flag.FlagType == "multivariate" && len(variants) > 0 { - hash := hashUser(flagKey+":variant", userID) - cumulative := 0 - for _, v := range variants { - cumulative += v.RolloutPct - if hash <= cumulative { - result.Variant = v.Key - break - } - } - if result.Variant == "" { - result.Variant = variants[0].Key - } + r := EvaluateDefinition(def, userID, userCtx) + result := &r + if !result.Enabled { + return result, nil } // Fire-and-forget: evaluation tracking is best-effort, must not block @@ -494,11 +419,8 @@ func (s *FlagService) quarantine(flag FeatureFlag, part string, err error) *Eval } } -func hashUser(flagKey, userID string) int { - h := sha256.Sum256([]byte(flagKey + ":" + userID)) - val := int(h[0])<<8 | int(h[1]) - return int(math.Abs(float64(val%100))) + 1 -} +// hashUser is the pre-groups name for Bucket, kept for existing tests. +func hashUser(flagKey, userID string) int { return Bucket(flagKey, userID) } func genID() string { b := make([]byte, 16) From 90f848c87971157f5c3e8c2904fc397d06786f02 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:43:36 +0000 Subject: [PATCH 21/65] wip --- cmd/observe/main.go | 1 + cmd/observe/sentry_routes.go | 43 +++ internal/sentrycompat/envelope.go | 248 +++++++++++++++ internal/sentrycompat/handler.go | 402 ++++++++++++++++++++++++ internal/sentrycompat/mapping.go | 495 ++++++++++++++++++++++++++++++ 5 files changed, 1189 insertions(+) create mode 100644 cmd/observe/sentry_routes.go create mode 100644 internal/sentrycompat/envelope.go create mode 100644 internal/sentrycompat/handler.go create mode 100644 internal/sentrycompat/mapping.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..4d907b57 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -888,6 +888,7 @@ func main() { // (O01 ADR §5.2) stamps Retry-After on the durable-mode 503s (disk // high-water refusal, WAL latch) — the OTLP handlers' convention. apiKeyMW := auth.APIKeyAuthMiddleware(authSvc) + registerSentryRoutes(r, authSvc, errorBuf, rateLimiter, logger) ingestGroup := r.Group("/api/v1", ingestCORS, apiKeyMW, rateLimiter.Middleware, neutron.BodyLimit(2<<20), ingest.RetryAfterOnUnavailable(5*time.Second)) neutron.Post(ingestGroup, "/events", ingest.Handler(buf, cfg.SessionSalt, siteSvc), neutron.WithTags("ingest"), diff --git a/cmd/observe/sentry_routes.go b/cmd/observe/sentry_routes.go new file mode 100644 index 00000000..f6176c4d --- /dev/null +++ b/cmd/observe/sentry_routes.go @@ -0,0 +1,43 @@ +package main + +import ( + "log/slog" + "net/http" + "time" + + "github.com/useteploy/teploy-observe/internal/auth" + obserrors "github.com/useteploy/teploy-observe/internal/errors" + "github.com/useteploy/teploy-observe/internal/ingest" + "github.com/useteploy/teploy-observe/internal/sentrycompat" +) + +// registerSentryRoutes mounts the Sentry wire-protocol ingest so stock Sentry +// SDKs can use an Observe DSN: https://@/. +// Authentication, the BoundSite invariant, rate limiting and body caps are +// all enforced inside the handler (the DSN key can arrive in the query, the +// X-Sentry-Auth header or the envelope header, so the X-API-Key middleware +// does not apply). Patterns end in {$} so they match the exact path and do +// not claim the /api/ subtree. +func registerSentryRoutes(r interface { + Handle(pattern string, handler http.Handler) +}, authSvc *auth.AuthService, buf *obserrors.ErrorBuffer, limiter *ingest.RateLimiter, logger *slog.Logger) { + h := &sentrycompat.Handler{ + Keys: authSvc, + Sink: buf, + Limiter: limiter, + Logger: logger, + RetryAfter: 2 * time.Second, + } + for _, p := range []struct { + path string + fn http.HandlerFunc + }{ + {"/api/{project_id}/envelope/{$}", h.Envelope}, + {"/api/{project_id}/envelope{$}", h.Envelope}, + {"/api/{project_id}/store/{$}", h.Store}, + {"/api/{project_id}/store{$}", h.Store}, + } { + r.Handle("POST "+p.path, p.fn) + r.Handle("OPTIONS "+p.path, http.HandlerFunc(h.Preflight)) + } +} diff --git a/internal/sentrycompat/envelope.go b/internal/sentrycompat/envelope.go new file mode 100644 index 00000000..111453f5 --- /dev/null +++ b/internal/sentrycompat/envelope.go @@ -0,0 +1,248 @@ +package sentrycompat + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "compress/zlib" + "encoding/json" + "errors" + "io" + "strconv" + "strings" +) + +// Wire limits. The caps apply to DECOMPRESSED bytes: a 5 MiB gzip body can +// expand a thousandfold, so every decoder is wrapped in an io.LimitReader and +// the rejection is a 413, never an allocation. +const ( + // MaxEnvelopeBytes bounds the compressed body and, independently, the + // decompressed envelope. + MaxEnvelopeBytes = 5 << 20 + // MaxItemBytes bounds one item payload (after decompression). + MaxItemBytes = 1 << 20 + // maxItems bounds item headers walked per envelope (parse-cost bound; + // the 5 MiB cap already limits it, this keeps the loop obviously finite). + maxItems = 1000 + // maxEventsPerEnvelope bounds durable error admissions one request can + // cause (each Push is a WAL append). Extra events are counted, dropped. + maxEventsPerEnvelope = 100 + // maxHeaderLine bounds an envelope/item header line. + maxHeaderLine = 64 << 10 +) + +var ( + // errTooLarge marks a decompressed (or raw) body over the cap -> 413. + errTooLarge = errors.New("sentrycompat: payload too large") + // errMalformed marks an undecodable envelope -> 400. + errMalformed = errors.New("sentrycompat: malformed envelope") + // errEncoding marks an unsupported Content-Encoding -> 415. + errEncoding = errors.New("sentrycompat: unsupported content encoding") +) + +// decodeBody turns the raw (already size-capped) request bytes into the +// plain envelope bytes. encoding is the Content-Encoding header value. +func decodeBody(raw []byte, encoding string) ([]byte, error) { + enc := effectiveEncoding(raw, encoding) + switch enc { + case "": + if len(raw) > MaxEnvelopeBytes { + return nil, errTooLarge + } + return raw, nil + case "gzip": + zr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, errMalformed + } + defer zr.Close() + return readCapped(zr) + case "deflate": + // HTTP "deflate" is zlib-wrapped, but plenty of clients send raw + // DEFLATE. Try zlib first, fall back to raw on a bad header. + if zr, err := zlib.NewReader(bytes.NewReader(raw)); err == nil { + defer zr.Close() + return readCapped(zr) + } + fr := flate.NewReader(bytes.NewReader(raw)) + defer fr.Close() + return readCapped(fr) + default: + return nil, errEncoding + } +} + +// effectiveEncoding normalizes the header to "", "gzip", "deflate" or the +// (unsupported) original. It sniffs the gzip magic when no encoding is +// declared: some proxies strip Content-Encoding but leave the body. +func effectiveEncoding(raw []byte, header string) string { + enc := strings.ToLower(strings.TrimSpace(header)) + switch enc { + case "", "identity": + if len(raw) >= 2 && raw[0] == 0x1f && raw[1] == 0x8b { + return "gzip" + } + return "" + case "x-gzip": + return "gzip" + } + return enc +} + +// readCapped reads at most MaxEnvelopeBytes of decompressed data; one byte +// more is a bomb/oversize (413). A truncated stream is malformed (400). +func readCapped(r io.Reader) ([]byte, error) { + out, err := io.ReadAll(io.LimitReader(r, MaxEnvelopeBytes+1)) + if len(out) > MaxEnvelopeBytes { + return nil, errTooLarge + } + if err != nil { + return nil, errMalformed + } + return out, nil +} + +// peekHeaderLine decompresses at most limit bytes and returns the first line. +// Used to read the envelope-header DSN BEFORE authentication without paying +// for (or trusting) the rest of the body. +func peekHeaderLine(raw []byte, encoding string, limit int) []byte { + var r io.Reader + switch effectiveEncoding(raw, encoding) { + case "": + r = bytes.NewReader(raw) + case "gzip": + zr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil + } + defer zr.Close() + r = zr + case "deflate": + if zr, err := zlib.NewReader(bytes.NewReader(raw)); err == nil { + defer zr.Close() + r = zr + } else { + fr := flate.NewReader(bytes.NewReader(raw)) + defer fr.Close() + r = fr + } + default: + return nil + } + buf, _ := io.ReadAll(io.LimitReader(r, int64(limit))) + if i := bytes.IndexByte(buf, '\n'); i >= 0 { + buf = buf[:i] + } + return buf +} + +// envelopeHeader is the first line of an envelope. +type envelopeHeader struct { + EventID string + DSN string +} + +// item is one parsed envelope item. +type item struct { + Type string + Payload []byte +} + +// parsedEnvelope is the outcome of splitting an envelope. Per-item problems +// are counted, not fatal, so one bad item never costs the good ones. +type parsedEnvelope struct { + Header envelopeHeader + Items []item + Malformed int // items skipped: undecodable header, lying/truncated length + Oversize int // items skipped: payload over MaxItemBytes + Dropped int // items beyond maxItems +} + +// parseEnvelope splits newline-delimited envelope bytes. A bad envelope +// header (or empty input) is errMalformed; everything after is best-effort. +// +// Item framing: header line, then payload. With a header "length" the payload +// is exactly that many bytes (so it may contain newlines); without it the +// payload runs to the next newline. A length that overruns the body, or an +// unparseable header, ends parsing (no resync is possible) and counts one +// malformed item. +func parseEnvelope(data []byte) (*parsedEnvelope, error) { + line, pos := readLine(data, 0) + if len(bytes.TrimSpace(line)) == 0 || len(line) > maxHeaderLine { + return nil, errMalformed + } + var hdr map[string]json.RawMessage + if err := json.Unmarshal(line, &hdr); err != nil || hdr == nil { + return nil, errMalformed + } + pe := &parsedEnvelope{} + pe.Header.EventID = rawString(hdr["event_id"]) + pe.Header.DSN = rawString(hdr["dsn"]) + + for pos < len(data) { + if len(pe.Items) >= maxItems { + pe.Dropped++ + break + } + line, pos = readLine(data, pos) + if len(bytes.TrimSpace(line)) == 0 { + continue // tolerate blank separators + } + if len(line) > maxHeaderLine { + pe.Malformed++ + break + } + var ih map[string]json.RawMessage + if err := json.Unmarshal(line, &ih); err != nil || ih == nil { + pe.Malformed++ + break + } + typ := rawString(ih["type"]) + + var payload []byte + if lraw, ok := ih["length"]; ok && string(lraw) != "null" { + n, err := strconv.ParseInt(string(bytes.TrimSpace(lraw)), 10, 64) + if err != nil || n < 0 || n > int64(len(data)-pos) { + pe.Malformed++ + break + } + payload = data[pos : pos+int(n)] + pos += int(n) + if pos < len(data) && data[pos] == '\r' { + pos++ + } + if pos < len(data) && data[pos] == '\n' { + pos++ + } + } else { + payload, pos = readLine(data, pos) + } + if len(payload) > MaxItemBytes { + pe.Oversize++ + continue + } + pe.Items = append(pe.Items, item{Type: typ, Payload: payload}) + } + return pe, nil +} + +// readLine returns the line starting at pos (without \n or a trailing \r) and +// the position after it. +func readLine(data []byte, pos int) (line []byte, next int) { + if pos >= len(data) { + return nil, len(data) + } + i := bytes.IndexByte(data[pos:], '\n') + if i < 0 { + return bytes.TrimSuffix(data[pos:], []byte("\r")), len(data) + } + return bytes.TrimSuffix(data[pos:pos+i], []byte("\r")), pos + i + 1 +} + +func rawString(raw json.RawMessage) string { + var s string + if len(raw) == 0 || json.Unmarshal(raw, &s) != nil { + return "" + } + return s +} diff --git a/internal/sentrycompat/handler.go b/internal/sentrycompat/handler.go new file mode 100644 index 00000000..ef2165e4 --- /dev/null +++ b/internal/sentrycompat/handler.go @@ -0,0 +1,402 @@ +// Package sentrycompat accepts the Sentry wire protocol (envelope + legacy +// store endpoints) so stock Sentry SDKs can point their DSN at Observe. +// +// DSN format: https://@/. The DSN public key +// is an Observe telemetry API key; the DSN "project" is the Observe site id +// (or any numeric alias, see checkProject). Only error events are applied; +// every other item type is acknowledged and dropped. +package sentrycompat + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/useteploy/teploy-observe/internal/auth" + obserrors "github.com/useteploy/teploy-observe/internal/errors" + "github.com/useteploy/teploy-observe/internal/ingest" +) + +// KeyValidator is *auth.AuthService's key check. +type KeyValidator interface { + ValidateAPIKey(ctx context.Context, key string) (auth.ValidatedKey, error) +} + +// ErrorSink is *errors.ErrorBuffer's admission method. +type ErrorSink interface { + Push(siteID string, input obserrors.ErrorInput) error +} + +// Limiter is *ingest.RateLimiter's admission check (per site and client IP). +type Limiter interface { + Allow(siteID, ip string) bool +} + +// knownIgnored is the closed set of item types counted by name. Anything +// else collapses into "unknown" so a client cannot grow the counter map +// with invented type names. +var knownIgnored = map[string]bool{ + "transaction": true, "span": true, "session": true, "sessions": true, + "attachment": true, "client_report": true, "profile": true, + "profile_chunk": true, "replay_event": true, "replay_recording": true, + "check_in": true, "log": true, "user_report": true, "feedback": true, + "statsd": true, "metric_buckets": true, "trace_metric": true, + "event_ignored": true, +} + +// Handler serves the Sentry ingest routes. +type Handler struct { + Keys KeyValidator + Sink ErrorSink + Limiter Limiter // nil disables per-site limiting + Logger *slog.Logger + // RetryAfter is advertised on 429/503 (default 5s). + RetryAfter time.Duration + + accepted atomic.Int64 + malformed atomic.Int64 + oversize atomic.Int64 + dedup atomic.Int64 + conflicts atomic.Int64 + mu sync.Mutex + ignored map[string]int64 +} + +// Stats is a point-in-time counter snapshot. +type Stats struct { + Accepted int64 `json:"accepted"` + Deduped int64 `json:"deduped"` + Conflicts int64 `json:"conflicts"` + Malformed int64 `json:"malformed"` + Oversize int64 `json:"oversize"` + IgnoredByType map[string]int64 `json:"ignored_by_type"` +} + +// Stats returns the counters. +func (h *Handler) Stats() Stats { + h.mu.Lock() + ign := make(map[string]int64, len(h.ignored)) + for k, v := range h.ignored { + ign[k] = v + } + h.mu.Unlock() + return Stats{ + Accepted: h.accepted.Load(), Deduped: h.dedup.Load(), Conflicts: h.conflicts.Load(), + Malformed: h.malformed.Load(), Oversize: h.oversize.Load(), IgnoredByType: ign, + } +} + +func (h *Handler) countIgnored(typ string) { + if !knownIgnored[typ] { + typ = "unknown" + } + h.mu.Lock() + if h.ignored == nil { + h.ignored = map[string]int64{} + } + h.ignored[typ]++ + h.mu.Unlock() +} + +func (h *Handler) retryAfter() time.Duration { + if h.RetryAfter > 0 { + return h.RetryAfter + } + return 5 * time.Second +} + +func (h *Handler) logger() *slog.Logger { + if h.Logger != nil { + return h.Logger + } + return slog.Default() +} + +// Envelope serves POST /api/{project_id}/envelope/. +func (h *Handler) Envelope(w http.ResponseWriter, r *http.Request) { h.serve(w, r, false) } + +// Store serves POST /api/{project_id}/store/ (legacy single-event JSON). +func (h *Handler) Store(w http.ResponseWriter, r *http.Request) { h.serve(w, r, true) } + +// Preflight answers CORS preflight for the Sentry routes. +func (h *Handler) Preflight(w http.ResponseWriter, r *http.Request) { + setCORS(w) + w.WriteHeader(http.StatusNoContent) +} + +func setCORS(w http.ResponseWriter) { + hd := w.Header() + hd.Set("Access-Control-Allow-Origin", "*") + hd.Set("Access-Control-Allow-Methods", "POST, OPTIONS") + hd.Set("Access-Control-Allow-Headers", "Content-Type, Content-Encoding, X-Sentry-Auth, X-API-Key") + hd.Set("Access-Control-Expose-Headers", "X-Sentry-Rate-Limits, Retry-After") + hd.Set("Access-Control-Max-Age", "86400") +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func fail(w http.ResponseWriter, status int, detail string) { + writeJSON(w, status, map[string]string{"detail": detail}) +} + +func (h *Handler) limited(w http.ResponseWriter, why string) { + secs := strconv.Itoa(int(h.retryAfter() / time.Second)) + w.Header().Set("Retry-After", secs) + // retry_after:categories:scope - empty categories means every category. + w.Header().Set("X-Sentry-Rate-Limits", secs+"::key") + fail(w, http.StatusTooManyRequests, why) +} + +func (h *Handler) serve(w http.ResponseWriter, r *http.Request, legacyStore bool) { + setCORS(w) + projectID := r.PathValue("project_id") + + // 1. Credentials, from the cheap places first (query, header). The DSN + // in the envelope header is the last resort and needs a bounded peek. + key := keyFromRequest(r) + + // 2. The body is read at most once, capped, and only after we know a key + // exists; with a header/query key we authenticate before touching it. + var raw []byte + readBody := func() bool { + if r.ContentLength > MaxEnvelopeBytes { + fail(w, http.StatusRequestEntityTooLarge, "payload too large") + return false + } + b, err := io.ReadAll(http.MaxBytesReader(w, r.Body, MaxEnvelopeBytes)) + if err != nil { + var mbe *http.MaxBytesError + if errors.As(err, &mbe) { + fail(w, http.StatusRequestEntityTooLarge, "payload too large") + } else { + fail(w, http.StatusBadRequest, "could not read body") + } + return false + } + raw = b + return true + } + bodyRead := false + if key == "" && !legacyStore { + if !readBody() { + return + } + bodyRead = true + line := peekHeaderLine(raw, r.Header.Get("Content-Encoding"), maxHeaderLine) + var hdr struct { + DSN string `json:"dsn"` + } + if json.Unmarshal(line, &hdr) == nil { + key = keyFromDSN(hdr.DSN) + } + } + if key == "" { + fail(w, http.StatusUnauthorized, "missing sentry key") + return + } + + validated, err := h.Keys.ValidateAPIKey(r.Context(), key) + if err != nil { + if errors.Is(err, auth.ErrAuthUnavailable) { + w.Header().Set("Retry-After", "5") + fail(w, http.StatusServiceUnavailable, "authentication temporarily unavailable") + return + } + fail(w, http.StatusUnauthorized, "invalid key") + return + } + if !auth.HasScope(validated.Scopes, auth.ScopeTelemetry) { + fail(w, http.StatusForbidden, "key lacks the telemetry capability") + return + } + // BoundSite invariant: the key's site is authoritative; a project that + // names another site is a cross-tenant write attempt. + ctx := ingest.WithSiteID(r.Context(), validated.SiteID) + siteID, err := boundProject(ctx, projectID) + if err != nil { + fail(w, http.StatusForbidden, "project does not match the key") + return + } + + if h.Limiter != nil { + ip := ingest.ClientIPFromContext(ctx) + if ip == "" { + ip = r.RemoteAddr + } + if !h.Limiter.Allow(siteID, ip) { + h.limited(w, "rate limited") + return + } + } + + if !bodyRead && !readBody() { + return + } + data, err := decodeBody(raw, r.Header.Get("Content-Encoding")) + if err != nil { + switch { + case errors.Is(err, errTooLarge): + fail(w, http.StatusRequestEntityTooLarge, "payload too large") + case errors.Is(err, errEncoding): + fail(w, http.StatusUnsupportedMediaType, "unsupported content encoding (use gzip, deflate or none)") + default: + h.malformed.Add(1) + fail(w, http.StatusBadRequest, "malformed body") + } + return + } + + var pe *parsedEnvelope + if legacyStore { + if len(data) > MaxItemBytes { + h.oversize.Add(1) + fail(w, http.StatusRequestEntityTooLarge, "event too large") + return + } + pe = &parsedEnvelope{Items: []item{{Type: "event", Payload: data}}} + } else { + pe, err = parseEnvelope(data) + if err != nil { + h.malformed.Add(1) + fail(w, http.StatusBadRequest, "malformed envelope") + return + } + } + h.oversize.Add(int64(pe.Oversize)) + h.malformed.Add(int64(pe.Malformed)) + + respID := pe.Header.EventID + goodItems, events := 0, 0 + for _, it := range pe.Items { + if it.Type != "event" { + goodItems++ + h.countIgnored(it.Type) + continue + } + if events >= maxEventsPerEnvelope { + h.malformed.Add(1) + continue + } + in, eventID, merr := mapEvent(it.Payload, pe.Header.EventID) + if merr != nil { + h.malformed.Add(1) + continue + } + goodItems++ + events++ + in.SiteID = siteID + switch perr := h.Sink.Push(siteID, in); { + case perr == nil: + h.accepted.Add(1) + case errors.Is(perr, obserrors.ErrAdmittedDuplicate): + h.dedup.Add(1) + case errors.Is(perr, obserrors.ErrEventIDConflict): + // A producer bug, not something a retry fixes: ack so the SDK + // does not loop, count it for the operator. + h.conflicts.Add(1) + case errors.Is(perr, obserrors.ErrInvalidEventID): + h.malformed.Add(1) + case errors.Is(perr, obserrors.ErrErrorBufferFull): + h.limited(w, "error buffer full") + return + default: + w.Header().Set("Retry-After", strconv.Itoa(int(h.retryAfter()/time.Second))) + fail(w, http.StatusServiceUnavailable, "error durability unavailable, retry later") + return + } + if respID == "" { + respID = eventID + } + } + + // Nothing usable: an envelope whose every item was oversize is a 413; + // one whose items were all undecodable is a 400. A header-only envelope + // (no items at all) is valid and acknowledged. + if goodItems == 0 && (pe.Oversize > 0 || pe.Malformed > 0 || pe.Dropped > 0 || legacyStore) { + if pe.Oversize > 0 && pe.Malformed == 0 { + fail(w, http.StatusRequestEntityTooLarge, "item too large") + return + } + fail(w, http.StatusBadRequest, "no valid items") + return + } + writeJSON(w, http.StatusOK, map[string]string{"id": respID}) +} + +// boundProject enforces the BoundSite invariant on the DSN project segment. +// The segment must be the key's own site id, or a short all-digit alias: +// the JavaScript Sentry SDKs refuse a non-numeric project id in a DSN, so +// `https://@host/1` must work. An alias names no site - the key alone +// decides the site - so it cannot address another tenant. (Real site ids are +// 32 hex characters or "default", never 1-12 digits.) +func boundProject(ctx context.Context, project string) (string, error) { + if isNumericAlias(project) { + return ingest.BoundSite(ctx, "") + } + return ingest.BoundSite(ctx, project) +} + +func isNumericAlias(s string) bool { + if s == "" || len(s) > 12 { + return false + } + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} + +// keyFromRequest reads the DSN public key from the sentry_key query +// parameter or the X-Sentry-Auth header (query wins, as in Relay). +func keyFromRequest(r *http.Request) string { + if k := strings.TrimSpace(r.URL.Query().Get("sentry_key")); k != "" { + return k + } + if k := authHeaderKey(r.Header.Get("X-Sentry-Auth")); k != "" { + return k + } + // Convenience for hand-rolled clients: the repo's standard header. + return strings.TrimSpace(r.Header.Get("X-API-Key")) +} + +// authHeaderKey parses `Sentry sentry_version=7, sentry_key=abc, ...`. +func authHeaderKey(v string) string { + v = strings.TrimSpace(v) + if len(v) < 7 || !strings.EqualFold(v[:6], "sentry") { + return "" + } + for _, part := range strings.Split(v[6:], ",") { + k, val, ok := strings.Cut(strings.TrimSpace(part), "=") + if ok && strings.TrimSpace(k) == "sentry_key" { + return strings.Trim(strings.TrimSpace(val), `"`) + } + } + return "" +} + +// keyFromDSN extracts the public key (userinfo) from a DSN string. +func keyFromDSN(dsn string) string { + if dsn == "" || len(dsn) > 2048 { + return "" + } + u, err := url.Parse(dsn) + if err != nil || u.User == nil { + return "" + } + return u.User.Username() +} diff --git a/internal/sentrycompat/mapping.go b/internal/sentrycompat/mapping.go new file mode 100644 index 00000000..eb4dd6d9 --- /dev/null +++ b/internal/sentrycompat/mapping.go @@ -0,0 +1,495 @@ +package sentrycompat + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + obserrors "github.com/useteploy/teploy-observe/internal/errors" +) + +// ProducerID namespaces Sentry event ids in Observe's error inbox so a Sentry +// event_id can never collide with an Observe-native one. +const ProducerID = "sentry" + +// Field caps. They keep one mapped record far below the error buffer's +// 256 KiB admission cap (over which Push answers "buffer full", i.e. 429 and +// an SDK retry for an event that can never be admitted). +const ( + maxFrames = 100 + maxBreadcrumbs = 100 + maxCrumbData = 2 << 10 + maxContextEntry = 8 << 10 + maxContextsTotal = 32 << 10 + maxExtraTotal = 16 << 10 + maxTags = 100 + maxStr = 2048 + maxMessage = 8192 + maxChain = 10 +) + +var ( + errNoContent = errors.New("event has no exception or message") + hexID = regexp.MustCompile(`^[0-9a-fA-F]{1,32}$`) + validEventID = regexp.MustCompile(`^[A-Za-z0-9_-]{8,64}$`) + validLevels = map[string]bool{"fatal": true, "error": true, "warning": true, "log": true, "info": true, "debug": true} +) + +// mapEvent converts one Sentry event payload (the JSON of an `event` item or +// a /store/ body) into Observe's error input. It returns the stable event id +// used for idempotency. It never reads the clock or randomness: the same +// bytes always map to the same ErrorInput, so a retried envelope has the same +// payload digest and dedupes instead of conflicting. +func mapEvent(payload []byte, headerEventID string) (obserrors.ErrorInput, string, error) { + var ev map[string]any + if err := json.Unmarshal(payload, &ev); err != nil || ev == nil { + return obserrors.ErrorInput{}, "", fmt.Errorf("event is not a JSON object") + } + + in := obserrors.ErrorInput{ + Level: "error", + Handled: true, + ProducerID: ProducerID, + } + + if l := strings.ToLower(str(ev["level"])); validLevels[l] { + in.Level = l + } + in.ReleaseTag = clip(str(ev["release"]), 200) + in.Environment = clip(str(ev["environment"]), 100) + + // Identity: the event's own id, else the envelope header's, else a digest + // of the payload (stable across retries of the same bytes). + eventID := strings.ReplaceAll(strings.ToLower(str(ev["event_id"])), "-", "") + if !validEventID.MatchString(eventID) { + eventID = strings.ReplaceAll(strings.ToLower(headerEventID), "-", "") + } + if !validEventID.MatchString(eventID) { + sum := sha256.Sum256(payload) + eventID = "d" + hex.EncodeToString(sum[:16]) + } + in.EventID = eventID + + // Exception: Sentry orders chained exceptions oldest-first, so the last + // value is the one that surfaced. + vals := exceptionValues(ev["exception"]) + var primary map[string]any + if len(vals) > 0 { + primary = vals[len(vals)-1] + } + switch { + case primary != nil: + in.ErrorType = clip(str(primary["type"]), 256) + in.ErrorValue = clip(str(primary["value"]), maxMessage) + if in.ErrorType == "" { + in.ErrorType = "Error" + } + if mech, ok := primary["mechanism"].(map[string]any); ok { + in.Mechanism = clip(str(mech["type"]), 100) + if h, ok := mech["handled"].(bool); ok { + in.Handled = h + } + } + if st, ok := primary["stacktrace"].(map[string]any); ok { + in.StackTrace = mapFrames(st["frames"]) + } + default: + msg := messageOf(ev) + if msg == "" { + return obserrors.ErrorInput{}, "", errNoContent + } + in.ErrorType = "Message" + in.ErrorValue = clip(msg, maxMessage) + if st, ok := ev["stacktrace"].(map[string]any); ok { + in.StackTrace = mapFrames(st["frames"]) + } + } + + // Request URL (the page, for browser SDKs). Sanitized again server-side + // by CapturedURL. + if req, ok := ev["request"].(map[string]any); ok { + in.URL = clip(str(req["url"]), 2048) + } + + // User: only the identifier crosses, as distinct_id, which the error + // service HASHES with the per-site salt. The raw user object (email, IP) + // is deliberately not kept in contexts - the shim does keep it; we do + // not, so hashing is not defeated by a verbatim copy. + if u, ok := ev["user"].(map[string]any); ok { + for _, k := range []string{"id", "username", "email"} { + if v := scalarStr(u[k]); v != "" { + in.DistinctID = clip(v, 200) + break + } + } + } + + // Contexts: sentry contexts + tags + sdk bookkeeping. + ctxs := map[string]any{} + if c, ok := ev["contexts"].(map[string]any); ok { + fillCapped(ctxs, c, maxContextEntry, maxContextsTotal) + in.Browser = nameVersion(c["browser"]) + in.OS = nameVersion(c["os"]) + in.Device = deviceName(c["device"]) + if tr, ok := c["trace"].(map[string]any); ok { + if t := str(tr["trace_id"]); hexID.MatchString(t) { + in.TraceID = t + } + if s := str(tr["span_id"]); hexID.MatchString(s) { + in.SpanID = s + } + } + } + if tags := mapTags(ev["tags"]); len(tags) > 0 { + ctxs["tags"] = tags + } + meta := map[string]any{} + for _, k := range []string{"platform", "logger", "transaction", "server_name", "dist"} { + if v := clip(str(ev[k]), 256); v != "" { + meta[k] = v + } + } + if sdk, ok := ev["sdk"].(map[string]any); ok { + meta["sdk"] = clip(str(sdk["name"]), 100) + "/" + clip(str(sdk["version"]), 50) + } + if len(vals) > 1 { + chain := []map[string]string{} + for _, v := range vals[:len(vals)-1] { + if len(chain) >= maxChain { + break + } + chain = append(chain, map[string]string{"type": clip(str(v["type"]), 128), "value": clip(str(v["value"]), 512)}) + } + meta["exception_chain"] = chain + } + if len(meta) > 0 { + ctxs["sentry"] = meta + } + if len(ctxs) > 0 { + in.Contexts = ctxs + } + + if ex, ok := ev["extra"].(map[string]any); ok { + extra := map[string]any{} + fillCapped(extra, ex, maxContextEntry, maxExtraTotal) + if len(extra) > 0 { + in.Extra = extra + } + } + + // Custom fingerprint. "{{ default }}" asks Sentry to splice its own + // default hash in; Observe's grouping differs, so any use falls back to + // Observe's default grouping rather than guessing. + if fp, ok := ev["fingerprint"].([]any); ok && len(fp) > 0 && len(fp) <= 20 { + parts := make([]string, 0, len(fp)) + usable := true + for _, p := range fp { + s := clip(scalarStr(p), 200) + if s == "" || strings.Contains(s, "{{") { + usable = false + break + } + parts = append(parts, s) + } + if usable { + in.Fingerprint = parts + } + } + + in.Breadcrumbs = mapBreadcrumbs(ev["breadcrumbs"]) + return in, eventID, nil +} + +// exceptionValues accepts {"values":[...]} and the bare-list form. +func exceptionValues(v any) []map[string]any { + var list []any + switch t := v.(type) { + case map[string]any: + list, _ = t["values"].([]any) + case []any: + list = t + } + var out []map[string]any + for _, e := range list { + if m, ok := e.(map[string]any); ok { + out = append(out, m) + } + } + if len(out) > 50 { + out = out[len(out)-50:] + } + return out +} + +func messageOf(ev map[string]any) string { + if le, ok := ev["logentry"].(map[string]any); ok { + if s := str(le["formatted"]); s != "" { + return s + } + if s := str(le["message"]); s != "" { + return s + } + } + switch m := ev["message"].(type) { + case string: + return m + case map[string]any: + if s := str(m["formatted"]); s != "" { + return s + } + return str(m["message"]) + } + return "" +} + +// mapFrames converts Sentry frames (oldest call first) to Observe frames +// (innermost first - IssueCulprit reads the first in-app frame). Works for +// python, node, browser (minified, filename is the script URL) and go frames. +func mapFrames(v any) []obserrors.StackFrame { + list, _ := v.([]any) + if len(list) == 0 { + return nil + } + if len(list) > maxFrames { + list = list[len(list)-maxFrames:] // keep the innermost frames + } + anyInApp := false + for _, f := range list { + if m, ok := f.(map[string]any); ok { + if _, has := m["in_app"].(bool); has { + anyInApp = true + break + } + } + } + out := make([]obserrors.StackFrame, 0, len(list)) + for i := len(list) - 1; i >= 0; i-- { + m, ok := list[i].(map[string]any) + if !ok { + continue + } + fn := firstNonEmpty(str(m["filename"]), str(m["abs_path"]), str(m["module"])) + fr := obserrors.StackFrame{ + Filename: clip(fn, 512), + Function: clip(firstNonEmpty(str(m["function"]), str(m["module"])), 256), + Lineno: clampInt(m["lineno"]), + Colno: clampInt(m["colno"]), + } + if b, has := m["in_app"].(bool); has { + fr.InApp = b + } else if !anyInApp { + // Browser SDKs omit in_app. Treat as app code unless it is + // plainly a library path. + fr.InApp = !libraryPath(firstNonEmpty(str(m["abs_path"]), fn)) + } + out = append(out, fr) + } + return out +} + +var libraryMarkers = []string{"node_modules", "site-packages", "dist-packages", "/usr/lib/", "/usr/local/lib/", "/vendor/", "/pkg/mod/", "/go/src/", "node:internal", "webpack/bootstrap"} + +func libraryPath(p string) bool { + for _, m := range libraryMarkers { + if strings.Contains(p, m) { + return true + } + } + return false +} + +func mapBreadcrumbs(v any) []obserrors.Breadcrumb { + var list []any + switch t := v.(type) { + case map[string]any: + list, _ = t["values"].([]any) + case []any: + list = t + } + if len(list) > maxBreadcrumbs { + list = list[len(list)-maxBreadcrumbs:] // keep the most recent + } + var out []obserrors.Breadcrumb + for _, c := range list { + m, ok := c.(map[string]any) + if !ok { + continue + } + b := obserrors.Breadcrumb{ + Type: clip(firstNonEmpty(str(m["type"]), "default"), 64), + Category: clip(str(m["category"]), 128), + Message: clip(str(m["message"]), 1024), + Timestamp: toMillis(m["timestamp"]), + } + if l := strings.ToLower(str(m["level"])); validLevels[l] { + b.Level = l + } + if d, ok := m["data"]; ok && d != nil { + if raw, err := json.Marshal(d); err == nil { + if len(raw) <= maxCrumbData { + b.Data = d + } else { + b.Data = map[string]any{"truncated": true} + } + } + } + out = append(out, b) + } + return out +} + +// mapTags accepts {"k":"v"} and [["k","v"],...] / [{"key","value"}]. +func mapTags(v any) map[string]string { + out := map[string]string{} + add := func(k, val string) { + if k != "" && len(out) < maxTags { + out[clip(k, 128)] = clip(val, 256) + } + } + switch t := v.(type) { + case map[string]any: + keys := make([]string, 0, len(t)) + for k := range t { + keys = append(keys, k) + } + sort.Strings(keys) // deterministic truncation + for _, k := range keys { + add(k, scalarStr(t[k])) + } + case []any: + for _, e := range t { + switch p := e.(type) { + case []any: + if len(p) == 2 { + add(scalarStr(p[0]), scalarStr(p[1])) + } + case map[string]any: + add(str(p["key"]), scalarStr(p["value"])) + } + } + } + return out +} + +// fillCapped copies src entries into dst in key order, replacing any entry +// over entryMax with a marker and stopping once totalMax is reached. +func fillCapped(dst, src map[string]any, entryMax, totalMax int) { + keys := make([]string, 0, len(src)) + for k := range src { + keys = append(keys, k) + } + sort.Strings(keys) + total := 0 + for _, k := range keys { + raw, err := json.Marshal(src[k]) + if err != nil { + continue + } + val := src[k] + if len(raw) > entryMax { + val = map[string]any{"truncated": true} + raw = []byte(`{"truncated":true}`) + } + if total+len(raw)+len(k) > totalMax { + break + } + total += len(raw) + len(k) + dst[clip(k, 128)] = val + } +} + +func nameVersion(v any) string { + m, _ := v.(map[string]any) + n := clip(str(m["name"]), 100) + if n == "" { + return "" + } + if ver := clip(scalarStr(m["version"]), 50); ver != "" { + return n + " " + ver + } + return n +} + +func deviceName(v any) string { + m, _ := v.(map[string]any) + return clip(firstNonEmpty(str(m["model"]), str(m["name"]), str(m["family"])), 100) +} + +// toMillis converts a Sentry timestamp (float seconds, ms, or RFC 3339) to +// epoch milliseconds, 0 when absent or unusable. +func toMillis(v any) int64 { + switch t := v.(type) { + case float64: + if math.IsNaN(t) || math.IsInf(t, 0) || t <= 0 { + return 0 + } + if t > 1e11 { // already milliseconds + return int64(t) + } + return int64(t * 1000) + case string: + if ts, err := time.Parse(time.RFC3339Nano, t); err == nil { + return ts.UnixMilli() + } + } + return 0 +} + +func clampInt(v any) int { + f, ok := v.(float64) + if !ok || math.IsNaN(f) || f < 0 || f > 1e9 { + return 0 + } + return int(f) +} + +func str(v any) string { + s, _ := v.(string) + return s +} + +// scalarStr renders strings and numbers/bools; nested values are ignored. +func scalarStr(v any) string { + switch t := v.(type) { + case string: + return t + case float64: + return strconv.FormatFloat(t, 'f', -1, 64) + case bool: + return strconv.FormatBool(t) + } + return "" +} + +func firstNonEmpty(ss ...string) string { + for _, s := range ss { + if s != "" { + return s + } + } + return "" +} + +// clip truncates to max bytes on a rune boundary and strips NULs. +func clip(s string, max int) string { + if strings.IndexByte(s, 0) >= 0 { + s = strings.ReplaceAll(s, "\x00", "") + } + if len(s) <= max { + return s + } + s = s[:max] + for !utf8.ValidString(s) && len(s) > 0 { + s = s[:len(s)-1] + } + return s +} From 51773cde8efddde705c3c5a18a76ea5ca8ecc158 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:43:43 +0000 Subject: [PATCH 22/65] wip --- cmd/observe/ingest_listener.go | 1 + cmd/observe/main.go | 7 +- cmd/observe/surveys_routes.go | 150 ++++++++ cmd/observe/tracker/observe-surveys.js | 491 +++++++++++++++++++++++++ internal/surveys/active.go | 107 ++++++ internal/surveys/surveys.go | 9 + internal/surveys/targeting.go | 371 +++++++++++++++++++ internal/surveys/targeting_test.go | 239 ++++++++++++ 8 files changed, 1371 insertions(+), 4 deletions(-) create mode 100644 cmd/observe/surveys_routes.go create mode 100644 cmd/observe/tracker/observe-surveys.js create mode 100644 internal/surveys/active.go create mode 100644 internal/surveys/targeting.go create mode 100644 internal/surveys/targeting_test.go diff --git a/cmd/observe/ingest_listener.go b/cmd/observe/ingest_listener.go index 7edb1cf9..26e97771 100644 --- a/cmd/observe/ingest_listener.go +++ b/cmd/observe/ingest_listener.go @@ -50,6 +50,7 @@ var ingestRoutes = map[string]bool{ "POST /api/v1/experiments/convert": true, "POST /api/v1/flags/evaluate": true, "GET /api/v1/surveys/active": true, + "POST /api/v1/surveys/expose": true, "POST /api/v1/surveys/respond": true, // CI uploads source maps so stack traces symbolicate; the handler accepts diff --git a/cmd/observe/main.go b/cmd/observe/main.go index ab90b0e2..f34ffa13 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -1425,10 +1425,9 @@ func main() { neutron.WithTags("surveys"), neutron.WithSummary("List survey responses")) neutron.Get(surveyGroup, "/{survey_id}/stats", surveyStatsHandler(surveySvc), neutron.WithTags("surveys"), neutron.WithSummary("Survey exposure/response stats")) - // Public: get active surveys, record exposure, submit response - r.HandleFunc("GET /api/v1/surveys/active", activeSurveysPublicHandler(surveySvc)) - r.HandleFunc("POST /api/v1/surveys/expose", surveyExposeHandler(surveySvc)) - r.HandleFunc("POST /api/v1/surveys/respond", surveyRespondHandler(surveySvc)) + // Public: get active surveys, record exposure, submit response, and the + // widget script (surveys_routes.go). + registerSurveyPublicRoutes(r, surveySvc) // --- Release health (JWT auth) --- releaseHealthSvc := obserrors.NewReleaseHealthService(db) diff --git a/cmd/observe/surveys_routes.go b/cmd/observe/surveys_routes.go new file mode 100644 index 00000000..c83ab2c6 --- /dev/null +++ b/cmd/observe/surveys_routes.go @@ -0,0 +1,150 @@ +package main + +import ( + "bytes" + _ "embed" + "encoding/json" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/useteploy/teploy-observe/internal/ingest" + "github.com/useteploy/teploy-observe/internal/surveys" +) + +// Browser survey widget (B5). Served like the other trackers; sites opt in +// with ", "\">", + "javascript:alert(1)", "", "{{constructor.constructor('window.pwned=1')()}}", + "<b>", "\u0000", "' onmouseover='window.pwned=1", + ]; + for (const p of payloads) { + const sv = SV({ + name: p, + questions: JSON.stringify([ + { id: "q1", type: "text", text: p, placeholder: p }, + { id: "q2", type: "choice", text: p, choices: [p, p + "2"] }, + { id: p, type: "rating", text: p }, + ]), + appearance: JSON.stringify({ accent: p, position: p }), + }); + const env = makeEnv({ surveys: [sv] }); + await settle(); + assert.equal(env.sandbox.pwned, undefined, "payload executed: " + p); + const root = env.root(); + assert.ok(root, "still renders: " + p); + // The payload exists only as text content, never as an element/attribute name. + const tags = new Set(root.findAll(() => true).map((c) => c.tagName)); + for (const t of tags) assert.ok(["DIV", "H2", "BUTTON", "FORM", "FIELDSET", "LEGEND", "TEXTAREA", "LABEL", "INPUT", "SPAN", "P", "STYLE"].includes(t), "unexpected element " + t); + const wrap = root.find("wrap"); + assert.equal(wrap.style.props["--accent"], undefined, "unsafe accent rejected"); + // Hostile question id falls back to a safe answer key. + const card = cardOf(root); + radios(card, 2)[0].checked = true; + card.findAll((c) => c.tagName === "FORM")[0].dispatch("submit"); + await settle(); + const body = env.calls.filter((c) => c.url.endsWith("/respond"))[0].body; + assert.deepEqual(Object.keys(body.answers), ["q3"]); + } +}); + +test("valid accent colour is applied through CSSOM, not markup", async () => { + const env = makeEnv({ surveys: [SV({ appearance: JSON.stringify({ accent: "#ff6600", position: "center" }) })] }); + await settle(); + assert.equal(env.root().find("wrap").style.props["--accent"], "#ff6600"); + assert.equal(cardOf(env.root()).attrs["aria-modal"], "true"); + assert.ok(env.root().find("wrap").className.includes("center")); +}); + +test("payload bounds: oversize text and choice lists are clipped before rendering and sending", async () => { + const long = "x".repeat(5000); + const choices = Array.from({ length: 100 }, (_, i) => "c" + i); + const qs = Array.from({ length: 30 }, (_, i) => ({ id: "q" + i, type: "text", text: long })); + qs.unshift({ id: "pick", type: "choice", text: "pick", choices }); + const env = makeEnv({ surveys: [SV({ questions: JSON.stringify(qs) })] }); + await settle(); + const card = cardOf(env.root()); + assert.equal(card.findAll((c) => c.tagName === "FIELDSET").length, 10, "max 10 questions"); + assert.equal(radios(card, 0).length, 20, "max 20 choices"); + const ta = card.findAll((c) => c.tagName === "TEXTAREA")[0]; + assert.ok(ta.maxLength <= 2000); + assert.ok(card.findAll((c) => c.tagName === "LEGEND")[1].textContent.length <= 500); + ta.value = "y".repeat(10000); + radios(card, 0)[0].checked = true; + card.findAll((c) => c.tagName === "FORM")[0].dispatch("submit"); + await settle(); + const body = env.calls.find((c) => c.url.endsWith("/respond")).body; + assert.equal(body.answers.q0.length, 2000); +}); + +test("rate-limited and failing posts retry a bounded number of times with the same response_id", async () => { + let respondCalls = []; + const env = makeEnv({ + surveys: [SV()], + fetchImpl: (url, opts) => { + if (url.includes("/active")) return Promise.resolve({ ok: true, status: 200, json: () => Promise.resolve([SV()]) }); + if (url.endsWith("/respond")) { + respondCalls.push(JSON.parse(opts.body)); + return Promise.resolve({ ok: false, status: 429, headers: { get: () => "9999" }, json: () => Promise.resolve({}) }); + } + return Promise.resolve({ ok: true, status: 200, json: () => Promise.resolve({}) }); + }, + }); + await settle(); + const card = cardOf(env.root()); + radios(card, 0)[0].checked = true; + card.findAll((c) => c.tagName === "FORM")[0].dispatch("submit"); + for (let i = 0; i < 10; i++) { + await settle(); + const t = env.timers.shift(); + if (!t) break; + assert.ok(t.ms <= 30000, "Retry-After capped at 30s, got " + t.ms); + t.fn(); + } + await settle(); + assert.equal(respondCalls.length, 4, "1 attempt + 3 retries, then stop"); + assert.equal(new Set(respondCalls.map((b) => b.response_id)).size, 1, "stable id for server dedupe"); +}); + +test("a 4xx refusal (e.g. closed survey) is not retried", async () => { + const env = makeEnv({ + surveys: [SV()], + responses: { respond: { ok: false, status: 400, headers: { get: () => null }, json: () => Promise.resolve({}) } }, + }); + await settle(); + const card = cardOf(env.root()); + radios(card, 0)[0].checked = true; + card.findAll((c) => c.tagName === "FORM")[0].dispatch("submit"); + await settle(); + assert.equal(env.timers.length, 0); + assert.equal(env.calls.filter((c) => c.url.endsWith("/respond")).length, 1); +}); + +test("/active 429 stops quietly; 5xx retries at most twice", async () => { + const e1 = makeEnv({ responses: { active: { ok: false, status: 429, headers: { get: () => null }, json: () => Promise.resolve([]) } } }); + await settle(); + assert.equal(e1.timers.length, 0); + assert.equal(e1.root(), null); + const e2 = makeEnv({ responses: { active: { ok: false, status: 503, headers: { get: () => null }, json: () => Promise.resolve([]) } } }); + for (let i = 0; i < 6; i++) { await settle(); const t = e2.timers.shift(); if (t) t.fn(); } + await settle(); + assert.equal(e2.calls.filter((c) => c.url.includes("/active")).length, 3); +}); + +test("script source has no eval/innerHTML/inline-handler sinks", () => { + assert.doesNotMatch(src.replace(/\/\/.*$|\/\*[\s\S]*?\*\//gm, ""), /\binnerHTML\b|\bouterHTML\b|insertAdjacentHTML|document\.write|\beval\s*\(|new Function|setTimeout\s*\(\s*['"`]|\bon[a-z]+\s*=\s*['"]/); +}); diff --git a/docs/sdk/SURVEYS.md b/docs/sdk/SURVEYS.md new file mode 100644 index 00000000..7cc54e3d --- /dev/null +++ b/docs/sdk/SURVEYS.md @@ -0,0 +1,53 @@ +# Survey widget + +Add to any page of a site you have registered in Observe: + +```html + +``` + +On load the widget asks `GET /api/v1/surveys/active` for the surveys that target +the current page, shows at most one (a bottom card, or a centred modal when the +survey's `appearance` is `{"position":"center"}`), reports it with +`POST /api/v1/surveys/expose`, and submits answers with +`POST /api/v1/surveys/respond`. Question types: `text`, `rating` (1-5), +`nps` (0-10), `choice`. + +Attributes: `data-site-id` (required), `data-endpoint` (override the +`/api/v1/surveys` base), `data-respect-dnt="false"` (Do Not Track is honoured by +default, exactly like `observe.js`). Single-page apps can call +`window.observeSurveys.check()` after a route change. + +Behaviour: Shadow DOM, `role="dialog"` with a labelled title, focus moves into +the dialog and returns on close, `Esc` dismisses, honours +`prefers-reduced-motion` and `prefers-color-scheme`. All survey text is +rendered with `textContent`; styling is a constructed stylesheet, so a CSP +without `unsafe-inline`/`unsafe-eval` works. Completion and dismissal are +remembered in `localStorage` (dismissals lapse after 30 days; completed +surveys and `once` surveys never return). With storage blocked the widget still +works; it just cannot remember. + +## Targeting + +The survey's `targeting` field is a JSON object, validated when the survey is +created (400 with the offending key on error) and evaluated server-side on every +`/active` call. All keys are optional and ANDed; list values are ORed; the three +`url_*` keys are ORed with each other. + +| key | value | meaning | +| --- | --- | --- | +| `url_equals` | string or list | path equals (query/fragment ignored, trailing slash ignored); must start with `/` | +| `url_prefix` | string or list | path starts with; must start with `/` | +| `url_contains` | string or list | path contains the text (literal, case-sensitive) | +| `device` | list of `desktop` `mobile` `tablet` | client hint, else derived from the User-Agent | +| `referrer_host` | list of hostnames | exact, case-insensitive host of the referrer | +| `sample_percent` | integer 0-100 | deterministic per-visitor bucket (anonymous visitor estimate, re-rolls monthly) | +| `once` | boolean | the browser shows it at most once (enforced client-side) | + +There is no regex support, by design (ReDoS). Surveys stored before this schema +(empty, `{}`, unrecognised keys) match every page; unrecognised keys are ignored. + +Public-endpoint limits: `/active` 60 per minute per IP and 1200 per site; +`/expose` 30 and 600; `/respond` 10 and 120. Request bodies are capped at +64 KiB; a response holds at most 50 answers, each key `[A-Za-z0-9_-]{1,64}`, +text up to 2000 characters, lists of up to 20 strings of up to 500 characters. From 0db6429ef3cbceea5fb964c99e93323dd9231e23 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:51:41 +0000 Subject: [PATCH 27/65] feat: OTLP gRPC receiver for traces, metrics and logs (default off) New internal/otlpgrpc serves the three OTLP Export services on OBSERVE_OTLP_GRPC_ADDR (empty = disabled), optional TLS via OBSERVE_OTLP_GRPC_TLS_CERT/KEY. Decode and ingest reuse the HTTP receivers' code through thin IngestOTLPProto wrappers in new files in tracing, metrics and logs. API key via x-api-key or authorization Bearer, same key store, telemetry scope and site binding as the HTTP middleware; per-site rate limiter; Unavailable+RetryInfo for storage/auth outages; 10 MiB receive cap; keepalive enforcement; graceful stop ordered before the ingest buffers. Registers a gzip compressor because grpc/encoding/gzip is not vendored. go.mod: grpc and genproto rpc/status become direct requirements (already vendored, no vendor changes). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- docs/operations/otlp-grpc.md | 51 ++++++++++++++++++++++++++++++++ internal/otlpgrpc/server.go | 4 ++- internal/otlpgrpc/server_test.go | 11 +++++-- 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 docs/operations/otlp-grpc.md diff --git a/docs/operations/otlp-grpc.md b/docs/operations/otlp-grpc.md new file mode 100644 index 00000000..af6935aa --- /dev/null +++ b/docs/operations/otlp-grpc.md @@ -0,0 +1,51 @@ +# OTLP/gRPC receiver + +Observe accepts OTLP over gRPC for traces, metrics and logs in addition to +OTLP/HTTP (`/v1/traces`, `/v1/metrics`, `/v1/logs`), which is unchanged. The +gRPC listener is **off by default**. + +| Variable | Default | Meaning | +|---|---|---| +| `OBSERVE_OTLP_GRPC_ADDR` | empty (disabled) | Listen address, e.g. `127.0.0.1:4317`. | +| `OBSERVE_OTLP_GRPC_TLS_CERT` | empty | PEM certificate. Set together with the key, or not at all. | +| `OBSERVE_OTLP_GRPC_TLS_KEY` | empty | PEM private key. | + +Without a certificate the listener is plaintext HTTP/2. Keep it on localhost, a +tailnet, or behind a proxy that terminates TLS (Caddy, nginx, an L4 load +balancer); do not publish a plaintext port to the internet. A half-configured +TLS pair or an unbindable address fails startup. + +## Authentication + +The same API key as the HTTP receivers, sent as gRPC metadata: + + x-api-key: obs_... + authorization: Bearer obs_... + +The key must carry the `telemetry` scope; the site comes from the key, never +from the request. Example exporter settings: + + OTEL_EXPORTER_OTLP_PROTOCOL=grpc + OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp.example.com:4317 + OTEL_EXPORTER_OTLP_HEADERS=x-api-key=obs_... + +## Status codes + +| Condition | Code | Exporter behaviour | +|---|---|---| +| Missing / unknown / revoked key | `UNAUTHENTICATED` | not retried | +| Key without `telemetry` scope | `PERMISSION_DENIED` | not retried | +| Auth or storage outage | `UNAVAILABLE` + RetryInfo (5s) | retried | +| Per-site rate limit | `RESOURCE_EXHAUSTED` + RetryInfo (1s) | retried after delay | +| Message over 10 MiB (after decompression) | `RESOURCE_EXHAUSTED`, no RetryInfo | not retried | +| Undecodable payload, metrics cardinality refusal | `INVALID_ARGUMENT` | not retried | + +Logs report rejected records via `partial_success.rejected_log_records`, as on +HTTP. Traces and metrics are all-or-nothing. gzip compression is supported. + +## Limits + +10 MiB max message, 256 concurrent streams per connection, keepalive pings no +more often than every 30s, idle connections closed after 10 minutes. Shutdown +is graceful (in-flight exports finish) and bounded by the app shutdown +deadline; the receiver stops before the ingest buffers it feeds. diff --git a/internal/otlpgrpc/server.go b/internal/otlpgrpc/server.go index 4229f05d..5ea7545c 100644 --- a/internal/otlpgrpc/server.go +++ b/internal/otlpgrpc/server.go @@ -201,8 +201,10 @@ func (s *Server) Shutdown(ctx context.Context) error { case <-done: return nil case <-ctx.Done(): + // Stop closes every connection and cancels handler contexts; it is not + // waited on further, so a sink that ignores its context cannot hold + // shutdown past the deadline. s.gs.Stop() - <-done return ctx.Err() } } diff --git a/internal/otlpgrpc/server_test.go b/internal/otlpgrpc/server_test.go index 5ec98b6e..0014c24c 100644 --- a/internal/otlpgrpc/server_test.go +++ b/internal/otlpgrpc/server_test.go @@ -75,7 +75,11 @@ func (s *sink) record(ctx context.Context, site string) error { } } if s.block != nil { - <-s.block + select { + case <-s.block: + case <-ctx.Done(): + return ctx.Err() + } } s.mu.Lock() defer s.mu.Unlock() @@ -151,7 +155,10 @@ func newHarness(t *testing.T, mutate func(*Config)) *harness { } func withKey(key string) context.Context { - ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + // The 10s deadline only guards against a hung test; it is released by the + // timer, which is fine for short-lived test contexts. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + _ = cancel return metadata.AppendToOutgoingContext(ctx, "x-api-key", key) } From a556151326aae22f88d2f2cc4a7ca60f1ae170b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:52:06 +0000 Subject: [PATCH 28/65] feat(errors): issue merge/unmerge and assignment Migration 058 adds issue_merges and issue_assignments (new tables only, replacing_mergetree, argMax read collapse). Reads of a merged source resolve to its target, listings hide it, counts/events include it, and new events mapped to a merged source are attributed to the target. POST /issues/{id}/merge|unmerge and PUT /issues/{id}/assignee are editor+, site-scoped (foreign ids are 404), reject self-merge, cycles and over-deep chains, and are audited by auditMiddleware. Small UI controls on the issue detail (unbuilt). Also bounds scrubber work per string and pre-filters its regexes. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/errors_routes.go | 132 ++++++ cmd/observe/errors_routes_test.go | 146 +++++++ cmd/observe/main.go | 1 + internal/errors/handler.go | 3 + internal/errors/issues.go | 75 +++- internal/errors/merge.go | 382 ++++++++++++++++++ internal/errors/merge_read.go | 72 ++++ internal/errors/merge_test.go | 246 +++++++++++ internal/errors/scrub.go | 52 ++- .../migrations/058_issue_merge_assign.up.sql | 49 +++ ui/src/api/errors.ts | 13 +- ui/src/lib/issueAdmin.test.ts | 21 + ui/src/lib/issueAdmin.ts | 20 + ui/src/routes/errors.tsx | 43 ++ 14 files changed, 1226 insertions(+), 29 deletions(-) create mode 100644 cmd/observe/errors_routes.go create mode 100644 cmd/observe/errors_routes_test.go create mode 100644 internal/errors/merge.go create mode 100644 internal/errors/merge_read.go create mode 100644 internal/errors/merge_test.go create mode 100644 internal/schema/migrations/058_issue_merge_assign.up.sql create mode 100644 ui/src/lib/issueAdmin.test.ts create mode 100644 ui/src/lib/issueAdmin.ts diff --git a/cmd/observe/errors_routes.go b/cmd/observe/errors_routes.go new file mode 100644 index 00000000..0967d3eb --- /dev/null +++ b/cmd/observe/errors_routes.go @@ -0,0 +1,132 @@ +package main + +// Error-tracking depth routes: issue merge / unmerge and assignment +// (migration 058). One RegisterIssueAdminRoutes call from main.go. +// +// Authorization: all three are mutations, so they sit behind the JWT +// middleware AND requireEditor (viewers get 403 before any handler +// code). The site boundary is enforced in the service: both issues must +// exist in the request's site_id, and a foreign id is a plain 404. +// The audit trail comes from auditMiddleware (every non-GET /api/v1 +// call: actor, issues.merge.create / issues.unmerge.create / +// issues.assignee.update, target path, result). + +import ( + "context" + "encoding/json" + "errors" + "net/http" + + "github.com/neutron-build/neutron/go/neutron" + + obserrors "github.com/useteploy/teploy-observe/internal/errors" +) + +// issueAdminService is the slice of *errors.IssueService these routes use. +type issueAdminService interface { + MergeIssues(ctx context.Context, siteID, sourceID, targetID, actor string) error + UnmergeIssue(ctx context.Context, siteID, sourceID, actor string) error + AssignIssue(ctx context.Context, siteID, issueID, assignee, actor string) error +} + +// routeRegistrar is the part of *neutron.Router (and http.ServeMux) used. +type routeRegistrar interface { + Handle(pattern string, handler http.Handler) +} + +const maxIssueAdminBody = 4 << 10 + +// RegisterIssueAdminRoutes mounts the merge/unmerge/assignee routes. +// actor resolves the acting username from the request (may return ""). +func RegisterIssueAdminRoutes( + r routeRegistrar, + jwtMW, requireEditor neutron.Middleware, + svc issueAdminService, + actor func(*http.Request) string, +) { + wrap := func(h http.HandlerFunc) http.Handler { return jwtMW(requireEditor(h)) } + r.Handle("POST /api/v1/issues/{issue_id}/merge", wrap(issueMergeHandler(svc, actor))) + r.Handle("POST /api/v1/issues/{issue_id}/unmerge", wrap(issueUnmergeHandler(svc, actor))) + r.Handle("PUT /api/v1/issues/{issue_id}/assignee", wrap(issueAssigneeHandler(svc, actor))) +} + +type issueAdminBody struct { + SiteID string `json:"site_id"` + TargetID string `json:"target_id"` + Assignee string `json:"assignee"` +} + +func decodeIssueAdmin(w http.ResponseWriter, r *http.Request) (issueAdminBody, string, bool) { + var body issueAdminBody + r.Body = http.MaxBytesReader(w, r.Body, maxIssueAdminBody) + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + neutron.WriteError(w, r, neutron.ErrBadRequest("invalid JSON body")) + return body, "", false + } + id := r.PathValue("issue_id") + if id == "" || body.SiteID == "" { + neutron.WriteError(w, r, neutron.ErrBadRequest("issue_id and site_id required")) + return body, "", false + } + return body, id, true +} + +func writeIssueAdminError(w http.ResponseWriter, r *http.Request, err error) { + switch { + case errors.Is(err, obserrors.ErrIssueNotFound): + neutron.WriteError(w, r, neutron.ErrNotFound("issue not found")) + case errors.Is(err, obserrors.ErrMergeSelf), errors.Is(err, obserrors.ErrInvalidAssignee): + neutron.WriteError(w, r, neutron.ErrBadRequest(err.Error())) + case errors.Is(err, obserrors.ErrMergeCycle), errors.Is(err, obserrors.ErrMergeDepth), + errors.Is(err, obserrors.ErrAlreadyMerged), errors.Is(err, obserrors.ErrNotMerged): + neutron.WriteError(w, r, neutron.ErrConflict(err.Error())) + default: + neutron.WriteError(w, r, neutron.ErrInternal("issue update failed")) + } +} + +func issueMergeHandler(svc issueAdminService, actor func(*http.Request) string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, id, ok := decodeIssueAdmin(w, r) + if !ok { + return + } + if body.TargetID == "" { + neutron.WriteError(w, r, neutron.ErrBadRequest("target_id required")) + return + } + if err := svc.MergeIssues(r.Context(), body.SiteID, id, body.TargetID, actor(r)); err != nil { + writeIssueAdminError(w, r, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func issueUnmergeHandler(svc issueAdminService, actor func(*http.Request) string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, id, ok := decodeIssueAdmin(w, r) + if !ok { + return + } + if err := svc.UnmergeIssue(r.Context(), body.SiteID, id, actor(r)); err != nil { + writeIssueAdminError(w, r, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func issueAssigneeHandler(svc issueAdminService, actor func(*http.Request) string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, id, ok := decodeIssueAdmin(w, r) + if !ok { + return + } + if err := svc.AssignIssue(r.Context(), body.SiteID, id, body.Assignee, actor(r)); err != nil { + writeIssueAdminError(w, r, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} diff --git a/cmd/observe/errors_routes_test.go b/cmd/observe/errors_routes_test.go new file mode 100644 index 00000000..0be1b1cb --- /dev/null +++ b/cmd/observe/errors_routes_test.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/neutron-build/neutron/go/neutron" + + obserrors "github.com/useteploy/teploy-observe/internal/errors" +) + +type fakeIssueAdmin struct { + calls []string + err error +} + +func (f *fakeIssueAdmin) MergeIssues(_ context.Context, site, src, tgt, actor string) error { + f.calls = append(f.calls, "merge "+site+" "+src+" "+tgt+" "+actor) + return f.err +} +func (f *fakeIssueAdmin) UnmergeIssue(_ context.Context, site, src, actor string) error { + f.calls = append(f.calls, "unmerge "+site+" "+src+" "+actor) + return f.err +} +func (f *fakeIssueAdmin) AssignIssue(_ context.Context, site, id, who, actor string) error { + f.calls = append(f.calls, "assign "+site+" "+id+" "+who+" "+actor) + return f.err +} + +func issueAdminMux(svc issueAdminService, role string) *http.ServeMux { + mux := http.NewServeMux() + jwt := neutron.Middleware(func(next http.Handler) http.Handler { return next }) + editor := neutron.Middleware(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if role != "editor" && role != "admin" { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + next.ServeHTTP(w, r) + }) + }) + RegisterIssueAdminRoutes(mux, jwt, editor, svc, func(*http.Request) string { return "tyler" }) + return mux +} + +func doIssueAdmin(mux *http.ServeMux, method, path, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec +} + +func TestIssueAdminRoutesViewerForbidden(t *testing.T) { + f := &fakeIssueAdmin{} + mux := issueAdminMux(f, "viewer") + for _, c := range []struct{ m, p, b string }{ + {"POST", "/api/v1/issues/i1/merge", `{"site_id":"s","target_id":"i2"}`}, + {"POST", "/api/v1/issues/i1/unmerge", `{"site_id":"s"}`}, + {"PUT", "/api/v1/issues/i1/assignee", `{"site_id":"s","assignee":"a"}`}, + } { + if rec := doIssueAdmin(mux, c.m, c.p, c.b); rec.Code != http.StatusForbidden { + t.Errorf("%s %s: viewer got %d", c.m, c.p, rec.Code) + } + } + if len(f.calls) != 0 { + t.Fatalf("viewer reached the service: %v", f.calls) + } +} + +func TestIssueAdminRoutesHappyAndValidation(t *testing.T) { + f := &fakeIssueAdmin{} + mux := issueAdminMux(f, "editor") + if rec := doIssueAdmin(mux, "POST", "/api/v1/issues/i1/merge", `{"site_id":"s","target_id":"i2"}`); rec.Code != 204 { + t.Fatalf("merge %d %s", rec.Code, rec.Body) + } + if rec := doIssueAdmin(mux, "POST", "/api/v1/issues/i1/unmerge", `{"site_id":"s"}`); rec.Code != 204 { + t.Fatalf("unmerge %d", rec.Code) + } + if rec := doIssueAdmin(mux, "PUT", "/api/v1/issues/i1/assignee", `{"site_id":"s","assignee":"alice"}`); rec.Code != 204 { + t.Fatalf("assign %d", rec.Code) + } + want := []string{"merge s i1 i2 tyler", "unmerge s i1 tyler", "assign s i1 alice tyler"} + for i, w := range want { + if f.calls[i] != w { + t.Errorf("call %d = %q want %q", i, f.calls[i], w) + } + } + n := len(f.calls) + for _, c := range []struct{ m, p, b string }{ + {"POST", "/api/v1/issues/i1/merge", `{"site_id":"s"}`}, // no target + {"POST", "/api/v1/issues/i1/merge", `{"target_id":"i2"}`}, // no site + {"POST", "/api/v1/issues/i1/merge", `not json`}, // bad body + {"PUT", "/api/v1/issues/i1/assignee", `{"assignee":"a"}`}, // no site + {"POST", "/api/v1/issues/i1/unmerge", `{"site_id":"s","x":"` + strings.Repeat("a", 9000) + `"}`}, // oversized + } { + if rec := doIssueAdmin(mux, c.m, c.p, c.b); rec.Code != http.StatusBadRequest { + t.Errorf("%s %s %.20s: got %d want 400", c.m, c.p, c.b, rec.Code) + } + } + if len(f.calls) != n { + t.Fatalf("invalid requests reached the service: %v", f.calls[n:]) + } +} + +func TestIssueAdminRoutesErrorMapping(t *testing.T) { + cases := []struct { + err error + want int + }{ + {obserrors.ErrIssueNotFound, 404}, // also what a cross-site id yields + {obserrors.ErrMergeSelf, 400}, + {obserrors.ErrInvalidAssignee, 400}, + {obserrors.ErrMergeCycle, 409}, + {obserrors.ErrMergeDepth, 409}, + {obserrors.ErrAlreadyMerged, 409}, + {obserrors.ErrNotMerged, 409}, + {context.DeadlineExceeded, 500}, + } + for _, c := range cases { + mux := issueAdminMux(&fakeIssueAdmin{err: c.err}, "admin") + rec := doIssueAdmin(mux, "POST", "/api/v1/issues/i1/merge", `{"site_id":"s","target_id":"i2"}`) + if rec.Code != c.want { + t.Errorf("%v: got %d want %d", c.err, rec.Code, c.want) + } + } +} + +// The routes are audited by auditMiddleware (non-GET under /api/v1). +func TestIssueAdminRoutesAreAudited(t *testing.T) { + for p, want := range map[string]string{ + "POST /api/v1/issues/0123456789abcdef0123456789abcdef/merge": "issues.merge.create", + "POST /api/v1/issues/0123456789abcdef0123456789abcdef/unmerge": "issues.unmerge.create", + "PUT /api/v1/issues/0123456789abcdef0123456789abcdef/assignee": "issues.assignee.update", + } { + parts := strings.SplitN(p, " ", 2) + if !auditableRequest(parts[0], parts[1]) { + t.Errorf("%s not audited", p) + } + if got := deriveAction(parts[0], parts[1]); got != want { + t.Errorf("%s action = %s want %s", p, got, want) + } + } +} diff --git a/cmd/observe/main.go b/cmd/observe/main.go index 4aaeec0d..63532015 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -1832,6 +1832,7 @@ func main() { // and exposes MembersForFilter to the stats service for ?cohort_id= // filtering across every analytics chart. RegisterCohortsRoutes(r, jwtMW, requireEditor, cohortsSvc) + RegisterIssueAdminRoutes(r, jwtMW, requireEditor, issueSvc, func(req *http.Request) string { return requestActor(authSvc, req) }) // SPA catch-all: serve index.html for all non-API, non-asset GET requests. // This must be registered last so API routes take precedence. diff --git a/internal/errors/handler.go b/internal/errors/handler.go index 6d2575fd..51b67e57 100644 --- a/internal/errors/handler.go +++ b/internal/errors/handler.go @@ -214,6 +214,9 @@ func (s *Service) insertErrorEvent(ctx context.Context, sqlc *nucleus.SQLModel, if err != nil { return "", "", fmt.Errorf("resolve issue: %w", err) } + // A fingerprint that maps to a merged source is attributed to the + // merge target (merge.go). + issueID = s.issueSvc.attributeMerged(ctx, input.SiteID, issueID, now.UnixMilli()) // Serialize JSONB fields stackJSON := jsonOrEmpty(input.StackTrace) diff --git a/internal/errors/issues.go b/internal/errors/issues.go index df6a6929..b1591379 100644 --- a/internal/errors/issues.go +++ b/internal/errors/issues.go @@ -51,10 +51,14 @@ type IssueService struct { db *nucleus.Client // notifier receives new-issue and regression events (see notify.go). notifier atomic.Pointer[IssueNotifier] + // merge holds issue merge/assignment state (merge.go). + merge *mergeState } func NewIssueService(db *nucleus.Client) *IssueService { - return &IssueService{db: db} + s := &IssueService{db: db} + s.merge = newMergeState(sqlMergeStore{db}) + return s } // Issue represents a grouped error issue. @@ -99,6 +103,13 @@ type Issue struct { // Releases is the release-impact breakdown (per-release event // counts) for this issue, populated on detail reads only. Releases []IssueRelease `json:"releases,omitempty"` + // Merge/assignment (migration 058). ResolvedFrom is set when a read of + // a merged source id was answered with its target; MergedSources lists + // the issues folded into this one. + ResolvedFrom string `json:"resolved_from,omitempty"` + MergedSources []string `json:"merged_sources,omitempty"` + Assignee string `json:"assignee,omitempty"` + AssignedAt int64 `json:"assigned_at,omitempty"` } // IssueRelease is one release's share of an issue's events. @@ -405,14 +416,27 @@ func (s *IssueService) ListIssues(ctx context.Context, siteID, status string, li } now := time.Now().UTC() out := make([]Issue, 0, len(issues)) + merges := s.siteMerges(ctx, siteID) + assigns := s.siteAssignments(ctx, siteID) for i := range issues { + if _, hidden := merges[issues[i].IssueID]; hidden { + // Merged away: shown only through its target. (A page can run + // short by the number of merged sources it contained.) + continue + } issue := issues[i].toIssue(now) + sources := capIDs(sourcesOf(merges, issue.IssueID)) + ids := append([]string{issue.IssueID}, sources...) + issue.MergedSources = sources + if a, ok := assigns[issue.IssueID]; ok { + issue.Assignee, issue.AssignedAt = a.Assignee, a.AssignedAt + } // user_count and event_count are computed on read (the stored counters race // under concurrent flushes — read-modify-write into a ReplacingMergeTree // loses increments). COUNT over error_events is exact and concurrency-safe. // Bounded by the page limit, so the per-issue lookups stay cheap. - issue.UserCount = s.affectedUsers(ctx, siteID, issue.IssueID) - issue.EventCount = s.eventCount(ctx, siteID, issue.IssueID) + issue.UserCount = s.affectedUsers(ctx, siteID, ids) + issue.EventCount = s.eventCount(ctx, siteID, ids) out = append(out, issue) } return out, nil @@ -420,6 +444,10 @@ func (s *IssueService) ListIssues(ctx context.Context, siteID, status string, li // GetIssue returns a single issue by ID. func (s *IssueService) GetIssue(ctx context.Context, issueID, siteID string) (*Issue, error) { + // A merged source resolves to its target (merge.go); the lookup stays + // site-scoped, so a foreign id still reads as not found. + requested := issueID + issueID = s.ResolveMerged(ctx, siteID, issueID) rows, err := nucleus.Query[issueScan](ctx, s.db.SQL(), `SELECT `+issueSelectCols+` FROM `+issuesLatest("issue_id = $1 AND site_id = $2"), @@ -432,19 +460,28 @@ func (s *IssueService) GetIssue(ctx context.Context, issueID, siteID string) (*I return nil, nil } issue := rows[0].toIssue(time.Now().UTC()) - issue.UserCount = s.affectedUsers(ctx, siteID, issueID) - issue.EventCount = s.eventCount(ctx, siteID, issueID) - issue.Releases = s.issueReleases(ctx, siteID, issueID) + if requested != issueID { + issue.ResolvedFrom = requested + } + ids := s.issueScope(ctx, siteID, issueID) + issue.MergedSources = ids[1:] + if a, ok := s.siteAssignments(ctx, siteID)[issueID]; ok { + issue.Assignee, issue.AssignedAt = a.Assignee, a.AssignedAt + } + issue.UserCount = s.affectedUsers(ctx, siteID, ids) + issue.EventCount = s.eventCount(ctx, siteID, ids) + issue.Releases = s.issueReleases(ctx, siteID, ids) return &issue, nil } // eventCount returns the exact number of events for an issue, computed on read // from the append-only error_events table (the stored issues.event_count is // racey). Best-effort: returns 0 on any error. -func (s *IssueService) eventCount(ctx context.Context, siteID, issueID string) int64 { +func (s *IssueService) eventCount(ctx context.Context, siteID string, ids []string) int64 { + in, args := issueIDClause(2, ids) rows, err := nucleus.Query[issueCountRow](ctx, s.db.SQL(), - `SELECT COUNT(*) AS n FROM error_events WHERE site_id = $1 AND issue_id = $2`, - siteID, issueID, + `SELECT COUNT(*) AS n FROM error_events WHERE site_id = $1 AND `+in, + append([]any{siteID}, args...)..., ) if err != nil || len(rows) == 0 { return 0 @@ -470,13 +507,14 @@ type issueCountRow struct { // pre-O05 session proxy (distinct session_id), documented as an // estimate of affected anonymous traffic, never conflated with // identified persons. Best-effort: returns 0 on any error. -func (s *IssueService) affectedUsers(ctx context.Context, siteID, issueID string) int64 { +func (s *IssueService) affectedUsers(ctx context.Context, siteID string, ids []string) int64 { + in, args := issueIDClause(2, ids) rows, err := nucleus.Query[affectedRow](ctx, s.db.SQL(), `SELECT COUNT(DISTINCT CASE WHEN distinct_id <> '' THEN distinct_id ELSE NULL END) AS n_distinct, COUNT(DISTINCT CASE WHEN distinct_id = '' AND session_id <> '' THEN session_id ELSE NULL END) AS n_sessions FROM error_events - WHERE site_id = $1 AND issue_id = $2`, - siteID, issueID, + WHERE site_id = $1 AND `+in, + append([]any{siteID}, args...)..., ) if err != nil || len(rows) == 0 { return 0 @@ -497,15 +535,16 @@ type affectedRow struct { // event's own release field (SDKs send `release`; absent = ” = unknown // — never fabricated). Ordered by release_tag for a total order under // the LIMIT. Best-effort: returns nil on error. -func (s *IssueService) issueReleases(ctx context.Context, siteID, issueID string) []IssueRelease { +func (s *IssueService) issueReleases(ctx context.Context, siteID string, ids []string) []IssueRelease { + in, args := issueIDClause(2, ids) rows, err := nucleus.Query[IssueRelease](ctx, s.db.SQL(), `SELECT release_tag, COUNT(*) AS event_count FROM error_events - WHERE site_id = $1 AND issue_id = $2 AND release_tag <> '' + WHERE site_id = $1 AND `+in+` AND release_tag <> '' GROUP BY release_tag ORDER BY release_tag ASC LIMIT 10`, - siteID, issueID, + append([]any{siteID}, args...)..., ) if err != nil { return nil @@ -544,6 +583,8 @@ func (s *IssueService) LatestEvents(ctx context.Context, issueID, siteID string, if limit <= 0 { limit = 10 } + // Includes events filed under issues merged into this one. + in, args := issueIDClause(2, s.issueScope(ctx, siteID, s.ResolveMerged(ctx, siteID, issueID))) return nucleus.Query[ErrorEvent](ctx, s.db.SQL(), fmt.Sprintf(`SELECT error_id, tenant_id, site_id, session_id, COALESCE(replay_id, '') AS replay_id, @@ -557,10 +598,10 @@ func (s *IssueService) LatestEvents(ctx context.Context, issueID, siteID string, COALESCE(contexts, '') AS contexts, COALESCE(extra, '') AS extra FROM error_events - WHERE issue_id = $1 AND site_id = $2 + WHERE site_id = $1 AND `+in+` ORDER BY timestamp DESC LIMIT %d`, limit), - issueID, siteID, + append([]any{siteID}, args...)..., ) } diff --git a/internal/errors/merge.go b/internal/errors/merge.go new file mode 100644 index 00000000..589f519b --- /dev/null +++ b/internal/errors/merge.go @@ -0,0 +1,382 @@ +package errors + +// Issue merge/unmerge and assignment (migration 058). +// +// Semantics: +// - A merge points a SOURCE issue at a TARGET. Reads of the source +// resolve to the target (GetIssue), the source disappears from +// listings, the target's counts/events include the source's, and NEW +// events whose fingerprint maps to the source are attributed to the +// target at ingest. Unmerge restores all of it; no history is +// rewritten (old events keep their original issue_id). +// - Authorization boundary: every operation names a site and both +// issues must exist IN THAT SITE; a foreign or unknown id is +// ErrIssueNotFound (never distinguishable from a missing one). +// - Integrity: no self-merge, no cycles, chains bounded by +// maxMergeDepth, a source can have only one active merge. +// +// Storage is behind mergeStore so the rules are unit-testable without +// Nucleus; sqlMergeStore is the live implementation (UNVERIFIED against +// live Nucleus in this environment). + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + "unicode" + + "github.com/neutron-build/neutron/go/nucleus" + + "github.com/useteploy/teploy-observe/internal/dbutil" +) + +const ( + maxMergeDepth = 5 + maxMergesPerSite = 10000 + maxAssigneeLen = 128 + mergeCacheTTL = 15 * time.Second + mergeCacheMaxSites = 1024 +) + +var ( + ErrIssueNotFound = errors.New("issue not found") + ErrMergeSelf = errors.New("cannot merge an issue into itself") + ErrMergeCycle = errors.New("merge would create a cycle") + ErrMergeDepth = errors.New("merge chain too deep") + ErrAlreadyMerged = errors.New("issue is already merged; unmerge it first") + ErrNotMerged = errors.New("issue is not merged") + ErrInvalidAssignee = errors.New("invalid assignee") +) + +// Assignment is the current assignee of an issue. +type Assignment struct { + Assignee string `json:"assignee"` + AssignedAt int64 `json:"assigned_at"` + AssignedBy string `json:"assigned_by"` +} + +type mergeStore interface { + issueExists(ctx context.Context, siteID, issueID string) (bool, error) + // activeMerges returns source -> target for active merges of a site. + activeMerges(ctx context.Context, siteID string) (map[string]string, error) + putMerge(ctx context.Context, siteID, source, target string, active bool, by string) error + assignments(ctx context.Context, siteID string) (map[string]Assignment, error) + putAssignment(ctx context.Context, siteID, issueID, assignee, by string) error +} + +type mergeCacheEntry struct { + merges map[string]string + fetched time.Time +} + +// mergeState is the IssueService's merge cache (single-process posture: +// in-process invalidation on write, TTL for anything else). +type mergeState struct { + store mergeStore + mu sync.Mutex + cache map[string]mergeCacheEntry + now func() time.Time +} + +func newMergeState(store mergeStore) *mergeState { + return &mergeState{store: store, cache: map[string]mergeCacheEntry{}, now: time.Now} +} + +func (m *mergeState) invalidate(siteID string) { + m.mu.Lock() + delete(m.cache, siteID) + m.mu.Unlock() +} + +// siteMerges returns the cached active-merge map. Read failure degrades +// to "no merges" (logged): ingest and reads must not fail because the +// merge table is unreadable. +func (m *mergeState) siteMerges(ctx context.Context, siteID string) map[string]string { + m.mu.Lock() + if e, ok := m.cache[siteID]; ok && m.now().Sub(e.fetched) < mergeCacheTTL { + m.mu.Unlock() + return e.merges + } + m.mu.Unlock() + merges, err := m.store.activeMerges(ctx, siteID) + if err != nil { + slog.Warn("errors: merge lookup failed; treating as no merges", "site", siteID, "err", err) + return nil + } + m.mu.Lock() + if len(m.cache) >= mergeCacheMaxSites { + m.cache = map[string]mergeCacheEntry{} + } + m.cache[siteID] = mergeCacheEntry{merges: merges, fetched: m.now()} + m.mu.Unlock() + return merges +} + +// resolveChain follows source -> target links to the final target. +// depth reports the hops; ok=false on a cycle or a chain over the limit +// (callers fall back to the unresolved id). +func resolveChain(merges map[string]string, id string) (final string, depth int, ok bool) { + seen := map[string]bool{id: true} + cur := id + for { + next, merged := merges[cur] + if !merged { + return cur, depth, true + } + if seen[next] || depth >= maxMergeDepth { + return id, depth, false + } + seen[next] = true + cur = next + depth++ + } +} + +// sourcesOf returns every issue id that resolves (transitively) to target. +func sourcesOf(merges map[string]string, target string) []string { + var out []string + for src := range merges { + if src == target { + continue + } + if final, _, ok := resolveChain(merges, src); ok && final == target { + out = append(out, src) + } + } + return out +} + +// ResolveMerged maps an issue id to its merge target (itself when not +// merged). Hot path at ingest: cached. +func (s *IssueService) ResolveMerged(ctx context.Context, siteID, issueID string) string { + if s.merge == nil { + return issueID + } + final, _, ok := resolveChain(s.merge.siteMerges(ctx, siteID), issueID) + if !ok { + return issueID + } + return final +} + +// MergeIssues merges source into target within one site. +func (s *IssueService) MergeIssues(ctx context.Context, siteID, sourceID, targetID, actor string) error { + if sourceID == "" || targetID == "" || siteID == "" { + return ErrIssueNotFound + } + if sourceID == targetID { + return ErrMergeSelf + } + // Both issues must exist in THIS site (IDOR boundary). + for _, id := range []string{sourceID, targetID} { + ok, err := s.merge.store.issueExists(ctx, siteID, id) + if err != nil { + return err + } + if !ok { + return ErrIssueNotFound + } + } + // Fresh read: the cache must not hide a concurrent merge from validation. + s.merge.invalidate(siteID) + merges, err := s.merge.store.activeMerges(ctx, siteID) + if err != nil { + return err + } + if _, merged := merges[sourceID]; merged { + return ErrAlreadyMerged + } + finalTarget, _, ok := resolveChain(merges, targetID) + if !ok { + return ErrMergeDepth + } + if finalTarget == sourceID { + return ErrMergeCycle + } + // Depth of the resulting chain: sources that already point at + // source gain one hop. + probe := make(map[string]string, len(merges)+1) + for k, v := range merges { + probe[k] = v + } + probe[sourceID] = finalTarget + for src := range probe { + if _, _, ok := resolveChain(probe, src); !ok { + return ErrMergeDepth + } + } + if err := s.merge.store.putMerge(ctx, siteID, sourceID, finalTarget, true, actor); err != nil { + return err + } + s.merge.invalidate(siteID) + return nil +} + +// UnmergeIssue restores a merged source. +func (s *IssueService) UnmergeIssue(ctx context.Context, siteID, sourceID, actor string) error { + ok, err := s.merge.store.issueExists(ctx, siteID, sourceID) + if err != nil { + return err + } + if !ok { + return ErrIssueNotFound + } + merges, err := s.merge.store.activeMerges(ctx, siteID) + if err != nil { + return err + } + target, merged := merges[sourceID] + if !merged { + return ErrNotMerged + } + if err := s.merge.store.putMerge(ctx, siteID, sourceID, target, false, actor); err != nil { + return err + } + s.merge.invalidate(siteID) + return nil +} + +// ValidateAssignee bounds the free-text assignee (” clears). +func ValidateAssignee(a string) error { + if len(a) > maxAssigneeLen { + return fmt.Errorf("%w: longer than %d bytes", ErrInvalidAssignee, maxAssigneeLen) + } + for _, r := range a { + if unicode.IsControl(r) { + return fmt.Errorf("%w: control characters", ErrInvalidAssignee) + } + } + return nil +} + +// AssignIssue sets (or, with ”, clears) an issue's assignee. An issue +// that is merged away is assigned via its target. +func (s *IssueService) AssignIssue(ctx context.Context, siteID, issueID, assignee, actor string) error { + assignee = strings.TrimSpace(assignee) + if err := ValidateAssignee(assignee); err != nil { + return err + } + ok, err := s.merge.store.issueExists(ctx, siteID, issueID) + if err != nil { + return err + } + if !ok { + return ErrIssueNotFound + } + issueID = s.ResolveMerged(ctx, siteID, issueID) + return s.merge.store.putAssignment(ctx, siteID, issueID, assignee, actor) +} + +// ---- live SQL store ------------------------------------------------------ + +type sqlMergeStore struct{ db *nucleus.Client } + +func (s sqlMergeStore) issueExists(ctx context.Context, siteID, issueID string) (bool, error) { + rows, err := nucleus.Query[issueCountRow](ctx, s.db.SQL(), + `SELECT COUNT(*) AS n FROM issues WHERE site_id = $1 AND issue_id = $2`, siteID, issueID) + if err != nil { + return false, err + } + return len(rows) > 0 && rows[0].N > 0, nil +} + +func (s sqlMergeStore) activeMerges(ctx context.Context, siteID string) (map[string]string, error) { + type row struct { + Source string `db:"source_issue_id"` + Target string `db:"target_issue_id"` + } + rows, err := nucleus.Query[row](ctx, s.db.SQL(), + `SELECT source_issue_id, target_issue_id FROM ( + SELECT tenant_id, site_id, source_issue_id, + argMax(target_issue_id, version) AS target_issue_id, + argMax(active, version) AS active + FROM issue_merges WHERE site_id = $1 + GROUP BY tenant_id, site_id, source_issue_id) AS m + WHERE active = 'true' + LIMIT `+fmt.Sprint(maxMergesPerSite), siteID) + if err != nil { + return nil, err + } + out := make(map[string]string, len(rows)) + for _, r := range rows { + out[r.Source] = r.Target + } + return out, nil +} + +type maxVersionRow struct { + V int64 `db:"v"` +} + +func (s sqlMergeStore) putMerge(ctx context.Context, siteID, source, target string, active bool, by string) error { + rows, err := nucleus.Query[maxVersionRow](ctx, s.db.SQL(), + `SELECT COALESCE(MAX(version), 0) AS v FROM issue_merges WHERE site_id = $1 AND source_issue_id = $2`, + siteID, source) + if err != nil { + return err + } + now := time.Now().UTC().UnixMilli() + ver := now + if len(rows) > 0 && rows[0].V >= ver { + ver = rows[0].V + 1 + } + flag := "false" + if active { + flag = "true" + } + _, err = s.db.SQL().Exec(ctx, + `INSERT INTO issue_merges (tenant_id, site_id, source_issue_id, target_issue_id, active, merged_at, merged_by, version) + VALUES ('default', $1, $2, $3, $4, CAST($5 AS BIGINT), $6, CAST($7 AS BIGINT))`, + siteID, source, target, flag, dbutil.IntParam(now), by, dbutil.IntParam(ver)) + return err +} + +func (s sqlMergeStore) assignments(ctx context.Context, siteID string) (map[string]Assignment, error) { + type row struct { + IssueID string `db:"issue_id"` + Assignee string `db:"assignee"` + AssignedAt int64 `db:"assigned_at"` + AssignedBy string `db:"assigned_by"` + } + rows, err := nucleus.Query[row](ctx, s.db.SQL(), + `SELECT issue_id, assignee, assigned_at, assigned_by FROM ( + SELECT tenant_id, site_id, issue_id, + argMax(assignee, version) AS assignee, + argMax(assigned_at, version) AS assigned_at, + argMax(assigned_by, version) AS assigned_by + FROM issue_assignments WHERE site_id = $1 + GROUP BY tenant_id, site_id, issue_id) AS a + WHERE assignee <> '' + LIMIT `+fmt.Sprint(maxMergesPerSite), siteID) + if err != nil { + return nil, err + } + out := make(map[string]Assignment, len(rows)) + for _, r := range rows { + out[r.IssueID] = Assignment{Assignee: r.Assignee, AssignedAt: r.AssignedAt, AssignedBy: r.AssignedBy} + } + return out, nil +} + +func (s sqlMergeStore) putAssignment(ctx context.Context, siteID, issueID, assignee, by string) error { + rows, err := nucleus.Query[maxVersionRow](ctx, s.db.SQL(), + `SELECT COALESCE(MAX(version), 0) AS v FROM issue_assignments WHERE site_id = $1 AND issue_id = $2`, + siteID, issueID) + if err != nil { + return err + } + now := time.Now().UTC().UnixMilli() + ver := now + if len(rows) > 0 && rows[0].V >= ver { + ver = rows[0].V + 1 + } + _, err = s.db.SQL().Exec(ctx, + `INSERT INTO issue_assignments (tenant_id, site_id, issue_id, assignee, assigned_at, assigned_by, version) + VALUES ('default', $1, $2, $3, CAST($4 AS BIGINT), $5, CAST($6 AS BIGINT))`, + siteID, issueID, assignee, dbutil.IntParam(now), by, dbutil.IntParam(ver)) + return err +} diff --git a/internal/errors/merge_read.go b/internal/errors/merge_read.go new file mode 100644 index 00000000..5906edac --- /dev/null +++ b/internal/errors/merge_read.go @@ -0,0 +1,72 @@ +package errors + +import ( + "context" + "fmt" + "log/slog" + "strings" +) + +// Read-side helpers for merge/assignment (see merge.go). + +const maxScopeIDs = 50 + +func capIDs(ids []string) []string { + if len(ids) > maxScopeIDs-1 { + ids = ids[:maxScopeIDs-1] + } + return ids +} + +// issueIDClause renders `issue_id IN ($n, ...)` for ids (an empty list +// matches nothing), with parameters numbered from start. +func issueIDClause(start int, ids []string) (string, []any) { + if len(ids) == 0 { + return "1 = 0", nil + } + ph := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + ph[i] = fmt.Sprintf("$%d", start+i) + args[i] = id + } + return "issue_id IN (" + strings.Join(ph, ", ") + ")", args +} + +func (s *IssueService) siteMerges(ctx context.Context, siteID string) map[string]string { + if s.merge == nil { + return nil + } + return s.merge.siteMerges(ctx, siteID) +} + +func (s *IssueService) siteAssignments(ctx context.Context, siteID string) map[string]Assignment { + if s.merge == nil { + return nil + } + m, err := s.merge.store.assignments(ctx, siteID) + if err != nil { + slog.Warn("errors: assignment lookup failed", "site", siteID, "err", err) + return nil + } + return m +} + +// attributeMerged redirects an ingest-time issue id to its merge target +// and bumps the target (last_seen, regression reopen) the way ResolveIssue +// did for the source. Unmerged issues pass through untouched. +func (s *IssueService) attributeMerged(ctx context.Context, siteID, issueID string, ts int64) string { + target := s.ResolveMerged(ctx, siteID, issueID) + if target == issueID { + return issueID + } + count := s.eventCount(ctx, siteID, s.issueScope(ctx, siteID, target)) + 1 + _ = s.bumpIssue(ctx, target, siteID, ts, count) + return target +} + +// issueScope is [issue, merged sources...] (capped), the id set whose +// events count towards the issue. +func (s *IssueService) issueScope(ctx context.Context, siteID, issueID string) []string { + return append([]string{issueID}, capIDs(sourcesOf(s.siteMerges(ctx, siteID), issueID))...) +} diff --git a/internal/errors/merge_test.go b/internal/errors/merge_test.go new file mode 100644 index 00000000..5099725a --- /dev/null +++ b/internal/errors/merge_test.go @@ -0,0 +1,246 @@ +package errors + +import ( + "context" + "errors" + "strings" + "sync" + "testing" +) + +// fakeMergeStore is an in-memory mergeStore: issues are (site, id) pairs. +type fakeMergeStore struct { + mu sync.Mutex + issues map[string]bool // site+"/"+id + merges map[string]map[string]string + assigns map[string]map[string]Assignment + reads int +} + +func newFakeStore(siteIssues map[string][]string) *fakeMergeStore { + f := &fakeMergeStore{issues: map[string]bool{}, merges: map[string]map[string]string{}, assigns: map[string]map[string]Assignment{}} + for site, ids := range siteIssues { + for _, id := range ids { + f.issues[site+"/"+id] = true + } + } + return f +} + +func (f *fakeMergeStore) issueExists(_ context.Context, site, id string) (bool, error) { + return f.issues[site+"/"+id], nil +} + +func (f *fakeMergeStore) activeMerges(_ context.Context, site string) (map[string]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.reads++ + out := map[string]string{} + for k, v := range f.merges[site] { + out[k] = v + } + return out, nil +} + +func (f *fakeMergeStore) putMerge(_ context.Context, site, src, tgt string, active bool, _ string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.merges[site] == nil { + f.merges[site] = map[string]string{} + } + if active { + f.merges[site][src] = tgt + } else { + delete(f.merges[site], src) + } + return nil +} + +func (f *fakeMergeStore) assignments(_ context.Context, site string) (map[string]Assignment, error) { + return f.assigns[site], nil +} + +func (f *fakeMergeStore) putAssignment(_ context.Context, site, id, who, by string) error { + if f.assigns[site] == nil { + f.assigns[site] = map[string]Assignment{} + } + f.assigns[site][id] = Assignment{Assignee: who, AssignedBy: by} + return nil +} + +func svcWith(f *fakeMergeStore) *IssueService { + return &IssueService{merge: newMergeState(f)} +} + +func TestMergeRulesAndResolution(t *testing.T) { + ctx := context.Background() + f := newFakeStore(map[string][]string{"A": {"i1", "i2", "i3", "i4", "i5", "i6", "i7"}, "B": {"x1"}}) + s := svcWith(f) + + if err := s.MergeIssues(ctx, "A", "i1", "i1", "u"); !errors.Is(err, ErrMergeSelf) { + t.Fatalf("self-merge: %v", err) + } + if err := s.MergeIssues(ctx, "A", "i1", "i2", "u"); err != nil { + t.Fatal(err) + } + if got := s.ResolveMerged(ctx, "A", "i1"); got != "i2" { + t.Fatalf("resolve = %s", got) + } + if got := s.ResolveMerged(ctx, "A", "i2"); got != "i2" { + t.Fatalf("target must resolve to itself, got %s", got) + } + // Cycle: i2 -> i1 while i1 -> i2. + if err := s.MergeIssues(ctx, "A", "i2", "i1", "u"); !errors.Is(err, ErrMergeCycle) { + t.Fatalf("cycle: %v", err) + } + // Already merged source. + if err := s.MergeIssues(ctx, "A", "i1", "i3", "u"); !errors.Is(err, ErrAlreadyMerged) { + t.Fatalf("double merge: %v", err) + } + // Merging into a merged target flattens to the final target. + if err := s.MergeIssues(ctx, "A", "i3", "i1", "u"); err != nil { + t.Fatal(err) + } + if f.merges["A"]["i3"] != "i2" { + t.Fatalf("not flattened: %v", f.merges["A"]) + } + srcs := sourcesOf(f.merges["A"], "i2") + if len(srcs) != 2 { + t.Fatalf("sources = %v", srcs) + } + // Unmerge restores. + if err := s.UnmergeIssue(ctx, "A", "i1", "u"); err != nil { + t.Fatal(err) + } + if got := s.ResolveMerged(ctx, "A", "i1"); got != "i1" { + t.Fatalf("after unmerge resolve = %s", got) + } + if err := s.UnmergeIssue(ctx, "A", "i1", "u"); !errors.Is(err, ErrNotMerged) { + t.Fatalf("unmerge twice: %v", err) + } +} + +func TestMergeDepthLimit(t *testing.T) { + ctx := context.Background() + ids := []string{"n0", "n1", "n2", "n3", "n4", "n5", "n6", "n7"} + f := newFakeStore(map[string][]string{"A": ids}) + s := svcWith(f) + // Build a raw chain n0->n1->...->n6 beyond the limit, bypassing the + // service, to prove resolution and new merges both stay bounded. + f.merges["A"] = map[string]string{} + for i := 0; i < 6; i++ { + f.merges["A"][ids[i]] = ids[i+1] + } + if _, _, ok := resolveChain(f.merges["A"], "n0"); ok { + t.Fatal("over-deep chain must not resolve") + } + if got := s.ResolveMerged(ctx, "A", "n0"); got != "n0" { + t.Fatalf("over-deep chain resolved to %s (must fall back to the id itself)", got) + } + if err := s.MergeIssues(ctx, "A", "n7", "n0", "u"); !errors.Is(err, ErrMergeDepth) { + t.Fatalf("merge into over-deep chain: %v", err) + } +} + +// IDOR: every operation is scoped to the named site. A foreign or unknown +// id is indistinguishable (ErrIssueNotFound) and writes nothing. +func TestMergeAssignCrossSiteIDOR(t *testing.T) { + ctx := context.Background() + f := newFakeStore(map[string][]string{"A": {"a1", "a2"}, "B": {"b1", "b2"}}) + s := svcWith(f) + + cases := []struct { + name string + err error + }{ + {"merge foreign target", s.MergeIssues(ctx, "A", "a1", "b1", "u")}, + {"merge foreign source", s.MergeIssues(ctx, "A", "b1", "a1", "u")}, + {"merge both foreign", s.MergeIssues(ctx, "A", "b1", "b2", "u")}, + {"unmerge foreign", s.UnmergeIssue(ctx, "A", "b1", "u")}, + {"assign foreign", s.AssignIssue(ctx, "A", "b1", "eve", "u")}, + {"merge unknown", s.MergeIssues(ctx, "A", "a1", "nope", "u")}, + } + for _, c := range cases { + if !errors.Is(c.err, ErrIssueNotFound) { + t.Errorf("%s: want ErrIssueNotFound, got %v", c.name, c.err) + } + } + if len(f.merges) != 0 || len(f.assigns) != 0 { + t.Fatalf("cross-site attempts wrote state: %v %v", f.merges, f.assigns) + } + // A merge in site B never affects resolution in site A. + if err := s.MergeIssues(ctx, "B", "b1", "b2", "u"); err != nil { + t.Fatal(err) + } + if got := s.ResolveMerged(ctx, "A", "b1"); got != "b1" { + t.Fatalf("site A resolved a site B merge: %s", got) + } +} + +func TestAssignIssue(t *testing.T) { + ctx := context.Background() + f := newFakeStore(map[string][]string{"A": {"a1", "a2"}}) + s := svcWith(f) + if err := s.AssignIssue(ctx, "A", "a1", " alice@example.com ", "bob"); err != nil { + t.Fatal(err) + } + if a := f.assigns["A"]["a1"]; a.Assignee != "alice@example.com" || a.AssignedBy != "bob" { + t.Fatalf("assignment = %+v", a) + } + if err := s.AssignIssue(ctx, "A", "a1", "", "bob"); err != nil { // clear + t.Fatal(err) + } + for _, bad := range []string{strings.Repeat("x", maxAssigneeLen+1), "a\x00b", "a\nb"} { + if err := s.AssignIssue(ctx, "A", "a1", bad, "bob"); !errors.Is(err, ErrInvalidAssignee) { + t.Errorf("assignee %q accepted: %v", bad, err) + } + } + // Assigning a merged source lands on its target. + if err := s.MergeIssues(ctx, "A", "a1", "a2", "u"); err != nil { + t.Fatal(err) + } + if err := s.AssignIssue(ctx, "A", "a1", "carol", "u"); err != nil { + t.Fatal(err) + } + if f.assigns["A"]["a2"].Assignee != "carol" { + t.Fatalf("assignment not redirected: %v", f.assigns["A"]) + } +} + +func TestMergeCacheInvalidatedOnWrite(t *testing.T) { + ctx := context.Background() + f := newFakeStore(map[string][]string{"A": {"a1", "a2"}}) + s := svcWith(f) + if got := s.ResolveMerged(ctx, "A", "a1"); got != "a1" { // primes cache (negative) + t.Fatal(got) + } + if err := s.MergeIssues(ctx, "A", "a1", "a2", "u"); err != nil { + t.Fatal(err) + } + if got := s.ResolveMerged(ctx, "A", "a1"); got != "a2" { + t.Fatalf("stale cache after merge: %s", got) + } + before := f.reads + for i := 0; i < 50; i++ { + s.ResolveMerged(ctx, "A", "a1") + } + if f.reads != before { + t.Fatalf("hot path re-read the store %d times", f.reads-before) + } + if err := s.UnmergeIssue(ctx, "A", "a1", "u"); err != nil { + t.Fatal(err) + } + if got := s.ResolveMerged(ctx, "A", "a1"); got != "a1" { + t.Fatalf("stale cache after unmerge: %s", got) + } +} + +func TestIssueIDClause(t *testing.T) { + in, args := issueIDClause(2, []string{"a", "b"}) + if in != "issue_id IN ($2, $3)" || len(args) != 2 { + t.Fatalf("%q %v", in, args) + } + if in, _ := issueIDClause(2, nil); in != "1 = 0" { + t.Fatal(in) + } +} diff --git a/internal/errors/scrub.go b/internal/errors/scrub.go index fc684f1e..1a456e60 100644 --- a/internal/errors/scrub.go +++ b/internal/errors/scrub.go @@ -44,6 +44,8 @@ const truncated = "[Truncated]" const ( scrubMaxDepth = 16 scrubMaxNodes = 20000 + // scrubMaxString bounds the bytes examined per string value. + scrubMaxString = 64 << 10 ) // Substring-matched key fragments (compared against the key lowercased @@ -178,23 +180,51 @@ func luhnValid(digits string) bool { return len(digits) > 0 && sum%10 == 0 } +func countDigits(v string) int { + n := 0 + for i := 0; i < len(v); i++ { + if v[i] >= '0' && v[i] <= '9' { + n++ + } + } + return n +} + // scrubString applies the value-pattern rules to one string. func (s *Scrubber) scrubString(v string) string { if v == "" { return v } - v = jwtRe.ReplaceAllString(v, Filtered) - v = bearerRe.ReplaceAllString(v, "$1 "+Filtered) - v = queryParamRe.ReplaceAllStringFunc(v, func(m string) string { - sub := queryParamRe.FindStringSubmatch(m) - name := sub[2] - ln := strings.ToLower(name) - if s.sensitiveKey(name) || ln == "key" || ln == "sig" || ln == "signature" { - return sub[1] + name + "=" + Filtered + if len(v) > scrubMaxString { + // Bounded work per string; the unexamined tail is dropped, not + // passed through. + v = v[:scrubMaxString] + truncated + } + // Cheap substring pre-filters keep the regexes off strings that cannot + // match (the counted-repeat card pattern is the expensive one). + if strings.Contains(v, "eyJ") { + v = jwtRe.ReplaceAllString(v, Filtered) + } + if strings.Contains(strings.ToLower(v), "bearer") { + v = bearerRe.ReplaceAllString(v, "$1 "+Filtered) + } + if strings.Contains(v, "=") { + if strings.ContainsAny(v, "?&;") { + v = queryParamRe.ReplaceAllStringFunc(v, func(m string) string { + sub := queryParamRe.FindStringSubmatch(m) + name := sub[2] + ln := strings.ToLower(name) + if s.sensitiveKey(name) || ln == "key" || ln == "sig" || ln == "signature" { + return sub[1] + name + "=" + Filtered + } + return m + }) } - return m - }) - v = kvSecretRe.ReplaceAllString(v, "$1="+Filtered) + v = kvSecretRe.ReplaceAllString(v, "$1="+Filtered) + } + if countDigits(v) < 14 { + return v + } v = cardRe.ReplaceAllStringFunc(v, func(m string) string { digits := make([]byte, 0, 19) for i := 0; i < len(m); i++ { diff --git a/internal/schema/migrations/058_issue_merge_assign.up.sql b/internal/schema/migrations/058_issue_merge_assign.up.sql new file mode 100644 index 00000000..ac99021b --- /dev/null +++ b/internal/schema/migrations/058_issue_merge_assign.up.sql @@ -0,0 +1,49 @@ +-- 058: error-tracking depth - issue merge/unmerge and assignment. +-- +-- Two NEW tables only (CREATE TABLE IF NOT EXISTS); no existing table is +-- altered. Both are ReplacingMergeTree keyed on the logical row and are +-- collapsed at read time with argMax over version, the 034/047 pattern. +-- Nothing is ever deleted: an unmerge writes a newer version with +-- active = 'false', clearing an assignee writes assignee = ''. +-- +-- issue_merges one logical row per SOURCE issue. active = 'true' +-- means events and reads for source_issue_id resolve +-- to target_issue_id. New events whose fingerprint +-- maps to a merged source are attributed to the +-- target at ingest. Chains are followed to a bounded +-- depth; cycles are rejected by the service. +-- issue_assignments one logical row per issue; assignee '' = unassigned. +-- +-- version is a strictly increasing unix-ms stamp chosen by the writer +-- (GREATEST(now, previous + 1)). merged_at / assigned_at are unix ms. +-- +-- Comments pure ASCII (038 rule). + +CREATE TABLE IF NOT EXISTS issue_merges ( + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + source_issue_id TEXT NOT NULL, + target_issue_id TEXT NOT NULL, + active TEXT NOT NULL DEFAULT 'true', + merged_at BIGINT NOT NULL DEFAULT 0, + merged_by TEXT NOT NULL DEFAULT '', + version BIGINT NOT NULL DEFAULT 0 +) WITH ( + engine = 'replacing_mergetree', + version_column = 'version' +) +ORDER BY (tenant_id, site_id, source_issue_id); + +CREATE TABLE IF NOT EXISTS issue_assignments ( + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + issue_id TEXT NOT NULL, + assignee TEXT NOT NULL DEFAULT '', + assigned_at BIGINT NOT NULL DEFAULT 0, + assigned_by TEXT NOT NULL DEFAULT '', + version BIGINT NOT NULL DEFAULT 0 +) WITH ( + engine = 'replacing_mergetree', + version_column = 'version' +) +ORDER BY (tenant_id, site_id, issue_id); diff --git a/ui/src/api/errors.ts b/ui/src/api/errors.ts index 50902d1f..bf38ec51 100644 --- a/ui/src/api/errors.ts +++ b/ui/src/api/errors.ts @@ -1,6 +1,6 @@ // Error tracking API — issues, events, search, releases. -import { get, post } from "./helpers.js"; +import { get, post, put } from "./helpers.js"; const BASE = "/api/v1"; @@ -25,6 +25,11 @@ export interface Issue { snooze_active?: boolean; // Per-release event counts, detail reads only. releases?: IssueRelease[]; + // Merge/assignment (additive). + resolved_from?: string; + merged_sources?: string[]; + assignee?: string; + assigned_at?: number; } export interface IssueRelease { @@ -109,6 +114,12 @@ export const errorsApi = { post<{ ok: boolean }>(`${BASE}/issues/${issueId}/status`, { site_id: siteId, status, ...(until ? { until } : {}), }), + mergeIssue: (issueId: string, siteId: string, targetId: string) => + post(`${BASE}/issues/${issueId}/merge`, { site_id: siteId, target_id: targetId }), + unmergeIssue: (issueId: string, siteId: string) => + post(`${BASE}/issues/${issueId}/unmerge`, { site_id: siteId }), + setAssignee: (issueId: string, siteId: string, assignee: string) => + put(`${BASE}/issues/${issueId}/assignee`, { site_id: siteId, assignee }), search: (siteId: string, query: string) => get(`${BASE}/issues/search?site_id=${siteId}&q=${encodeURIComponent(query)}`), releases: (siteId: string) => diff --git a/ui/src/lib/issueAdmin.test.ts b/ui/src/lib/issueAdmin.test.ts new file mode 100644 index 00000000..c2134075 --- /dev/null +++ b/ui/src/lib/issueAdmin.test.ts @@ -0,0 +1,21 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { normalizeAssignee, canMerge, MAX_ASSIGNEE_LEN } from "./issueAdmin.ts"; + +test("normalizeAssignee trims and allows empty", () => { + assert.equal(normalizeAssignee(" alice@example.com "), "alice@example.com"); + assert.equal(normalizeAssignee(" "), ""); +}); + +test("normalizeAssignee rejects control chars and oversize", () => { + assert.equal(normalizeAssignee("a\u0000b"), null); + assert.equal(normalizeAssignee("a\nb"), null); + assert.equal(normalizeAssignee("x".repeat(MAX_ASSIGNEE_LEN + 1)), null); + assert.equal(normalizeAssignee("x".repeat(MAX_ASSIGNEE_LEN)), "x".repeat(MAX_ASSIGNEE_LEN)); +}); + +test("canMerge refuses empty and self", () => { + assert.equal(canMerge("a", ""), false); + assert.equal(canMerge("a", " a "), false); + assert.equal(canMerge("a", "b"), true); +}); diff --git a/ui/src/lib/issueAdmin.ts b/ui/src/lib/issueAdmin.ts new file mode 100644 index 00000000..bef6b573 --- /dev/null +++ b/ui/src/lib/issueAdmin.ts @@ -0,0 +1,20 @@ +// Pure helpers for the issue assignee / merge controls (errors route). + +export const MAX_ASSIGNEE_LEN = 128; + +// normalizeAssignee trims the input and returns null when the server would +// reject it (too long or containing control characters). "" is valid: it +// clears the assignee. +export function normalizeAssignee(raw: string): string | null { + const v = raw.trim(); + if (new TextEncoder().encode(v).length > MAX_ASSIGNEE_LEN) return null; + // eslint-disable-next-line no-control-regex + if (/[\u0000-\u001f\u007f]/.test(v)) return null; + return v; +} + +// canMerge: a merge needs a target id that differs from the issue itself. +export function canMerge(issueId: string, targetId: string): boolean { + const t = targetId.trim(); + return t.length > 0 && t !== issueId; +} diff --git a/ui/src/routes/errors.tsx b/ui/src/routes/errors.tsx index 2c51895c..118c33f7 100644 --- a/ui/src/routes/errors.tsx +++ b/ui/src/routes/errors.tsx @@ -10,6 +10,7 @@ import ExportButton from "../components/shared/ExportButton.js"; import EmptyState from "../components/shared/EmptyState.js"; import "../styles/errors.css"; import { useFilters } from "../hooks/useFilters.js"; +import { normalizeAssignee, canMerge } from "../lib/issueAdmin.js"; export const config = { mode: "app" }; @@ -163,6 +164,35 @@ function IssueDetail({ issue, siteId, onBack }: { issue: Issue; siteId: string; finally { setUpdatingStatus(false); } }; + const [assignee, setAssignee] = useState(issue.assignee ?? ""); + const [mergeTarget, setMergeTarget] = useState(""); + const [adminMsg, setAdminMsg] = useState(""); + + const handleAssign = async () => { + const v = normalizeAssignee(assignee); + if (v === null) { setAdminMsg("Invalid assignee"); return; } + try { + await errorsApi.setAssignee(issue.issue_id, siteId, v); + setAssignee(v); + setAdminMsg(v ? `Assigned to ${v}` : "Unassigned"); + } catch (err) { setAdminMsg(String(err)); } + }; + + const handleMerge = async () => { + if (!canMerge(issue.issue_id, mergeTarget)) { setAdminMsg("Enter another issue id to merge into"); return; } + try { + await errorsApi.mergeIssue(issue.issue_id, siteId, mergeTarget.trim()); + setAdminMsg("Merged. This issue now resolves to the target."); + } catch (err) { setAdminMsg(String(err)); } + }; + + const handleUnmerge = async (sourceId: string) => { + try { + await errorsApi.unmergeIssue(sourceId, siteId); + setAdminMsg("Unmerged " + sourceId); + } catch (err) { setAdminMsg(String(err)); } + }; + const handleSnooze = async () => { const until = new Date(Date.now() + 7 * 86400000).toISOString(); await handleStatusChange("resolved", until); @@ -217,6 +247,19 @@ function IssueDetail({ issue, siteId, onBack }: { issue: Issue; siteId: string; +
+ setAssignee((e.target as HTMLInputElement).value)} /> + + setMergeTarget((e.target as HTMLInputElement).value)} /> + + {(issue.merged_sources ?? []).map(src => ( + + ))} + {adminMsg && {adminMsg}} +
+
Events From bf57d07ab366fcf4146a5f2df731fe525118a22e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 23:43:18 +0000 Subject: [PATCH 29/65] wip --- internal/experiments/analysis.go | 179 +++++++++++++++-- internal/experiments/bayesian.go | 32 ++- internal/experiments/bayesian_test.go | 6 +- internal/experiments/compute.go | 279 ++++++++++++++++++++++++++ internal/experiments/config.go | 208 +++++++++++++++++++ internal/experiments/continuous.go | 260 ++++++++++++++++++++++++ 6 files changed, 938 insertions(+), 26 deletions(-) create mode 100644 internal/experiments/compute.go create mode 100644 internal/experiments/config.go create mode 100644 internal/experiments/continuous.go diff --git a/internal/experiments/analysis.go b/internal/experiments/analysis.go index ca2a3d0d..8468fc71 100644 --- a/internal/experiments/analysis.go +++ b/internal/experiments/analysis.go @@ -47,6 +47,10 @@ type PairwiseResult struct { HolmAdjustedP float64 `json:"holm_adjusted_p"` Significant bool `json:"significant"` // survives Holm at alphaOmnibus UsedFisher bool `json:"used_fisher"` + // T and DF are set for continuous metrics (Welch's t statistic and its + // Welch-Satterthwaite degrees of freedom); absent for binary arms. + T float64 `json:"t,omitempty"` + DF float64 `json:"df,omitempty"` } // AnalysisResult is the honest reporting layer: every gate that stands @@ -63,6 +67,50 @@ type AnalysisResult struct { SRM SRMDiagnostic `json:"srm"` Pairwise []PairwiseResult `json:"pairwise_vs_control"` WinnerRule string `json:"winner_rule"` + // PlannedSamplePerArm is the design-time per-arm n from the MDE + // sample-size endpoint, when the experiment declares one. The horizon is + // the larger of it and min_sample unless HorizonOverridden. + PlannedSamplePerArm int `json:"planned_sample_per_arm,omitempty"` + // HorizonOverridden is true when the planned-sample part of the horizon + // was waived (explicit override). The min_sample floor is never waived. + HorizonOverridden bool `json:"horizon_overridden,omitempty"` +} + +// horizonSpec is the fixed-horizon contract for one analysis: the winner gate +// stays shut until every arm reaches Required(). +type horizonSpec struct { + MinSample int // experiment min_sample, per arm; never waived + Planned int // design-time per-arm n (0 = none declared) + Floor int // metric-kind floor (continuous metrics) + Override bool +} + +// Required is the per-arm n at which the horizon is met. +func (h horizonSpec) Required() int { + n := h.MinSample + if h.Floor > n { + n = h.Floor + } + if !h.Override && h.Planned > n { + n = h.Planned + } + return n +} + +// peekingWarning is the fixed-horizon anti-peeking message, empty once the +// horizon is met or when no arm has data yet. +func peekingWarning(minArm int64, anyData bool, h horizonSpec) string { + req := h.Required() + if !anyData || minArm >= int64(req) { + return "" + } + msg := fmt.Sprintf("results requested before the planned sample size (smallest arm %d of %d per arm). "+ + "This analysis is fixed-horizon, not sequential: repeatedly checking p-values and stopping when one dips below 0.05 inflates the false-positive rate. "+ + "Treat these numbers as a progress report and decide at the planned horizon.", minArm, req) + if h.Override { + msg += " The planned-sample gate is overridden; the min_sample floor still applies." + } + return msg } // analyze runs the full decided pipeline over per-arm counts, with the @@ -73,7 +121,14 @@ type AnalysisResult struct { // badge before every arm reaches minSamplePerArm. Estimates and intervals // are always computed and displayed regardless. func analyze(arms []VariantResult, weights []float64, minSamplePerArm int) AnalysisResult { - res := AnalysisResult{MinSamplePerArm: minSamplePerArm, Test: "none"} + return analyzeWithHorizon(arms, weights, horizonSpec{MinSample: minSamplePerArm}) +} + +// analyzeWithHorizon is analyze with the full horizon contract (planned +// sample size and its override). +func analyzeWithHorizon(arms []VariantResult, weights []float64, h horizonSpec) AnalysisResult { + minSamplePerArm := h.Required() + res := AnalysisResult{MinSamplePerArm: minSamplePerArm, Test: "none", PlannedSamplePerArm: h.Planned, HorizonOverridden: h.Override && h.Planned > 0} k := len(arms) if k < 2 { res.WinnerRule = "fewer than two arms have exposures" @@ -86,23 +141,7 @@ func analyze(arms []VariantResult, weights []float64, minSamplePerArm int) Analy exposures[i] = a.Exposures conversions[i] = a.Conversions } - res.MinArmExposures = exposures[0] - for _, n := range exposures { - if n < res.MinArmExposures { - res.MinArmExposures = n - } - } - res.HorizonMet = res.MinArmExposures >= int64(minSamplePerArm) - - // SRM: goodness-of-fit against the declared allocation. - if w := append([]float64(nil), weights...); len(w) == k { - stat, p := srmGoodnessOfFit(exposures, w) - res.SRM = SRMDiagnostic{ChiSquare: stat, PValue: p} - if p < alphaSRM { - res.SRM.Detected = true - res.SRM.Note = "assignment is broken; do not trust these results" - } - } + applyHorizonAndSRM(&res, exposures, weights, minSamplePerArm) // Omnibus test with Fisher fallback for two arms and small cells. omni := chiSquareOmnibus(exposures, conversions) @@ -178,18 +217,118 @@ func analyze(arms []VariantResult, weights []float64, minSamplePerArm int) Analy return res } +// applyHorizonAndSRM fills the per-arm horizon gate and the SRM diagnostic, +// which depend only on the exposure counts (shared by binary and continuous). +func applyHorizonAndSRM(res *AnalysisResult, exposures []int64, weights []float64, required int) { + k := len(exposures) + res.MinArmExposures = exposures[0] + for _, n := range exposures { + if n < res.MinArmExposures { + res.MinArmExposures = n + } + } + res.HorizonMet = res.MinArmExposures >= int64(required) + + // SRM: goodness-of-fit against the declared allocation. + if w := append([]float64(nil), weights...); len(w) == k { + stat, p := srmGoodnessOfFit(exposures, w) + res.SRM = SRMDiagnostic{ChiSquare: stat, PValue: p} + if p < alphaSRM { + res.SRM.Detected = true + res.SRM.Note = "assignment is broken; do not trust these results" + } + } +} + +// analyzeContinuous is the continuous-metric (count / mean) counterpart of +// analyzeWithHorizon: the same horizon and SRM gates, then Welch's t-test of +// every arm against the control (arms[0]) with Holm correction across the +// arms. sums are the per-arm value summaries aligned with arms. There is no +// omnibus test: Holm over the pairwise family already controls the +// family-wise error, and the headline p_value is the smallest Holm-adjusted +// p. Higher is better; the winner is the highest-mean arm whose comparison +// survives Holm. +func analyzeContinuous(arms []VariantResult, sums []contSummary, weights []float64, h horizonSpec) AnalysisResult { + h.Floor = minContinuousN + res := AnalysisResult{MinSamplePerArm: h.Required(), Test: "none", PlannedSamplePerArm: h.Planned, HorizonOverridden: h.Override && h.Planned > 0} + k := len(arms) + if k < 2 { + res.WinnerRule = "fewer than two arms have exposures" + return res + } + exposures := make([]int64, k) + for i, a := range arms { + exposures[i] = a.Exposures + } + applyHorizonAndSRM(&res, exposures, weights, res.MinSamplePerArm) + res.Test = "welch-t" + + rawP := make([]float64, 0, k-1) + welch := make([]WelchResult, 0, k-1) + for i := 1; i < k; i++ { + w := welchTest(sums[0], sums[i], alphaOmnibus) + welch = append(welch, w) + rawP = append(rawP, w.PValue) + } + adj := holmAdjusted(rawP) + minAdj := 1.0 + for i, w := range welch { + pr := PairwiseResult{ + Variant: arms[i+1].Variant, + LiftAbsolute: w.Diff, + CiLow: w.CILow, + CiHigh: w.CIHigh, + PValue: w.PValue, + HolmAdjustedP: adj[i], + Significant: adj[i] <= alphaOmnibus, + T: w.T, + DF: w.DF, + } + if sums[0].Mean != 0 { + pr.LiftRelative = w.Diff / sums[0].Mean + } + if adj[i] < minAdj { + minAdj = adj[i] + } + res.Pairwise = append(res.Pairwise, pr) + } + res.PValue = minAdj + + res.WinnerRule = "winner requires: per-arm horizon (min " + strconv.Itoa(res.MinSamplePerArm) + + " per arm), no SRM, winner's Welch t-test vs control surviving Holm at " + strconv.FormatFloat(alphaOmnibus, 'f', -1, 64) + switch { + case !res.HorizonMet: + res.WinnerRule += fmt.Sprintf(" - waiting for horizon (min arm %d of %d)", res.MinArmExposures, res.MinSamplePerArm) + case res.SRM.Detected: + res.WinnerRule += " - SRM detected: " + res.SRM.Note + case res.PValue > alphaOmnibus: + res.WinnerRule += fmt.Sprintf(" - no arm distinguishable from control (smallest Holm p=%.4g)", res.PValue) + } + return res +} + // winnerFrom returns the winner arm given the analysis and arms (control at // index 0): the best observed conversion-rate arm whose pairwise-vs-control // comparison survived Holm. Empty string when no winner may be claimed. If // the single best-rate arm fails Holm there is no winner: the omnibus fired // but no specific arm is distinguishable from control. func winnerFrom(res AnalysisResult, arms []VariantResult) string { + scores := make([]float64, len(arms)) + for i, a := range arms { + scores[i] = a.ConversionRate + } + return winnerByScore(res, arms, scores) +} + +// winnerByScore is winnerFrom over an arbitrary higher-is-better score per arm +// (conversion rate for binary goals, the arm mean for continuous goals). +func winnerByScore(res AnalysisResult, arms []VariantResult, scores []float64) string { if !res.HorizonMet || res.SRM.Detected || res.Test == "none" || res.PValue >= alphaOmnibus { return "" } bestIdx := -1 - for i, v := range arms[1:] { - if bestIdx == -1 || v.ConversionRate > arms[1+bestIdx].ConversionRate { + for i := range arms[1:] { + if bestIdx == -1 || scores[1+i] > scores[1+bestIdx] { bestIdx = i } } diff --git a/internal/experiments/bayesian.go b/internal/experiments/bayesian.go index c992551e..2400cbe8 100644 --- a/internal/experiments/bayesian.go +++ b/internal/experiments/bayesian.go @@ -1,19 +1,22 @@ package experiments import ( + "crypto/sha256" + "encoding/binary" "math" "math/rand" ) // computeBayesianProbabilities fills in ProbBeatControl for each variant using // a Beta(1+conversions, 1+nonconversions) prior and 4000 Monte Carlo samples. -// The first variant is treated as the control. -func computeBayesianProbabilities(variants []VariantResult) { +// The first variant is treated as the control. seed fixes the Monte Carlo +// stream; see bayesianSeed. +func computeBayesianProbabilities(variants []VariantResult, seed int64) { if len(variants) < 2 { return } const samples = 4000 - rng := rand.New(rand.NewSource(42)) + rng := rand.New(rand.NewSource(seed)) controlSamples := drawBeta(rng, float64(1+variants[0].Conversions), float64(1+variants[0].Exposures-variants[0].Conversions), samples) @@ -32,6 +35,29 @@ func computeBayesianProbabilities(variants []VariantResult) { variants[0].ProbBeatControl = 0 } +// bayesianSeed derives the Monte Carlo seed from the experiment id and a hash +// of the arm counts. Identical calls on identical data return identical +// probabilities (a page refresh never jitters the number), while different +// experiments - and different data of the same experiment - draw independent +// streams, so a shared constant seed cannot correlate the noise of every +// experiment's estimate. The probability stays display-only: it never gates +// the winner. +func bayesianSeed(experimentID string, variants []VariantResult) int64 { + h := sha256.New() + h.Write([]byte(experimentID)) + var buf [8]byte + for _, v := range variants { + h.Write([]byte{0}) + h.Write([]byte(v.Variant)) + binary.BigEndian.PutUint64(buf[:], uint64(v.Exposures)) + h.Write(buf[:]) + binary.BigEndian.PutUint64(buf[:], uint64(v.Conversions)) + h.Write(buf[:]) + } + sum := h.Sum(nil) + return int64(binary.BigEndian.Uint64(sum[:8]) & math.MaxInt64) +} + // drawBeta returns n samples from Beta(alpha, beta) via the ratio of two Gammas. func drawBeta(rng *rand.Rand, alpha, beta float64, n int) []float64 { out := make([]float64, n) diff --git a/internal/experiments/bayesian_test.go b/internal/experiments/bayesian_test.go index 93e39f20..a0fb39ab 100644 --- a/internal/experiments/bayesian_test.go +++ b/internal/experiments/bayesian_test.go @@ -9,7 +9,7 @@ func TestBayesianWinnerClear(t *testing.T) { {Variant: "control", Exposures: 1000, Conversions: 100}, {Variant: "A", Exposures: 1000, Conversions: 300}, } - computeBayesianProbabilities(v) + computeBayesianProbabilities(v, bayesianSeed("test-exp", v)) if v[1].ProbBeatControl < 0.99 { t.Errorf("expected P(A > control) ≈ 1, got %v", v[1].ProbBeatControl) } @@ -24,7 +24,7 @@ func TestBayesianWinnerTight(t *testing.T) { {Variant: "control", Exposures: 1000, Conversions: 100}, {Variant: "A", Exposures: 1000, Conversions: 100}, } - computeBayesianProbabilities(v) + computeBayesianProbabilities(v, bayesianSeed("test-exp", v)) if v[1].ProbBeatControl < 0.35 || v[1].ProbBeatControl > 0.65 { t.Errorf("expected P(A > control) ~0.5, got %v", v[1].ProbBeatControl) } @@ -36,7 +36,7 @@ func TestBayesianLoser(t *testing.T) { {Variant: "control", Exposures: 1000, Conversions: 150}, {Variant: "A", Exposures: 1000, Conversions: 50}, } - computeBayesianProbabilities(v) + computeBayesianProbabilities(v, bayesianSeed("test-exp", v)) if v[1].ProbBeatControl > 0.01 { t.Errorf("expected P(A > control) ~0, got %v", v[1].ProbBeatControl) } diff --git a/internal/experiments/compute.go b/internal/experiments/compute.go new file mode 100644 index 00000000..5c4d35fa --- /dev/null +++ b/internal/experiments/compute.go @@ -0,0 +1,279 @@ +package experiments + +// The pure core of Results: raw per-user rows in, ExperimentResults out. The +// service fetches the rows (SQL, unverified offline); everything that decides +// a number or a gate happens here so it is unit-testable without a database. + +import ( + "sort" +) + +// exposureRow is one user's exposure to one variant inside the running +// interval, with the earliest such exposure time (epoch ms). +type exposureRow struct { + User string + Variant string + First int64 +} + +// userVariant is a distinct (user, variant) conversion attribution row. +type userVariant struct { + User string + Variant string +} + +// metricEvent is one count/mean/secondary observation. +type metricEvent struct { + User string + TS int64 + Value float64 +} + +// ResultsOptions are per-request analysis options. +type ResultsOptions struct { + // AllowEarly waives the planned-sample part of the horizon for this + // request (the winner gate and the peeking warning). The min_sample floor + // is never waived. The experiment's allow_early_winner setting has the + // same effect persistently. + AllowEarly bool +} + +type resultInputs struct { + Exp Experiment + Cfg ExperimentConfig + Opts ResultsOptions + WindowMs int64 + // Exposures: distinct (user, variant) rows in the running interval. + Exposures []exposureRow + // BinaryConversions: distinct (user, variant) rows of the legacy + // experiment_conversions join, used when the primary kind is binary. + BinaryConversions []userVariant + // Events by metric key ("primary" or a secondary key). + Events map[string][]metricEvent + // Truncated marks metric keys whose event read hit the row cap. + Truncated map[string]bool +} + +// MetricResult is one evaluated goal (primary-continuous detail or a +// secondary). Secondary metrics are informational and never gate the winner. +type MetricResult struct { + Key string `json:"key"` + Name string `json:"name,omitempty"` + Kind string `json:"kind"` + // Role is "primary" or "secondary". + Role string `json:"role"` + // GatesWinner is true only for the primary metric. + GatesWinner bool `json:"gates_winner"` + Variants []VariantResult `json:"variants"` + Analysis AnalysisResult `json:"analysis"` + Note string `json:"note,omitempty"` +} + +// cohortOf collapses exposure rows into one entry per user. A user exposed to +// more than one variant is contaminated: excluded from every statistic and +// counted separately. Returns the clean cohort and the contaminated count. +func cohortOf(rows []exposureRow) (map[string]exposureRow, int) { + first := make(map[string]exposureRow, len(rows)) + bad := make(map[string]bool) + for _, r := range rows { + prev, ok := first[r.User] + switch { + case !ok: + first[r.User] = r + case prev.Variant != r.Variant: + bad[r.User] = true + case r.First < prev.First: + first[r.User] = r + } + } + for u := range bad { + delete(first, u) + } + return first, len(bad) +} + +// qualifyingValues maps each cohort user with at least one event inside its +// attribution window [first exposure, first exposure + window] to its metric +// value: 1 for binary, the event count for count, the value sum for mean. +func qualifyingValues(kind string, events []metricEvent, cohort map[string]exposureRow, windowMs int64) map[string]float64 { + out := make(map[string]float64) + for _, e := range events { + c, ok := cohort[e.User] + if !ok || e.TS < c.First || e.TS > c.First+windowMs { + continue + } + switch kind { + case KindBinary: + out[e.User] = 1 + case KindCount: + out[e.User]++ + default: + out[e.User] += e.Value + } + } + return out +} + +// evaluateMetric builds the per-arm results and analysis for one goal over +// the clean cohort. variants is the ordered arm list (control first) with +// Exposures filled; vals maps users to qualifying values (see +// qualifyingValues). Binary goals use the chi-square/Fisher pipeline; +// count/mean goals use Welch with optional winsorizing. +func evaluateMetric(kind string, winsorPct float64, variants []VariantResult, cohort map[string]exposureRow, vals map[string]float64, weights []float64, h horizonSpec) ([]VariantResult, AnalysisResult) { + arms := append([]VariantResult(nil), variants...) + idx := make(map[string]int, len(arms)) + for i, a := range arms { + idx[a.Variant] = i + } + for u := range vals { + arms[idx[cohort[u].Variant]].Conversions++ + } + for i := range arms { + a := &arms[i] + if a.Exposures > 0 { + a.ConversionRate = float64(a.Conversions) / float64(a.Exposures) + } + a.WilsonLow, a.WilsonHigh = wilsonInterval(a.Conversions, a.Exposures, zAlphaTwoSided005) + } + if kind == KindBinary { + return arms, analyzeWithHorizon(arms, weights, h) + } + + // Continuous: every exposed user contributes (zero when no events). + perArm := make([][]float64, len(arms)) + for u, c := range cohort { + perArm[idx[c.Variant]] = append(perArm[idx[c.Variant]], vals[u]) + } + if winsorPct > 0 { + winsorizeArms(perArm, winsorPct) + } + sums := make([]contSummary, len(arms)) + for i := range arms { + sums[i] = summarize(perArm[i]) + m, sd := sums[i].Mean, sums[i].stdDev() + arms[i].Mean, arms[i].StdDev = &m, &sd + } + an := analyzeContinuous(arms, sums, weights, h) + return arms, an +} + +// continuousScores returns each arm's mean for winnerByScore. +func continuousScores(arms []VariantResult) []float64 { + s := make([]float64, len(arms)) + for i, a := range arms { + if a.Mean != nil { + s[i] = *a.Mean + } + } + return s +} + +func computeResults(in resultInputs) *ExperimentResults { + exp, cfg := in.Exp, in.Cfg + cohort, contaminated := cohortOf(in.Exposures) + + // Per-variant clean exposure counts, sorted by variant name (the legacy + // ORDER BY variant) before the declared control is moved to the front. + counts := make(map[string]int64) + for _, c := range cohort { + counts[c.Variant]++ + } + names := make([]string, 0, len(counts)) + for v := range counts { + names = append(names, v) + } + sort.Strings(names) + base := make([]VariantResult, len(names)) + for i, v := range names { + base[i] = VariantResult{Variant: v, Exposures: counts[v]} + } + orderControlFirst(base, controlKey(exp.Variants)) + + minSample := exp.MinSample + if minSample <= 0 { + minSample = 100 + } + h := horizonSpec{ + MinSample: minSample, + Planned: cfg.PlannedSamplePerArm, + Override: in.Opts.AllowEarly || cfg.AllowEarlyWinner, + } + weights := allocationWeights(exp.Variants, len(base)) + + // Primary goal. + var primaryVals map[string]float64 + if cfg.MetricKind == KindBinary { + primaryVals = make(map[string]float64) + for _, c := range in.BinaryConversions { + if cc, ok := cohort[c.User]; ok && cc.Variant == c.Variant { + primaryVals[c.User] = 1 + } + } + } else { + primaryVals = qualifyingValues(cfg.MetricKind, in.Events[PrimaryMetricKey], cohort, in.WindowMs) + } + variants, analysis := evaluateMetric(cfg.MetricKind, cfg.WinsorizePct, base, cohort, primaryVals, weights, h) + + var winner string + var significant bool + if cfg.MetricKind == KindBinary { + // Bayesian: probability each variant beats the control (index 0), + // Beta(1+conv, 1+nonconv) with a Monte Carlo seeded from the experiment + // id and the data (see bayesianSeed). Display-only: it never gates the + // winner (2026-09-23 decision: the fixed-horizon frequentist gates do). + if len(variants) >= 2 { + computeBayesianProbabilities(variants, bayesianSeed(exp.ExperimentID, variants)) + } + winner = winnerFrom(analysis, variants) + significant = analysis.HorizonMet && !analysis.SRM.Detected && + analysis.Test != "none" && analysis.PValue < alphaOmnibus + } else { + winner = winnerByScore(analysis, variants, continuousScores(variants)) + significant = analysis.HorizonMet && !analysis.SRM.Detected && + analysis.Test != "none" && analysis.PValue < alphaOmnibus + } + if in.Truncated[PrimaryMetricKey] { + winner, significant = "", false + analysis.WinnerRule += " - metric events were truncated at the read cap; results are not reliable" + } + + res := &ExperimentResults{ + Experiment: exp, + Variants: variants, + Significant: significant, + Winner: winner, + Analysis: analysis, + MetricKind: cfg.MetricKind, + ContaminatedUsers: contaminated, + PeekingWarning: peekingWarning(analysis.MinArmExposures, len(variants) >= 2, horizonSpec{MinSample: minSample, Planned: h.Planned, Floor: floorFor(cfg.MetricKind), Override: h.Override}), + } + if cfg.MetricKind != KindBinary || cfg.WinsorizePct > 0 || cfg.PlannedSamplePerArm > 0 || cfg.AllowEarlyWinner || len(cfg.Secondary) > 0 { + c := cfg + res.Settings = &c + } + + // Secondary goals: same cohort, same machinery, never gating. + for _, g := range cfg.Secondary { + vals := qualifyingValues(g.Kind, in.Events[g.Key], cohort, in.WindowMs) + arms, an := evaluateMetric(g.Kind, cfg.WinsorizePct, base, cohort, vals, weights, h) + an.WinnerRule = "secondary metric - informational only, does not gate the winner; " + an.WinnerRule + mr := MetricResult{ + Key: g.Key, Name: g.Name, Kind: g.Kind, Role: "secondary", + Variants: arms, Analysis: an, + Note: "secondary: excluded from winner selection; uncorrected across metrics", + } + if in.Truncated[g.Key] { + mr.Note += "; events truncated at the read cap, results are not reliable" + } + res.Secondary = append(res.Secondary, mr) + } + res.MultipleComparisonNote = cfg.multipleComparisonNote() + return res +} + +func floorFor(kind string) int { + if kind == KindBinary { + return 0 + } + return minContinuousN +} diff --git a/internal/experiments/config.go b/internal/experiments/config.go new file mode 100644 index 00000000..84070d5f --- /dev/null +++ b/internal/experiments/config.go @@ -0,0 +1,208 @@ +package experiments + +// Per-experiment analysis configuration (migration 063). The experiments +// table cannot take new columns (ALTER ADD COLUMN corrupts populated tables, +// open P1 L9), so metric kind, secondary goals, winsorizing and the planned +// horizon live in an append-only experiment_settings table holding one JSON +// document per write; the newest write wins. An experiment with no settings +// row behaves exactly as before 063: one binary conversion goal. + +import ( + "context" + "encoding/json" + "fmt" + "math" + "regexp" + "strconv" + "time" + + "github.com/neutron-build/neutron/go/nucleus" +) + +// Metric kinds. Binary is "did the user ever convert"; count is the number of +// events per exposed user; mean is the summed event value per exposed user +// (revenue per exposed user, so non-purchasers contribute zero). +const ( + KindBinary = "binary" + KindCount = "count" + KindMean = "mean" + + // PrimaryMetricKey is the metric_events key of the primary goal when it is + // continuous. A binary primary keeps using experiment_conversions. + PrimaryMetricKey = "primary" + + maxSecondaryGoals = 3 + maxMetricValueAbs = 1e12 + maxPlannedSample = 1_000_000_000 +) + +var metricKeyRE = regexp.MustCompile(`^[a-z0-9_]{1,32}$`) + +// SecondaryGoal is one informational metric evaluated beside the primary. It +// never gates the winner. +type SecondaryGoal struct { + Key string `json:"key"` + Name string `json:"name,omitempty"` + Kind string `json:"kind"` +} + +// ExperimentConfig is the analysis configuration document. +type ExperimentConfig struct { + // MetricKind of the primary goal: binary (default), count or mean. + MetricKind string `json:"metric_kind"` + // WinsorizePct clips continuous values to the pooled [p, 100-p] + // percentiles; 0 (default) is off. Valid range (0, 50). + WinsorizePct float64 `json:"winsorize_pct,omitempty"` + // PlannedSamplePerArm is the design-time n from the sample-size endpoint. + // Until every arm reaches it, results carry a peeking warning and no + // winner is declared (unless AllowEarlyWinner). + PlannedSamplePerArm int `json:"planned_sample_per_arm,omitempty"` + // AllowEarlyWinner is the explicit override of the planned-sample gate. + AllowEarlyWinner bool `json:"allow_early_winner,omitempty"` + // Secondary goals, at most three. + Secondary []SecondaryGoal `json:"secondary_goals,omitempty"` +} + +// DefaultConfig is the pre-063 behaviour. +func DefaultConfig() ExperimentConfig { return ExperimentConfig{MetricKind: KindBinary} } + +func validKind(k string) bool { return k == KindBinary || k == KindCount || k == KindMean } + +// Normalize fills defaults and validates. It returns a copy. +func (c ExperimentConfig) Normalize() (ExperimentConfig, error) { + if c.MetricKind == "" { + c.MetricKind = KindBinary + } + if !validKind(c.MetricKind) { + return c, fmt.Errorf("metric_kind must be binary, count or mean") + } + if math.IsNaN(c.WinsorizePct) || c.WinsorizePct < 0 || c.WinsorizePct >= 50 { + return c, fmt.Errorf("winsorize_pct must be 0 (off) or in (0, 50)") + } + if c.WinsorizePct > 0 && c.MetricKind == KindBinary { + return c, fmt.Errorf("winsorize_pct applies to count and mean metrics only") + } + if c.PlannedSamplePerArm < 0 || c.PlannedSamplePerArm > maxPlannedSample { + return c, fmt.Errorf("planned_sample_per_arm out of range") + } + if len(c.Secondary) > maxSecondaryGoals { + return c, fmt.Errorf("at most %d secondary goals", maxSecondaryGoals) + } + seen := map[string]bool{PrimaryMetricKey: true} + sec := make([]SecondaryGoal, len(c.Secondary)) + for i, g := range c.Secondary { + if !metricKeyRE.MatchString(g.Key) { + return c, fmt.Errorf("secondary goal key must match %s", metricKeyRE.String()) + } + if seen[g.Key] { + return c, fmt.Errorf("duplicate or reserved secondary goal key %q", g.Key) + } + seen[g.Key] = true + if g.Kind == "" { + g.Kind = KindBinary + } + if !validKind(g.Kind) { + return c, fmt.Errorf("secondary goal %q: kind must be binary, count or mean", g.Key) + } + if len(g.Name) > 120 { + return c, fmt.Errorf("secondary goal %q: name too long", g.Key) + } + sec[i] = g + } + c.Secondary = sec + return c, nil +} + +// SecondaryEvaluationsNote is the multiple-comparison caveat attached to any +// result that evaluates more than one metric. +func (c ExperimentConfig) multipleComparisonNote() string { + if len(c.Secondary) == 0 { + return "" + } + return fmt.Sprintf("%d secondary metric(s) are shown beside the primary. Only the primary gates the winner. "+ + "Holm correction applies across the arms of each metric, not across metrics, so a secondary metric crossing p<0.05 "+ + "is exploratory: with several metrics some will cross by chance. Treat secondary results as hypotheses for a follow-up experiment.", + len(c.Secondary)) +} + +// --------------------------------------------------------------------------- +// Storage. + +// SaveConfig validates and appends a settings document for an experiment of +// the site. The experiment must exist for that site. +func (s *ExperimentService) SaveConfig(ctx context.Context, experimentID, siteID string, cfg ExperimentConfig) (ExperimentConfig, error) { + cfg, err := cfg.Normalize() + if err != nil { + return cfg, err + } + if _, err := s.loadExperiment(ctx, experimentID, siteID); err != nil { + return cfg, err + } + doc, err := json.Marshal(cfg) + if err != nil { + return cfg, err + } + _, err = s.db.SQL().Exec(ctx, + `INSERT INTO experiment_settings (setting_id, tenant_id, experiment_id, site_id, config, timestamp) + VALUES ($1, 'default', $2, $3, $4, $5)`, + genID(), experimentID, siteID, string(doc), time.Now().UTC().UnixMilli()) + if err != nil { + return cfg, fmt.Errorf("save experiment settings: %w", err) + } + return cfg, nil +} + +// LoadConfig returns the newest settings document, or DefaultConfig when the +// experiment has none (every pre-063 experiment). +func (s *ExperimentService) LoadConfig(ctx context.Context, experimentID, siteID string) (ExperimentConfig, error) { + type row struct { + Config string `db:"config"` + } + rows, err := nucleus.Query[row](ctx, s.db.SQL(), + `SELECT config FROM experiment_settings + WHERE experiment_id = $1 AND site_id = $2 + ORDER BY timestamp DESC, setting_id DESC LIMIT 1`, + experimentID, siteID) + if err != nil { + return DefaultConfig(), err + } + if len(rows) == 0 { + return DefaultConfig(), nil + } + var cfg ExperimentConfig + if err := json.Unmarshal([]byte(rows[0].Config), &cfg); err != nil { + return DefaultConfig(), fmt.Errorf("experiment settings corrupt: %w", err) + } + return cfg.Normalize() +} + +// RecordMetric appends one metric event (a count/mean/secondary observation) +// for an exposed user. Like RecordConversion, an event from a user with no +// exposure is ignored. value must be finite; binary and count goals ignore it +// at analysis time but it is still validated and stored. +func (s *ExperimentService) RecordMetric(ctx context.Context, experimentID, siteID, userID, metric string, value float64) error { + if metric != PrimaryMetricKey && !metricKeyRE.MatchString(metric) { + return fmt.Errorf("metric must match %s", metricKeyRE.String()) + } + if math.IsNaN(value) || math.IsInf(value, 0) || math.Abs(value) > maxMetricValueAbs { + return fmt.Errorf("value must be a finite number with magnitude <= %g", maxMetricValueAbs) + } + type vrow struct { + Variant string `db:"variant"` + } + rows, err := nucleus.Query[vrow](ctx, s.db.SQL(), + `SELECT variant FROM experiment_exposures + WHERE experiment_id = $1 AND site_id = $2 AND user_id = $3 LIMIT 1`, + experimentID, siteID, userID) + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + _, err = s.db.SQL().Exec(ctx, + `INSERT INTO experiment_metric_events (event_id, tenant_id, experiment_id, site_id, user_id, metric, value, timestamp) + VALUES ($1, 'default', $2, $3, $4, $5, $6, $7)`, + genID(), experimentID, siteID, userID, metric, strconv.FormatFloat(value, 'g', -1, 64), time.Now().UTC().UnixMilli()) + return err +} diff --git a/internal/experiments/continuous.go b/internal/experiments/continuous.go new file mode 100644 index 00000000..ee7f27de --- /dev/null +++ b/internal/experiments/continuous.go @@ -0,0 +1,260 @@ +package experiments + +// Continuous-metric statistics (count and mean/revenue goals): Welch's +// unequal-variance t-test with Welch-Satterthwaite degrees of freedom, the +// mean-difference confidence interval, and percentile winsorizing. Pure +// functions, pinned by scipy goldens in continuous_test.go. + +import ( + "math" + "sort" +) + +// minContinuousN floors the per-arm sample for continuous metrics: a t-test +// on a handful of heavy-tailed revenue values is not trustworthy even when +// the experiment's min_sample is set lower. +const minContinuousN = 30 + +// --------------------------------------------------------------------------- +// Regularized incomplete beta I_x(a, b) and the Student t distribution. + +// betaContinuedFraction is the Lentz evaluation of the incomplete-beta +// continued fraction (Numerical Recipes betacf). +func betaContinuedFraction(a, b, x float64) float64 { + const ( + maxIter = 500 + eps = 1e-15 + tiny = 1e-300 + ) + qab, qap, qam := a+b, a+1, a-1 + c := 1.0 + d := 1 - qab*x/qap + if math.Abs(d) < tiny { + d = tiny + } + d = 1 / d + h := d + for m := 1; m <= maxIter; m++ { + fm := float64(m) + m2 := 2 * fm + aa := fm * (b - fm) * x / ((qam + m2) * (a + m2)) + d = 1 + aa*d + if math.Abs(d) < tiny { + d = tiny + } + c = 1 + aa/c + if math.Abs(c) < tiny { + c = tiny + } + d = 1 / d + h *= d * c + aa = -(a + fm) * (qab + fm) * x / ((a + m2) * (qap + m2)) + d = 1 + aa*d + if math.Abs(d) < tiny { + d = tiny + } + c = 1 + aa/c + if math.Abs(c) < tiny { + c = tiny + } + d = 1 / d + del := d * c + h *= del + if math.Abs(del-1) < eps { + break + } + } + return h +} + +// regIncBeta is the regularized incomplete beta function I_x(a, b). +func regIncBeta(a, b, x float64) float64 { + if x <= 0 { + return 0 + } + if x >= 1 { + return 1 + } + lbeta := lnGamma(a+b) - lnGamma(a) - lnGamma(b) + front := math.Exp(lbeta + a*math.Log(x) + b*math.Log1p(-x)) + if x < (a+1)/(a+b+2) { + return front * betaContinuedFraction(a, b, x) / a + } + return 1 - front*betaContinuedFraction(b, a, 1-x)/b +} + +// studentTTwoSidedP is P(|T| >= |t|) for Student's t with df degrees of +// freedom (df may be fractional, as Welch-Satterthwaite produces). +func studentTTwoSidedP(t, df float64) float64 { + if df <= 0 || math.IsNaN(t) { + return 1 + } + if math.IsInf(t, 0) { + return 0 + } + if t == 0 { + return 1 + } + x := df / (df + t*t) + p := regIncBeta(df/2, 0.5, x) + return clamp01(p) +} + +// studentTQuantileUpper returns the t such that P(T > t) = tail for +// 0 < tail < 0.5 (bisection on the monotone two-sided p; 200 halvings reach +// double precision). +func studentTQuantileUpper(tail, df float64) float64 { + if tail <= 0 || tail >= 0.5 || df <= 0 { + return math.NaN() + } + target := 2 * tail + lo, hi := 0.0, 1.0 + for studentTTwoSidedP(hi, df) > target && hi < 1e12 { + hi *= 2 + } + for i := 0; i < 200; i++ { + mid := (lo + hi) / 2 + if studentTTwoSidedP(mid, df) > target { + lo = mid + } else { + hi = mid + } + } + return (lo + hi) / 2 +} + +// --------------------------------------------------------------------------- +// Per-arm summary and Welch's test. + +// contSummary is the sufficient statistic of one arm's per-user values. +type contSummary struct { + N int64 + Mean float64 + Variance float64 // unbiased sample variance (n-1); 0 when N < 2 +} + +func summarize(values []float64) contSummary { + n := len(values) + if n == 0 { + return contSummary{} + } + var sum float64 + for _, v := range values { + sum += v + } + mean := sum / float64(n) + s := contSummary{N: int64(n), Mean: mean} + if n > 1 { + var ss float64 + for _, v := range values { + d := v - mean + ss += d * d + } + s.Variance = ss / float64(n-1) + } + return s +} + +func (s contSummary) stdDev() float64 { return math.Sqrt(s.Variance) } + +// WelchResult is one two-sample Welch comparison (b minus a). +type WelchResult struct { + Diff float64 + T float64 + DF float64 + PValue float64 + CILow float64 + CIHigh float64 +} + +// welchTest compares arm b against arm a (diff = mean_b - mean_a) with the +// two-sided p-value and the 1-alpha confidence interval of the difference, +// using Welch-Satterthwaite degrees of freedom. Fewer than two users in an +// arm gives no inference (p = 1, unbounded-width CI is reported as the point +// difference with p = 1 so callers never claim significance). +func welchTest(a, b contSummary, alpha float64) WelchResult { + diff := b.Mean - a.Mean + res := WelchResult{Diff: diff, PValue: 1, CILow: diff, CIHigh: diff} + if a.N < 2 || b.N < 2 { + return res + } + va := a.Variance / float64(a.N) + vb := b.Variance / float64(b.N) + se2 := va + vb + if se2 <= 0 { + // Both arms are constant: the difference is exact. Identical + // constants are no evidence; different constants are certain. + res.DF = float64(a.N + b.N - 2) + if diff != 0 { + res.PValue = 0 + } + return res + } + se := math.Sqrt(se2) + res.T = diff / se + res.DF = se2 * se2 / (va*va/float64(a.N-1) + vb*vb/float64(b.N-1)) + res.PValue = studentTTwoSidedP(res.T, res.DF) + q := studentTQuantileUpper(alpha/2, res.DF) + res.CILow = diff - q*se + res.CIHigh = diff + q*se + return res +} + +// --------------------------------------------------------------------------- +// Winsorizing. + +// percentileLinear is the linear-interpolation percentile (numpy.percentile +// default) of an ascending-sorted slice, q in [0, 100]. +func percentileLinear(sorted []float64, q float64) float64 { + n := len(sorted) + if n == 0 { + return 0 + } + if n == 1 { + return sorted[0] + } + pos := q / 100 * float64(n-1) + lo := int(math.Floor(pos)) + if lo < 0 { + lo = 0 + } + if lo >= n-1 { + return sorted[n-1] + } + frac := pos - float64(lo) + return sorted[lo] + frac*(sorted[lo+1]-sorted[lo]) +} + +// winsorizeBounds returns the [pct, 100-pct] percentile clip bounds of the +// pooled values. pct must be in (0, 50); otherwise there is nothing to clip. +func winsorizeBounds(pooled []float64, pct float64) (lo, hi float64, ok bool) { + if !(pct > 0 && pct < 50) || len(pooled) == 0 { + return 0, 0, false + } + sorted := append([]float64(nil), pooled...) + sort.Float64s(sorted) + return percentileLinear(sorted, pct), percentileLinear(sorted, 100-pct), true +} + +// winsorizeArms clips every arm's values to the pooled [pct, 100-pct] +// percentile bounds, in place. Bounds are pooled across arms so the clip is +// identical for every arm (a per-arm clip would bias the comparison). +func winsorizeArms(arms [][]float64, pct float64) { + var pooled []float64 + for _, a := range arms { + pooled = append(pooled, a...) + } + lo, hi, ok := winsorizeBounds(pooled, pct) + if !ok { + return + } + for _, a := range arms { + for i, v := range a { + if v < lo { + a[i] = lo + } else if v > hi { + a[i] = hi + } + } + } +} From aab5fa5b2d5a3c3c3cbeee2eb3d345b3baae832b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:04:12 +0000 Subject: [PATCH 30/65] feat: Sentry-protocol ingest so stock Sentry SDKs can use an Observe DSN POST /api/{project}/envelope/ and /store/ authenticate the DSN key (query, X-Sentry-Auth or envelope dsn) against Observe telemetry API keys, enforce BoundSite, cap decompressed sizes, map event items to the error pipeline and acknowledge-and-drop every other item type. Docs updated to the new truth. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/sentry_routes.go | 12 +- cmd/observe/sentry_routes_test.go | 28 ++ docs/migrations/from-sentry.md | 37 +- docs/sdk/COMPATIBILITY.md | 73 ++++ internal/sentrycompat/handler.go | 14 +- internal/sentrycompat/handler_test.go | 591 ++++++++++++++++++++++++++ 6 files changed, 724 insertions(+), 31 deletions(-) create mode 100644 cmd/observe/sentry_routes_test.go create mode 100644 internal/sentrycompat/handler_test.go diff --git a/cmd/observe/sentry_routes.go b/cmd/observe/sentry_routes.go index f6176c4d..bfd372b1 100644 --- a/cmd/observe/sentry_routes.go +++ b/cmd/observe/sentry_routes.go @@ -16,8 +16,11 @@ import ( // Authentication, the BoundSite invariant, rate limiting and body caps are // all enforced inside the handler (the DSN key can arrive in the query, the // X-Sentry-Auth header or the envelope header, so the X-API-Key middleware -// does not apply). Patterns end in {$} so they match the exact path and do -// not claim the /api/ subtree. +// does not apply). Patterns are exact paths (no subtree claim). No OPTIONS +// route: "OPTIONS /api/{project_id}/..." conflicts with the existing +// "OPTIONS /api/v1/{path...}" preflight handler at boot, and Sentry SDKs send +// simple requests (text/plain, DSN key in the query string), which never +// preflight. func registerSentryRoutes(r interface { Handle(pattern string, handler http.Handler) }, authSvc *auth.AuthService, buf *obserrors.ErrorBuffer, limiter *ingest.RateLimiter, logger *slog.Logger) { @@ -33,11 +36,10 @@ func registerSentryRoutes(r interface { fn http.HandlerFunc }{ {"/api/{project_id}/envelope/{$}", h.Envelope}, - {"/api/{project_id}/envelope{$}", h.Envelope}, + {"/api/{project_id}/envelope", h.Envelope}, {"/api/{project_id}/store/{$}", h.Store}, - {"/api/{project_id}/store{$}", h.Store}, + {"/api/{project_id}/store", h.Store}, } { r.Handle("POST "+p.path, p.fn) - r.Handle("OPTIONS "+p.path, http.HandlerFunc(h.Preflight)) } } diff --git a/cmd/observe/sentry_routes_test.go b/cmd/observe/sentry_routes_test.go new file mode 100644 index 00000000..23cce585 --- /dev/null +++ b/cmd/observe/sentry_routes_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// The Sentry patterns must coexist with the existing /api/v1 routes in one +// mux (a pattern conflict panics at registration, i.e. at boot). +func TestSentryRoutesNoPatternConflict(t *testing.T) { + mux := http.NewServeMux() + noop := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }) + mux.Handle("POST /api/v1/events", noop) + mux.Handle("POST /api/v1/errors", noop) + mux.Handle("OPTIONS /api/v1/{path...}", noop) + mux.Handle("GET /api/v1/{path...}", noop) + mux.Handle("POST /api/v1/infra/report", noop) + registerSentryRoutes(mux, nil, nil, nil, nil) + + for _, p := range []string{"/api/1/envelope/", "/api/1/envelope", "/api/abc/store/", "/api/abc/store"} { + rr := httptest.NewRecorder() + mux.ServeHTTP(rr, httptest.NewRequest("POST", p, nil)) + if rr.Code != 401 { // reached the handler (no key), not the v1 routes + t.Errorf("%s: %d", p, rr.Code) + } + } +} diff --git a/docs/migrations/from-sentry.md b/docs/migrations/from-sentry.md index e7379766..3851c288 100644 --- a/docs/migrations/from-sentry.md +++ b/docs/migrations/from-sentry.md @@ -4,25 +4,24 @@ Observe's error tracking covers the same core surface as Sentry: grouped issues, stack traces with source maps, breadcrumbs, releases, and webhook alerts. This guide walks through the SDK-level changes and the concept mapping. -> **TL;DR** — Replace your Sentry SDK with one of Observe's SDKs (or the -> `@teploy/observe-sentry-shim` package). The payload shapes map almost -> 1-to-1; the only concept Sentry has that Observe doesn't is "organization" -> (Observe uses sites instead). - -> **Stock Sentry SDKs cannot be pointed at Observe by changing the DSN.** -> Observe has no Sentry-protocol ingest: there is no `/envelope` or `/store` -> route (nor `/api//envelope/`). Its error endpoint is -> `POST /api/v1/errors` with Observe's own JSON body (`internal/errors/handler.go`). -> A stock `@sentry/*`, `sentry-sdk` or `sentry-go` client given an Observe URL -> will get 404s and drop events. You must swap the SDK (or use the shim, which -> speaks Observe's API, not Sentry's). +> **TL;DR** - Stock Sentry SDKs (`sentry-sdk`, `@sentry/browser`, +> `@sentry/node`, `sentry-go`) can point their DSN at Observe: +> `https://@/`. Error events are ingested; +> transactions, sessions, replays, profiles, attachments, check-ins and logs +> are acknowledged with 200 and dropped. For the full Observe feature set +> (or if you prefer a smaller dependency) swap to an Observe SDK or the +> `@teploy/observe-sentry-shim` package. The only Sentry concept Observe lacks +> is "organization" (Observe uses sites instead). +> +> The key must be an Observe API key with the telemetry capability, bound to +> the site named in the DSN. Details: [Sentry wire protocol](../sdk/COMPATIBILITY.md#sentry-wire-protocol-stock-sentry-sdks-no-shim). ## Concept mapping | Sentry | Observe | |------------------------------|-------------------------------------------------| | Project | Site | -| DSN | Endpoint URL + API key (headers, not URL) | +| DSN | `https://@host/` (wire-compatible), or endpoint URL + API key | | Issue | Issue | | Event | Error event (`error_events` table) | | Release | Release tag (string field) | @@ -181,10 +180,14 @@ Issues are grouped by `group_hash` on ingest, so duplicates merge automatically. ## What doesn't port cleanly -- **Performance / tracing on the Sentry side** — use Observe's OTLP trace - endpoint instead; Sentry's performance SDK doesn't map 1-to-1. -- **Sentry Replay** — Observe has its own replay format (`observe-replay.js`) - rather than rrweb. +- **Performance / tracing on the Sentry side** — transactions and spans sent by + a stock Sentry SDK are acknowledged and dropped; use Observe's OTLP trace + endpoint instead. +- **Release-health sessions, profiles, crons, attachments, user feedback** — + acknowledged and dropped (stock SDK path). +- **Sentry Replay** — dropped on the Sentry wire; Observe has its own replay + format (`observe-replay.js`) rather than rrweb. +- **Event timestamps** — events are timestamped at ingest. - **Organization / team permissions** — Observe is single-tenant per deployment. ## Checklist diff --git a/docs/sdk/COMPATIBILITY.md b/docs/sdk/COMPATIBILITY.md index 7875705f..10e1b129 100644 --- a/docs/sdk/COMPATIBILITY.md +++ b/docs/sdk/COMPATIBILITY.md @@ -108,6 +108,79 @@ back to the shared global scope and **warns once** — in that mode, attribute per-request data through `withScope` (synchronous) or capture-call hints instead. +## Sentry wire protocol (stock Sentry SDKs, no shim) + +Observe speaks the Sentry ingest protocol, so an unmodified Sentry SDK can +send **error events** to Observe by changing only its DSN. This is separate +from the shim above, which is an Observe-native client with a Sentry-shaped API. + +### DSN format + +``` +https://@/ +``` + +- `` is a **telemetry-scoped** Observe API key + (`obs_...`, Settings > API keys). A publish-only key is rejected (403). +- `` is the Observe site id. The key must belong to that site; + naming another site is rejected with 403 (BoundSite invariant). +- The JavaScript SDKs reject a non-numeric project id in a DSN. For them (or + anyone) a numeric id of 1-12 digits is accepted as an **alias**: it names no + site, the key alone decides the site. `https://@host/1` therefore works. + +Routes: `POST /api//envelope/` and `POST /api//store/` +(trailing slash optional). The key is read from `sentry_key` (query), +`X-Sentry-Auth`, or the `dsn` field of the envelope header, in that order. +`gzip` and `deflate` bodies are accepted. Authentication happens before the +body is decoded. + +```python +import sentry_sdk +sentry_sdk.init(dsn="https://obs_xxx@observe.example.com/1", release="v1.4.2") +``` +```js +import * as Sentry from "@sentry/browser"; // or @sentry/node +Sentry.init({ dsn: "https://obs_xxx@observe.example.com/1", release: "v1.4.2" }); +``` +```go +sentry.Init(sentry.ClientOptions{Dsn: "https://obs_xxx@observe.example.com/1"}) +``` + +### What is accepted + +| Item | Status | Notes | +|---|---|---| +| `event` (exceptions, messages / `logentry`) | Accepted | Applied through the normal error pipeline: issues, grouping, FTS, source maps. | +| `/store/` single-event JSON | Accepted | Same mapping. | +| exception chain | Changed semantics | The last (outermost) exception is the event; earlier ones ride in `contexts.sentry.exception_chain`. | +| stack frames incl. `in_app` | Accepted | python / node / browser / go frames. Reordered innermost-first. If no frame carries `in_app` (browser SDKs), it is inferred from the path. | +| `release`, `environment`, `level`, `request.url` | Accepted | URL is stripped of query, fragment and credentials. | +| tags, `contexts`, `extra`, `trace` ids | Accepted | Tags land in `contexts.tags`. Oversized values are replaced by a truncation marker. | +| breadcrumbs | Accepted | Newest 100 kept. | +| `user` | Changed semantics | Only `id` (else `username`, `email`) is used, as `distinct_id`, **hashed server-side** with the site salt. Email, IP and the raw user object are not stored. | +| `fingerprint` | Changed semantics | Honored unless it uses `{{ default }}`, in which case Observe's default grouping applies. | +| `event_id` | Accepted | Idempotency key (namespaced `sentry`); resending the same event applies once. | +| event `timestamp` | Unsupported | Events are stamped at ingest time. | +| `transaction`, `span`, `session`, `sessions`, `attachment`, `client_report`, `profile`, `replay_event`, `check_in`, `log`, and any unknown type | Acknowledged and dropped | 200, counted as `ignored_by_type`. Never an error, so SDKs do not retry. | + +### Status codes + +200 `{"id": ""}` accepted (including dropped types and duplicates); +400 malformed envelope; 401 missing or invalid key; 403 key lacks telemetry +scope or names another site; 413 body over 5 MiB (compressed or decompressed) +or only oversize items (limit 1 MiB per item); 415 unsupported +`Content-Encoding`; 429 with `Retry-After` and `X-Sentry-Rate-Limits` when the +site rate limit or error buffer is hit; 503 with `Retry-After` when error +durability or the auth store is unavailable. At most 100 events are applied per +envelope. + +### Operational notes + +- Not served on the separate ingest listener (`OBSERVE_INGEST_ADDR`) until its + allowlist gains the `/api//envelope|store` routes. +- No CORS preflight route is registered; Sentry SDKs use simple requests. + Responses carry `Access-Control-Allow-Origin: *`. + ## PostHog (`posthog-js` → `@teploy/observe-browser`) | posthog-js API | Status | Notes | diff --git a/internal/sentrycompat/handler.go b/internal/sentrycompat/handler.go index ef2165e4..d5c5a3c8 100644 --- a/internal/sentrycompat/handler.go +++ b/internal/sentrycompat/handler.go @@ -127,12 +127,6 @@ func (h *Handler) Envelope(w http.ResponseWriter, r *http.Request) { h.serve(w, // Store serves POST /api/{project_id}/store/ (legacy single-event JSON). func (h *Handler) Store(w http.ResponseWriter, r *http.Request) { h.serve(w, r, true) } -// Preflight answers CORS preflight for the Sentry routes. -func (h *Handler) Preflight(w http.ResponseWriter, r *http.Request) { - setCORS(w) - w.WriteHeader(http.StatusNoContent) -} - func setCORS(w http.ResponseWriter) { hd := w.Header() hd.Set("Access-Control-Allow-Origin", "*") @@ -279,7 +273,7 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, legacyStore bool h.malformed.Add(int64(pe.Malformed)) respID := pe.Header.EventID - goodItems, events := 0, 0 + goodItems, events, bad := 0, 0, 0 for _, it := range pe.Items { if it.Type != "event" { goodItems++ @@ -288,11 +282,13 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, legacyStore bool } if events >= maxEventsPerEnvelope { h.malformed.Add(1) + bad++ continue } in, eventID, merr := mapEvent(it.Payload, pe.Header.EventID) if merr != nil { h.malformed.Add(1) + bad++ continue } goodItems++ @@ -325,8 +321,8 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, legacyStore bool // Nothing usable: an envelope whose every item was oversize is a 413; // one whose items were all undecodable is a 400. A header-only envelope // (no items at all) is valid and acknowledged. - if goodItems == 0 && (pe.Oversize > 0 || pe.Malformed > 0 || pe.Dropped > 0 || legacyStore) { - if pe.Oversize > 0 && pe.Malformed == 0 { + if goodItems == 0 && (pe.Oversize > 0 || pe.Malformed > 0 || bad > 0 || pe.Dropped > 0 || legacyStore) { + if pe.Oversize > 0 && pe.Malformed == 0 && bad == 0 { fail(w, http.StatusRequestEntityTooLarge, "item too large") return } diff --git a/internal/sentrycompat/handler_test.go b/internal/sentrycompat/handler_test.go new file mode 100644 index 00000000..552c16db --- /dev/null +++ b/internal/sentrycompat/handler_test.go @@ -0,0 +1,591 @@ +package sentrycompat + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "compress/zlib" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/useteploy/teploy-observe/internal/auth" + obserrors "github.com/useteploy/teploy-observe/internal/errors" +) + +type fakeKeys struct{} + +func (fakeKeys) ValidateAPIKey(_ context.Context, key string) (auth.ValidatedKey, error) { + switch key { + case "obs_siteA": + return auth.ValidatedKey{SiteID: "siteaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Scopes: "telemetry"}, nil + case "obs_siteB": + return auth.ValidatedKey{SiteID: "sitebbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", Scopes: "telemetry"}, nil + case "obs_publish": + return auth.ValidatedKey{SiteID: "siteaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Scopes: "publish"}, nil + case "obs_down": + return auth.ValidatedKey{}, fmt.Errorf("%w: boom", auth.ErrAuthUnavailable) + } + return auth.ValidatedKey{}, errors.New("auth: invalid api key") +} + +const siteA = "siteaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const siteB = "sitebbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +type sinkRec struct { + site string + in obserrors.ErrorInput +} + +// fakeSink mimics the buffer's idempotency: same (site,event_id) twice is a +// duplicate. +type fakeSink struct { + mu sync.Mutex + seen map[string]bool + got []sinkRec + err error +} + +func (s *fakeSink) Push(site string, in obserrors.ErrorInput) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.err != nil { + return s.err + } + k := site + "|" + in.ProducerID + "|" + in.EventID + if s.seen == nil { + s.seen = map[string]bool{} + } + if s.seen[k] { + return obserrors.ErrAdmittedDuplicate + } + s.seen[k] = true + s.got = append(s.got, sinkRec{site, in}) + return nil +} + +type denyLimiter struct{ deny bool } + +func (d denyLimiter) Allow(string, string) bool { return !d.deny } + +func newH(sink *fakeSink, lim Limiter) http.Handler { + h := &Handler{Keys: fakeKeys{}, Sink: sink, Limiter: lim} + mux := http.NewServeMux() + mux.HandleFunc("POST /api/{project_id}/envelope/{$}", h.Envelope) + mux.HandleFunc("POST /api/{project_id}/envelope", h.Envelope) + mux.HandleFunc("POST /api/{project_id}/store/{$}", h.Store) + return mux +} + +func post(h http.Handler, path string, body []byte, hdr map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", path, bytes.NewReader(body)) + for k, v := range hdr { + req.Header.Set(k, v) + } + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + return rr +} + +func gz(t testing.TB, b []byte) []byte { + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + w.Write(b) + w.Close() + return buf.Bytes() +} + +func envelope(items ...string) []byte { + return []byte(`{"event_id":"9ec79c33ec9942ab8353589fcb2e04dc","sent_at":"2026-01-01T00:00:00Z"}` + "\n" + strings.Join(items, "\n") + "\n") +} + +func evItem(payload string) string { + return fmt.Sprintf(`{"type":"event","length":%d}`, len(payload)) + "\n" + payload +} + +// ---- real-shaped fixtures (hand-written replicas of SDK output) ---- + +const pyEvent = `{"event_id":"9ec79c33ec9942ab8353589fcb2e04dc","level":"error","platform":"python","release":"app@1.2.3","environment":"production","server_name":"web-1","timestamp":"2026-01-01T00:00:00.000000Z","sdk":{"name":"sentry.python","version":"2.8.0"},"user":{"id":"u-42","email":"a@b.co","ip_address":"1.2.3.4"},"tags":{"region":"eu"},"fingerprint":["{{ default }}"],"exception":{"values":[{"type":"ZeroDivisionError","value":"division by zero","mechanism":{"type":"excepthook","handled":false},"stacktrace":{"frames":[{"filename":"app/main.py","abs_path":"/srv/app/main.py","function":"","lineno":10,"in_app":true},{"filename":"app/calc.py","abs_path":"/srv/app/calc.py","function":"div","lineno":3,"in_app":true},{"filename":"requests/api.py","abs_path":"/usr/lib/python3/site-packages/requests/api.py","function":"get","lineno":5,"in_app":false}]}}]},"breadcrumbs":{"values":[{"type":"log","category":"app","message":"started","level":"info","timestamp":1767225600.5}]},"contexts":{"runtime":{"name":"CPython","version":"3.12"},"trace":{"trace_id":"771a43a4192642f0b136d5159a501700","span_id":"a1b2c3d4e5f60708"}}}` + +const browserEvent = `{"event_id":"11111111111111111111111111111111","platform":"javascript","level":"error","release":"web@9","request":{"url":"https://app.example.com/page?token=SECRET#x","headers":{"User-Agent":"x"}},"exception":{"values":[{"type":"TypeError","value":"x is undefined","mechanism":{"type":"onerror","handled":false},"stacktrace":{"frames":[{"filename":"https://app.example.com/static/vendor.js","function":"a","lineno":1,"colno":100},{"filename":"https://app.example.com/static/app.js","function":"onClick","lineno":1,"colno":2000}]}}]},"breadcrumbs":[{"category":"ui.click","message":"button","timestamp":1767225600.1}],"contexts":{"browser":{"name":"Chrome","version":"120"},"os":{"name":"Windows","version":"11"}},"user":{"id":7}}` + +const nodeEvent = `{"event_id":"22222222222222222222222222222222","platform":"node","level":"fatal","exception":{"values":[{"type":"Error","value":"boom","stacktrace":{"frames":[{"filename":"/app/node_modules/express/lib/router.js","function":"handle","lineno":1,"in_app":false},{"filename":"/app/server.js","function":"handler","lineno":42,"colno":9,"in_app":true}]}}]}}` + +const goEvent = `{"event_id":"33333333333333333333333333333333","platform":"go","level":"error","message":"plain message","exception":[{"type":"*errors.errorString","value":"oops","stacktrace":{"frames":[{"function":"main.main","module":"main","abs_path":"/src/app/main.go","lineno":12,"in_app":true}]}}]}` + +const sessionPayload = `{"sid":"a1b2c3","did":"u","init":true,"started":"2026-01-01T00:00:00Z","status":"ok","attrs":{"release":"x"}}` +const txPayload = `{"type":"transaction","transaction":"/home","spans":[],"start_timestamp":1,"timestamp":2}` + +func TestRealFixturesMapped(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + env := envelope(evItem(pyEvent), evItem(browserEvent), evItem(nodeEvent), evItem(goEvent), + `{"type":"session"}`+"\n"+sessionPayload, `{"type":"transaction"}`+"\n"+txPayload) + rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", env, nil) + if rr.Code != 200 { + t.Fatalf("code %d %s", rr.Code, rr.Body) + } + if len(sink.got) != 4 { + t.Fatalf("pushed %d", len(sink.got)) + } + py := sink.got[0].in + if sink.got[0].site != siteA || py.SiteID != siteA { + t.Fatal("site not bound to key") + } + if py.ErrorType != "ZeroDivisionError" || py.Handled || py.Mechanism != "excepthook" || py.ReleaseTag != "app@1.2.3" { + t.Fatalf("py: %+v", py) + } + // innermost first: library frame first (reversed), in_app preserved + if len(py.StackTrace) != 3 || py.StackTrace[0].Filename != "requests/api.py" || py.StackTrace[0].InApp || !py.StackTrace[2].InApp || py.StackTrace[2].Function != "" { + t.Fatalf("py frames: %+v", py.StackTrace) + } + if py.DistinctID != "u-42" || py.EventID != "9ec79c33ec9942ab8353589fcb2e04dc" || py.ProducerID != "sentry" { + t.Fatalf("identity: %+v", py) + } + if len(py.Fingerprint) != 0 { + t.Fatal("{{ default }} fingerprint must fall back") + } + if py.TraceID != "771a43a4192642f0b136d5159a501700" || len(py.Breadcrumbs) != 1 || py.Breadcrumbs[0].Timestamp != 1767225600500 { + t.Fatalf("trace/crumbs: %+v", py) + } + raw, _ := json.Marshal(py.Contexts) + if strings.Contains(string(raw), "a@b.co") || strings.Contains(string(raw), "1.2.3.4") { + t.Fatalf("raw user data leaked into contexts: %s", raw) + } + br := sink.got[1].in + if br.Browser != "Chrome 120" || br.OS != "Windows 11" || br.DistinctID != "7" || len(br.Breadcrumbs) != 1 { + t.Fatalf("browser: %+v", br) + } + if !br.StackTrace[0].InApp { // browser frames lack in_app -> inferred + t.Fatalf("browser in_app not inferred: %+v", br.StackTrace) + } + nd := sink.got[2].in + if nd.Level != "fatal" || nd.StackTrace[0].Function != "handler" || !nd.StackTrace[0].InApp || nd.StackTrace[1].InApp { + t.Fatalf("node: %+v", nd) + } + g := sink.got[3].in + if g.ErrorType != "*errors.errorString" || g.StackTrace[0].Filename != "/src/app/main.go" { + t.Fatalf("go: %+v", g) + } + st := h.(*http.ServeMux) + _ = st +} + +func TestIgnoredTypesCounted(t *testing.T) { + sink := &fakeSink{} + hd := &Handler{Keys: fakeKeys{}, Sink: sink} + mux := http.NewServeMux() + mux.HandleFunc("POST /api/{project_id}/envelope/{$}", hd.Envelope) + env := envelope(`{"type":"session"}`+"\n"+sessionPayload, `{"type":"transaction"}`+"\n"+txPayload, + `{"type":"weird_made_up_type_1"}`+"\n{}", `{"type":"weird_made_up_type_2"}`+"\n{}", `{"type":"check_in"}`+"\n{}") + rr := post(mux, "/api/"+siteA+"/envelope/", env, map[string]string{"X-Sentry-Auth": "Sentry sentry_version=7, sentry_key=obs_siteA, sentry_client=x/1"}) + if rr.Code != 200 || len(sink.got) != 0 { + t.Fatalf("code %d pushed %d", rr.Code, len(sink.got)) + } + s := hd.Stats() + if s.IgnoredByType["session"] != 1 || s.IgnoredByType["transaction"] != 1 || s.IgnoredByType["unknown"] != 2 || s.IgnoredByType["check_in"] != 1 { + t.Fatalf("stats %+v", s) + } + if len(s.IgnoredByType) != 4 { + t.Fatal("type cardinality not bounded") + } +} + +func TestAuth(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + body := envelope(evItem(nodeEvent)) + cases := []struct { + name string + path string + hdr map[string]string + want int + }{ + {"missing key", "/api/1/envelope/", nil, 401}, + {"wrong key", "/api/1/envelope/?sentry_key=nope", nil, 401}, + {"publish-only key", "/api/1/envelope/?sentry_key=obs_publish", nil, 403}, + {"auth store down", "/api/1/envelope/?sentry_key=obs_down", nil, 503}, + {"key A project B", "/api/" + siteB + "/envelope/?sentry_key=obs_siteA", nil, 403}, + {"key B project A", "/api/" + siteA + "/envelope/", map[string]string{"X-Sentry-Auth": "Sentry sentry_key=obs_siteB"}, 403}, + {"project traversal-ish", "/api/..%2f" + siteB + "/envelope/?sentry_key=obs_siteA", nil, 403}, + {"ok numeric alias", "/api/1/envelope/?sentry_key=obs_siteA", nil, 200}, + {"ok own site", "/api/" + siteA + "/envelope?sentry_key=obs_siteA", nil, 200}, + {"empty key param", "/api/1/envelope/?sentry_key=", nil, 401}, + {"garbled auth header", "/api/1/envelope/", map[string]string{"X-Sentry-Auth": "Sentry sentry_key"}, 401}, + } + for _, c := range cases { + sink.got = nil + sink.seen = nil + rr := post(h, c.path, body, c.hdr) + if rr.Code != c.want { + t.Errorf("%s: got %d want %d (%s)", c.name, rr.Code, c.want, rr.Body) + } + if c.want != 200 && len(sink.got) != 0 { + t.Errorf("%s: wrote despite failure", c.name) + } + if strings.Contains(rr.Body.String(), "obs_") { + t.Errorf("%s: key echoed", c.name) + } + } +} + +func TestAuthBeforeParse(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + // A bomb with a bad key: must be 401, not 413 (the body is never decoded). + rr := post(h, "/api/1/envelope/?sentry_key=bad", gz(t, bytes.Repeat([]byte("a"), 8<<20)), map[string]string{"Content-Encoding": "gzip"}) + if rr.Code != 401 { + t.Fatalf("got %d", rr.Code) + } +} + +func TestDSNInEnvelopeHeader(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + env := []byte(`{"event_id":"9ec79c33ec9942ab8353589fcb2e04dc","dsn":"https://obs_siteA@observe.example.com/1"}` + "\n" + evItem(nodeEvent) + "\n") + if rr := post(h, "/api/1/envelope/", env, nil); rr.Code != 200 || len(sink.got) != 1 { + t.Fatalf("code %d pushed %d", rr.Code, len(sink.got)) + } + // gzip body, DSN of a bad key + env2 := []byte(`{"dsn":"https://nope@h/1"}` + "\n" + evItem(nodeEvent)) + if rr := post(h, "/api/1/envelope/", gz(t, env2), map[string]string{"Content-Encoding": "gzip"}); rr.Code != 401 { + t.Fatalf("got %d", rr.Code) + } + // header key site A, DSN in header for B is ignored (query/header key wins) + sink.seen = nil + env3 := []byte(`{"dsn":"https://obs_siteB@h/1"}` + "\n" + evItem(nodeEvent)) + post(h, "/api/1/envelope/?sentry_key=obs_siteA", env3, nil) + if sink.got[len(sink.got)-1].site != siteA { + t.Fatal("header DSN overrode explicit key") + } +} + +func TestEncodings(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + env := envelope(evItem(nodeEvent)) + var zl bytes.Buffer + zw := zlib.NewWriter(&zl) + zw.Write(env) + zw.Close() + var raw bytes.Buffer + fw, _ := flate.NewWriter(&raw, 6) + fw.Write(env) + fw.Close() + cases := []struct { + name string + body []byte + enc string + want int + }{ + {"gzip", gz(t, env), "gzip", 200}, + {"gzip sniffed", gz(t, env), "", 200}, + {"deflate zlib", zl.Bytes(), "deflate", 200}, + {"deflate raw", raw.Bytes(), "deflate", 200}, + {"truncated gzip", gz(t, env)[:20], "gzip", 400}, + {"garbage gzip", []byte("not gzip at all"), "gzip", 400}, + {"unsupported br", env, "br", 415}, + } + for _, c := range cases { + sink.seen = nil + hdr := map[string]string{} + if c.enc != "" { + hdr["Content-Encoding"] = c.enc + } + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", c.body, hdr); rr.Code != c.want { + t.Errorf("%s: got %d want %d", c.name, rr.Code, c.want) + } + } +} + +func TestBombsAndOversize(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + q := "/api/1/envelope/?sentry_key=obs_siteA" + // gzip bomb: 64 MiB of zeros compresses to ~64 KiB. + bomb := gz(t, bytes.Repeat([]byte{0}, 12<<20)) + if rr := post(h, q, bomb, map[string]string{"Content-Encoding": "gzip"}); rr.Code != 413 { + t.Fatalf("bomb: %d", rr.Code) + } + var zl bytes.Buffer + zw := zlib.NewWriter(&zl) + zw.Write(bytes.Repeat([]byte{0}, 12<<20)) + zw.Close() + if rr := post(h, q, zl.Bytes(), map[string]string{"Content-Encoding": "deflate"}); rr.Code != 413 { + t.Fatalf("deflate bomb: %d", rr.Code) + } + // compressed body over cap + if rr := post(h, q, bytes.Repeat([]byte("x"), MaxEnvelopeBytes+10), nil); rr.Code != 413 { + t.Fatalf("raw oversize: %d", rr.Code) + } + // single oversize item (2 MiB) -> 413, nothing applied + big := `{"event_id":"44444444444444444444444444444444","message":"` + strings.Repeat("a", 2<<20) + `"}` + if rr := post(h, q, envelope(evItem(big)), nil); rr.Code != 413 { + t.Fatalf("oversize item: %d", rr.Code) + } + // oversize item next to a good one: good applied, 200 + if rr := post(h, q, envelope(evItem(big), evItem(nodeEvent)), nil); rr.Code != 200 || len(sink.got) != 1 { + t.Fatalf("mixed: %d pushed %d", rr.Code, len(sink.got)) + } + if len(sink.got) > 0 && h != nil { + _ = 0 + } +} + +func TestMalformedShapes(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + q := "/api/1/envelope/?sentry_key=obs_siteA" + cases := []struct { + name string + body []byte + want int + }{ + {"empty body", nil, 400}, + {"whitespace body", []byte("\n\n \n"), 400}, + {"header not json", []byte("garbage\n{}\n"), 400}, + {"header is array", []byte("[1,2]\n"), 400}, + {"header only", []byte(`{"event_id":"9ec79c33ec9942ab8353589fcb2e04dc"}` + "\n"), 200}, + {"header only no newline", []byte(`{}`), 200}, + {"length too long", []byte("{}\n" + `{"type":"event","length":99999}` + "\n{}\n"), 400}, + {"negative length", []byte("{}\n" + `{"type":"event","length":-5}` + "\n{}\n"), 400}, + {"length not a number", []byte("{}\n" + `{"type":"event","length":"x"}` + "\n{}\n"), 400}, + {"item header garbage", []byte("{}\n" + "not json\n{}\n"), 400}, + {"payload not json", []byte("{}\n" + `{"type":"event"}` + "\nnot-json\n"), 400}, + {"event with nothing", []byte("{}\n" + `{"type":"event"}` + "\n{}\n"), 400}, + {"payload is array", []byte("{}\n" + `{"type":"event"}` + "\n[1]\n"), 400}, + {"missing type, ignored", []byte("{}\n" + "{}\n{}\n"), 200}, + } + for _, c := range cases { + rr := post(h, q, c.body, nil) + if rr.Code != c.want { + t.Errorf("%s: got %d want %d (%s)", c.name, rr.Code, c.want, rr.Body) + } + } + if len(sink.got) != 0 { + t.Fatal("malformed input was applied") + } + // length lies short: payload cut mid-JSON, then trailing bytes parse as a bad header. + short := `{"event_id":"55555555555555555555555555555555","message":"hello"}` + body := "{}\n" + fmt.Sprintf(`{"type":"event","length":%d}`, len(short)-10) + "\n" + short + "\n" + if rr := post(h, q, []byte(body), nil); rr.Code != 400 || len(sink.got) != 0 { + t.Fatalf("short length: %d", rr.Code) + } + // length lies long but within the body: swallows the next header; no 5xx. + body = "{}\n" + fmt.Sprintf(`{"type":"event","length":%d}`, len(short)+5) + "\n" + short + "\n{}\n" + if rr := post(h, q, []byte(body), nil); rr.Code >= 500 { + t.Fatalf("long length: %d", rr.Code) + } +} + +func TestNoLengthItems(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + body := "{}\n" + `{"type":"event"}` + "\n" + nodeEvent + "\n" + `{"type":"session"}` + "\n" + sessionPayload + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", []byte(body), nil); rr.Code != 200 || len(sink.got) != 1 { + t.Fatalf("%d %d", rr.Code, len(sink.got)) + } +} + +func TestIdempotentDuplicate(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + env := envelope(evItem(pyEvent)) + for i := 0; i < 3; i++ { + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", env, nil); rr.Code != 200 { + t.Fatal(rr.Code) + } + } + if len(sink.got) != 1 { + t.Fatalf("applied %d times", len(sink.got)) + } + // Same event_id from another site is NOT a duplicate (site-scoped). + post(h, "/api/1/envelope/?sentry_key=obs_siteB", env, nil) + if len(sink.got) != 2 { + t.Fatal("dedupe leaked across sites") + } +} + +func TestMappingDeterministic(t *testing.T) { + a, ida, _ := mapEvent([]byte(pyEvent), "") + b, idb, _ := mapEvent([]byte(pyEvent), "") + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + if !bytes.Equal(ja, jb) || ida != idb { + t.Fatal("mapping not deterministic (retries would conflict)") + } + // no event_id anywhere: stable derived id + noid := `{"message":"hi"}` + _, i1, _ := mapEvent([]byte(noid), "") + _, i2, _ := mapEvent([]byte(noid), "") + if i1 != i2 || !validEventID.MatchString(i1) { + t.Fatalf("derived id %q", i1) + } + // hostile event id falls back to a valid one + in, id, _ := mapEvent([]byte(`{"event_id":"../../etc/passwd'--","message":"x"}`), "") + if !validEventID.MatchString(id) || in.EventID != id { + t.Fatalf("hostile id kept: %q", id) + } +} + +func TestHundredItems(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + var items []string + for i := 0; i < 100; i++ { + items = append(items, evItem(fmt.Sprintf(`{"event_id":"%032x","message":"m%d"}`, i+1, i))) + } + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", envelope(items...), nil); rr.Code != 200 || len(sink.got) != 100 { + t.Fatalf("%d %d", rr.Code, len(sink.got)) + } + // 150 events: capped at maxEventsPerEnvelope, still 200 + sink = &fakeSink{} + h = newH(sink, nil) + items = nil + for i := 0; i < 150; i++ { + items = append(items, evItem(fmt.Sprintf(`{"event_id":"%032x","message":"m%d"}`, i+1, i))) + } + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", envelope(items...), nil); rr.Code != 200 || len(sink.got) != maxEventsPerEnvelope { + t.Fatalf("%d %d", rr.Code, len(sink.got)) + } + // thousands of tiny items: bounded, no 5xx + var sb strings.Builder + sb.WriteString("{}\n") + for i := 0; i < 5000; i++ { + sb.WriteString(`{"type":"client_report"}` + "\n{}\n") + } + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", []byte(sb.String()), nil); rr.Code >= 500 { + t.Fatalf("many items: %d", rr.Code) + } +} + +func TestRateLimitAndBackpressure(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, denyLimiter{deny: true}) + rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", envelope(evItem(nodeEvent)), nil) + if rr.Code != 429 || rr.Header().Get("Retry-After") == "" || !strings.Contains(rr.Header().Get("X-Sentry-Rate-Limits"), "::key") { + t.Fatalf("%d %v", rr.Code, rr.Header()) + } + if len(sink.got) != 0 { + t.Fatal("applied while limited") + } + // buffer full -> 429; durability error -> 503 with Retry-After + sink = &fakeSink{err: obserrors.ErrErrorBufferFull} + h = newH(sink, nil) + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", envelope(evItem(nodeEvent)), nil); rr.Code != 429 || rr.Header().Get("X-Sentry-Rate-Limits") == "" { + t.Fatalf("%d", rr.Code) + } + sink = &fakeSink{err: errors.New("disk")} + h = newH(sink, nil) + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", envelope(evItem(nodeEvent)), nil); rr.Code != 503 || rr.Header().Get("Retry-After") == "" { + t.Fatalf("%d", rr.Code) + } + // conflicting reuse is acked (no retry loop) + sink = &fakeSink{err: obserrors.ErrEventIDConflict} + h = newH(sink, nil) + if rr := post(h, "/api/1/envelope/?sentry_key=obs_siteA", envelope(evItem(nodeEvent)), nil); rr.Code != 200 { + t.Fatalf("%d", rr.Code) + } +} + +func TestStoreEndpoint(t *testing.T) { + sink := &fakeSink{} + h := newH(sink, nil) + rr := post(h, "/api/1/store/?sentry_key=obs_siteA", []byte(goEvent), nil) + if rr.Code != 200 || len(sink.got) != 1 { + t.Fatalf("%d %s", rr.Code, rr.Body) + } + var out map[string]string + json.Unmarshal(rr.Body.Bytes(), &out) + if out["id"] != "33333333333333333333333333333333" { + t.Fatalf("id %v", out) + } + if rr := post(h, "/api/1/store/?sentry_key=obs_siteA", []byte("nope"), nil); rr.Code != 400 { + t.Fatalf("%d", rr.Code) + } + if rr := post(h, "/api/1/store/", []byte(goEvent), nil); rr.Code != 401 { + t.Fatalf("%d", rr.Code) + } +} + +func TestCORSHeaders(t *testing.T) { + h := newH(&fakeSink{}, nil) + rr := post(h, "/api/1/envelope/", nil, map[string]string{"Content-Type": "text/plain;charset=UTF-8"}) + if rr.Header().Get("Access-Control-Allow-Origin") != "*" || !strings.Contains(rr.Header().Get("Access-Control-Expose-Headers"), "X-Sentry-Rate-Limits") { + t.Fatal("error responses must carry CORS headers") + } +} + +func TestOversizedFieldsStayUnderRecordCap(t *testing.T) { + huge := strings.Repeat("x", 6000) + var crumbs []string + for i := 0; i < 500; i++ { + crumbs = append(crumbs, fmt.Sprintf(`{"message":"%s","data":{"k":"%s"}}`, huge, huge)) + } + ctxs := `{"a":{"v":"` + strings.Repeat("y", 100000) + `"},"b":{"v":1}}` + p := fmt.Sprintf(`{"message":"m","breadcrumbs":[%s],"contexts":%s,"extra":%s}`, strings.Join(crumbs, ","), ctxs, ctxs) + // the raw payload is > MaxItemBytes; shrink to the cap by dropping crumbs + if len(p) > MaxItemBytes { + p = fmt.Sprintf(`{"message":"m","breadcrumbs":[%s],"contexts":%s,"extra":%s}`, strings.Join(crumbs[:100], ","), ctxs, ctxs) + } + in, _, err := mapEvent([]byte(p), "") + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(in) + if len(raw) > 200<<10 { + t.Fatalf("mapped record %d bytes risks the 256KiB admission cap", len(raw)) + } +} + +func TestConcurrentRace(t *testing.T) { + sink := &fakeSink{} + hd := &Handler{Keys: fakeKeys{}, Sink: sink} + mux := http.NewServeMux() + mux.HandleFunc("POST /api/{project_id}/envelope/{$}", hd.Envelope) + var wg sync.WaitGroup + for g := 0; g < 32; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 20; i++ { + ev := fmt.Sprintf(`{"event_id":"%032x","message":"c"}`, (g%4)*100+i) // overlapping ids + env := envelope(evItem(ev), `{"type":"session"}`+"\n"+sessionPayload) + key := "obs_siteA" + if g%2 == 1 { + key = "obs_siteB" + } + post(mux, "/api/1/envelope/?sentry_key="+key, gz(t, env), map[string]string{"Content-Encoding": "gzip"}) + hd.Stats() + } + }(g) + } + wg.Wait() + // sites alternate with g%2 and ids with g%4: 2 id groups x 20 ids x 2 sites = 80 + if len(sink.got) != 80 { + t.Fatalf("applied %d, want 80 (idempotency under concurrency)", len(sink.got)) + } + for _, g := range sink.got { + if g.in.SiteID != g.site { + t.Fatal("site mismatch") + } + } +} + +func TestKeyParsers(t *testing.T) { + if k := authHeaderKey(`Sentry sentry_version=7,sentry_key=abc,sentry_secret=zzz`); k != "abc" { + t.Fatal(k) + } + if authHeaderKey(`Bearer abc`) != "" || authHeaderKey("") != "" { + t.Fatal("non-sentry scheme accepted") + } + if keyFromDSN("https://abc:secret@h/1") != "abc" || keyFromDSN("://bad") != "" || keyFromDSN("https://h/1") != "" { + t.Fatal("dsn parse") + } +} From e0c58afa34e7f8c15254156f344ddfa3c8239094 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:04:50 +0000 Subject: [PATCH 31/65] feat: trace search attribute filters, orphan traces, query guard - Repeated attr=key:op:value (eq, neq, exists, contains), max 5; keys validated by ^[A-Za-z0-9_.\-/]{1,128}$, values only as bound params. Two-stage: superset LIKE narrowing in SQL, exact JSON verification in Go over a bounded candidate set (cap 2000); `truncated` reported when hit. - Opt-in include_orphans=true lists traces with no root span, labeled root_missing, bounded. - Trace read paths (ListServices, SearchTraces, GetTrace, dependencies) go through the guardmap admission guard with labeled 429/504. - Span-count aggregate restricted to the page's trace ids. - UI: attribute chips, root_missing badge, truncated notice, pure helper with node tests. UI UNBUILT. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/main.go | 124 +----- cmd/observe/trace_handlers.go | 279 ++++++++++++ cmd/observe/trace_handlers_test.go | 171 ++++++++ docs/operations/trace-search.md | 68 +++ internal/tracing/query.go | 149 +++---- internal/tracing/search.go | 669 +++++++++++++++++++++++++++++ internal/tracing/search_test.go | 538 +++++++++++++++++++++++ ui/src/api/traces.ts | 14 + ui/src/lib/traceSearch.test.ts | 87 ++++ ui/src/lib/traceSearch.ts | 91 ++++ ui/src/routes/traces.tsx | 86 +++- 11 files changed, 2060 insertions(+), 216 deletions(-) create mode 100644 cmd/observe/trace_handlers.go create mode 100644 cmd/observe/trace_handlers_test.go create mode 100644 docs/operations/trace-search.md create mode 100644 internal/tracing/search.go create mode 100644 internal/tracing/search_test.go create mode 100644 ui/src/lib/traceSearch.test.ts create mode 100644 ui/src/lib/traceSearch.ts diff --git a/cmd/observe/main.go b/cmd/observe/main.go index 7f415e4f..daaef856 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -458,6 +458,8 @@ func main() { // The same limiter and budgets guard the log and metric read paths. logSvc.WithQueryGuard(queryLimiter, queryBudgets) metricsSvc.WithQueryGuard(queryLimiter, queryBudgets) + // O12: trace reads share the stats limiter and budgets. + traceQuery.WithQueryGuard(queryLimiter, queryBudgets) uptimeSvc := monitoring.NewUptimeService(db, logger) cronSvc := monitoring.NewCronService(db, logger) linkSvc := tracking.NewLinkService(db) @@ -1056,6 +1058,7 @@ func main() { neutron.WithTags("traces"), neutron.WithSummary("Get service dependency graph"), ) + RegisterTraceSearchRoutes(traceGroup, traceQuery) // --- Platform API (JWT auth, admin-only for writes) --- // AUD-008: the stream-ticket mint sits here (JWT-authed) rather than on @@ -5785,127 +5788,6 @@ func otlpTraceHandler(svc *tracing.IngestService) neutron.HandlerFunc[tracing.Ex } } -// --- Trace query handlers --- - -type listServicesInput struct { - SiteID string `query:"site_id"` - From string `query:"from"` - To string `query:"to"` -} - -func listServicesHandler(svc *tracing.QueryService) neutron.HandlerFunc[listServicesInput, []tracing.ServiceSummary] { - return func(ctx context.Context, input listServicesInput) ([]tracing.ServiceSummary, error) { - if input.SiteID == "" { - return nil, neutron.ErrBadRequest("site_id required") - } - from, to, err := parseTimeRange(input.From, input.To) - if err != nil { - return nil, neutron.ErrBadRequest(err.Error()) - } - return emptyOnNil(svc.ListServices(ctx, input.SiteID, from, to)) - } -} - -type listOpsInput struct { - SiteID string `query:"site_id"` - Service string `path:"service"` - From string `query:"from"` - To string `query:"to"` -} - -func listOperationsHandler(svc *tracing.QueryService) neutron.HandlerFunc[listOpsInput, []tracing.OperationSummary] { - return func(ctx context.Context, input listOpsInput) ([]tracing.OperationSummary, error) { - if input.SiteID == "" || input.Service == "" { - return nil, neutron.ErrBadRequest("site_id and service required") - } - from, to, err := parseTimeRange(input.From, input.To) - if err != nil { - return nil, neutron.ErrBadRequest(err.Error()) - } - return emptyOnNil(svc.ListOperations(ctx, input.SiteID, input.Service, from, to)) - } -} - -type searchTracesInput struct { - SiteID string `query:"site_id"` - From string `query:"from"` - To string `query:"to"` - Service string `query:"service"` - Operation string `query:"operation"` - Offset int `query:"offset"` - Status string `query:"status"` - MinDuration int64 `query:"min_duration"` - MaxDuration int64 `query:"max_duration"` - Limit int `query:"limit"` -} - -func searchTracesHandler(svc *tracing.QueryService) neutron.HandlerFunc[searchTracesInput, []tracing.TraceSummary] { - return func(ctx context.Context, input searchTracesInput) ([]tracing.TraceSummary, error) { - if input.SiteID == "" { - return nil, neutron.ErrBadRequest("site_id required") - } - from, to, err := parseTimeRange(input.From, input.To) - if err != nil { - return nil, neutron.ErrBadRequest(err.Error()) - } - return emptyOnNil(svc.SearchTraces(ctx, input.SiteID, from, to, input.Service, input.Operation, input.Status, input.MinDuration, input.MaxDuration, input.Limit, input.Offset)) - } -} - -type getTraceInput struct { - TraceID string `path:"trace_id"` - SiteID string `query:"site_id"` -} - -func getTraceHandler(svc *tracing.QueryService) neutron.HandlerFunc[getTraceInput, []tracing.Span] { - return func(ctx context.Context, input getTraceInput) ([]tracing.Span, error) { - if input.SiteID == "" || input.TraceID == "" { - return nil, neutron.ErrBadRequest("site_id and trace_id required") - } - return emptyOnNil(svc.GetTrace(ctx, input.TraceID, input.SiteID)) - } -} - -type traceErrorsInput struct { - TraceID string `path:"trace_id"` - SiteID string `query:"site_id"` -} - -func traceErrorsHandler(svc *tracing.QueryService) neutron.HandlerFunc[traceErrorsInput, []tracing.TraceErrorHit] { - return func(ctx context.Context, input traceErrorsInput) ([]tracing.TraceErrorHit, error) { - if input.SiteID == "" || input.TraceID == "" { - return nil, neutron.ErrBadRequest("site_id and trace_id required") - } - hits, err := svc.TraceErrors(ctx, input.TraceID, input.SiteID) - if err != nil { - return nil, err - } - if hits == nil { - hits = []tracing.TraceErrorHit{} - } - return hits, nil - } -} - -type serviceDepsInput struct { - SiteID string `query:"site_id"` - From string `query:"from"` - To string `query:"to"` -} - -func serviceDepsHandler(svc *tracing.QueryService) neutron.HandlerFunc[serviceDepsInput, []tracing.Dependency] { - return func(ctx context.Context, input serviceDepsInput) ([]tracing.Dependency, error) { - if input.SiteID == "" { - return nil, neutron.ErrBadRequest("site_id required") - } - from, to, err := parseTimeRange(input.From, input.To) - if err != nil { - return nil, neutron.ErrBadRequest(err.Error()) - } - return emptyOnNil(svc.ServiceDependencies(ctx, input.SiteID, from, to)) - } -} - // parseTimeRange resolves a from/to window. Omitted values default (last 24 // hours / now). R35 (round 4): an explicitly supplied but malformed timestamp // is a 400-grade error, not a silent fallback to the default window — the old diff --git a/cmd/observe/trace_handlers.go b/cmd/observe/trace_handlers.go new file mode 100644 index 00000000..63c7c479 --- /dev/null +++ b/cmd/observe/trace_handlers.go @@ -0,0 +1,279 @@ +package main + +// Trace query handlers, moved out of main.go so the O12 admission mapping and +// the attribute/orphan search live in one file. Behavior of the original +// handlers is unchanged except that query-guard refusals now keep their +// labeled 429/504 problem response (guardmap.HTTPError). + +import ( + "context" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/guardmap" + "github.com/useteploy/teploy-observe/internal/tracing" +) + +type listServicesInput struct { + SiteID string `query:"site_id"` + From string `query:"from"` + To string `query:"to"` +} + +func listServicesHandler(svc *tracing.QueryService) neutron.HandlerFunc[listServicesInput, []tracing.ServiceSummary] { + return func(ctx context.Context, input listServicesInput) ([]tracing.ServiceSummary, error) { + if input.SiteID == "" { + return nil, neutron.ErrBadRequest("site_id required") + } + from, to, err := parseTimeRange(input.From, input.To) + if err != nil { + return nil, neutron.ErrBadRequest(err.Error()) + } + rows, err := svc.ListServices(ctx, input.SiteID, from, to) + return emptyOnNil(rows, guardmap.HTTPError(err)) + } +} + +type listOpsInput struct { + SiteID string `query:"site_id"` + Service string `path:"service"` + From string `query:"from"` + To string `query:"to"` +} + +func listOperationsHandler(svc *tracing.QueryService) neutron.HandlerFunc[listOpsInput, []tracing.OperationSummary] { + return func(ctx context.Context, input listOpsInput) ([]tracing.OperationSummary, error) { + if input.SiteID == "" || input.Service == "" { + return nil, neutron.ErrBadRequest("site_id and service required") + } + from, to, err := parseTimeRange(input.From, input.To) + if err != nil { + return nil, neutron.ErrBadRequest(err.Error()) + } + rows, err := svc.ListOperations(ctx, input.SiteID, input.Service, from, to) + return emptyOnNil(rows, guardmap.HTTPError(err)) + } +} + +type searchTracesInput struct { + SiteID string `query:"site_id"` + From string `query:"from"` + To string `query:"to"` + Service string `query:"service"` + Operation string `query:"operation"` + Offset int `query:"offset"` + Status string `query:"status"` + MinDuration int64 `query:"min_duration"` + MaxDuration int64 `query:"max_duration"` + Limit int `query:"limit"` +} + +// searchTracesHandler is the original root-only search. Attribute filters and +// the orphan mode are on searchTracesAdvancedHandler; this route ignores them +// (callers that need them use /search-advanced). +func searchTracesHandler(svc *tracing.QueryService) neutron.HandlerFunc[searchTracesInput, []tracing.TraceSummary] { + return func(ctx context.Context, input searchTracesInput) ([]tracing.TraceSummary, error) { + if input.SiteID == "" { + return nil, neutron.ErrBadRequest("site_id required") + } + from, to, err := parseTimeRange(input.From, input.To) + if err != nil { + return nil, neutron.ErrBadRequest(err.Error()) + } + rows, err := svc.SearchTraces(ctx, input.SiteID, from, to, input.Service, input.Operation, input.Status, input.MinDuration, input.MaxDuration, input.Limit, input.Offset) + return emptyOnNil(rows, guardmap.HTTPError(err)) + } +} + +type getTraceInput struct { + TraceID string `path:"trace_id"` + SiteID string `query:"site_id"` +} + +func getTraceHandler(svc *tracing.QueryService) neutron.HandlerFunc[getTraceInput, []tracing.Span] { + return func(ctx context.Context, input getTraceInput) ([]tracing.Span, error) { + if input.SiteID == "" || input.TraceID == "" { + return nil, neutron.ErrBadRequest("site_id and trace_id required") + } + rows, err := svc.GetTrace(ctx, input.TraceID, input.SiteID) + return emptyOnNil(rows, guardmap.HTTPError(err)) + } +} + +type traceErrorsInput struct { + TraceID string `path:"trace_id"` + SiteID string `query:"site_id"` +} + +func traceErrorsHandler(svc *tracing.QueryService) neutron.HandlerFunc[traceErrorsInput, []tracing.TraceErrorHit] { + return func(ctx context.Context, input traceErrorsInput) ([]tracing.TraceErrorHit, error) { + if input.SiteID == "" || input.TraceID == "" { + return nil, neutron.ErrBadRequest("site_id and trace_id required") + } + hits, err := svc.TraceErrors(ctx, input.TraceID, input.SiteID) + if err != nil { + return nil, err + } + if hits == nil { + hits = []tracing.TraceErrorHit{} + } + return hits, nil + } +} + +type serviceDepsInput struct { + SiteID string `query:"site_id"` + From string `query:"from"` + To string `query:"to"` +} + +func serviceDepsHandler(svc *tracing.QueryService) neutron.HandlerFunc[serviceDepsInput, []tracing.Dependency] { + return func(ctx context.Context, input serviceDepsInput) ([]tracing.Dependency, error) { + if input.SiteID == "" { + return nil, neutron.ErrBadRequest("site_id required") + } + from, to, err := parseTimeRange(input.From, input.To) + if err != nil { + return nil, neutron.ErrBadRequest(err.Error()) + } + rows, err := svc.ServiceDependencies(ctx, input.SiteID, from, to) + return emptyOnNil(rows, guardmap.HTTPError(err)) + } +} + +// RegisterTraceSearchRoutes wires GET /search-advanced onto the trace group +// (JWT). It is an untyped handler because the typed binder reads only the +// first value of a query key and comma-splits slices, which would corrupt +// repeated attr= parameters whose values may contain commas. +func RegisterTraceSearchRoutes(r *neutron.Router, svc *tracing.QueryService) { + r.Handle("GET /search-advanced", searchTracesAdvancedHandler(svc)) +} + +// traceSearchRequest is a validated /search-advanced request. +type traceSearchRequest struct { + siteID string + from, to time.Time + opts tracing.SearchOptions +} + +// parseTraceSearch validates the query string. Every failure is a client +// error (400); nothing here touches the database. +// +// attr repeated key:op[:value], op in eq|neq|exists|contains +// http_status_code first-class shortcut for attr=http.status_code:eq: +// http_method first-class shortcut for attr=http.method:eq: +// include_orphans true to also list traces whose root span is missing +// +// At most tracing.MaxAttrFilters filters in total (shortcuts included). +func parseTraceSearch(q url.Values) (traceSearchRequest, error) { + var req traceSearchRequest + req.siteID = q.Get("site_id") + if req.siteID == "" { + return req, neutron.ErrBadRequest("site_id required") + } + from, to, err := parseTimeRange(q.Get("from"), q.Get("to")) + if err != nil { + return req, neutron.ErrBadRequest(err.Error()) + } + req.from, req.to = from, to + + o := &req.opts + o.Service, o.Operation, o.Status = q.Get("service"), q.Get("operation"), q.Get("status") + for _, f := range []struct { + name string + dst *int64 + }{{"min_duration", &o.MinDuration}, {"max_duration", &o.MaxDuration}} { + if v := q.Get(f.name); v != "" { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil || n < 0 { + return req, neutron.ErrBadRequest(f.name + " must be a non-negative integer") + } + *f.dst = n + } + } + for _, f := range []struct { + name string + dst *int + }{{"limit", &o.Limit}, {"offset", &o.Offset}} { + if v := q.Get(f.name); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return req, neutron.ErrBadRequest(f.name + " must be a non-negative integer") + } + *f.dst = n + } + } + if v := q.Get("include_orphans"); v != "" { + b, err := strconv.ParseBool(v) + if err != nil { + return req, neutron.ErrBadRequest("include_orphans must be true or false") + } + o.IncludeOrphans = b + } + + raw := q["attr"] + shortcuts := 0 + if q.Get("http_status_code") != "" { + shortcuts++ + } + if q.Get("http_method") != "" { + shortcuts++ + } + // Count before parsing so an oversized request is refused cheaply. + if len(raw)+shortcuts > tracing.MaxAttrFilters { + return req, neutron.ErrBadRequest("at most " + strconv.Itoa(tracing.MaxAttrFilters) + " attribute filters are allowed") + } + for _, s := range raw { + f, err := tracing.ParseAttrFilter(s) + if err != nil { + return req, neutron.ErrBadRequest(err.Error()) + } + o.Attrs = append(o.Attrs, f) + } + if v := q.Get("http_status_code"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 100 || n > 599 { + return req, neutron.ErrBadRequest("http_status_code must be a status code between 100 and 599") + } + o.Attrs = append(o.Attrs, tracing.AttrFilter{Key: "http.status_code", Op: tracing.AttrEq, Value: strconv.Itoa(n)}) + } + if v := q.Get("http_method"); v != "" { + v = strings.ToUpper(v) + if len(v) > 16 { + return req, neutron.ErrBadRequest("http_method is not a valid HTTP method") + } + for _, c := range v { + if c < 'A' || c > 'Z' { + return req, neutron.ErrBadRequest("http_method is not a valid HTTP method") + } + } + o.Attrs = append(o.Attrs, tracing.AttrFilter{Key: "http.method", Op: tracing.AttrEq, Value: v}) + } + if err := tracing.ValidateAttrFilters(o.Attrs); err != nil { + return req, neutron.ErrBadRequest(err.Error()) + } + return req, nil +} + +func searchTracesAdvancedHandler(svc *tracing.QueryService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + req, err := parseTraceSearch(r.URL.Query()) + if err != nil { + neutron.WriteError(w, r, err) + return + } + res, err := svc.SearchTracesEx(r.Context(), req.siteID, req.from, req.to, req.opts) + if err != nil { + // Refusals keep their labeled 429/504; anything else keeps the + // framework's problem mapping. + neutron.WriteError(w, r, guardmap.HTTPError(err)) + return + } + neutron.JSON(w, http.StatusOK, res) + } +} diff --git a/cmd/observe/trace_handlers_test.go b/cmd/observe/trace_handlers_test.go new file mode 100644 index 00000000..1a8cff49 --- /dev/null +++ b/cmd/observe/trace_handlers_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/queryguard" + "github.com/useteploy/teploy-observe/internal/tracing" +) + +func TestParseTraceSearchValidation(t *testing.T) { + good := func(extra url.Values) url.Values { + v := url.Values{"site_id": {"s1"}} + for k, vs := range extra { + v[k] = vs + } + return v + } + bad := map[string]url.Values{ + "missing site": {}, + "bad op": good(url.Values{"attr": {"k:regex:x"}}), + "bad key": good(url.Values{"attr": {"a b:eq:x"}}), + "sql key": good(url.Values{"attr": {"a';DROP TABLE spans;--:eq:x"}}), + "eq without value": good(url.Values{"attr": {"k:eq"}}), + "no op": good(url.Values{"attr": {"k"}}), + "too many": good(url.Values{"attr": { + "a:exists", "b:exists", "c:exists", "d:exists", "e:exists", "f:exists"}}), + "too many with shortcuts": good(url.Values{ + "attr": {"a:exists", "b:exists", "c:exists", "d:exists"}, + "http_status_code": {"500"}, "http_method": {"GET"}}), + "bad status shortcut": good(url.Values{"http_status_code": {"99"}}), + "status injection": good(url.Values{"http_status_code": {"500 OR 1=1"}}), + "bad method": good(url.Values{"http_method": {"GET;--"}}), + "bad orphans flag": good(url.Values{"include_orphans": {"maybe"}}), + "bad limit": good(url.Values{"limit": {"-1"}}), + "bad min duration": good(url.Values{"min_duration": {"x"}}), + "bad time": good(url.Values{"from": {"yesterday"}}), + } + for name, v := range bad { + _, err := parseTraceSearch(v) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + var app *neutron.AppError + if !errors.As(err, &app) || app.Status != http.StatusBadRequest { + t.Errorf("%s: not a 400: %v", name, err) + } + } + + req, err := parseTraceSearch(good(url.Values{ + "attr": {"db.system:eq:postgres", "http.route:contains:/a,b"}, + "http_status_code": {"503"}, + "http_method": {"get"}, + "include_orphans": {"true"}, + "limit": {"25"}, + })) + if err != nil { + t.Fatal(err) + } + o := req.opts + if !o.IncludeOrphans || o.Limit != 25 || len(o.Attrs) != 4 { + t.Fatalf("opts = %+v", o) + } + // Repeated attr values keep their commas (the typed binder would split). + if o.Attrs[1].Value != "/a,b" { + t.Errorf("comma value mangled: %+v", o.Attrs[1]) + } + if o.Attrs[2] != (tracing.AttrFilter{Key: "http.status_code", Op: tracing.AttrEq, Value: "503"}) || + o.Attrs[3] != (tracing.AttrFilter{Key: "http.method", Op: tracing.AttrEq, Value: "GET"}) { + t.Errorf("shortcuts = %+v %+v", o.Attrs[2], o.Attrs[3]) + } + + // Exactly at the bound is fine. + if _, err := parseTraceSearch(good(url.Values{"attr": {"a:exists", "b:exists", "c:exists", "d:exists", "e:exists"}})); err != nil { + t.Errorf("five filters refused: %v", err) + } +} + +// heldTraceSvc returns a guarded service whose only concurrency slot is taken, +// so any read is refused before it could reach the (nil) database. +func heldTraceSvc(t *testing.T) (*tracing.QueryService, func()) { + t.Helper() + l := queryguard.NewLimiter(1, 1) + rel, err := l.Acquire(context.Background(), "s1") + if err != nil { + t.Fatal(err) + } + return tracing.NewQueryService(nil).WithQueryGuard(l, queryguard.DefaultBudgets()), rel +} + +// A saturated limiter comes back as the labeled 429, not a bare 500. +func TestAdvancedSearchGuardRefusalIs429(t *testing.T) { + svc, release := heldTraceSvc(t) + defer release() + for _, qs := range []string{"site_id=s1", "site_id=s1&include_orphans=true", "site_id=s1&attr=k:exists"} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/api/v1/traces/search-advanced?"+qs, nil) + searchTracesAdvancedHandler(svc)(w, r) + if w.Code != http.StatusTooManyRequests { + t.Errorf("%s: status %d body %s", qs, w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "query_concurrency") { + t.Errorf("%s: unlabeled body %s", qs, w.Body.String()) + } + } +} + +func TestAdvancedSearchBadParamsAre400(t *testing.T) { + svc, release := heldTraceSvc(t) + defer release() + for _, qs := range []string{ + "site_id=s1&attr=k:nope:x", + "site_id=s1&attr=" + url.QueryEscape("a b:eq:x"), + "site_id=s1&attr=a:exists&attr=b:exists&attr=c:exists&attr=d:exists&attr=e:exists&attr=f:exists", + } { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/api/v1/traces/search-advanced?"+qs, nil) + searchTracesAdvancedHandler(svc)(w, r) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status %d body %s", qs, w.Code, w.Body.String()) + } + } +} + +// The typed trace reads map refusals to a 429 AppError too. +func TestTypedTraceHandlersMapRefusals(t *testing.T) { + svc, release := heldTraceSvc(t) + defer release() + ctx := context.Background() + from := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + errs := map[string]error{} + _, errs["services"] = listServicesHandler(svc)(ctx, listServicesInput{SiteID: "s1"}) + _, errs["operations"] = listOperationsHandler(svc)(ctx, listOpsInput{SiteID: "s1", Service: "x"}) + _, errs["search"] = searchTracesHandler(svc)(ctx, searchTracesInput{SiteID: "s1"}) + _, errs["trace"] = getTraceHandler(svc)(ctx, getTraceInput{SiteID: "s1", TraceID: "t"}) + _, errs["deps"] = serviceDepsHandler(svc)(ctx, serviceDepsInput{SiteID: "s1", From: from}) + for name, err := range errs { + var app *neutron.AppError + if !errors.As(err, &app) || app.Status != http.StatusTooManyRequests { + t.Errorf("%s: want 429 AppError, got %v", name, err) + } + } +} + +// The advanced route must coexist with the typed trace routes (/{trace_id}, +// /{trace_id}/errors, /dependencies, /search) without a pattern collision. +func TestTraceSearchRouteRegistersAlongsideTraceRoutes(t *testing.T) { + r := neutron.New().Router() + g := r.Group("/api/v1/traces") + svc := tracing.NewQueryService(nil) + neutron.Get(g, "/search", searchTracesHandler(svc)) + neutron.Get(g, "/{trace_id}", getTraceHandler(svc)) + neutron.Get(g, "/{trace_id}/errors", traceErrorsHandler(svc)) + neutron.Get(g, "/dependencies", serviceDepsHandler(svc)) + RegisterTraceSearchRoutes(g, svc) + + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/v1/traces/search-advanced?attr=a+b:eq:x", nil)) + if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "site_id") { + t.Fatalf("advanced route not reached: %d %s", w.Code, w.Body.String()) + } +} diff --git a/docs/operations/trace-search.md b/docs/operations/trace-search.md new file mode 100644 index 00000000..2bd1f01b --- /dev/null +++ b/docs/operations/trace-search.md @@ -0,0 +1,68 @@ +# Trace search: attribute filters and orphan traces + +`GET /api/v1/traces/search-advanced` (JWT) extends `GET /api/v1/traces/search`, +which is unchanged (root spans only, array response, ignores the parameters +below). Everything here is **UNVERIFIED against a live Nucleus**: it is covered +by unit tests with stubbed queries only. + +## Parameters + +| Param | Meaning | +|---|---| +| `site_id`, `from`, `to`, `service`, `operation`, `status`, `min_duration`, `max_duration`, `limit`, `offset` | As `/search`. `limit` is capped at 200. | +| `attr=key:op[:value]` | Repeatable, at most 5 in total. `op` is `eq`, `neq`, `exists` or `contains`. Keys match `^[A-Za-z0-9_.\-/]{1,128}$`; values are at most 256 bytes. | +| `http_status_code=500` | Shortcut for `attr=http.status_code:eq:500` (counts toward the 5). | +| `http_method=GET` | Shortcut for `attr=http.method:eq:GET` (counts toward the 5). | +| `include_orphans=true` | Also list traces with no root span (default off). | + +`neq` means "present with a different value" (an absent key does not match). +`contains` is case-sensitive. Filters apply to the listed span: the root span, +or for an orphan trace its earliest span. A bad key, op, value, flag or more +than five filters is a 400. + +The response is an envelope, not an array: + +```json +{"traces": [...], "truncated": false, "truncated_reason": "", "candidate_cap": 2000} +``` + +`root_missing: true` on a trace means it has no root span and the row is its +earliest span (span count is the spans found; duration is the trace extent). + +## How attribute filtering works + +`spans.attributes` is a JSONB column (migration 003) holding a string map. The +repo has no verified JSON extraction over it (`internal/metrics/attrs.go` +records that Nucleus has no JSONB extract; `internal/query/goals.go` uses `->>` +only on `events.properties`), so no key is ever interpolated into SQL. Search +is two-stage: + +1. SQL narrows by the existing predicates plus, per filter, + `CAST(attributes AS TEXT) LIKE $n` with a bound pattern `%"key"%` and, for + `eq`/`contains`, `%value%`. Patterns are built to only ever over-select: + `_` stays a one-character wildcard (a superset of itself), and a value + containing `%`, `\`, a quote, a control byte or non-ASCII skips the value + pattern entirely, so the result never depends on how the engine escapes or + re-serialises JSON. Values and keys are always bound parameters. +2. Go decodes the attributes of up to 2000 candidate root spans and checks + every filter exactly. + +If the candidate set hits 2000 the response says `truncated: true` with a +reason; narrow the time range or filters. It is never silent. + +## Orphan mode + +With `include_orphans=true` the newest 2000 non-root spans in the window are +inspected (at most 200 distinct traces, 5000 spans loaded). A trace is +reported when it has no span with an empty parent and its earliest span's +parent is not in the trace. This is best effort on a busy site and will +usually be `truncated`; the cost of an exact answer would be a window-wide +anti-join that this repo has no precedent for. + +## Query admission (O12) + +Service list, operation list, trace search, trace fetch and the dependency +graph now take a slot from the shared limiter and run under the time budget +(see `capacity.md`). A full limiter is a labeled 429 and a blown budget a 504, +with the refusal code in the problem body. The per-page span count is now +restricted to the page's trace ids instead of grouping the whole window. diff --git a/internal/tracing/query.go b/internal/tracing/query.go index 78f6a3ea..6c197503 100644 --- a/internal/tracing/query.go +++ b/internal/tracing/query.go @@ -11,6 +11,8 @@ import ( "github.com/neutron-build/neutron/go/nucleus" "github.com/useteploy/teploy-observe/internal/dbutil" + "github.com/useteploy/teploy-observe/internal/guardmap" + "github.com/useteploy/teploy-observe/internal/queryguard" ) // servicesCacheTTL is how long a services rollup stays servable. The RED @@ -41,6 +43,22 @@ type QueryService struct { // so the map stays bounded by the windows queried within one TTL. servicesMu sync.Mutex servicesCache map[string]servicesEntry + + // qguard is the O12 read-path admission (concurrency slot + time/row + // budgets). The nil Guard is valid: no concurrency bound, default budgets. + qguard *guardmap.Guard + + // Test seams for the search pipeline; nil means query the database. + querySpans func(ctx context.Context, sql string, args ...any) ([]Span, error) + querySummaries func(ctx context.Context, sql string, args ...any) ([]TraceSummary, error) + queryCounts func(ctx context.Context, sql string, args ...any) ([]spanCountRow, error) +} + +// WithQueryGuard installs the O12 read-path admission state. A nil limiter +// disables concurrency admission but keeps the budgets. +func (q *QueryService) WithQueryGuard(l *queryguard.Limiter, b queryguard.Budgets) *QueryService { + q.qguard = guardmap.NewGuard(l, b) + return q } func NewQueryService(db *nucleus.Client) *QueryService { @@ -131,6 +149,14 @@ func (q *QueryService) ListServices(ctx context.Context, siteID string, from, to fromMs := dbutil.IntParam(from.UnixMilli()) toMs := dbutil.IntParam(to.UnixMilli()) + // O12: a cache hit above costs nothing and takes no slot; the scan below + // does, and runs under the time budget. + qctx, release, err := q.qguard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + // Compute RED metrics from raw spans, NOT the service_stats rollup. The // rollup is written once per ingest batch with that batch's counts into a // ReplacingMergeTree keyed on (service, operation, bucket); read-time dedup @@ -158,7 +184,7 @@ func (q *QueryService) ListServices(ctx context.Context, siteID string, from, to } satisfiedMs := dbutil.IntParam(apdexThresholdMs) toleratedMs := dbutil.IntParam(4 * apdexThresholdMs) - rows, err := nucleus.Query[rawStat](ctx, q.db.SQL(), + rows, err := nucleus.Query[rawStat](qctx, q.db.SQL(), `SELECT service_name, COUNT(*) AS request_count, SUM(CASE WHEN status_code = 'error' THEN 1 ELSE 0 END) AS error_count, @@ -175,7 +201,7 @@ func (q *QueryService) ListServices(ctx context.Context, siteID string, from, to siteID, fromMs, toMs, satisfiedMs, toleratedMs, ) if err != nil { - return nil, err + return nil, q.qguard.DeadlineError(ctx, err) } result := make([]ServiceSummary, 0, len(rows)) @@ -219,6 +245,12 @@ func (q *QueryService) ListOperations(ctx context.Context, siteID, service strin fromMs := dbutil.IntParam(from.UnixMilli()) toMs := dbutil.IntParam(to.UnixMilli()) + qctx, release, err := q.qguard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + // RED per operation from raw spans (see ListServices for why the // service_stats rollup is not the source of truth). type rawStat struct { @@ -230,7 +262,7 @@ func (q *QueryService) ListOperations(ctx context.Context, siteID, service strin P95 int64 `db:"p95_ms"` P99 int64 `db:"p99_ms"` } - rows, err := nucleus.Query[rawStat](ctx, q.db.SQL(), + rows, err := nucleus.Query[rawStat](qctx, q.db.SQL(), `SELECT operation_name, COUNT(*) AS request_count, SUM(CASE WHEN status_code = 'error' THEN 1 ELSE 0 END) AS error_count, @@ -244,7 +276,7 @@ func (q *QueryService) ListOperations(ctx context.Context, siteID, service strin siteID, service, fromMs, toMs, ) if err != nil { - return nil, err + return nil, q.qguard.DeadlineError(ctx, err) } result := make([]OperationSummary, 0, len(rows)) @@ -275,88 +307,20 @@ type TraceSummary struct { DurationMs int64 `json:"duration_ms"` SpanCount int64 `json:"span_count"` StatusCode string `json:"status_code"` + // RootMissing marks a trace listed by include_orphans: it has no root + // span and the row shown is its earliest span. Omitted otherwise. + RootMissing bool `json:"root_missing,omitempty"` } -// SearchTraces finds traces matching filters. +// SearchTraces finds root-span traces matching filters. Guarded (O12); the +// attribute and orphan modes live on SearchTracesEx. func (q *QueryService) SearchTraces(ctx context.Context, siteID string, from, to time.Time, service, operation, status string, minDuration, maxDuration int64, limit, offset int) ([]TraceSummary, error) { - fromMs := dbutil.IntParam(from.UnixMilli()) - toMs := dbutil.IntParam(to.UnixMilli()) - if limit <= 0 { - limit = 20 - } - if offset < 0 { - offset = 0 - } - - // start_time is stored as TEXT (digit-string) and Nucleus doesn't coerce it - // to BIGINT for the comparison — both sides need an explicit cast. - where := "site_id = $1 AND start_time >= $2 AND start_time < $3" - params := []any{siteID, fromMs, toMs} - // Site+time-only filter for the span-count aggregate, so a status/service - // filter on the root-span search doesn't undercount a trace's total spans. - baseWhere := where - baseParams := []any{siteID, fromMs, toMs} - idx := 4 - - if service != "" { - where += fmt.Sprintf(" AND service_name = $%d", idx) - params = append(params, service) - idx++ - } - if operation != "" { - where += fmt.Sprintf(" AND operation_name = $%d", idx) - params = append(params, operation) - idx++ - } - if status != "" { - where += fmt.Sprintf(" AND status_code = $%d", idx) - params = append(params, status) - idx++ - } - if minDuration > 0 { - where += fmt.Sprintf(" AND CAST(duration_ms AS BIGINT) >= %d", minDuration) - } - if maxDuration > 0 { - where += fmt.Sprintf(" AND CAST(duration_ms AS BIGINT) <= %d", maxDuration) - } - - q2 := fmt.Sprintf(`SELECT trace_id, - service_name AS root_service, - operation_name AS root_op, - CAST(start_time AS TEXT) AS start_time, - CAST(duration_ms AS TEXT) AS duration_ms, - '0' AS span_count, - status_code - FROM spans - WHERE %s AND parent_span_id = '' - ORDER BY start_time DESC - LIMIT %d OFFSET %d`, where, limit, offset) - - summaries, err := nucleus.Query[TraceSummary](ctx, q.db.SQL(), q2, params...) - if err != nil || len(summaries) == 0 { - return summaries, err - } - - // span_count was previously hardcoded to 1. Compute the real per-trace span - // count with a second aggregate and fold it in by trace_id (Nucleus rejects - // some single-query aggregate shapes, hence the separate pass). - type spanCountRow struct { - TraceID string `db:"trace_id"` - N int64 `db:"n"` - } - counts, cErr := nucleus.Query[spanCountRow](ctx, q.db.SQL(), - fmt.Sprintf(`SELECT trace_id, COUNT(*) AS n FROM spans WHERE %s GROUP BY trace_id`, baseWhere), - baseParams...) - if cErr == nil { - byTrace := make(map[string]int64, len(counts)) - for _, c := range counts { - byTrace[c.TraceID] = c.N - } - for i := range summaries { - summaries[i].SpanCount = byTrace[summaries[i].TraceID] - } - } - return summaries, nil + res, err := q.SearchTracesEx(ctx, siteID, from, to, SearchOptions{ + Service: service, Operation: operation, Status: status, + MinDuration: minDuration, MaxDuration: maxDuration, + Limit: limit, Offset: offset, + }) + return res.Traces, err } // Span is a stored span for the waterfall view. @@ -381,11 +345,16 @@ type Span struct { // GetTrace returns all spans for a trace, ordered for waterfall rendering. func (q *QueryService) GetTrace(ctx context.Context, traceID, siteID string) ([]Span, error) { - spans, err := q.getTraceSpans(ctx, traceID, siteID) + qctx, release, err := q.qguard.Begin(ctx, siteID) if err != nil { return nil, err } - q.attachLinks(ctx, traceID, siteID, spans) + defer release() + spans, err := q.getTraceSpans(qctx, traceID, siteID) + if err != nil { + return nil, q.qguard.DeadlineError(ctx, err) + } + q.attachLinks(qctx, traceID, siteID, spans) return spans, nil } @@ -426,6 +395,12 @@ func (q *QueryService) ServiceDependencies(ctx context.Context, siteID string, f fromMs := dbutil.IntParam(from.UnixMilli()) toMs := dbutil.IntParam(to.UnixMilli()) + qctx, release, err := q.qguard.Begin(ctx, siteID) + if err != nil { + return nil, err + } + defer release() + // Dependency edges from raw spans (parent span's service -> child span's // service) via self-join, NOT the service_dependencies rollup — that is // written per ingest batch and deduped to the newest version per edge, so @@ -437,7 +412,7 @@ func (q *QueryService) ServiceDependencies(ctx context.Context, siteID string, f ErrorCount int64 `db:"error_count"` DurationSum int64 `db:"duration_sum"` } - rows, err := nucleus.Query[rawDep](ctx, q.db.SQL(), + rows, err := nucleus.Query[rawDep](qctx, q.db.SQL(), `SELECT p.service_name AS src_service, c.service_name AS dst_service, COUNT(*) AS call_count, SUM(CASE WHEN c.status_code = 'error' THEN 1 ELSE 0 END) AS error_count, @@ -449,7 +424,7 @@ func (q *QueryService) ServiceDependencies(ctx context.Context, siteID string, f siteID, fromMs, toMs, ) if err != nil { - return nil, err + return nil, q.qguard.DeadlineError(ctx, err) } out := make([]Dependency, 0, len(rows)) diff --git a/internal/tracing/search.go b/internal/tracing/search.go new file mode 100644 index 00000000..6de78355 --- /dev/null +++ b/internal/tracing/search.go @@ -0,0 +1,669 @@ +package tracing + +// Trace search depth: span-attribute filters and an opt-in orphan mode. +// +// Attribute filters are a TWO-STAGE design because the repo has no verified +// JSON extraction over spans.attributes (a JSONB column, migration 003; +// metrics/attrs.go records "Nucleus has no JSONB extract", while +// internal/query/goals.go uses `->>` on events.properties). Stage one is SQL: +// the existing indexed predicates plus a conservative LIKE over +// CAST(attributes AS TEXT) that can only over-select. Stage two is Go: the +// attribute JSON of every candidate is decoded and each filter is verified +// exactly. The candidate set is bounded (searchCandidateCap) and hitting the +// bound is reported as `truncated`, never hidden. +// +// UNVERIFIED against live Nucleus: that CAST(jsonb AS TEXT) LIKE behaves as +// Postgres does. Correctness does not depend on the LIKE being selective, only +// on it not rejecting a true match, so the patterns are built to be supersets +// regardless of how the engine serialises the JSON (see likeLiteral). + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "github.com/neutron-build/neutron/go/nucleus" + + "github.com/useteploy/teploy-observe/internal/dbutil" +) + +const ( + // MaxAttrFilters bounds the attribute filters one search may carry. + MaxAttrFilters = 5 + // MaxAttrValueLen bounds a filter value in bytes. + MaxAttrValueLen = 256 + + // searchCandidateCap is the most root spans one attribute/orphan search + // verifies in Go. One past it is fetched to detect overflow. + searchCandidateCap = 2000 + // orphanTraceCap is the most distinct candidate traces the orphan pass + // inspects, and orphanSpanCap the most spans it loads for them. + orphanTraceCap = 200 + orphanSpanCap = 5000 + // maxSearchLimit is the largest page size; it also bounds the trace-id + // list used to restrict the span-count aggregate. + maxSearchLimit = 200 +) + +// attrKeyRe is the only shape of attribute key the search accepts. It is +// matched before a key reaches any query text or LIKE pattern. +var attrKeyRe = regexp.MustCompile(`^[A-Za-z0-9_.\-/]{1,128}$`) + +// AttrOp is a comparison applied to one span attribute. +type AttrOp string + +const ( + AttrEq AttrOp = "eq" + AttrNeq AttrOp = "neq" + AttrExists AttrOp = "exists" + AttrContains AttrOp = "contains" +) + +// AttrFilter is one `key:op:value` attribute predicate. neq means "present +// with a different value" (an absent key does not match); contains is +// case-sensitive. +type AttrFilter struct { + Key string + Op AttrOp + Value string +} + +// ValidAttrKey reports whether key is acceptable as a filter key. +func ValidAttrKey(key string) bool { return attrKeyRe.MatchString(key) } + +// Validate checks a filter without trusting any field. +func (f AttrFilter) Validate() error { + if !ValidAttrKey(f.Key) { + return fmt.Errorf("invalid attribute key %q (allowed: A-Z a-z 0-9 _ . - /, 1-128 chars)", truncateForError(f.Key)) + } + switch f.Op { + case AttrExists: + if f.Value != "" { + return fmt.Errorf("attribute op exists takes no value") + } + case AttrEq, AttrNeq, AttrContains: + if f.Value == "" { + return fmt.Errorf("attribute op %s requires a value", f.Op) + } + if len(f.Value) > MaxAttrValueLen { + return fmt.Errorf("attribute value exceeds %d bytes", MaxAttrValueLen) + } + if !utf8.ValidString(f.Value) || strings.ContainsRune(f.Value, 0) { + return fmt.Errorf("attribute value must be valid text without NUL") + } + default: + return fmt.Errorf("invalid attribute op %q (allowed: eq, neq, exists, contains)", truncateForError(string(f.Op))) + } + return nil +} + +func truncateForError(s string) string { + if len(s) > 40 { + return s[:40] + "..." + } + return s +} + +// ParseAttrFilter parses `key:op[:value]`. The key regex excludes ':' so the +// first colon always ends the key; the value may itself contain colons. +func ParseAttrFilter(s string) (AttrFilter, error) { + parts := strings.SplitN(s, ":", 3) + if len(parts) < 2 { + return AttrFilter{}, fmt.Errorf("attr must be key:op[:value], got %q", truncateForError(s)) + } + f := AttrFilter{Key: parts[0], Op: AttrOp(parts[1])} + if len(parts) == 3 { + f.Value = parts[2] + } + if err := f.Validate(); err != nil { + return AttrFilter{}, err + } + return f, nil +} + +// ValidateAttrFilters enforces the count bound and validates each filter. +func ValidateAttrFilters(fs []AttrFilter) error { + if len(fs) > MaxAttrFilters { + return fmt.Errorf("at most %d attribute filters are allowed, got %d", MaxAttrFilters, len(fs)) + } + for _, f := range fs { + if err := f.Validate(); err != nil { + return err + } + } + return nil +} + +// likeLiteral renders s for use inside a LIKE pattern. It never emits an +// escape sequence (the engine's escape handling is unverified); instead it +// guarantees the result can only match MORE than the literal: +// - '_' is kept as is (a single-character wildcard, a superset of itself); +// - '%' and '\' cannot be neutralised that way, and any byte that JSON may +// serialise differently (quote, control, non-ASCII) cannot be matched +// textually, so ok=false and the caller skips that narrowing and leaves it +// to the exact Go-side check. +// +// The caller's `%` wildcards are added around the result, never inside it. +func likeLiteral(s string) (string, bool) { + for i := 0; i < len(s); i++ { + c := s[i] + if c < 0x20 || c >= 0x7f || c == '%' || c == '\\' || c == '"' { + return "", false + } + } + return s, true +} + +// SearchOptions are the filters of one trace search. Service, Operation, +// Status and the duration bounds apply to the listed (root) span. +type SearchOptions struct { + Service string + Operation string + Status string + MinDuration int64 + MaxDuration int64 + Limit int + Offset int + Attrs []AttrFilter + IncludeOrphans bool +} + +// SearchResult is a page of trace summaries plus the honesty labels. +type SearchResult struct { + Traces []TraceSummary `json:"traces"` + // Truncated is true when a bound cut the candidate set, so the page may be + // missing matches that exist. TruncatedReason names which bound. + Truncated bool `json:"truncated"` + TruncatedReason string `json:"truncated_reason,omitempty"` + CandidateCap int `json:"candidate_cap,omitempty"` +} + +// buildSearchWhere assembles the shared WHERE clause and its bound params. +// siteID and every filter value travel as bound parameters; the only +// interpolations are integers from int64 (%d) and the placeholder numbers. +// Attribute LIKE narrowing is included only when narrowAttrs is set. +func buildSearchWhere(siteID string, from, to time.Time, o SearchOptions, narrowAttrs bool) (string, []any) { + where := "site_id = $1 AND start_time >= $2 AND start_time < $3" + params := []any{siteID, dbutil.IntParam(from.UnixMilli()), dbutil.IntParam(to.UnixMilli())} + idx := 4 + if o.Service != "" { + where += fmt.Sprintf(" AND service_name = $%d", idx) + params = append(params, o.Service) + idx++ + } + if o.Operation != "" { + where += fmt.Sprintf(" AND operation_name = $%d", idx) + params = append(params, o.Operation) + idx++ + } + if o.Status != "" { + where += fmt.Sprintf(" AND status_code = $%d", idx) + params = append(params, o.Status) + idx++ + } + if o.MinDuration > 0 { + where += fmt.Sprintf(" AND CAST(duration_ms AS BIGINT) >= %d", o.MinDuration) + } + if o.MaxDuration > 0 { + where += fmt.Sprintf(" AND CAST(duration_ms AS BIGINT) <= %d", o.MaxDuration) + } + if narrowAttrs { + for _, f := range o.Attrs { + // Every op needs the key present. Key chars are regex-validated, so + // the quoted form is what JSON would hold verbatim. + if k, ok := likeLiteral(f.Key); ok { + where += fmt.Sprintf(" AND CAST(attributes AS TEXT) LIKE $%d", idx) + params = append(params, `%"`+k+`"%`) + idx++ + } + if f.Op == AttrEq || f.Op == AttrContains { + if v, ok := likeLiteral(f.Value); ok { + where += fmt.Sprintf(" AND CAST(attributes AS TEXT) LIKE $%d", idx) + params = append(params, "%"+v+"%") + idx++ + } + } + } + } + return where, params +} + +// spanSelectCols is the column list of the Go-verified search path. +const spanSelectCols = `trace_id, span_id, parent_span_id, service_name, operation_name, + span_kind, + CAST(start_time AS TEXT) AS start_time, + CAST(end_time AS TEXT) AS end_time, + CAST(duration_ms AS TEXT) AS duration_ms, + status_code, status_message, + COALESCE(attributes, '') AS attributes` + +// buildRootCandidateSQL selects up to cap+1 root spans, newest first. +func buildRootCandidateSQL(where string, cap int) string { + return fmt.Sprintf(`SELECT %s + FROM spans + WHERE %s AND parent_span_id = '' + ORDER BY start_time DESC + LIMIT %d`, spanSelectCols, where, cap+1) +} + +// buildOrphanCandidateSQL selects trace ids of non-root spans in the window. +func buildOrphanCandidateSQL(cap int) string { + return fmt.Sprintf(`SELECT trace_id + FROM spans + WHERE site_id = $1 AND start_time >= $2 AND start_time < $3 AND parent_span_id <> '' + ORDER BY start_time DESC + LIMIT %d`, cap+1) +} + +// buildTraceSpansSQL loads the spans of the given trace ids ($2..). Ordered by +// trace so a row-cap cut only ever splits the final trace. +func buildTraceSpansSQL(n, cap int) string { + ph := make([]string, n) + for i := range ph { + ph[i] = fmt.Sprintf("$%d", i+2) + } + return fmt.Sprintf(`SELECT %s + FROM spans + WHERE site_id = $1 AND trace_id IN (%s) + ORDER BY trace_id, start_time ASC + LIMIT %d`, spanSelectCols, strings.Join(ph, ","), cap+1) +} + +// buildSpanCountSQL counts spans per trace for exactly the page's trace ids +// ($4..), inside the same site+window as the search (precedent for bound IN +// lists: internal/jobs/rollups.go, funnels.go). +func buildSpanCountSQL(n int) string { + ph := make([]string, n) + for i := range ph { + ph[i] = fmt.Sprintf("$%d", i+4) + } + return fmt.Sprintf(`SELECT trace_id, COUNT(*) AS n FROM spans + WHERE site_id = $1 AND start_time >= $2 AND start_time < $3 AND trace_id IN (%s) + GROUP BY trace_id`, strings.Join(ph, ",")) +} + +// attrText renders one decoded JSON attribute value the way the filters +// compare it. Stored values are strings (AttrsToMap); other scalars are +// tolerated for rows written by other paths. +func attrText(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + case json.Number: + return t.String() + case bool: + if t { + return "true" + } + return "false" + default: + b, err := json.Marshal(t) + if err != nil { + return "" + } + return string(b) + } +} + +// decodeAttrs parses a stored attributes JSON object; anything else is nil. +func decodeAttrs(raw string) map[string]any { + if raw == "" { + return nil + } + dec := json.NewDecoder(bytes.NewReader([]byte(raw))) + dec.UseNumber() + var m map[string]any + if err := dec.Decode(&m); err != nil { + return nil + } + return m +} + +// matchAttrs is the exact check: every filter must hold. +func matchAttrs(raw string, fs []AttrFilter) bool { + if len(fs) == 0 { + return true + } + m := decodeAttrs(raw) + if m == nil { + return false + } + for _, f := range fs { + v, present := m[f.Key] + if !present { + return false + } + s := attrText(v) + switch f.Op { + case AttrExists: + case AttrEq: + if s != f.Value { + return false + } + case AttrNeq: + if s == f.Value { + return false + } + case AttrContains: + if !strings.Contains(s, f.Value) { + return false + } + default: + return false + } + } + return true +} + +type spanCountRow struct { + TraceID string `db:"trace_id"` + N int64 `db:"n"` +} + +func (q *QueryService) runSpans(ctx context.Context, sql string, args ...any) ([]Span, error) { + if q.querySpans != nil { + return q.querySpans(ctx, sql, args...) + } + return nucleus.Query[Span](ctx, q.db.SQL(), sql, args...) +} + +func (q *QueryService) runSummaries(ctx context.Context, sql string, args ...any) ([]TraceSummary, error) { + if q.querySummaries != nil { + return q.querySummaries(ctx, sql, args...) + } + return nucleus.Query[TraceSummary](ctx, q.db.SQL(), sql, args...) +} + +func (q *QueryService) runCounts(ctx context.Context, sql string, args ...any) ([]spanCountRow, error) { + if q.queryCounts != nil { + return q.queryCounts(ctx, sql, args...) + } + return nucleus.Query[spanCountRow](ctx, q.db.SQL(), sql, args...) +} + +// fillSpanCounts sets SpanCount for the page's traces that lack one, using one +// aggregate restricted to those trace ids. A failure leaves counts at zero +// (the page is still correct without them), as before. +func (q *QueryService) fillSpanCounts(ctx context.Context, siteID string, from, to time.Time, page []TraceSummary) { + ids := make([]string, 0, len(page)) + for _, t := range page { + if t.SpanCount == 0 { + ids = append(ids, t.TraceID) + } + } + if len(ids) == 0 { + return + } + params := []any{siteID, dbutil.IntParam(from.UnixMilli()), dbutil.IntParam(to.UnixMilli())} + for _, id := range ids { + params = append(params, id) + } + counts, err := q.runCounts(ctx, buildSpanCountSQL(len(ids)), params...) + if err != nil { + return + } + byTrace := make(map[string]int64, len(counts)) + for _, c := range counts { + byTrace[c.TraceID] = c.N + } + for i := range page { + if page[i].SpanCount == 0 { + page[i].SpanCount = byTrace[page[i].TraceID] + } + } +} + +func summaryOf(s Span) TraceSummary { + return TraceSummary{ + TraceID: s.TraceID, RootService: s.ServiceName, RootOp: s.OperationName, + StartTime: s.StartTime, DurationMs: s.DurationMs, StatusCode: s.StatusCode, + } +} + +// SearchTracesEx is SearchTraces with attribute filters and the optional +// orphan mode. Guarded (O12): concurrency slot, time budget, page-depth cap. +// With no attribute filters and no orphans it runs the original SQL-paged +// root search; otherwise it verifies a bounded candidate set in Go. +func (q *QueryService) SearchTracesEx(ctx context.Context, siteID string, from, to time.Time, o SearchOptions) (SearchResult, error) { + if err := ValidateAttrFilters(o.Attrs); err != nil { + return SearchResult{}, err + } + if o.Limit <= 0 { + o.Limit = 20 + } + if o.Limit > maxSearchLimit { + o.Limit = maxSearchLimit + } + if o.Offset < 0 { + o.Offset = 0 + } + if int64(o.Limit)+int64(o.Offset) > q.qguard.MaxRows() { + return SearchResult{}, q.qguard.RowRefusal() + } + qctx, release, err := q.qguard.Begin(ctx, siteID) + if err != nil { + return SearchResult{}, err + } + defer release() + + var res SearchResult + if len(o.Attrs) == 0 && !o.IncludeOrphans { + res, err = q.searchRootsPaged(qctx, siteID, from, to, o) + } else { + res, err = q.searchVerified(qctx, siteID, from, to, o) + } + if err != nil { + return SearchResult{}, q.qguard.DeadlineError(ctx, err) + } + if res.Traces == nil { + res.Traces = []TraceSummary{} + } + return res, nil +} + +// searchRootsPaged is the original root-only search, SQL-paged. +func (q *QueryService) searchRootsPaged(ctx context.Context, siteID string, from, to time.Time, o SearchOptions) (SearchResult, error) { + where, params := buildSearchWhere(siteID, from, to, o, false) + sql := fmt.Sprintf(`SELECT trace_id, + service_name AS root_service, + operation_name AS root_op, + CAST(start_time AS TEXT) AS start_time, + CAST(duration_ms AS TEXT) AS duration_ms, + '0' AS span_count, + status_code + FROM spans + WHERE %s AND parent_span_id = '' + ORDER BY start_time DESC + LIMIT %d OFFSET %d`, where, o.Limit, o.Offset) + page, err := q.runSummaries(ctx, sql, params...) + if err != nil || len(page) == 0 { + return SearchResult{Traces: page}, err + } + q.fillSpanCounts(ctx, siteID, from, to, page) + return SearchResult{Traces: page}, nil +} + +// searchVerified is the Go-paged path: bounded root candidates, exact +// attribute verification, optional orphan traces, then one merged page. +func (q *QueryService) searchVerified(ctx context.Context, siteID string, from, to time.Time, o SearchOptions) (SearchResult, error) { + res := SearchResult{CandidateCap: searchCandidateCap} + where, params := buildSearchWhere(siteID, from, to, o, true) + cands, err := q.runSpans(ctx, buildRootCandidateSQL(where, searchCandidateCap), params...) + if err != nil { + return SearchResult{}, err + } + if len(cands) > searchCandidateCap { + cands = cands[:searchCandidateCap] + res.Truncated = true + res.TruncatedReason = "root candidate cap reached; narrow the time range or filters" + } + var all []TraceSummary + for _, s := range cands { + if matchAttrs(s.Attributes, o.Attrs) { + all = append(all, summaryOf(s)) + } + } + if o.IncludeOrphans { + orphans, trunc, reason, err := q.findOrphans(ctx, siteID, from, to, o) + if err != nil { + return SearchResult{}, err + } + if trunc && !res.Truncated { + res.Truncated, res.TruncatedReason = true, reason + } + all = append(all, orphans...) + } + sort.SliceStable(all, func(i, j int) bool { + if !all[i].StartTime.Equal(all[j].StartTime) { + return all[i].StartTime.After(all[j].StartTime) + } + return all[i].TraceID < all[j].TraceID + }) + if o.Offset >= len(all) { + return res, nil + } + end := o.Offset + o.Limit + if end > len(all) { + end = len(all) + } + res.Traces = all[o.Offset:end] + q.fillSpanCounts(ctx, siteID, from, to, res.Traces) + return res, nil +} + +// findOrphans surfaces traces in the window with no root span whose earliest +// span points at a parent that is not in the trace (root missing or late). +// Best effort over the newest searchCandidateCap non-root spans and at most +// orphanTraceCap traces; hitting either bound is reported via truncated. +// Service/operation/status/duration/attribute filters apply to the earliest +// span (the span listed in place of the root). +func (q *QueryService) findOrphans(ctx context.Context, siteID string, from, to time.Time, o SearchOptions) ([]TraceSummary, bool, string, error) { + base := []any{siteID, dbutil.IntParam(from.UnixMilli()), dbutil.IntParam(to.UnixMilli())} + cands, err := q.runSpans(ctx, buildOrphanCandidateSQL(searchCandidateCap), base...) + if err != nil { + return nil, false, "", err + } + truncated, reason := false, "" + if len(cands) > searchCandidateCap { + cands = cands[:searchCandidateCap] + truncated, reason = true, "orphan candidate cap reached; only the newest non-root spans were inspected" + } + var ids []string + seen := map[string]bool{} + for _, c := range cands { + if c.TraceID == "" || seen[c.TraceID] { + continue + } + if len(ids) == orphanTraceCap { + truncated, reason = true, "orphan trace cap reached; only the newest traces were inspected" + break + } + seen[c.TraceID] = true + ids = append(ids, c.TraceID) + } + if len(ids) == 0 { + return nil, truncated, reason, nil + } + params := []any{siteID} + for _, id := range ids { + params = append(params, id) + } + spans, err := q.runSpans(ctx, buildTraceSpansSQL(len(ids), orphanSpanCap), params...) + if err != nil { + return nil, false, "", err + } + if len(spans) > orphanSpanCap { + // The cut can only split the final trace (ordered by trace_id); drop it + // rather than classify a partial span set. + spans = spans[:orphanSpanCap] + last := spans[len(spans)-1].TraceID + for len(spans) > 0 && spans[len(spans)-1].TraceID == last { + spans = spans[:len(spans)-1] + } + truncated, reason = true, "orphan span cap reached; some candidate traces were not inspected" + } + byTrace := map[string][]Span{} + for _, s := range spans { + byTrace[s.TraceID] = append(byTrace[s.TraceID], s) + } + var out []TraceSummary + for _, id := range ids { + sum, earliest, ok := classifyOrphan(byTrace[id]) + if !ok || !orphanMatches(sum, earliest, from, to, o) { + continue + } + out = append(out, sum) + } + return out, truncated, reason, nil +} + +// classifyOrphan returns the summary for a trace with no root span whose +// earliest span's parent is absent from the trace. +func classifyOrphan(spans []Span) (TraceSummary, Span, bool) { + if len(spans) == 0 { + return TraceSummary{}, Span{}, false + } + ids := make(map[string]bool, len(spans)) + earliest := spans[0] + for _, s := range spans { + if s.ParentSpanID == "" { + return TraceSummary{}, Span{}, false // has a root: listed by the root search + } + ids[s.SpanID] = true + if s.StartTime.Before(earliest.StartTime) || + (s.StartTime.Equal(earliest.StartTime) && s.SpanID < earliest.SpanID) { + earliest = s + } + } + if ids[earliest.ParentSpanID] { + return TraceSummary{}, Span{}, false + } + sum := summaryOf(earliest) + sum.RootMissing = true + sum.SpanCount = int64(len(spans)) + // Extent of the whole trace, since there is no root duration to show. + var end time.Time + for _, s := range spans { + if s.EndTime.After(end) { + end = s.EndTime + } + if s.StatusCode == "error" { + sum.StatusCode = "error" + } + } + if end.After(earliest.StartTime) { + sum.DurationMs = end.Sub(earliest.StartTime).Milliseconds() + } + return sum, earliest, true +} + +// orphanMatches applies the search filters to an orphan trace's earliest span. +func orphanMatches(sum TraceSummary, earliest Span, from, to time.Time, o SearchOptions) bool { + if sum.StartTime.Before(from) || !sum.StartTime.Before(to) { + return false + } + if o.Service != "" && sum.RootService != o.Service { + return false + } + if o.Operation != "" && sum.RootOp != o.Operation { + return false + } + if o.Status != "" && sum.StatusCode != o.Status { + return false + } + if o.MinDuration > 0 && sum.DurationMs < o.MinDuration { + return false + } + if o.MaxDuration > 0 && sum.DurationMs > o.MaxDuration { + return false + } + return matchAttrs(earliest.Attributes, o.Attrs) +} diff --git a/internal/tracing/search_test.go b/internal/tracing/search_test.go new file mode 100644 index 00000000..9974b393 --- /dev/null +++ b/internal/tracing/search_test.go @@ -0,0 +1,538 @@ +package tracing + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +func TestParseAttrFilter(t *testing.T) { + ok := []struct { + in string + want AttrFilter + }{ + {"http.status_code:eq:500", AttrFilter{"http.status_code", AttrEq, "500"}}, + {"http.route:contains:/api/v1", AttrFilter{"http.route", AttrContains, "/api/v1"}}, + {"db.system:exists", AttrFilter{"db.system", AttrExists, ""}}, + {"db.system:exists:", AttrFilter{"db.system", AttrExists, ""}}, + {"url.full:eq:http://x:80/a", AttrFilter{"url.full", AttrEq, "http://x:80/a"}}, + {"a-b/c_d.e:neq:x y", AttrFilter{"a-b/c_d.e", AttrNeq, "x y"}}, + } + for _, c := range ok { + got, err := ParseAttrFilter(c.in) + if err != nil || got != c.want { + t.Errorf("%q: got %+v, %v; want %+v", c.in, got, err, c.want) + } + } + bad := []string{ + "", + "key", + "key:eq", // eq needs a value + ":eq:x", // empty key + "key:regex:x", // unknown op + "key:EQ:x", // ops are lowercase + "key:exists:nope", // exists takes no value + "a'b:eq:x", // quote in key + "a b:eq:x", // space in key + "a\"b:eq:x", // json quote in key + "a;DROP TABLE spans:eq:x", // statement in key + "a%:eq:x", // LIKE wildcard in key + "key\n:eq:x", // control char + "kéy:eq:x", // non-ASCII key + strings.Repeat("k", 129) + ":eq:x", + "key:eq:" + strings.Repeat("v", MaxAttrValueLen+1), + "key:eq:a\x00b", + "key:eq:\xff\xfe", + } + for _, in := range bad { + if f, err := ParseAttrFilter(in); err == nil { + t.Errorf("%q: accepted as %+v", truncateForError(in), f) + } + } +} + +func TestValidateAttrFiltersBound(t *testing.T) { + f := AttrFilter{"k", AttrExists, ""} + if err := ValidateAttrFilters(make5(f, MaxAttrFilters)); err != nil { + t.Fatalf("at the bound: %v", err) + } + if err := ValidateAttrFilters(make5(f, MaxAttrFilters+1)); err == nil { + t.Fatal("one past the bound accepted") + } +} + +func make5(f AttrFilter, n int) []AttrFilter { + out := make([]AttrFilter, n) + for i := range out { + out[i] = f + } + return out +} + +// Hostile values and keys never reach the SQL text: the WHERE clause holds +// placeholders only, and every user string travels as a bound parameter. +func TestBuildSearchWhereNeverInterpolatesUserText(t *testing.T) { + hostile := []string{ + "x' OR '1'='1", `"; DROP TABLE spans; --`, "100%", "a_b", `back\slash`, + "x') UNION SELECT 1 --", "é中", "$1", "'", + } + from, to := time.Unix(100, 0), time.Unix(200, 0) + for _, v := range hostile { + o := SearchOptions{ + Service: v, Operation: v, Status: v, + Attrs: []AttrFilter{{"http.route", AttrEq, v}, {"k_1", AttrContains, v}}, + } + where, params := buildSearchWhere("site", from, to, o, true) + if strings.Contains(where, v) && len(v) > 2 { + t.Errorf("value %q leaked into SQL: %s", v, where) + } + if strings.Contains(where, "'") { + t.Errorf("quote in SQL for %q: %s", v, where) + } + // Placeholders are dense and match the param count. + for i := 1; i <= len(params); i++ { + if !strings.Contains(where, fmt.Sprintf("$%d", i)) { + t.Errorf("param $%d unused in %s", i, where) + } + } + if strings.Contains(where, fmt.Sprintf("$%d", len(params)+1)) { + t.Errorf("placeholder beyond params in %s", where) + } + } +} + +func TestBuildSearchWhereLikePatterns(t *testing.T) { + from, to := time.Unix(100, 0), time.Unix(200, 0) + cases := []struct { + name string + f AttrFilter + wantParams []string // trailing LIKE params + }{ + {"exists", AttrFilter{"k.a", AttrExists, ""}, []string{`%"k.a"%`}}, + {"eq plain", AttrFilter{"k.a", AttrEq, "GET"}, []string{`%"k.a"%`, "%GET%"}}, + {"neq narrows by key only", AttrFilter{"k.a", AttrNeq, "GET"}, []string{`%"k.a"%`}}, + {"contains", AttrFilter{"k.a", AttrContains, "/api"}, []string{`%"k.a"%`, "%/api%"}}, + // Wildcards that cannot be neutralised drop the value narrowing; the + // exact Go-side check still enforces the value. + {"percent value", AttrFilter{"k.a", AttrEq, "100%"}, []string{`%"k.a"%`}}, + {"backslash value", AttrFilter{"k.a", AttrEq, `a\b`}, []string{`%"k.a"%`}}, + {"quote value", AttrFilter{"k.a", AttrEq, `a"b`}, []string{`%"k.a"%`}}, + {"non-ascii value", AttrFilter{"k.a", AttrEq, "café"}, []string{`%"k.a"%`}}, + // '_' stays a single-char wildcard: a superset of the literal, safe. + {"underscore value", AttrFilter{"k_a", AttrEq, "a_b"}, []string{`%"k_a"%`, "%a_b%"}}, + } + for _, c := range cases { + _, params := buildSearchWhere("s", from, to, SearchOptions{Attrs: []AttrFilter{c.f}}, true) + got := params[3:] + if len(got) != len(c.wantParams) { + t.Errorf("%s: params %v, want %v", c.name, got, c.wantParams) + continue + } + for i := range got { + if got[i] != c.wantParams[i] { + t.Errorf("%s: param %d = %v, want %v", c.name, i, got[i], c.wantParams[i]) + } + } + } + // narrowing off: no attribute SQL at all. + where, params := buildSearchWhere("s", from, to, SearchOptions{Attrs: []AttrFilter{{"k", AttrEq, "v"}}}, false) + if strings.Contains(where, "attributes") || len(params) != 3 { + t.Errorf("narrowing leaked when off: %s %v", where, params) + } +} + +func TestLikeLiteral(t *testing.T) { + for _, s := range []string{"%", "a%b", `\`, `"`, "\n", "é"} { + if _, ok := likeLiteral(s); ok { + t.Errorf("%q should not be narrowable", s) + } + } + for _, s := range []string{"GET", "a_b", "/api/v1", "x y", "a.b-c"} { + if got, ok := likeLiteral(s); !ok || got != s { + t.Errorf("%q: %q %v", s, got, ok) + } + } +} + +func TestMatchAttrs(t *testing.T) { + raw := `{"http.status_code":"500","http.route":"/api/users","empty":"","n":42,"b":true}` + cases := []struct { + f AttrFilter + want bool + }{ + {AttrFilter{"http.status_code", AttrEq, "500"}, true}, + {AttrFilter{"http.status_code", AttrEq, "50"}, false}, + {AttrFilter{"http.status_code", AttrNeq, "200"}, true}, + {AttrFilter{"http.status_code", AttrNeq, "500"}, false}, + {AttrFilter{"missing", AttrNeq, "x"}, false}, // neq needs presence + {AttrFilter{"missing", AttrExists, ""}, false}, + {AttrFilter{"empty", AttrExists, ""}, true}, + {AttrFilter{"http.route", AttrContains, "users"}, true}, + {AttrFilter{"http.route", AttrContains, "Users"}, false}, // case-sensitive + {AttrFilter{"n", AttrEq, "42"}, true}, + {AttrFilter{"b", AttrEq, "true"}, true}, + {AttrFilter{"http.route", AttrEq, "%"}, false}, // value wildcards are literal in Go + {AttrFilter{"http.route", AttrContains, "_"}, false}, + } + for _, c := range cases { + if got := matchAttrs(raw, []AttrFilter{c.f}); got != c.want { + t.Errorf("%+v: got %v want %v", c.f, got, c.want) + } + } + both := []AttrFilter{{"http.status_code", AttrEq, "500"}, {"http.route", AttrContains, "nope"}} + if matchAttrs(raw, both) { + t.Error("all filters must hold") + } + for _, bad := range []string{"", "not json", "[]", "null"} { + if matchAttrs(bad, []AttrFilter{{"k", AttrExists, ""}}) { + t.Errorf("%q matched", bad) + } + } + if !matchAttrs("", nil) { + t.Error("no filters matches everything") + } +} + +// --- search pipeline through the test seams --- + +func tsAt(sec int64) time.Time { return time.Unix(sec, 0) } + +func rootSpan(trace string, start int64, attrs string) Span { + return Span{TraceID: trace, SpanID: trace + "-root", ServiceName: "svc", OperationName: "op", + StartTime: tsAt(start), EndTime: tsAt(start + 1), DurationMs: 1000, StatusCode: "ok", Attributes: attrs} +} + +func TestSearchTracesExAttributeVerifyAndPaging(t *testing.T) { + q := &QueryService{} + var gotSQL string + var gotParams []any + q.querySpans = func(_ context.Context, sql string, args ...any) ([]Span, error) { + gotSQL, gotParams = sql, args + // The LIKE stage is a superset; Go must drop the false positives. + return []Span{ + rootSpan("t5", 50, `{"http.status_code":"500"}`), + rootSpan("t4", 40, `{"http.status_code":"5000"}`), // contains "500" textually, not equal + rootSpan("t3", 30, `{"http.status_code":"500"}`), + rootSpan("t2", 20, ``), + rootSpan("t1", 10, `{"http.status_code":"500"}`), + }, nil + } + var countSQL string + var countArgs []any + q.queryCounts = func(_ context.Context, sql string, args ...any) ([]spanCountRow, error) { + countSQL, countArgs = sql, args + return []spanCountRow{{"t5", 7}, {"t3", 3}}, nil + } + o := SearchOptions{Limit: 2, Attrs: []AttrFilter{{"http.status_code", AttrEq, "500"}}} + res, err := q.SearchTracesEx(context.Background(), "site", tsAt(0), tsAt(100), o) + if err != nil { + t.Fatal(err) + } + if len(res.Traces) != 2 || res.Traces[0].TraceID != "t5" || res.Traces[1].TraceID != "t3" { + t.Fatalf("page = %+v", res.Traces) + } + if res.Truncated { + t.Error("not truncated") + } + if res.Traces[0].SpanCount != 7 || res.Traces[1].SpanCount != 3 { + t.Errorf("span counts = %d %d", res.Traces[0].SpanCount, res.Traces[1].SpanCount) + } + if !strings.Contains(gotSQL, "parent_span_id = ''") || !strings.Contains(gotSQL, "LIKE") || + !strings.Contains(gotSQL, fmt.Sprintf("LIMIT %d", searchCandidateCap+1)) { + t.Errorf("candidate SQL: %s", gotSQL) + } + if len(gotParams) < 5 { + t.Errorf("params %v", gotParams) + } + // The count aggregate is restricted to the page's ids, not the window. + if !strings.Contains(countSQL, "trace_id IN ($4,$5)") || len(countArgs) != 5 || + countArgs[3] != "t5" || countArgs[4] != "t3" { + t.Errorf("count SQL/args: %s %v", countSQL, countArgs) + } + + // Second page. + o.Offset = 2 + res, _ = q.SearchTracesEx(context.Background(), "site", tsAt(0), tsAt(100), o) + if len(res.Traces) != 1 || res.Traces[0].TraceID != "t1" { + t.Fatalf("page 2 = %+v", res.Traces) + } +} + +func TestSearchTracesExTruncatesAtCandidateCap(t *testing.T) { + q := &QueryService{} + q.querySpans = func(context.Context, string, ...any) ([]Span, error) { + out := make([]Span, searchCandidateCap+1) + for i := range out { + out[i] = rootSpan(fmt.Sprintf("t%04d", i), int64(10000-i), `{"k":"v"}`) + } + return out, nil + } + q.queryCounts = func(context.Context, string, ...any) ([]spanCountRow, error) { return nil, nil } + res, err := q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(20000), + SearchOptions{Limit: 5, Attrs: []AttrFilter{{"k", AttrExists, ""}}}) + if err != nil { + t.Fatal(err) + } + if !res.Truncated || res.TruncatedReason == "" || res.CandidateCap != searchCandidateCap { + t.Fatalf("truncation not labeled: %+v", res) + } + if len(res.Traces) != 5 { + t.Fatalf("page = %d", len(res.Traces)) + } +} + +func TestSearchTracesExRejectsBadFilters(t *testing.T) { + q := &QueryService{} + _, err := q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(1), + SearchOptions{Attrs: []AttrFilter{{"bad key", AttrEq, "v"}}}) + if err == nil { + t.Fatal("bad key accepted") + } + _, err = q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(1), + SearchOptions{Attrs: make5(AttrFilter{"k", AttrExists, ""}, MaxAttrFilters+1)}) + if err == nil { + t.Fatal("too many filters accepted") + } +} + +func TestLegacySearchStaysRootOnlyAndSQLPaged(t *testing.T) { + q := &QueryService{} + var sql string + q.querySummaries = func(_ context.Context, s string, _ ...any) ([]TraceSummary, error) { + sql = s + return []TraceSummary{{TraceID: "a"}, {TraceID: "b"}}, nil + } + var countSQL string + q.queryCounts = func(_ context.Context, s string, _ ...any) ([]spanCountRow, error) { + countSQL = s + return []spanCountRow{{"a", 2}, {"b", 9}}, nil + } + out, err := q.SearchTraces(context.Background(), "s", tsAt(0), tsAt(10), "", "", "", 0, 0, 0, 0) + if err != nil || len(out) != 2 || out[1].SpanCount != 9 { + t.Fatalf("%v %+v", err, out) + } + if !strings.Contains(sql, "parent_span_id = ''") || !strings.Contains(sql, "LIMIT 20 OFFSET 0") || strings.Contains(sql, "attributes") { + t.Errorf("legacy SQL changed: %s", sql) + } + if strings.Contains(countSQL, "FROM spans WHERE site_id = $1 AND start_time >= $2 AND start_time < $3 GROUP BY") || + !strings.Contains(countSQL, "trace_id IN") { + t.Errorf("count aggregate not restricted to page ids: %s", countSQL) + } +} + +// --- orphans --- + +func child(trace, id, parent string, start int64, status string) Span { + return Span{TraceID: trace, SpanID: id, ParentSpanID: parent, ServiceName: "svc", OperationName: "op-" + id, + StartTime: tsAt(start), EndTime: tsAt(start + 2), DurationMs: 2000, StatusCode: status} +} + +func TestClassifyOrphan(t *testing.T) { + // Missing root: earliest span's parent "r" is absent. + spans := []Span{child("t", "b", "a", 12, "ok"), child("t", "a", "r", 10, "error")} + sum, earliest, ok := classifyOrphan(spans) + if !ok || !sum.RootMissing || sum.RootOp != "op-a" || earliest.SpanID != "a" || + sum.SpanCount != 2 || sum.StatusCode != "error" || sum.DurationMs != 4000 { + t.Fatalf("%v %+v", ok, sum) + } + // A trace with a root is not an orphan. + if _, _, ok := classifyOrphan([]Span{rootSpan("t", 1, ""), child("t", "c", "t-root", 2, "ok")}); ok { + t.Error("trace with root classified as orphan") + } + // Earliest span's parent present in the trace: not a missing root. + if _, _, ok := classifyOrphan([]Span{child("t", "a", "b", 10, "ok"), child("t", "b", "a", 11, "ok")}); ok { + t.Error("cycle classified as orphan") + } + if _, _, ok := classifyOrphan(nil); ok { + t.Error("empty trace") + } +} + +func TestSearchTracesExIncludeOrphans(t *testing.T) { + q := &QueryService{} + var traceSQL string + q.querySpans = func(_ context.Context, sql string, args ...any) ([]Span, error) { + switch { + case strings.Contains(sql, "parent_span_id = ''"): // root candidates + return []Span{rootSpan("rooted", 60, "")}, nil + case strings.Contains(sql, "parent_span_id <> ''") && !strings.Contains(sql, "trace_id IN"): + // orphan candidates: two orphan traces and one that has a root + return []Span{{TraceID: "late"}, {TraceID: "late"}, {TraceID: "gone"}, {TraceID: "rooted"}}, nil + default: + traceSQL = sql + return []Span{ + child("gone", "g1", "missing", 30, "ok"), + child("late", "l1", "root-not-yet", 50, "ok"), + child("late", "l2", "l1", 51, "ok"), + rootSpan("rooted", 60, ""), + child("rooted", "x", "rooted-root", 61, "ok"), + }, nil + } + } + q.queryCounts = func(context.Context, string, ...any) ([]spanCountRow, error) { + return []spanCountRow{{"rooted", 2}}, nil + } + res, err := q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(100), SearchOptions{IncludeOrphans: true, Limit: 10}) + if err != nil { + t.Fatal(err) + } + var ids []string + for _, tr := range res.Traces { + ids = append(ids, fmt.Sprintf("%s:%v", tr.TraceID, tr.RootMissing)) + } + if strings.Join(ids, ",") != "rooted:false,late:true,gone:true" { + t.Fatalf("traces = %v", ids) + } + if !strings.Contains(traceSQL, "trace_id IN ($2,$3,$4)") || !strings.Contains(traceSQL, fmt.Sprintf("LIMIT %d", orphanSpanCap+1)) { + t.Errorf("trace span SQL: %s", traceSQL) + } + + // Default mode never reports orphans. + q.querySummaries = func(context.Context, string, ...any) ([]TraceSummary, error) { + return []TraceSummary{{TraceID: "rooted"}}, nil + } + res, _ = q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(100), SearchOptions{Limit: 10}) + for _, tr := range res.Traces { + if tr.RootMissing { + t.Fatalf("orphan leaked into default mode: %+v", tr) + } + } + + // Filters apply to the orphan's earliest span. + res, _ = q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(100), SearchOptions{IncludeOrphans: true, Limit: 10, Operation: "op-l1"}) + // (the stub returns the same root candidate regardless of SQL filters) + var orphans []string + for _, tr := range res.Traces { + if tr.RootMissing { + orphans = append(orphans, tr.TraceID) + } + } + if strings.Join(orphans, ",") != "late" { + t.Fatalf("filtered orphans = %v", orphans) + } +} + +func TestOrphanTraceCapIsLabeled(t *testing.T) { + q := &QueryService{} + q.querySpans = func(_ context.Context, sql string, _ ...any) ([]Span, error) { + if strings.Contains(sql, "parent_span_id = ''") { + return nil, nil + } + if !strings.Contains(sql, "trace_id IN") { + out := make([]Span, orphanTraceCap+10) + for i := range out { + out[i] = Span{TraceID: fmt.Sprintf("t%03d", i)} + } + return out, nil + } + return nil, nil + } + res, err := q.SearchTracesEx(context.Background(), "s", tsAt(0), tsAt(100), SearchOptions{IncludeOrphans: true}) + if err != nil || !res.Truncated || res.TruncatedReason == "" { + t.Fatalf("%v %+v", err, res) + } +} + +// --- O12 guard --- + +func refusalOf(t *testing.T, err error) *queryguard.Refusal { + t.Helper() + var ref *queryguard.Refusal + if !errors.As(err, &ref) { + t.Fatalf("want a *queryguard.Refusal, got %v", err) + } + return ref +} + +func guardedTracing(rows int64, timeout time.Duration, l *queryguard.Limiter) *QueryService { + b := queryguard.DefaultBudgets() + b.MaxScanRows = rows + b.Timeout = timeout + return (&QueryService{}).WithQueryGuard(l, b) +} + +// Concurrency admission refuses before any engine access (the service has no +// db, so reaching a query would panic). +func TestTraceReadsConcurrencyRefusal(t *testing.T) { + l := queryguard.NewLimiter(1, 1) + hold, err := l.Acquire(context.Background(), "site") + if err != nil { + t.Fatal(err) + } + defer hold() + q := guardedTracing(100, time.Second, l) + ctx := context.Background() + from, to := tsAt(0), tsAt(100) + + if _, err := q.ListServices(ctx, "site", from, to); refusalOf(t, err).Status != 429 { + t.Errorf("ListServices not refused: %v", err) + } + if _, err := q.ListOperations(ctx, "site", "svc", from, to); refusalOf(t, err).Status != 429 { + t.Errorf("ListOperations not refused: %v", err) + } + if _, err := q.SearchTraces(ctx, "site", from, to, "", "", "", 0, 0, 10, 0); refusalOf(t, err).Status != 429 { + t.Errorf("SearchTraces not refused: %v", err) + } + if _, err := q.SearchTracesEx(ctx, "site", from, to, SearchOptions{IncludeOrphans: true}); refusalOf(t, err).Status != 429 { + t.Errorf("SearchTracesEx not refused: %v", err) + } + if _, err := q.GetTrace(ctx, "trace", "site"); refusalOf(t, err).Status != 429 { + t.Errorf("GetTrace not refused: %v", err) + } + if _, err := q.ServiceDependencies(ctx, "site", from, to); refusalOf(t, err).Status != 429 { + t.Errorf("ServiceDependencies not refused: %v", err) + } +} + +// A page reaching past the row budget is refused, not shortened. +func TestSearchTracesPageBeyondBudgetRefuses(t *testing.T) { + q := guardedTracing(100, time.Second, nil) // nil db: must refuse before querying + for _, c := range []struct{ limit, offset int }{{50, 51}, {10, 1 << 40}} { + _, err := q.SearchTracesEx(context.Background(), "site", tsAt(0), tsAt(100), SearchOptions{Limit: c.limit, Offset: c.offset}) + if ref := refusalOf(t, err); ref.Code != queryguard.CodeBudgetRows || ref.Status != 429 { + t.Fatalf("limit=%d offset=%d: %+v", c.limit, c.offset, ref) + } + } +} + +func TestSearchTracesTimeBudgetIs504(t *testing.T) { + q := guardedTracing(1000, 20*time.Millisecond, nil) + q.querySummaries = func(ctx context.Context, _ string, _ ...any) ([]TraceSummary, error) { + <-ctx.Done() + return nil, ctx.Err() + } + _, err := q.SearchTraces(context.Background(), "site", tsAt(0), tsAt(100), "", "", "", 0, 0, 10, 0) + if ref := refusalOf(t, err); ref.Code != queryguard.CodeBudgetTime || ref.Status != 504 { + t.Fatalf("refusal = %+v", ref) + } + // The verified path honors the same budget. + q.querySpans = func(ctx context.Context, _ string, _ ...any) ([]Span, error) { + <-ctx.Done() + return nil, ctx.Err() + } + _, err = q.SearchTracesEx(context.Background(), "site", tsAt(0), tsAt(100), + SearchOptions{Attrs: []AttrFilter{{"k", AttrExists, ""}}}) + if ref := refusalOf(t, err); ref.Status != 504 { + t.Fatalf("refusal = %+v", ref) + } +} + +// A caller that cancels is not mislabeled as a time-budget refusal. +func TestSearchTracesCallerCancelPropagates(t *testing.T) { + q := guardedTracing(1000, time.Minute, nil) + ctx, cancel := context.WithCancel(context.Background()) + q.querySummaries = func(c context.Context, _ string, _ ...any) ([]TraceSummary, error) { + cancel() + return nil, context.Canceled + } + _, err := q.SearchTraces(ctx, "site", tsAt(0), tsAt(100), "", "", "", 0, 0, 10, 0) + var ref *queryguard.Refusal + if errors.As(err, &ref) { + t.Fatalf("cancellation reported as refusal: %v", err) + } +} diff --git a/ui/src/api/traces.ts b/ui/src/api/traces.ts index aaa1ee10..977f652d 100644 --- a/ui/src/api/traces.ts +++ b/ui/src/api/traces.ts @@ -1,6 +1,8 @@ // Distributed tracing API — services, operations, waterfall, dependencies. import { get, post, del } from "./helpers.js"; +import { buildTraceSearchQuery } from "../lib/traceSearch.js"; +import type { TraceSearchParams } from "../lib/traceSearch.js"; const BASE = "/api/v1/traces"; @@ -33,6 +35,16 @@ export interface TraceSummary { duration_ms: number; span_count: number; status_code: string; + /** Set by include_orphans: no root span; the row is the trace's earliest span. */ + root_missing?: boolean; +} + +/** Response of /search-advanced. `truncated` means a bound cut the candidates. */ +export interface TraceSearchResult { + traces: TraceSummary[]; + truncated: boolean; + truncated_reason?: string; + candidate_cap?: number; } export interface Span { @@ -117,6 +129,8 @@ export const tracesApi = { if (opts?.max_duration) q += `&max_duration=${opts.max_duration}`; return get(`${BASE}/search?${q}`); }, + searchAdvanced: (params: TraceSearchParams) => + get(`${BASE}/search-advanced?${buildTraceSearchQuery(params)}`), trace: (traceId: string, siteId: string) => get(`${BASE}/${traceId}?site_id=${siteId}`), traceErrors: (traceId: string, siteId: string) => diff --git a/ui/src/lib/traceSearch.test.ts b/ui/src/lib/traceSearch.test.ts new file mode 100644 index 00000000..2e3ce9e9 --- /dev/null +++ b/ui/src/lib/traceSearch.test.ts @@ -0,0 +1,87 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + buildTraceSearchQuery, + validateChip, + canAddChip, + chipToParam, + chipLabel, + MAX_ATTR_FILTERS, +} from "./traceSearch.ts"; +import type { AttrChip } from "./traceSearch.ts"; + +const base = { siteId: "s1", from: "2026-01-01T00:00:00Z", to: "2026-01-02T00:00:00Z" }; + +test("minimal query carries site and window only", () => { + const q = new URLSearchParams(buildTraceSearchQuery(base)); + assert.equal(q.get("site_id"), "s1"); + assert.equal(q.get("from"), base.from); + assert.equal(q.has("attr"), false); + assert.equal(q.has("include_orphans"), false); +}); + +test("attr chips become repeated attr params in order", () => { + const attrs: AttrChip[] = [ + { key: "http.status_code", op: "eq", value: "500" }, + { key: "db.system", op: "exists", value: "" }, + { key: "http.route", op: "contains", value: "/api/v1" }, + { key: "http.method", op: "neq", value: "GET" }, + ]; + const q = new URLSearchParams(buildTraceSearchQuery({ ...base, attrs })); + assert.deepEqual(q.getAll("attr"), [ + "http.status_code:eq:500", + "db.system:exists", + "http.route:contains:/api/v1", + "http.method:neq:GET", + ]); +}); + +test("values with colons, commas, ampersands and hashes round-trip", () => { + const value = "a:b,c&d=e#f g"; + const qs = buildTraceSearchQuery({ ...base, attrs: [{ key: "k", op: "eq", value }] }); + assert.equal(new URLSearchParams(qs).get("attr"), "k:eq:" + value); + assert.equal(qs.includes("&d=e"), false); // the raw ampersand must be encoded + assert.equal(qs.includes("#"), false); +}); + +test("invalid chips are dropped and the list is capped", () => { + const attrs: AttrChip[] = [ + { key: "bad key", op: "eq", value: "x" }, + { key: "k", op: "eq", value: "" }, + ...Array.from({ length: 8 }, (_, i): AttrChip => ({ key: "k" + i, op: "exists", value: "" })), + ]; + const got = new URLSearchParams(buildTraceSearchQuery({ ...base, attrs })).getAll("attr"); + assert.equal(got.length, MAX_ATTR_FILTERS); + assert.equal(got[0], "k0:exists"); +}); + +test("validateChip mirrors the server rules", () => { + assert.equal(validateChip({ key: "a.b-c/d_e", op: "eq", value: "x" }), null); + assert.equal(validateChip({ key: "k", op: "exists", value: "" }), null); + assert.notEqual(validateChip({ key: "", op: "eq", value: "x" }), null); + assert.notEqual(validateChip({ key: "a'b", op: "eq", value: "x" }), null); + assert.notEqual(validateChip({ key: "a:b", op: "eq", value: "x" }), null); + assert.notEqual(validateChip({ key: "a".repeat(129), op: "eq", value: "x" }), null); + assert.notEqual(validateChip({ key: "k", op: "contains", value: "" }), null); + assert.notEqual(validateChip({ key: "k", op: "eq", value: "v".repeat(257) }), null); + assert.notEqual(validateChip({ key: "k", op: "regex" as never, value: "x" }), null); +}); + +test("canAddChip enforces the five-filter bound", () => { + const full: AttrChip[] = Array.from({ length: MAX_ATTR_FILTERS }, (_, i) => ({ key: "k" + i, op: "exists", value: "" })); + assert.notEqual(canAddChip(full, { key: "x", op: "exists", value: "" }), null); + assert.equal(canAddChip(full.slice(1), { key: "x", op: "exists", value: "" }), null); +}); + +test("orphans flag, durations and labels", () => { + const q = new URLSearchParams( + buildTraceSearchQuery({ ...base, includeOrphans: true, minDuration: 10.9, maxDuration: 0, service: "api" }), + ); + assert.equal(q.get("include_orphans"), "true"); + assert.equal(q.get("min_duration"), "10"); + assert.equal(q.has("max_duration"), false); + assert.equal(q.get("service"), "api"); + assert.equal(chipToParam({ key: "k", op: "exists", value: "ignored" }), "k:exists"); + assert.equal(chipLabel({ key: "k", op: "neq", value: "v" }), "k != v"); + assert.equal(chipLabel({ key: "k", op: "exists", value: "" }), "k exists"); +}); diff --git a/ui/src/lib/traceSearch.ts b/ui/src/lib/traceSearch.ts new file mode 100644 index 00000000..8c47ffdb --- /dev/null +++ b/ui/src/lib/traceSearch.ts @@ -0,0 +1,91 @@ +// Pure query-string builder for trace search with span-attribute filters. +// Mirrors the server contract in internal/tracing/search.go (GET +// /api/v1/traces/search-advanced): repeated attr=key:op[:value], at most five +// filters, keys limited to [A-Za-z0-9_.\-/]{1,128}. The server re-validates +// everything; this only keeps obviously bad chips from being sent. + +export const MAX_ATTR_FILTERS = 5; +export const MAX_ATTR_VALUE_LEN = 256; + +export type AttrOp = "eq" | "neq" | "exists" | "contains"; +export const ATTR_OPS: AttrOp[] = ["eq", "neq", "exists", "contains"]; + +export interface AttrChip { + key: string; + op: AttrOp; + value: string; +} + +export interface TraceSearchParams { + siteId: string; + from: string; + to: string; + service?: string; + operation?: string; + status?: string; + minDuration?: number; + maxDuration?: number; + attrs?: AttrChip[]; + includeOrphans?: boolean; +} + +const KEY_RE = /^[A-Za-z0-9_.\-/]{1,128}$/; + +/** Returns an error message for an unusable chip, or null when it is valid. */ +export function validateChip(chip: AttrChip): string | null { + if (!KEY_RE.test(chip.key)) return "Key may use letters, digits, _ . - / (max 128)"; + if (!ATTR_OPS.includes(chip.op)) return "Unknown operator"; + if (chip.op === "exists") return null; + if (chip.value === "") return "A value is required for " + chip.op; + if (new TextEncoder().encode(chip.value).length > MAX_ATTR_VALUE_LEN) { + return "Value is too long (max " + MAX_ATTR_VALUE_LEN + " bytes)"; + } + return null; +} + +/** Returns an error when the chip cannot be added to the current list. */ +export function canAddChip(existing: AttrChip[], chip: AttrChip): string | null { + if (existing.length >= MAX_ATTR_FILTERS) return "At most " + MAX_ATTR_FILTERS + " attribute filters"; + return validateChip(chip); +} + +/** The wire form of one chip: key:op or key:op:value. */ +export function chipToParam(chip: AttrChip): string { + return chip.op === "exists" ? chip.key + ":exists" : chip.key + ":" + chip.op + ":" + chip.value; +} + +/** Short label for a chip, e.g. `http.status_code = 500`. */ +export function chipLabel(chip: AttrChip): string { + switch (chip.op) { + case "eq": return chip.key + " = " + chip.value; + case "neq": return chip.key + " != " + chip.value; + case "contains": return chip.key + " contains " + chip.value; + default: return chip.key + " exists"; + } +} + +/** + * Builds the query string (no leading "?"). Invalid chips are dropped and the + * list is capped at MAX_ATTR_FILTERS so the request is never refused for size. + * Values are percent-encoded, so ':' ',' '&' and '#' survive intact. + */ +export function buildTraceSearchQuery(p: TraceSearchParams): string { + const q = new URLSearchParams(); + q.set("site_id", p.siteId); + q.set("from", p.from); + q.set("to", p.to); + if (p.service) q.set("service", p.service); + if (p.operation) q.set("operation", p.operation); + if (p.status) q.set("status", p.status); + if (p.minDuration && p.minDuration > 0) q.set("min_duration", String(Math.floor(p.minDuration))); + if (p.maxDuration && p.maxDuration > 0) q.set("max_duration", String(Math.floor(p.maxDuration))); + let n = 0; + for (const chip of p.attrs ?? []) { + if (n >= MAX_ATTR_FILTERS) break; + if (validateChip(chip) !== null) continue; + q.append("attr", chipToParam(chip)); + n++; + } + if (p.includeOrphans) q.set("include_orphans", "true"); + return q.toString(); +} diff --git a/ui/src/routes/traces.tsx b/ui/src/routes/traces.tsx index 0162d485..7604b7de 100644 --- a/ui/src/routes/traces.tsx +++ b/ui/src/routes/traces.tsx @@ -1,6 +1,8 @@ import { useState, useEffect, useCallback } from "preact/hooks"; import { tracesApi } from "../api/traces.js"; import type { Service, Operation, Span, TraceSummary, TraceError, ServiceDependency, PerformanceIssue, FunnelStep, FunnelResult, SavedFunnel } from "../api/traces.js"; +import { ATTR_OPS, canAddChip, chipLabel, MAX_ATTR_FILTERS } from "../lib/traceSearch.js"; +import type { AttrChip, AttrOp } from "../lib/traceSearch.js"; import SearchInput from "../components/shared/SearchInput.js"; import StatusBadge from "../components/shared/StatusBadge.js"; import CodeBlock from "../components/shared/CodeBlock.js"; @@ -474,6 +476,11 @@ function TraceListView({ traces, onSelectTrace }: { traces: TraceSummary[]; onSe {t.span_count} spans {formatDuration(t.duration_ms)} {hasError && } + {t.root_missing && ( + root missing + )} {formatDate(t.start_time)}
@@ -495,21 +502,43 @@ function SearchFilters({ siteId, from, to, services, onSelectTrace }: { const [filterMinDuration, setFilterMinDuration] = useState(""); const [filterMaxDuration, setFilterMaxDuration] = useState(""); const [results, setResults] = useState([]); + const [truncatedNote, setTruncatedNote] = useState(""); + const [chips, setChips] = useState([]); + const [chipKey, setChipKey] = useState(""); + const [chipOp, setChipOp] = useState("eq"); + const [chipValue, setChipValue] = useState(""); + const [chipError, setChipError] = useState(""); + const [includeOrphans, setIncludeOrphans] = useState(false); const [loading, setLoading] = useState(false); const [searched, setSearched] = useState(false); + const addChip = () => { + const chip: AttrChip = { key: chipKey.trim(), op: chipOp, value: chipOp === "exists" ? "" : chipValue }; + const err = canAddChip(chips, chip); + if (err) { setChipError(err); return; } + setChips([...chips, chip]); + setChipKey(""); setChipValue(""); setChipError(""); + }; + const handleSearch = async () => { setLoading(true); setSearched(true); + setTruncatedNote(""); try { - const opts: any = {}; - if (filterService) opts.service = filterService; - if (filterOperation) opts.operation = filterOperation; - if (filterStatus) opts.status = filterStatus; - if (filterMinDuration) opts.min_duration = parseInt(filterMinDuration); - if (filterMaxDuration) opts.max_duration = parseInt(filterMaxDuration); - const data = await tracesApi.search(siteId, from, to, opts); - setResults(data || []); + const data = await tracesApi.searchAdvanced({ + siteId, from, to, + service: filterService || undefined, + operation: filterOperation || undefined, + status: filterStatus || undefined, + minDuration: filterMinDuration ? parseInt(filterMinDuration) : undefined, + maxDuration: filterMaxDuration ? parseInt(filterMaxDuration) : undefined, + attrs: chips, + includeOrphans, + }); + setResults(data?.traces || []); + if (data?.truncated) { + setTruncatedNote(data.truncated_reason || "Results were cut by a search bound and may be incomplete"); + } } catch { setResults([]); } finally { setLoading(false); } }; @@ -544,11 +573,52 @@ function SearchFilters({ siteId, from, to, services, onSelectTrace }: { onInput={(e) => setFilterMaxDuration((e.target as HTMLInputElement).value)} style={{ flex: 1 }} /> +
+
+ setChipKey((e.target as HTMLInputElement).value)} + style={{ flex: 2 }} /> + + {chipOp !== "exists" && ( + setChipValue((e.target as HTMLInputElement).value)} + style={{ flex: 2 }} /> + )} + +
+ {chipError &&
{chipError}
} + {chips.length > 0 && ( +
+ {chips.map((c, i) => ( + setChips(chips.filter((_, j) => j !== i))}> + {chipLabel(c)} x + + ))} +
+ )} + +
+ {truncatedNote && ( +
+ Truncated: {truncatedNote} +
+ )} + {loading ? : searched && results.length === 0 ? (
No traces match the filters
) : results.length > 0 ? ( From 5ff1b906c55c3b0c06b72e5af4fddff00e4e5498 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:04:54 +0000 Subject: [PATCH 32/65] feat: person properties, explicit alias/merge and erasure tombstones (C3) Wires the predecessor's persons identity work (migration 061: person_properties, person_aliases, person_tombstones) into the app: POST /api/v1/persons/properties (telemetry key), /persons/merge (editor+), /persons/erase (admin), alias-resolved list/detail, persons UI additions, and ADR section 8 (supported/not-supported). Events are not deleted; erasure is tombstone + exclusion, documented as a limit. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/main.go | 8 +- cmd/observe/persons_c3_test.go | 203 ++++++++++++++++++++++ cmd/observe/persons_handlers.go | 221 +++++++++++++++++++++--- docs/IDENTITY_MODEL_ADR.md | 67 +++++++- internal/persons/identity_test.go | 275 ++++++++++++++++++++++++++++++ internal/persons/memory.go | 174 +++++++++++++++++++ ui/src/api/persons.ts | 12 ++ ui/src/routes/persons.tsx | 16 ++ 8 files changed, 951 insertions(+), 25 deletions(-) create mode 100644 cmd/observe/persons_c3_test.go create mode 100644 internal/persons/identity_test.go create mode 100644 internal/persons/memory.go diff --git a/cmd/observe/main.go b/cmd/observe/main.go index 7f415e4f..efe2c4f9 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -1830,7 +1830,13 @@ func main() { // C2 Wave 4: aggregate over events.distinct_id. Read-only; no editor // gating needed. Single-line wiring matches the metrics / boards // convention above. - RegisterPersonsRoutes(r, jwtMW, personsSvc) + // C3: properties (telemetry key), merge (editor+), erase (admin). + RegisterPersonsRoutes(r, PersonsRouteDeps{ + JWT: jwtMW, Editor: requireEditor, Admin: requireAdmin, + Ingest: ingestGroup, Svc: personsSvc, + Privacy: siteSvc, GlobalSalt: cfg.SessionSalt, + Actor: func(req *http.Request) string { return requestActor(authSvc, req) }, + }) // --- Cohorts API --- // C2 Wave 4: behavioural grouping. Owns its own table (migration 023) diff --git a/cmd/observe/persons_c3_test.go b/cmd/observe/persons_c3_test.go new file mode 100644 index 00000000..5476388e --- /dev/null +++ b/cmd/observe/persons_c3_test.go @@ -0,0 +1,203 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/auth" + "github.com/useteploy/teploy-observe/internal/identity" + "github.com/useteploy/teploy-observe/internal/ingest" + "github.com/useteploy/teploy-observe/internal/persons" +) + +// testRoleMW stands in for JWTAuthMiddleware: the role comes from a header. +func testRoleMW(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + role := r.Header.Get("X-Test-Role") + if role == "" { + neutron.WriteError(w, r, neutron.ErrUnauthorized("no role")) + return + } + next.ServeHTTP(w, r.WithContext(auth.WithRole(r.Context(), role))) + }) +} + +func testRequire(allowed ...string) neutron.Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + role := auth.RoleFromContext(r.Context()) + for _, a := range allowed { + if a == role { + next.ServeHTTP(w, r) + return + } + } + neutron.WriteError(w, r, neutron.ErrForbidden("insufficient role")) + }) + } +} + +// testKeyMW stands in for apiKeyMW: the key-bound site comes from a header. +func testKeyMW(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + site := r.Header.Get("X-Test-Key-Site") + if site == "" { + neutron.WriteError(w, r, neutron.ErrUnauthorized("no key")) + return + } + next.ServeHTTP(w, r.WithContext(ingest.WithSiteID(r.Context(), site))) + }) +} + +const testSalt = "salt" + +func newPersonsTestRouter(m *persons.Memory) http.Handler { + app := neutron.New() + r := app.Router() + RegisterPersonsRoutes(r, PersonsRouteDeps{ + JWT: testRoleMW, + Editor: testRequire(auth.RoleAdmin, auth.RoleEditor), + Admin: testRequire(auth.RoleAdmin), + Ingest: r.Group("/api/v1", testKeyMW), + Svc: m.Service(), GlobalSalt: testSalt, + Actor: func(*http.Request) string { return "tester" }, + }) + return r +} + +func do(h http.Handler, method, path, body string, hdr map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + for k, v := range hdr { + req.Header.Set(k, v) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + return w +} + +func seedP(m *persons.Memory, site string, keys ...string) { + for _, k := range keys { + m.AddPerson(site, persons.Person{DistinctID: k, FirstSeenMs: 1, LastSeenMs: 2, EventCount: 1, SessionCount: 1}) + } +} + +func TestPersonsAuthzMatrix(t *testing.T) { + m := persons.NewMemory() + seedP(m, "s1", "a", "b", "c", "d") + h := newPersonsTestRouter(m) + + mergeBody := func(from string) string { + return `{"site_id":"s1","from_key":"` + from + `","into_key":"d"}` + } + eraseBody := `{"site_id":"s1","person_key":"c"}` + cases := []struct { + name string + method, path string + body string + role string + want int + }{ + {"read anon", "GET", "/api/v1/persons?site_id=s1", "", "", 401}, + {"read viewer", "GET", "/api/v1/persons?site_id=s1", "", "viewer", 200}, + {"detail viewer", "GET", "/api/v1/persons/a?site_id=s1", "", "viewer", 200}, + {"merge anon", "POST", "/api/v1/persons/merge", mergeBody("a"), "", 401}, + {"merge viewer", "POST", "/api/v1/persons/merge", mergeBody("a"), "viewer", 403}, + {"merge editor", "POST", "/api/v1/persons/merge", mergeBody("a"), "editor", 200}, + {"merge admin", "POST", "/api/v1/persons/merge", mergeBody("b"), "admin", 200}, + {"erase anon", "POST", "/api/v1/persons/erase", eraseBody, "", 401}, + {"erase viewer", "POST", "/api/v1/persons/erase", eraseBody, "viewer", 403}, + {"erase editor", "POST", "/api/v1/persons/erase", eraseBody, "editor", 403}, + {"erase admin", "POST", "/api/v1/persons/erase", eraseBody, "admin", 200}, + } + for _, c := range cases { + hdr := map[string]string{} + if c.role != "" { + hdr["X-Test-Role"] = c.role + } + if got := do(h, c.method, c.path, c.body, hdr).Code; got != c.want { + t.Errorf("%s: got %d want %d", c.name, got, c.want) + } + } + // A JWT role must not reach the telemetry-key route and vice versa. + if got := do(h, "POST", "/api/v1/persons/properties", `{"distinct_id":"u","properties":{"a":1}}`, map[string]string{"X-Test-Role": "admin"}).Code; got != 401 { + t.Errorf("properties without API key: %d", got) + } + if got := do(h, "POST", "/api/v1/persons/merge", mergeBody("a"), map[string]string{"X-Test-Key-Site": "s1"}).Code; got != 401 { + t.Errorf("merge with only an API key: %d", got) + } +} + +func TestPersonsMergeErrorsAndIDOR(t *testing.T) { + m := persons.NewMemory() + seedP(m, "s1", "a", "b") + seedP(m, "s2", "z") + h := newPersonsTestRouter(m) + ed := map[string]string{"X-Test-Role": "editor"} + + if got := do(h, "POST", "/api/v1/persons/merge", `{"site_id":"s1","from_key":"a","into_key":"a"}`, ed).Code; got != 400 { + t.Errorf("self-merge: %d", got) + } + if got := do(h, "POST", "/api/v1/persons/merge", `{"site_id":"s1","from_key":"z","into_key":"b"}`, ed).Code; got != 404 { + t.Errorf("cross-site key: %d", got) + } + if got := do(h, "POST", "/api/v1/persons/merge", `{"site_id":"s1","from_key":"a","into_key":"b"}`, ed).Code; got != 200 { + t.Errorf("merge: %d", got) + } + if got := do(h, "POST", "/api/v1/persons/merge", `{"site_id":"s1","from_key":"b","into_key":"a"}`, ed).Code; got != 409 { + t.Errorf("cycle: %d", got) + } + if got := do(h, "POST", "/api/v1/persons/merge", `not json`, ed).Code; got != 400 { + t.Errorf("bad json: %d", got) + } + // s2 listing is unaffected by s1's alias. + w := do(h, "GET", "/api/v1/persons?site_id=s2", "", map[string]string{"X-Test-Role": "viewer"}) + if w.Code != 200 || !strings.Contains(w.Body.String(), `"z"`) { + t.Errorf("s2 list: %d %s", w.Code, w.Body.String()) + } +} + +func TestPersonsPropertiesEndpoint(t *testing.T) { + m := persons.NewMemory() + h := newPersonsTestRouter(m) + key := map[string]string{"X-Test-Key-Site": "s1"} + hashed := identity.HashDistinctID("user-42", testSalt) + + w := do(h, "POST", "/api/v1/persons/properties", `{"distinct_id":"user-42","properties":{"plan":"pro"}}`, key) + if w.Code != 200 || !strings.Contains(w.Body.String(), hashed) || strings.Contains(w.Body.String(), "user-42") { + t.Fatalf("hashing/echo: %d %s", w.Code, w.Body.String()) + } + if raw, found, _ := m.GetProps(context.Background(), "s1", hashed); !found || !strings.Contains(raw, "pro") { + t.Fatalf("stored under hashed key expected: %q %v", raw, found) + } + // Body site disagreeing with the key's site is a cross-tenant write. + if got := do(h, "POST", "/api/v1/persons/properties", `{"site_id":"s2","distinct_id":"u","properties":{"a":1}}`, key).Code; got != 403 { + t.Errorf("site mismatch: %d", got) + } + if _, found, _ := m.GetProps(context.Background(), "s2", identity.HashDistinctID("u", testSalt)); found { + t.Error("write landed in another site") + } + for name, body := range map[string]string{ + "nested": `{"distinct_id":"u","properties":{"a":{"b":1}}}`, + "badkey": `{"distinct_id":"u","properties":{"a b":1}}`, + "reserved": `{"distinct_id":"u","properties":{"user_id":"raw"}}`, + "noid": `{"properties":{"a":1}}`, + "big": `{"distinct_id":"u","properties":{"a":"` + strings.Repeat("x", 2000) + `"}}`, + } { + if got := do(h, "POST", "/api/v1/persons/properties", body, key).Code; got != 400 { + t.Errorf("%s: %d", name, got) + } + } + // Erased persons refuse writes. + if got := do(h, "POST", "/api/v1/persons/erase", `{"site_id":"s1","person_key":"`+hashed+`"}`, map[string]string{"X-Test-Role": "admin"}).Code; got != 200 { + t.Fatalf("erase: %d", got) + } + if got := do(h, "POST", "/api/v1/persons/properties", `{"distinct_id":"user-42","properties":{"a":1}}`, key).Code; got != 409 { + t.Errorf("write to erased person: %d", got) + } +} diff --git a/cmd/observe/persons_handlers.go b/cmd/observe/persons_handlers.go index e3df36ec..d5d6b313 100644 --- a/cmd/observe/persons_handlers.go +++ b/cmd/observe/persons_handlers.go @@ -1,33 +1,113 @@ package main -// Persons API (C2 Wave 4) — read-only aggregate over events.distinct_id. +// Persons API. C2 (Wave 4): read-only aggregate over events.distinct_id. +// C3: person properties (identify traits), explicit alias/merge and GDPR +// erasure, within the privacy model of docs/IDENTITY_MODEL_ADR.md. // Routes live here so the integration into main.go is a single // RegisterPersonsRoutes call (matches the boards / metrics / cohorts // convention). +// +// Authorization: +// - reads (list, detail): JWT (viewer+) +// - POST /persons/merge: editor+ +// - POST /persons/erase: admin only +// - POST /persons/properties: telemetry (API) key, site bound by the key +// +// Merge and erase take their person key in the BODY, not the path, so a +// raw (raw_distinct_id opt-in) identifier never lands in the audit trail's +// target path. All mutating routes are recorded by auditMiddleware. import ( "context" + "encoding/json" + "errors" + "net/http" "strings" "github.com/neutron-build/neutron/go/neutron" + "github.com/useteploy/teploy-observe/internal/identity" + "github.com/useteploy/teploy-observe/internal/ingest" "github.com/useteploy/teploy-observe/internal/persons" ) +// personsPrivacy resolves a site's identity hashing config (satisfied by +// *sites.SiteService). +type personsPrivacy interface { + PrivacyConfig(ctx context.Context, siteID string) (salt string, rawOptIn bool, ok bool) +} + +// PersonsRouteDeps carries everything RegisterPersonsRoutes wires. +type PersonsRouteDeps struct { + JWT, Editor, Admin neutron.Middleware + // Ingest is the API-key authenticated group rooted at /api/v1. + Ingest *neutron.Router + Svc *persons.Service + Privacy personsPrivacy // nil: global salt only (tests) + GlobalSalt string + // Actor resolves the acting username from the request (JWT). + Actor func(*http.Request) string +} + // RegisterPersonsRoutes wires the persons endpoints onto the given router. -// JWT-only — there is no destructive endpoint here (read aggregates only), -// so editor / admin role enforcement is unnecessary. -func RegisterPersonsRoutes(r *neutron.Router, jwtMW neutron.Middleware, svc *persons.Service) { - api := r.Group("/api/v1/persons", jwtMW) +func RegisterPersonsRoutes(r *neutron.Router, d PersonsRouteDeps) { + api := r.Group("/api/v1/persons", d.JWT) - neutron.Get(api, "", listPersonsHandler(svc), + neutron.Get(api, "", listPersonsHandler(d.Svc), neutron.WithTags("persons"), - neutron.WithSummary("List identified users (aggregate of events.distinct_id)"), + neutron.WithSummary("List identified users (aliases resolved, erased persons excluded)"), ) - neutron.Get(api, "/{distinct_id}", personDetailHandler(svc), + neutron.Get(api, "/{distinct_id}", personDetailHandler(d.Svc), neutron.WithTags("persons"), - neutron.WithSummary("Person detail with last 100 events"), + neutron.WithSummary("Person detail with properties, aliases and last 100 events"), ) + api.Handle("POST /merge", d.Editor(personsMergeHandler(d.Svc, d.Actor))) + api.Handle("POST /erase", d.Admin(personsEraseHandler(d.Svc, d.Actor))) + + if d.Ingest != nil { + d.Ingest.Handle("POST /persons/properties", personsPropertiesHandler(d.Svc, d.Privacy, d.GlobalSalt)) + } +} + +// personsHTTPError maps service sentinels to HTTP errors. +func personsHTTPError(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, persons.ErrInvalid): + return neutron.ErrBadRequest(err.Error()) + case errors.Is(err, persons.ErrNotFound): + return neutron.ErrNotFound(err.Error()) + case errors.Is(err, persons.ErrConflict), errors.Is(err, persons.ErrLimit): + return neutron.ErrConflict(err.Error()) + case errors.Is(err, persons.ErrErased): + return neutron.ErrConflict("person has been erased") + } + return err +} + +func writePersonsJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writePersonsError(w http.ResponseWriter, r *http.Request, err error) { + var ae *neutron.AppError + if errors.As(err, &ae) { + neutron.WriteError(w, r, ae) + return + } + neutron.WriteError(w, r, neutron.ErrInternal("persons request failed")) +} + +func decodePersonsBody(w http.ResponseWriter, r *http.Request, v any, max int64) bool { + r.Body = http.MaxBytesReader(w, r.Body, max) + if err := json.NewDecoder(r.Body).Decode(v); err != nil { + neutron.WriteError(w, r, neutron.ErrBadRequest("invalid JSON body")) + return false + } + return true } // --------------------------------------------------------------------------- @@ -44,12 +124,14 @@ type listPersonsInput struct { } // listPersonsResult bundles the page + total so the UI can render -// pagination without a second round-trip. +// pagination without a second round-trip. Truncated is set when alias +// resolution scanned only the most recently active persons. type listPersonsResult struct { - Persons []persons.Person `json:"persons"` - Total int64 `json:"total"` - Limit int `json:"limit"` - Offset int `json:"offset"` + Persons []persons.Person `json:"persons"` + Total int64 `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Truncated bool `json:"truncated,omitempty"` } func listPersonsHandler(svc *persons.Service) neutron.HandlerFunc[listPersonsInput, listPersonsResult] { @@ -61,13 +143,11 @@ func listPersonsHandler(svc *persons.Service) neutron.HandlerFunc[listPersonsInp if err != nil { return listPersonsResult{}, neutron.ErrBadRequest(err.Error()) } - fromMs, toMs := from.UnixMilli(), to.UnixMilli() - - rows, err := svc.ListPersons(ctx, in.SiteID, fromMs, toMs, in.Limit, in.Offset, in.IncludeAnonymous) + res, err := svc.ListResolved(ctx, in.SiteID, from.UnixMilli(), to.UnixMilli(), in.Limit, in.Offset, in.IncludeAnonymous) if err != nil { - return listPersonsResult{}, err + return listPersonsResult{}, personsHTTPError(err) } - total, _ := svc.CountPersons(ctx, in.SiteID, fromMs, toMs, in.IncludeAnonymous) + rows := res.Persons if rows == nil { rows = []persons.Person{} } @@ -76,8 +156,8 @@ func listPersonsHandler(svc *persons.Service) neutron.HandlerFunc[listPersonsInp limit = 50 } return listPersonsResult{ - Persons: rows, Total: total, - Limit: limit, Offset: in.Offset, + Persons: rows, Total: res.Total, + Limit: limit, Offset: in.Offset, Truncated: res.Truncated, }, nil } } @@ -95,6 +175,103 @@ func personDetailHandler(svc *persons.Service) neutron.HandlerFunc[personDetailI if strings.TrimSpace(in.DistinctID) == "" { return persons.PersonDetail{}, neutron.ErrBadRequest("distinct_id required") } - return svc.PersonDetail(ctx, in.SiteID, in.DistinctID) + d, err := svc.PersonDetail(ctx, in.SiteID, in.DistinctID) + if errors.Is(err, persons.ErrErased) { + return persons.PersonDetail{}, neutron.ErrNotFound("person not found") + } + return d, personsHTTPError(err) + } +} + +type personsMergeBody struct { + SiteID string `json:"site_id"` + FromKey string `json:"from_key"` + IntoKey string `json:"into_key"` +} + +func personsMergeHandler(svc *persons.Service, actor func(*http.Request) string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var in personsMergeBody + if !decodePersonsBody(w, r, &in, 4<<10) { + return + } + res, err := svc.Merge(r.Context(), in.SiteID, in.FromKey, in.IntoKey, personsActor(actor, r)) + if err != nil { + writePersonsError(w, r, personsHTTPError(err)) + return + } + writePersonsJSON(w, http.StatusOK, res) + } +} + +type personsEraseBody struct { + SiteID string `json:"site_id"` + PersonKey string `json:"person_key"` +} + +func personsEraseHandler(svc *persons.Service, actor func(*http.Request) string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var in personsEraseBody + if !decodePersonsBody(w, r, &in, 4<<10) { + return + } + res, err := svc.Erase(r.Context(), in.SiteID, in.PersonKey, personsActor(actor, r)) + if err != nil { + writePersonsError(w, r, personsHTTPError(err)) + return + } + writePersonsJSON(w, http.StatusOK, res) + } +} + +func personsActor(actor func(*http.Request) string, r *http.Request) string { + if actor == nil { + return "" + } + return actor(r) +} + +type personsPropertiesBody struct { + SiteID string `json:"site_id"` + DistinctID string `json:"distinct_id"` + Properties map[string]any `json:"properties"` + Replace bool `json:"replace"` +} + +// personsPropertiesHandler stores identify() traits. The raw distinct_id is +// hashed exactly as the event ingest path does (per-site salt, or raw when +// the site opted in) and never stored; the key-bound site is authoritative. +func personsPropertiesHandler(svc *persons.Service, priv personsPrivacy, globalSalt string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var in personsPropertiesBody + if !decodePersonsBody(w, r, &in, 256<<10) { + return + } + site, err := ingest.BoundSite(r.Context(), in.SiteID) + if err != nil { + neutron.WriteError(w, r, neutron.ErrForbidden("site_id does not match API key")) + return + } + if site == "" { + neutron.WriteError(w, r, neutron.ErrBadRequest("site_id required")) + return + } + if in.DistinctID == "" || len(in.DistinctID) > 512 { + neutron.WriteError(w, r, neutron.ErrBadRequest("distinct_id required (max 512 bytes)")) + return + } + salt, raw := globalSalt, false + if priv != nil { + if s, ro, ok := priv.PrivacyConfig(r.Context(), site); ok { + salt, raw = s, ro + } + } + key := identity.MaybeHashDistinctID(in.DistinctID, salt, raw) + props, err := svc.SetProperties(r.Context(), site, key, in.Properties, in.Replace) + if err != nil { + writePersonsError(w, r, personsHTTPError(err)) + return + } + writePersonsJSON(w, http.StatusOK, map[string]any{"person_key": key, "properties": props}) } } diff --git a/docs/IDENTITY_MODEL_ADR.md b/docs/IDENTITY_MODEL_ADR.md index e7281bcd..2819c903 100644 --- a/docs/IDENTITY_MODEL_ADR.md +++ b/docs/IDENTITY_MODEL_ADR.md @@ -55,11 +55,14 @@ Query-visible entity per surface (pinned): estimate and the person exists or is reconstructed. Target: unchanged at the boundary; internal association becomes explicit and queryable (first-seen-anonymous linkage), with history left as-is. -- **alias** — NOT SUPPORTED today; two distinct_ids are two persons +- **alias** — SUPPORTED as an explicit editor action since C3 (see + section 8). Before C3: two distinct_ids are two persons forever. Target: an alias mapping (one canonical person id + alias table) resolved at query time; historical rows are never rewritten — queries resolve aliases forward from the mapping (D7-era rule). -- **merge** — NOT SUPPORTED. Target: merge is a versioned person-graph +- **merge** — SUPPORTED explicitly since C3 (section 8; `from` becomes + an alias of `into`, resolved at read time; no auto-merge on identify). + Original target text: merge is a versioned person-graph operation (two canonical ids become one going forward; prior rows keep their ids and the merge table unifies them at read time). Silent history rewriting is prohibited. @@ -145,6 +148,66 @@ ingestion-time behavior as a known limitation, not a decision. visible per surface (surface labels always name whether the grouped entity is estimate/session/person). +## 8. C3 additions: properties, alias/merge, erasure (migration 061) + +Status: IMPLEMENTED (2026-10-03), compile+unit tested; SQL UNVERIFIED +against a live Nucleus. Persons remain an aggregate over +`events.distinct_id`; three side tables keyed by the same `person_key` +(the HMAC'd id, or raw under the site's `raw_distinct_id` opt-in) add data +without touching events. + +Supported now: + +- **Person properties** (`person_properties`). `POST /api/v1/persons/properties` + (telemetry API key; site bound by the key, a disagreeing body `site_id` is + 403) takes the RAW identify value, hashes it exactly like event ingest + (per-site salt, or raw on opt-in) and stores traits under the hashed key. + The raw id is never stored. Bounds: <= 50 keys, key `^[A-Za-z_$][A-Za-z0-9_.$-]{0,63}$`, + scalar values only (string/number/bool; null removes in merge mode), + <= 1 KiB per encoded value, `user_id`/`distinct_id` keys rejected so the + raw id cannot be echoed back. There is NO PII denylist: what a customer + puts in a trait (including email) is the customer's choice and their + responsibility. The browser SDK's `identify()` still sends traits only as + `$identify` event properties (with its own email/user_id filter); wiring + it to this endpoint is a separate SDK change. Caveat: a browser-embedded + key lets any holder write properties for a person whose raw id they can + guess - the same trust level as posting events with a chosen distinct_id. + Person detail returns `properties`. +- **Alias/merge** (`person_aliases`). `POST /api/v1/persons/merge` + (editor+; body `site_id`, `from_key`, `into_key`). `from` becomes an alias + of the canonical root of `into`. Refused: self-merge, merging a key that is + already an alias, merges that close a cycle, tombstoned keys, keys with no + events in THIS site (404, so a key from another site cannot be linked), + more than 20 aliases per person or 10000 per site. List and detail resolve + aliases forward in Go over the bounded alias set; canonical properties win + over alias properties. Events are never rewritten (D5). Merged + `session_count` is an upper bound (a shared monthly estimate counts + twice). The merged list scans at most 5000 persons per window and reports + `truncated` beyond that. There is no un-merge endpoint yet (the table + supports it: `active='false'`). +- **Erasure** (`person_tombstones`). `POST /api/v1/persons/erase` (admin + only; key in the body so a raw id never lands in the audit path). Writes a + tombstone for the key and every alias merged into it, deactivates those + alias edges and blanks their stored properties (as new versions; no + DELETE). Tombstoned persons disappear from list and detail, cannot be + merged, and refuse property writes. +- All mutating persons routes are recorded by the audit middleware + (actor, `persons..create`, result). + +NOT supported / honest limits: + +- **Events are not deleted.** The repo has no per-key DELETE on `events` + (retention deletes by time column only) and per-person DELETE semantics on + Nucleus are unproven. Erasure therefore hides and de-identifies the person + at the persons surface only; the hashed `distinct_id` still sits on event + rows, and in session/funnel/retention aggregates, until retention expires + them. A new event with the same identify value re-creates a visible + person, because ingest does not consult tombstones (only property writes + do). A true erasure that removes event rows needs an upstream-verified + per-key DELETE plus ingest-side tombstone checks; neither is built. +- Auto-merge on identify, pre-identify anonymous linkage, un-merge, and + person-level funnels/retention remain residual scope (section 7). + ## 7. Residual O03 scope (next slices) The identity model implementation itself (client ID + era stamping + diff --git a/internal/persons/identity_test.go b/internal/persons/identity_test.go new file mode 100644 index 00000000..f93c71a1 --- /dev/null +++ b/internal/persons/identity_test.go @@ -0,0 +1,275 @@ +package persons + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" +) + +var bg = context.Background() + +func seed(m *Memory, site string, keys ...string) { + for i, k := range keys { + m.AddPerson(site, Person{ + DistinctID: k, FirstSeenMs: int64(100 + i), LastSeenMs: int64(1000 + i*10), + EventCount: 2, SessionCount: 1, TopCountry: "US", TopBrowser: "b" + k, + }) + } +} + +func TestValidateProperties_Bounds(t *testing.T) { + many := map[string]any{} + for i := 0; i < MaxPropertyKeys+1; i++ { + many[fmt.Sprintf("k%d", i)] = 1 + } + cases := map[string]map[string]any{ + "too many keys": many, + "bad key": {"bad key": 1}, + "key injection": {"a'; DROP TABLE events;--": 1}, + "reserved": {"distinct_id": "raw"}, + "reserved case": {"User_ID": "raw"}, + "nested object": {"a": map[string]any{"b": 1}}, + "array": {"a": []any{1}}, + "oversize value": {"a": strings.Repeat("x", MaxPropertyValueLen)}, + "long key": {strings.Repeat("a", MaxPropertyKeyLen+1): 1}, + } + for name, in := range cases { + if _, _, err := ValidateProperties(in); !errors.Is(err, ErrInvalid) { + t.Errorf("%s: want ErrInvalid, got %v", name, err) + } + } + set, rm, err := ValidateProperties(map[string]any{"plan": "pro", "n": 3.0, "ok": true, "gone": nil, "edge": strings.Repeat("x", MaxPropertyValueLen-2)}) + if err != nil || len(set) != 4 || len(rm) != 1 || rm[0] != "gone" { + t.Fatalf("valid batch: set=%v rm=%v err=%v", set, rm, err) + } +} + +func TestValidatePersonKey(t *testing.T) { + for _, k := range []string{"", "a b", "a/b", "a'b", strings.Repeat("a", MaxPersonKeyLen+1), "x;y"} { + if ValidatePersonKey(k) == nil { + t.Errorf("%q should be invalid", k) + } + } + for _, k := range []string{"0123456789abcdef", "u@x.io", "a_b-c.d:e"} { + if err := ValidatePersonKey(k); err != nil { + t.Errorf("%q should be valid: %v", k, err) + } + } +} + +func TestResolver_ChainAndCycle(t *testing.T) { + r := NewResolver([]AliasRow{{"a", "b"}, {"b", "c"}}) + if r.Canonical("a") != "c" || r.Canonical("c") != "c" || r.Canonical("zzz") != "zzz" { + t.Fatal("chain resolution wrong") + } + if m := r.Members("c"); len(m) != 2 || m[0] != "a" || m[1] != "b" { + t.Fatalf("members = %v", m) + } + cyc := NewResolver([]AliasRow{{"x", "y"}, {"y", "x"}}) + if cyc.Canonical("x") != "x" || cyc.Canonical("y") != "y" { + t.Fatal("corrupt cycle must resolve to self") + } +} + +func TestMerge_Protections(t *testing.T) { + m := NewMemory() + s := m.Service() + seed(m, "s1", "a", "b", "c") + seed(m, "s2", "z") + + if _, err := s.Merge(bg, "s1", "a", "a", "u"); !errors.Is(err, ErrInvalid) { + t.Errorf("self-merge: %v", err) + } + if _, err := s.Merge(bg, "s1", "a", "b", "u"); err != nil { + t.Fatalf("merge a->b: %v", err) + } + if _, err := s.Merge(bg, "s1", "a", "c", "u"); !errors.Is(err, ErrConflict) { + t.Errorf("re-merging an alias: %v", err) + } + // b -> a would close a cycle (a is already an alias of b). + if _, err := s.Merge(bg, "s1", "b", "a", "u"); !errors.Is(err, ErrConflict) { + t.Errorf("cycle: %v", err) + } + // into an alias resolves to its root: c -> a == c -> b. + res, err := s.Merge(bg, "s1", "c", "a", "u") + if err != nil || res.CanonicalKey != "b" { + t.Errorf("merge into alias should canonicalize: %+v %v", res, err) + } + // Cross-site key (IDOR): z exists only in s2. + if _, err := s.Merge(bg, "s1", "z", "b", "u"); !errors.Is(err, ErrNotFound) { + t.Errorf("cross-site key must be not found: %v", err) + } + if _, err := s.Merge(bg, "s1", "nope", "b", "u"); !errors.Is(err, ErrNotFound) { + t.Errorf("unknown key: %v", err) + } + if _, err := s.Merge(bg, "", "a", "b", "u"); !errors.Is(err, ErrInvalid) { + t.Errorf("empty site: %v", err) + } + if _, err := s.Merge(bg, "s1", "a;b", "b", "u"); !errors.Is(err, ErrInvalid) { + t.Errorf("bad key: %v", err) + } +} + +func TestMerge_AliasFanoutLimit(t *testing.T) { + m := NewMemory() + s := m.Service() + seed(m, "s1", "root") + for i := 0; i < MaxAliasesPerPerson+2; i++ { + seed(m, "s1", fmt.Sprintf("k%d", i)) + } + var lastErr error + for i := 0; i < MaxAliasesPerPerson+2; i++ { + _, lastErr = s.Merge(bg, "s1", fmt.Sprintf("k%d", i), "root", "u") + if lastErr != nil { + break + } + } + if !errors.Is(lastErr, ErrLimit) { + t.Fatalf("want ErrLimit, got %v", lastErr) + } +} + +func TestListResolved_FoldsAliasesAndPaginates(t *testing.T) { + m := NewMemory() + s := m.Service() + seed(m, "s1", "a", "b", "c", "d") // last_seen a key -> events + aggs map[string]map[string]Person // site -> key -> aggregate + props map[string]map[string]string + alias map[string]map[string]AliasRow // site -> alias -> edge (active only) + tomb map[string]map[string]bool +} + +// NewMemory returns an empty Memory. +func NewMemory() *Memory { + return &Memory{ + events: map[string]map[string][]PersonEvent{}, + aggs: map[string]map[string]Person{}, + props: map[string]map[string]string{}, + alias: map[string]map[string]AliasRow{}, + tomb: map[string]map[string]bool{}, + } +} + +// Service builds a Service over m. +func (m *Memory) Service() *Service { return NewServiceFromParts(m, m) } + +// AddPerson seeds a person aggregate (and one timeline event) for a site. +func (m *Memory) AddPerson(site string, p Person) { + m.mu.Lock() + defer m.mu.Unlock() + if m.aggs[site] == nil { + m.aggs[site] = map[string]Person{} + m.events[site] = map[string][]PersonEvent{} + } + m.aggs[site][p.DistinctID] = p + m.events[site][p.DistinctID] = []PersonEvent{{EventID: "e-" + p.DistinctID, EventType: "pageview", Timestamp: p.LastSeenMs}} +} + +func (m *Memory) List(_ context.Context, site string, _, _ int64, limit, offset int, anon bool) ([]Person, error) { + all, _ := m.ListAll(context.Background(), site, 0, 0, 1<<30, anon) + if offset >= len(all) { + return nil, nil + } + all = all[offset:] + if len(all) > limit { + all = all[:limit] + } + return all, nil +} + +func (m *Memory) Count(ctx context.Context, site string, _, _ int64, anon bool) (int64, error) { + all, _ := m.ListAll(ctx, site, 0, 0, 1<<30, anon) + return int64(len(all)), nil +} + +func (m *Memory) ListAll(_ context.Context, site string, _, _ int64, limit int, anon bool) ([]Person, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []Person + for _, p := range m.aggs[site] { + if p.DistinctID == "" && !anon { + continue + } + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].LastSeenMs > out[j].LastSeenMs }) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + +func (m *Memory) Aggregate(_ context.Context, site, key string) (Person, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + p, ok := m.aggs[site][key] + return p, ok, nil +} + +func (m *Memory) Timeline(_ context.Context, site, key string) ([]PersonEvent, error) { + m.mu.Lock() + defer m.mu.Unlock() + return append([]PersonEvent(nil), m.events[site][key]...), nil +} + +func (m *Memory) Exists(_ context.Context, site, key string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.aggs[site][key] + return ok, nil +} + +func (m *Memory) GetProps(_ context.Context, site, key string) (string, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.props[site][key] + return v, ok, nil +} + +func (m *Memory) PutProps(_ context.Context, site, key, p string) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.props[site] == nil { + m.props[site] = map[string]string{} + } + m.props[site][key] = p + return nil +} + +func (m *Memory) ListAliases(_ context.Context, site string, limit int) ([]AliasRow, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []AliasRow + for _, a := range m.alias[site] { + out = append(out, a) + } + sort.Slice(out, func(i, j int) bool { return out[i].AliasKey < out[j].AliasKey }) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + +func (m *Memory) PutAlias(_ context.Context, site, alias, canonical, _ string, active bool) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.alias[site] == nil { + m.alias[site] = map[string]AliasRow{} + } + if active { + m.alias[site][alias] = AliasRow{AliasKey: alias, CanonicalKey: canonical} + } else { + delete(m.alias[site], alias) + } + return nil +} + +func (m *Memory) ListTombstones(_ context.Context, site string, limit int) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for k := range m.tomb[site] { + out = append(out, k) + } + sort.Strings(out) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + +func (m *Memory) IsTombstoned(_ context.Context, site, key string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.tomb[site][key], nil +} + +func (m *Memory) PutTombstone(_ context.Context, site, key, _ string) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.tomb[site] == nil { + m.tomb[site] = map[string]bool{} + } + m.tomb[site][key] = true + return nil +} diff --git a/ui/src/api/persons.ts b/ui/src/api/persons.ts index 0a178d64..b571e0a5 100644 --- a/ui/src/api/persons.ts +++ b/ui/src/api/persons.ts @@ -33,6 +33,10 @@ export interface PersonEvent { export interface PersonDetail { person: Person; timeline: PersonEvent[]; + // C3: identify() traits (aliases merged, canonical wins), merged keys. + properties?: Record; + aliases?: string[]; + canonical_key?: string; } export interface PersonsListResult { @@ -111,6 +115,14 @@ export const personsApi = { if (opts?.includeAnonymous) q += `&include_anonymous=true`; return get(`${BASE}/persons?${q}`); }, + // C3: editor+ merge of fromKey into intoKey; admin-only erase. Keys go in + // the body so a raw identifier never appears in a URL/audit path. + merge: (siteId: string, fromKey: string, intoKey: string) => + post<{ site_id: string; from_key: string; canonical_key: string }>( + `${BASE}/persons/merge`, { site_id: siteId, from_key: fromKey, into_key: intoKey }), + erase: (siteId: string, personKey: string) => + post<{ site_id: string; erased_keys: string[]; events_deleted: boolean }>( + `${BASE}/persons/erase`, { site_id: siteId, person_key: personKey }), detail: (distinctId: string, siteId: string) => get(`${BASE}/persons/${encodeURIComponent(distinctId)}?site_id=${encodeURIComponent(siteId)}`), }; diff --git a/ui/src/routes/persons.tsx b/ui/src/routes/persons.tsx index 3b27c0a4..f883e5bd 100644 --- a/ui/src/routes/persons.tsx +++ b/ui/src/routes/persons.tsx @@ -86,6 +86,22 @@ function PersonDetailPanel({ distinctID, siteID, onBack }: + {data.properties && Object.keys(data.properties).length > 0 && ( +
+

Properties

+
+ {Object.entries(data.properties).map(([k, v]) => ( +
{k}: {String(v)}
+ ))} +
+
+ )} + {data.aliases && data.aliases.length > 0 && ( +
+ Merged identities ({data.aliases.length}): {data.aliases.map(truncateID).join(", ")} +
+ )} +

Recent activity ({data.timeline.length}) From fb2ec68873bc7faa445d7ba5c434bc8f142f9095 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:04:57 +0000 Subject: [PATCH 33/65] feat(experiments): C5 depth - continuous metrics, secondary goals, contamination, peeking guard Count/mean primary metrics (Welch t, Welch-Satterthwaite CI, Holm, min-n gate, optional winsorize), up to 3 secondary goals excluded from winner gating, contaminated_users (multi-arm users excluded), Bayesian seed from experiment id + data hash, peeking_warning and planned-sample winner gate with explicit override, settings + metric ingest routes, docs/experiments.md, UI surface (unbuilt). Migration 063 adds experiment_settings and experiment_metric_events because experiment_conversions stores no value and ALTER ADD is forbidden. UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/experiment_metrics_routes.go | 106 +++++ cmd/observe/main.go | 6 +- docs/experiments.md | 67 +++ internal/experiments/compute.go | 12 +- internal/experiments/compute_test.go | 426 ++++++++++++++++++ internal/experiments/continuous_test.go | 175 +++++++ internal/experiments/experiments.go | 218 +++++---- internal/experiments/results_test.go | 50 ++ .../testdata/legacy_results.golden | 3 + .../migrations/063_experiment_metrics.up.sql | 44 ++ ui/src/api/flags.ts | 28 ++ ui/src/routes/experiments.tsx | 47 +- ui/src/utils/experimentMetrics.test.ts | 22 + ui/src/utils/experimentMetrics.ts | 30 ++ 14 files changed, 1145 insertions(+), 89 deletions(-) create mode 100644 cmd/observe/experiment_metrics_routes.go create mode 100644 docs/experiments.md create mode 100644 internal/experiments/compute_test.go create mode 100644 internal/experiments/continuous_test.go create mode 100644 internal/experiments/testdata/legacy_results.golden create mode 100644 internal/schema/migrations/063_experiment_metrics.up.sql create mode 100644 ui/src/utils/experimentMetrics.test.ts create mode 100644 ui/src/utils/experimentMetrics.ts diff --git a/cmd/observe/experiment_metrics_routes.go b/cmd/observe/experiment_metrics_routes.go new file mode 100644 index 00000000..8e42c32f --- /dev/null +++ b/cmd/observe/experiment_metrics_routes.go @@ -0,0 +1,106 @@ +package main + +// Experiment depth routes (migration 063): the per-experiment analysis +// settings and the ingest endpoint for count / mean / secondary metric +// observations. Registered from main.go with one RegisterExperimentMetricRoutes +// call. + +import ( + "context" + "math" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/experiments" + "github.com/useteploy/teploy-observe/internal/ingest" +) + +// RegisterExperimentMetricRoutes wires GET/PUT settings (JWT; editor+ writes) +// and POST /experiments/metric (API key ingest). +func RegisterExperimentMetricRoutes( + r *neutron.Router, + ingestGroup *neutron.Router, + jwtMW neutron.Middleware, + requireEditor neutron.Middleware, + svc *experiments.ExperimentService, +) { + neutron.Post(ingestGroup, "/experiments/metric", experimentMetricHandler(svc), + neutron.WithTags("experiments"), + neutron.WithSummary("Record a count/mean/secondary experiment metric observation")) + + read := r.Group("/api/v1/experiments", jwtMW) + write := read.Group("", requireEditor) + neutron.Get(read, "/{experiment_id}/settings", getExperimentSettingsHandler(svc), + neutron.WithTags("experiments"), neutron.WithSummary("Get experiment analysis settings")) + neutron.Put(write, "/{experiment_id}/settings", putExperimentSettingsHandler(svc), + neutron.WithTags("experiments"), neutron.WithSummary("Set experiment analysis settings (metric kind, secondary goals, horizon)")) +} + +type experimentMetricInput struct { + ExperimentID string `json:"experiment_id"` + UserID string `json:"user_id"` + Metric string `json:"metric"` + Value float64 `json:"value"` +} + +func experimentMetricHandler(svc *experiments.ExperimentService) neutron.HandlerFunc[experimentMetricInput, neutron.Empty] { + return func(ctx context.Context, in experimentMetricInput) (neutron.Empty, error) { + siteID := ingest.SiteIDFromContext(ctx) + if siteID == "" || in.ExperimentID == "" || in.UserID == "" { + return neutron.Empty{}, neutron.ErrBadRequest("experiment_id and user_id required") + } + if in.Metric == "" { + in.Metric = experiments.PrimaryMetricKey + } + if math.IsNaN(in.Value) || math.IsInf(in.Value, 0) { + return neutron.Empty{}, neutron.ErrBadRequest("value must be finite") + } + if err := svc.RecordMetric(ctx, in.ExperimentID, siteID, in.UserID, in.Metric, in.Value); err != nil { + return neutron.Empty{}, neutron.ErrBadRequest(err.Error()) + } + return neutron.Empty{}, nil + } +} + +type experimentSettingsGetInput struct { + ExperimentID string `path:"experiment_id"` + SiteID string `query:"site_id"` +} + +func getExperimentSettingsHandler(svc *experiments.ExperimentService) neutron.HandlerFunc[experimentSettingsGetInput, experiments.ExperimentConfig] { + return func(ctx context.Context, in experimentSettingsGetInput) (experiments.ExperimentConfig, error) { + if in.SiteID == "" { + return experiments.ExperimentConfig{}, neutron.ErrBadRequest("site_id required") + } + return svc.LoadConfig(ctx, in.ExperimentID, in.SiteID) + } +} + +type experimentSettingsPutInput struct { + ExperimentID string `path:"experiment_id"` + SiteID string `json:"site_id"` + // Fields mirror experiments.ExperimentConfig (explicit, not embedded, so + // the binder sees plain json tags). + MetricKind string `json:"metric_kind"` + WinsorizePct float64 `json:"winsorize_pct"` + PlannedSamplePerArm int `json:"planned_sample_per_arm"` + AllowEarlyWinner bool `json:"allow_early_winner"` + Secondary []experiments.SecondaryGoal `json:"secondary_goals"` +} + +func putExperimentSettingsHandler(svc *experiments.ExperimentService) neutron.HandlerFunc[experimentSettingsPutInput, experiments.ExperimentConfig] { + return func(ctx context.Context, in experimentSettingsPutInput) (experiments.ExperimentConfig, error) { + if in.SiteID == "" { + return experiments.ExperimentConfig{}, neutron.ErrBadRequest("site_id required") + } + cfg, err := svc.SaveConfig(ctx, in.ExperimentID, in.SiteID, experiments.ExperimentConfig{ + MetricKind: in.MetricKind, WinsorizePct: in.WinsorizePct, + PlannedSamplePerArm: in.PlannedSamplePerArm, AllowEarlyWinner: in.AllowEarlyWinner, + Secondary: in.Secondary, + }) + if err != nil { + return experiments.ExperimentConfig{}, neutron.ErrBadRequest(err.Error()) + } + return cfg, nil + } +} diff --git a/cmd/observe/main.go b/cmd/observe/main.go index 7f415e4f..f7e7bb43 100644 --- a/cmd/observe/main.go +++ b/cmd/observe/main.go @@ -1837,6 +1837,7 @@ func main() { // and exposes MembersForFilter to the stats service for ?cohort_id= // filtering across every analytics chart. RegisterCohortsRoutes(r, jwtMW, requireEditor, cohortsSvc) + RegisterExperimentMetricRoutes(r, ingestGroup, jwtMW, requireEditor, experimentSvc) // SPA catch-all: serve index.html for all non-API, non-asset GET requests. // This must be registered last so API routes take precedence. @@ -3888,6 +3889,8 @@ func stopExperimentHandler(svc *experiments.ExperimentService) neutron.HandlerFu type experimentResultsInput struct { ExperimentID string `path:"experiment_id"` SiteID string `query:"site_id"` + // AllowEarly=true waives the planned-sample winner gate for this request. + AllowEarly string `query:"allow_early"` } type experimentSampleSizeInput struct { @@ -3943,7 +3946,8 @@ func experimentSampleSizeHandler() neutron.HandlerFunc[experimentSampleSizeInput func experimentResultsHandler(svc *experiments.ExperimentService) neutron.HandlerFunc[experimentResultsInput, experiments.ExperimentResults] { return func(ctx context.Context, input experimentResultsInput) (experiments.ExperimentResults, error) { - r, err := svc.Results(ctx, input.ExperimentID, input.SiteID) + r, err := svc.ResultsWithOptions(ctx, input.ExperimentID, input.SiteID, + experiments.ResultsOptions{AllowEarly: input.AllowEarly == "true"}) if err != nil { return experiments.ExperimentResults{}, err } diff --git a/docs/experiments.md b/docs/experiments.md new file mode 100644 index 00000000..ccf256b3 --- /dev/null +++ b/docs/experiments.md @@ -0,0 +1,67 @@ +# Experiments: analysis contract + +## Fixed horizon, not sequential + +Results are a **fixed-horizon** analysis (alpha 0.05 two-sided). Nothing here is +sequential-safe: looking at p-values repeatedly and stopping when one dips below +0.05 inflates the false-positive rate. The contract: + +1. Plan the sample size first (`GET /api/v1/experiments/sample-size?p1=&mde=`). + Record it with `PUT /api/v1/experiments/{id}/settings` + (`planned_sample_per_arm`). +2. Until **every** arm has `max(min_sample, planned_sample_per_arm)` exposed + users, results carry `peeking_warning` and the winner gate stays shut + (`analysis.horizon_met = false`, `winner = ""`, `significant = false`). + Estimates and intervals are still shown as a progress report. +3. Override: `?allow_early=true` on the results request, or + `allow_early_winner: true` in the settings. This waives only the + planned-sample part; the `min_sample` floor (and 30 per arm for count/mean + metrics) is never waived. The output records `horizon_overridden`. + +## Who is counted + +A user exposed to more than one variant inside the running interval is +**contaminated**: excluded from every count and statistic and reported in +`contaminated_users`. A rising count means assignment is leaking between arms; +investigate before trusting the result. + +## Metric kinds + +Settings (`PUT /api/v1/experiments/{id}/settings`, body also carries `site_id`): + +| field | meaning | +|---|---| +| `metric_kind` | `binary` (default), `count`, `mean` | +| `winsorize_pct` | 0 = off; else clip count/mean values to the pooled `[p, 100-p]` percentiles (linear interpolation, numpy default) | +| `planned_sample_per_arm` | design-time n, see above | +| `allow_early_winner` | persistent override of the planned-sample gate | +| `secondary_goals` | up to 3 of `{key, name, kind}`; `key` matches `^[a-z0-9_]{1,32}$` | + +- **binary**: conversions come from `/experiments/convert`; chi-square / Fisher / + Holm / Wilson / Newcombe as before, Bayesian probability-to-beat shown. +- **count / mean**: observations are posted to `POST /api/v1/experiments/metric` + (`{experiment_id, user_id, metric, value}`; `metric` is `primary` or a + secondary key; API-key auth, ignored for users with no exposure). The per-user + value is over events inside the conversion window after the user's first + exposure: the number of events (count) or the sum of `value` (mean, e.g. + revenue per exposed user; non-buyers contribute 0). Each arm is compared with + the control by Welch's t-test (Welch-Satterthwaite df), 95% CI of the mean + difference, Holm across arms. Higher is better. No Bayesian value is shown. + +## Secondary metrics + +Evaluated on the same clean cohort with the same machinery, labelled +`role: "secondary"`, `gates_winner: false`. They never produce a winner. +Holm applies across the arms of one metric, not across metrics, so a secondary +crossing 0.05 is exploratory (`multiple_comparison_note`). + +## Bayesian display value + +`prob_beat_control` is a Monte Carlo estimate seeded from the experiment id and a +hash of the arm counts: stable between identical calls, independent across +experiments. It is display-only and never gates the winner. + +## Not covered + +Auto-recording an exposure from `/flags/evaluate` is not implemented (it needs a +hook in `internal/flags`); exposures are still posted to `/experiments/expose`. diff --git a/internal/experiments/compute.go b/internal/experiments/compute.go index 5c4d35fa..43db579e 100644 --- a/internal/experiments/compute.go +++ b/internal/experiments/compute.go @@ -141,8 +141,16 @@ func evaluateMetric(kind string, winsorPct float64, variants []VariantResult, co // Continuous: every exposed user contributes (zero when no events). perArm := make([][]float64, len(arms)) - for u, c := range cohort { - perArm[idx[c.Variant]] = append(perArm[idx[c.Variant]], vals[u]) + // Sorted user order keeps float summation, and so the output, identical + // between identical calls. + users := make([]string, 0, len(cohort)) + for u := range cohort { + users = append(users, u) + } + sort.Strings(users) + for _, u := range users { + i := idx[cohort[u].Variant] + perArm[i] = append(perArm[i], vals[u]) } if winsorPct > 0 { winsorizeArms(perArm, winsorPct) diff --git a/internal/experiments/compute_test.go b/internal/experiments/compute_test.go new file mode 100644 index 00000000..66668ef0 --- /dev/null +++ b/internal/experiments/compute_test.go @@ -0,0 +1,426 @@ +package experiments + +import ( + "encoding/json" + "fmt" + "math" + "os" + "reflect" + "strings" + "testing" +) + +const hourMs = int64(3600 * 1000) + +// addUsers appends n clean users of a variant with first exposure at ts. +func addUsers(in *resultInputs, prefix, variant string, n int, ts int64) { + for i := 0; i < n; i++ { + in.Exposures = append(in.Exposures, exposureRow{User: fmt.Sprintf("%s%d", prefix, i), Variant: variant, First: ts}) + } +} + +func baseInputs(minSample int) resultInputs { + return resultInputs{ + Exp: Experiment{ExperimentID: "e1", SiteID: "s", MinSample: minSample, + Variants: `[{"key":"control"},{"key":"b"},{"key":"c"}]`}, + Cfg: DefaultConfig(), + WindowMs: 72 * hourMs, + Events: map[string][]metricEvent{}, + Truncated: map[string]bool{}, + } +} + +// Legacy binary experiments: the JSON is the pre-063 output byte for byte +// (golden produced by the pre-change code) once the added fields and the +// Bayesian display value (new seed) are set aside. +func TestLegacyBinaryJSONUnchanged(t *testing.T) { + raw, err := os.ReadFile("testdata/legacy_results.golden") + if err != nil { + t.Fatal(err) + } + goldens := strings.Split(strings.TrimSpace(string(raw)), "\n") + cases := [][][2]int64{{{4000, 400}, {4000, 480}}, {{3000, 300}, {3000, 360}, {3000, 330}}, {{50, 5}, {50, 9}}} + names := []string{"control", "b", "c"} + if len(goldens) != len(cases) { + t.Fatalf("golden count %d", len(goldens)) + } + for ci, arms := range cases { + in := baseInputs(100) + for ai, a := range arms { + addUsers(&in, names[ai], names[ai], int(a[0]), 1000) + for i := int64(0); i < a[1]; i++ { + in.BinaryConversions = append(in.BinaryConversions, userVariant{User: fmt.Sprintf("%s%d", names[ai], i), Variant: names[ai]}) + } + } + res := computeResults(in) + got := normalizeJSON(t, mustJSON(t, res), true) + want := normalizeJSON(t, []byte(goldens[ci]), false) + if len(arms) == 3 { + // The golden generator passed nil weights (no SRM block); the + // real pipeline derives uniform weights from the variants JSON. + delete(got["analysis"].(map[string]any), "srm") + delete(want["analysis"].(map[string]any), "srm") + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("case %d: legacy JSON changed\n got: %s\nwant: %s", ci, mustJSON(t, got), mustJSON(t, want)) + } + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return b +} + +// normalizeJSON drops the fields added in 063 (when stripAdded) and the +// seed-dependent Bayesian display value. +func normalizeJSON(t *testing.T, b []byte, stripAdded bool) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + if stripAdded { + for _, k := range []string{"metric_kind", "contaminated_users", "peeking_warning", "settings", "secondary", "multiple_comparison_note"} { + delete(m, k) + } + } + for _, v := range m["variants"].([]any) { + delete(v.(map[string]any), "prob_beat_control") + } + return m +} + +func TestLegacyAddedFields(t *testing.T) { + in := baseInputs(100) + addUsers(&in, "c", "control", 150, 1000) + addUsers(&in, "b", "b", 150, 1000) + res := computeResults(in) + m := normalizeJSON(t, mustJSON(t, res), false) + if m["metric_kind"] != "binary" || m["contaminated_users"].(float64) != 0 || m["peeking_warning"] != "" { + t.Fatalf("added fields: %v %v %v", m["metric_kind"], m["contaminated_users"], m["peeking_warning"]) + } + if _, ok := m["settings"]; ok { + t.Fatal("default config must not echo settings") + } +} + +func TestContaminatedUsersExcluded(t *testing.T) { + in := baseInputs(1) + addUsers(&in, "c", "control", 40, 1000) + addUsers(&in, "b", "b", 40, 1000) + // 5 users exposed to both arms (and one to all three) are contaminated. + for i := 0; i < 5; i++ { + u := fmt.Sprintf("x%d", i) + in.Exposures = append(in.Exposures, + exposureRow{User: u, Variant: "control", First: 1000}, + exposureRow{User: u, Variant: "b", First: 2000}) + in.BinaryConversions = append(in.BinaryConversions, + userVariant{User: u, Variant: "control"}, userVariant{User: u, Variant: "b"}) + } + in.Exposures = append(in.Exposures, + exposureRow{User: "y", Variant: "control", First: 1}, exposureRow{User: "y", Variant: "b", First: 2}, exposureRow{User: "y", Variant: "c", First: 3}) + // repeated same-arm exposure rows are not contamination. + in.Exposures = append(in.Exposures, exposureRow{User: "c0", Variant: "control", First: 5000}) + res := computeResults(in) + if res.ContaminatedUsers != 6 { + t.Fatalf("contaminated = %d, want 6", res.ContaminatedUsers) + } + for _, v := range res.Variants { + if v.Exposures != 40 { + t.Fatalf("%s exposures = %d, contaminated users must be excluded", v.Variant, v.Exposures) + } + if v.Conversions != 0 { + t.Fatalf("%s conversions = %d, contaminated conversions must be excluded", v.Variant, v.Conversions) + } + } + if len(res.Variants) != 2 { + t.Fatalf("variant c has only contaminated users and must vanish: %+v", res.Variants) + } +} + +func TestBayesianSeedDeterministicAndIndependent(t *testing.T) { + mk := func() []VariantResult { + return []VariantResult{{Variant: "control", Exposures: 500, Conversions: 50}, {Variant: "a", Exposures: 500, Conversions: 56}} + } + a, b := mk(), mk() + computeBayesianProbabilities(a, bayesianSeed("exp-1", a)) + computeBayesianProbabilities(b, bayesianSeed("exp-1", b)) + if a[1].ProbBeatControl != b[1].ProbBeatControl { + t.Fatal("identical calls must be identical") + } + if bayesianSeed("exp-1", mk()) == bayesianSeed("exp-2", mk()) { + t.Fatal("different experiments must get different seeds") + } + d := mk() + d[1].Conversions++ + if bayesianSeed("exp-1", mk()) == bayesianSeed("exp-1", d) { + t.Fatal("different data must change the seed") + } + if bayesianSeed("exp-1", mk()) == 42 { + t.Fatal("seed must not be the legacy constant") + } + // probabilities stay in [0,1]. + if a[1].ProbBeatControl < 0 || a[1].ProbBeatControl > 1 { + t.Fatal("prob out of range") + } +} + +func TestPeekingWarningAndOverride(t *testing.T) { + in := baseInputs(10) + in.Cfg.PlannedSamplePerArm = 400 + addUsers(&in, "c", "control", 200, 1000) + addUsers(&in, "b", "b", 200, 1000) + // Strong effect so the winner gate is the only thing in the way. + for i := 0; i < 200; i++ { + if i < 20 { + in.BinaryConversions = append(in.BinaryConversions, userVariant{User: fmt.Sprintf("c%d", i), Variant: "control"}) + } + if i < 90 { + in.BinaryConversions = append(in.BinaryConversions, userVariant{User: fmt.Sprintf("b%d", i), Variant: "b"}) + } + } + res := computeResults(in) + if res.PeekingWarning == "" || !strings.Contains(res.PeekingWarning, "200 of 400") { + t.Fatalf("expected peeking warning, got %q", res.PeekingWarning) + } + if res.Winner != "" || res.Significant || res.Analysis.HorizonMet { + t.Fatalf("winner must be refused before the planned sample: %+v", res.Analysis) + } + // Per-request override. + in.Opts.AllowEarly = true + over := computeResults(in) + if over.Winner != "b" || !over.Analysis.HorizonOverridden || over.PeekingWarning != "" { + t.Fatalf("override: winner=%q overridden=%v warn=%q", over.Winner, over.Analysis.HorizonOverridden, over.PeekingWarning) + } + // Config override has the same effect. + in.Opts.AllowEarly = false + in.Cfg.AllowEarlyWinner = true + if computeResults(in).Winner != "b" { + t.Fatal("allow_early_winner must waive the planned gate") + } + // The min_sample floor is never waived. + in.Exp.MinSample = 500 + if r := computeResults(in); r.Winner != "" || r.PeekingWarning == "" { + t.Fatalf("min_sample floor must hold under override: %+v", r.Analysis) + } +} + +// continuousInputs builds a mean-metric experiment with per-user purchases. +func continuousInputs(n int, effect float64) resultInputs { + in := baseInputs(1) + in.Cfg.MetricKind = KindMean + for _, arm := range []struct { + name string + mult float64 + }{{"control", 1}, {"b", effect}, {"c", 1}} { + addUsers(&in, arm.name, arm.name, n, 1000) + for i := 0; i < n; i++ { + // deterministic spread, 60% buyers + if i%5 < 3 { + in.Events[PrimaryMetricKey] = append(in.Events[PrimaryMetricKey], + metricEvent{User: fmt.Sprintf("%s%d", arm.name, i), TS: 2000, Value: arm.mult * (10 + float64(i%7))}) + } + } + } + return in +} + +func TestContinuousPrimaryMatchesWelch(t *testing.T) { + in := continuousInputs(100, 1.5) + res := computeResults(in) + if res.MetricKind != KindMean || res.Analysis.Test != "welch-t" { + t.Fatalf("kind/test: %s %s", res.MetricKind, res.Analysis.Test) + } + // Reference: per-user arrays recomputed independently. + arr := func(name string, mult float64) []float64 { + v := make([]float64, 100) + for i := range v { + if i%5 < 3 { + v[i] = mult * (10 + float64(i%7)) + } + } + return v + } + want := welchTest(summarize(arr("control", 1)), summarize(arr("b", 1.5)), 0.05) + var got *PairwiseResult + for i := range res.Analysis.Pairwise { + if res.Analysis.Pairwise[i].Variant == "b" { + got = &res.Analysis.Pairwise[i] + } + } + if got == nil || relErr(got.PValue, want.PValue) > 1e-9 || math.Abs(got.LiftAbsolute-want.Diff) > 1e-9 || relErr(got.DF, want.DF) > 1e-9 { + t.Fatalf("pairwise %+v want %+v", got, want) + } + if res.Winner != "b" || !res.Significant { + t.Fatalf("expected winner b: winner=%q sig=%v rule=%s", res.Winner, res.Significant, res.Analysis.WinnerRule) + } + // Holm across 2 arms: adjusted p = min(1, 2*p) for the smallest. + if got.HolmAdjustedP < got.PValue-1e-15 || got.HolmAdjustedP > 1 { + t.Fatalf("holm: %+v", got) + } + for _, v := range res.Variants { + if v.Mean == nil || v.StdDev == nil { + t.Fatalf("continuous variants carry mean/std_dev: %+v", v) + } + if v.ProbBeatControl != 0 { + t.Fatal("no Bayesian value for continuous metrics") + } + } +} + +func TestContinuousMinSampleGate(t *testing.T) { + // 20 per arm is below the continuous floor of 30 even with min_sample 1. + res := computeResults(continuousInputs(20, 5)) + if res.Analysis.HorizonMet || res.Winner != "" || res.Significant { + t.Fatalf("n=20 must not declare: %+v", res.Analysis) + } + if res.Analysis.MinSamplePerArm != minContinuousN { + t.Fatalf("required n = %d", res.Analysis.MinSamplePerArm) + } +} + +func TestContinuousWindowAndCount(t *testing.T) { + in := baseInputs(1) + in.Cfg.MetricKind = KindCount + addUsers(&in, "c", "control", 40, 1000) + addUsers(&in, "b", "b", 40, 1000) + in.Events[PrimaryMetricKey] = []metricEvent{ + {User: "c0", TS: 1500, Value: 999}, // counts as 1 event (value ignored) + {User: "c0", TS: 1600, Value: 1}, // second event + {User: "c1", TS: 999, Value: 1}, // before exposure: ignored + {User: "c2", TS: 1000 + 72*hourMs + 1, Value: 1}, // after window: ignored + {User: "c3", TS: 1000 + 72*hourMs, Value: 1}, // boundary inclusive + {User: "ghost", TS: 1500, Value: 1}, // never exposed: ignored + } + res := computeResults(in) + ctrl := res.Variants[0] + if ctrl.Variant != "control" || ctrl.Conversions != 2 { + t.Fatalf("responders: %+v", ctrl) + } + if want := 3.0 / 40; math.Abs(*ctrl.Mean-want) > 1e-12 { + t.Fatalf("mean = %v want %v", *ctrl.Mean, want) + } +} + +func TestWinsorizeOptionRobustToOutlier(t *testing.T) { + in := continuousInputs(100, 1.0) + // One absurd outlier in control swamps the mean. + in.Events[PrimaryMetricKey] = append(in.Events[PrimaryMetricKey], metricEvent{User: "control0", TS: 2000, Value: 1e6}) + raw := computeResults(in) + in.Cfg.WinsorizePct = 1 + win := computeResults(in) + if *raw.Variants[0].Mean < 1000 { + t.Fatalf("test setup: outlier should dominate raw mean, got %v", *raw.Variants[0].Mean) + } + if *win.Variants[0].Mean > 100 { + t.Fatalf("winsorized mean should be tame, got %v", *win.Variants[0].Mean) + } + if win.Settings == nil || win.Settings.WinsorizePct != 1 { + t.Fatal("settings echoed") + } +} + +func TestSecondaryNeverGatesWinner(t *testing.T) { + in := baseInputs(1) + in.Cfg.Secondary = []SecondaryGoal{{Key: "signup", Kind: KindBinary}, {Key: "rev", Name: "Revenue", Kind: KindMean}} + addUsers(&in, "c", "control", 100, 1000) + addUsers(&in, "b", "b", 100, 1000) + // Primary: no difference. Secondary signup: huge difference. + for i := 0; i < 100; i++ { + if i < 10 { + in.BinaryConversions = append(in.BinaryConversions, + userVariant{User: fmt.Sprintf("c%d", i), Variant: "control"}, userVariant{User: fmt.Sprintf("b%d", i), Variant: "b"}) + } + if i < 5 { + in.Events["signup"] = append(in.Events["signup"], metricEvent{User: fmt.Sprintf("c%d", i), TS: 1500}) + } + if i < 80 { + in.Events["signup"] = append(in.Events["signup"], metricEvent{User: fmt.Sprintf("b%d", i), TS: 1500}) + } + } + res := computeResults(in) + if res.Winner != "" { + t.Fatalf("secondary must not produce a winner: %q", res.Winner) + } + if len(res.Secondary) != 2 || res.Secondary[0].Role != "secondary" || res.Secondary[0].GatesWinner { + t.Fatalf("secondary labelling: %+v", res.Secondary) + } + sig := res.Secondary[0] + if len(sig.Analysis.Pairwise) != 1 || !sig.Analysis.Pairwise[0].Significant { + t.Fatalf("secondary signup should still be evaluated: %+v", sig.Analysis) + } + if !strings.Contains(sig.Analysis.WinnerRule, "does not gate the winner") || res.MultipleComparisonNote == "" { + t.Fatal("labelling / multiple-comparison note missing") + } + // Secondary metric uses the same clean cohort (exposures equal primary). + if sig.Variants[0].Exposures != res.Variants[0].Exposures { + t.Fatal("secondary must share the cohort") + } +} + +func TestConfigNormalize(t *testing.T) { + ok, err := ExperimentConfig{}.Normalize() + if err != nil || ok.MetricKind != KindBinary { + t.Fatalf("default: %+v %v", ok, err) + } + bad := []ExperimentConfig{ + {MetricKind: "median"}, + {MetricKind: KindMean, WinsorizePct: 50}, + {MetricKind: KindMean, WinsorizePct: -1}, + {MetricKind: KindBinary, WinsorizePct: 5}, + {PlannedSamplePerArm: -1}, + {Secondary: []SecondaryGoal{{Key: "a"}, {Key: "b"}, {Key: "c"}, {Key: "d"}}}, + {Secondary: []SecondaryGoal{{Key: "Bad Key"}}}, + {Secondary: []SecondaryGoal{{Key: "primary"}}}, + {Secondary: []SecondaryGoal{{Key: "a"}, {Key: "a"}}}, + {Secondary: []SecondaryGoal{{Key: "a", Kind: "x"}}}, + {Secondary: []SecondaryGoal{{Key: "a'; DROP TABLE x;--"}}}, + } + for i, c := range bad { + if _, err := c.Normalize(); err == nil { + t.Errorf("case %d should be rejected: %+v", i, c) + } + } + if c, err := (ExperimentConfig{Secondary: []SecondaryGoal{{Key: "a"}}}).Normalize(); err != nil || c.Secondary[0].Kind != KindBinary { + t.Fatalf("secondary default kind: %+v %v", c, err) + } +} + +// Property: every p-value and probability in the output is in [0,1] across +// random data, and no output field is NaN/Inf (JSON-encodable). +func TestResultsPropertyRanges(t *testing.T) { + for seed := 0; seed < 60; seed++ { + n := 30 + seed*7 + in := continuousInputs(n, 1+float64(seed%5)*0.2) + if seed%3 == 0 { + in.Cfg.MetricKind = KindCount + } + in.Cfg.Secondary = []SecondaryGoal{{Key: "s1", Kind: KindBinary}} + for i := 0; i < n; i += 1 + seed%4 { + in.Events["s1"] = append(in.Events["s1"], metricEvent{User: fmt.Sprintf("b%d", i), TS: 1500}) + } + res := computeResults(in) + if _, err := json.Marshal(res); err != nil { + t.Fatalf("seed %d: not JSON-encodable: %v", seed, err) + } + check := func(an AnalysisResult) { + if an.PValue < 0 || an.PValue > 1 { + t.Fatalf("seed %d: p=%v", seed, an.PValue) + } + for _, pw := range an.Pairwise { + if pw.PValue < 0 || pw.PValue > 1 || pw.HolmAdjustedP < pw.PValue-1e-15 || pw.HolmAdjustedP > 1 { + t.Fatalf("seed %d: pairwise %+v", seed, pw) + } + } + } + check(res.Analysis) + for _, s := range res.Secondary { + check(s.Analysis) + } + } +} diff --git a/internal/experiments/continuous_test.go b/internal/experiments/continuous_test.go new file mode 100644 index 00000000..5d8cfddf --- /dev/null +++ b/internal/experiments/continuous_test.go @@ -0,0 +1,175 @@ +package experiments + +import ( + "math" + "math/rand" + "sort" + "testing" +) + +// Goldens generated 2026-10-03 with scipy 1.17.1 / numpy 2.4.6 +// (scipy.stats.t, scipy.stats.ttest_ind(equal_var=False), numpy.percentile). + +var tTwoSidedGolden = []struct{ t, df, p float64 }{ + {0.5, 3, 0.651447964848151}, + {2.0, 10, 0.07338803477074037}, + {2.5, 7.3, 0.03965023466560043}, + {-1.96, 1000, 0.050273184955748736}, + {5.0, 2.5, 0.02345118997086185}, + {0.1, 50, 0.92074421365297}, + {12, 30, 5.580185415199261e-13}, +} + +func TestStudentTTwoSidedPGolden(t *testing.T) { + for _, g := range tTwoSidedGolden { + if e := relErr(studentTTwoSidedP(g.t, g.df), g.p); e > 1e-9 { + t.Errorf("p(t=%v, df=%v) = %v want %v (rel %g)", g.t, g.df, studentTTwoSidedP(g.t, g.df), g.p, e) + } + } +} + +func TestStudentTQuantileGolden(t *testing.T) { + for _, g := range []struct{ tail, df, q float64 }{ + {0.025, 5, 2.5705818356363155}, + {0.025, 12.5, 2.1691859427125406}, + {0.025, 100, 1.983971518523552}, + {0.005, 20, 2.8453397097861077}, + {0.1, 3.3, 1.5983669585600868}, + } { + if e := relErr(studentTQuantileUpper(g.tail, g.df), g.q); e > 1e-9 { + t.Errorf("q(%v, df=%v) = %v want %v", g.tail, g.df, studentTQuantileUpper(g.tail, g.df), g.q) + } + } +} + +var ( + welchA1 = []float64{12.1, 9.8, 11.4, 10.2, 13.5, 8.9, 10.7, 11.9, 12.8, 9.5} + welchA2 = []float64{14.2, 15.1, 12.9, 16.3, 13.8, 15.7, 14.9, 13.1, 16.8, 15.2, 14.4, 13.6} + // Revenue-like: mostly zeros with a heavy right tail. + welchB1 = []float64{0, 0, 0, 5.5, 0, 12.0, 0, 0, 3.2, 0, 0, 0, 40.0, 0, 0, 1.1, 0, 0, 7.7, 0} + welchB2 = []float64{0, 2.2, 0, 9.9, 0, 0, 15.5, 0, 0, 4.4, 0, 0, 0, 60.0, 0, 3.3, 0, 0, 0, 8.8, 0, 0} +) + +func TestWelchGolden(t *testing.T) { + cases := []struct { + name string + a, b []float64 + diff, tt, df, p, ciL, ciH float64 + }{ + {"A", welchA1, welchA2, 3.586666666666666, 6.040422007471008, 17.359434660535243, 1.2110889125450551e-05, 2.335878672714065, 4.837454660619267}, + {"B", welchB1, welchB2, 1.2568181818181814, 0.36400584042590584, 37.78032762998611, 0.7178829444952389, -5.734227067120866, 8.247863430757228}, + } + for _, c := range cases { + r := welchTest(summarize(c.a), summarize(c.b), 0.05) + chk := func(n string, got, want float64) { + if e := relErr(got, want); e > 1e-9 { + t.Errorf("%s %s = %v want %v (rel %g)", c.name, n, got, want, e) + } + } + chk("diff", r.Diff, c.diff) + chk("t", r.T, c.tt) + chk("df", r.DF, c.df) + chk("p", r.PValue, c.p) + chk("ci_low", r.CILow, c.ciL) + chk("ci_high", r.CIHigh, c.ciH) + } +} + +func TestWinsorizeGolden(t *testing.T) { + pooled := append(append([]float64(nil), welchB1...), welchB2...) + for _, g := range []struct { + pct, lo, hi, diff, p, ciL, ciH float64 + }{ + {5, 0, 15.324999999999985, 0.4519318181818175, 0.7581746661195907, -2.4945443577862987, 3.3984079941499337}, + {10, 0, 9.79, 0.33099999999999974, 0.7667082820919727, -1.9085608638852396, 2.5705608638852393}, + } { + lo, hi, ok := winsorizeBounds(pooled, g.pct) + if !ok || math.Abs(lo-g.lo) > 1e-12 || math.Abs(hi-g.hi) > 1e-9 { + t.Fatalf("bounds(%v) = %v,%v want %v,%v", g.pct, lo, hi, g.lo, g.hi) + } + arms := [][]float64{append([]float64(nil), welchB1...), append([]float64(nil), welchB2...)} + winsorizeArms(arms, g.pct) + r := welchTest(summarize(arms[0]), summarize(arms[1]), 0.05) + if relErr(r.Diff, g.diff) > 1e-9 || relErr(r.PValue, g.p) > 1e-9 || relErr(r.CILow, g.ciL) > 1e-9 || relErr(r.CIHigh, g.ciH) > 1e-9 { + t.Errorf("winsorized %v%%: %+v want diff=%v p=%v ci=[%v,%v]", g.pct, r, g.diff, g.p, g.ciL, g.ciH) + } + } + // off / invalid percentiles clip nothing. + arms := [][]float64{{1, 100}, {2, 3}} + winsorizeArms(arms, 0) + winsorizeArms(arms, 50) + if arms[0][1] != 100 { + t.Fatal("pct 0/50 must not clip") + } +} + +func TestPercentileLinearGolden(t *testing.T) { + pooled := append(append([]float64(nil), welchB1...), welchB2...) + for _, g := range []struct{ q, want float64 }{{0, 0}, {50, 0}, {87.5, 8.662500000000001}, {100, 60}} { + s := append([]float64(nil), pooled...) + sort.Float64s(s) + if got := percentileLinear(s, g.q); math.Abs(got-g.want) > 1e-12 { + t.Errorf("percentile(%v) = %v want %v", g.q, got, g.want) + } + } +} + +// Property: p in [0,1], symmetric under swapping arms, monotone in the mean +// shift, CI contains the point difference and brackets 0 iff p > alpha. +func TestWelchProperties(t *testing.T) { + rng := rand.New(rand.NewSource(99)) + gen := func(n int, mu, sd float64) []float64 { + v := make([]float64, n) + for i := range v { + v[i] = mu + sd*rng.NormFloat64() + } + return v + } + for trial := 0; trial < 200; trial++ { + a := gen(5+rng.Intn(60), 10, 1+3*rng.Float64()) + b := gen(5+rng.Intn(60), 10+rng.NormFloat64()*2, 1+3*rng.Float64()) + sa, sb := summarize(a), summarize(b) + r := welchTest(sa, sb, 0.05) + if r.PValue < 0 || r.PValue > 1 || math.IsNaN(r.PValue) { + t.Fatalf("p out of range: %+v", r) + } + rev := welchTest(sb, sa, 0.05) + if math.Abs(rev.PValue-r.PValue) > 1e-12 || math.Abs(rev.Diff+r.Diff) > 1e-12 { + t.Fatalf("not symmetric: %+v vs %+v", r, rev) + } + if r.CILow > r.Diff || r.CIHigh < r.Diff { + t.Fatalf("CI does not contain diff: %+v", r) + } + containsZero := r.CILow <= 0 && r.CIHigh >= 0 + if containsZero != (r.PValue >= 0.05) && math.Abs(r.PValue-0.05) > 1e-6 { + t.Fatalf("CI/p disagree: %+v", r) + } + // Monotone: shifting b further from a never raises p. + prev := 2.0 + sign := 1.0 + if sb.Mean < sa.Mean { + sign = -1 + } + for shift := 0.0; shift <= 20; shift += 2 { + s := sb + s.Mean += sign * shift + p := welchTest(sa, s, 0.05).PValue + if p > prev+1e-12 && math.Abs(s.Mean-sa.Mean) > math.Abs(sb.Mean-sa.Mean) { + t.Fatalf("p not monotone in shift: %v after %v", p, prev) + } + prev = p + } + } +} + +func TestWelchDegenerate(t *testing.T) { + if r := welchTest(summarize([]float64{1}), summarize([]float64{2, 3}), 0.05); r.PValue != 1 { + t.Fatalf("n<2 must give p=1: %+v", r) + } + if r := welchTest(summarize([]float64{2, 2, 2}), summarize([]float64{2, 2, 2}), 0.05); r.PValue != 1 { + t.Fatalf("identical constants: %+v", r) + } + if r := welchTest(summarize([]float64{2, 2, 2}), summarize([]float64{5, 5, 5}), 0.05); r.PValue != 0 || math.IsNaN(r.CILow) { + t.Fatalf("different constants: %+v", r) + } +} diff --git a/internal/experiments/experiments.go b/internal/experiments/experiments.go index 7afbc11a..30110055 100644 --- a/internal/experiments/experiments.go +++ b/internal/experiments/experiments.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "math" "strconv" "time" @@ -99,6 +100,19 @@ type ExperimentResults struct { // Winner additionally requires the winning arm's pairwise comparison to // survive Holm. Analysis AnalysisResult `json:"analysis"` + // MetricKind of the primary goal: binary | count | mean. + MetricKind string `json:"metric_kind"` + // ContaminatedUsers were exposed to more than one variant inside the + // running interval. They are excluded from every count and statistic. + ContaminatedUsers int `json:"contaminated_users"` + // PeekingWarning is set while any arm is below the planned horizon: the + // analysis is fixed-horizon, so early p-values are a progress report. + PeekingWarning string `json:"peeking_warning"` + // Settings echoes the analysis configuration when it is non-default. + Settings *ExperimentConfig `json:"settings,omitempty"` + // Secondary metrics: informational, never gate the winner. + Secondary []MetricResult `json:"secondary,omitempty"` + MultipleComparisonNote string `json:"multiple_comparison_note,omitempty"` } type VariantResult struct { @@ -113,6 +127,11 @@ type VariantResult struct { // arm's conversion rate (O09: estimates always carry uncertainty). WilsonLow float64 `json:"wilson_low"` WilsonHigh float64 `json:"wilson_high"` + // Mean/StdDev are the per-exposed-user mean and sample standard + // deviation for count and mean metrics (absent for binary goals). For + // those goals Conversions counts users with at least one event. + Mean *float64 `json:"mean,omitempty"` + StdDev *float64 `json:"std_dev,omitempty"` } func (s *ExperimentService) Create(ctx context.Context, siteID, name, flagKey, goalMetric, goalValue, variants string, minSample int) (*Experiment, error) { @@ -201,9 +220,9 @@ const firstExposureSQL = `SELECT variant FROM experiment_exposures // makes. It used to read the LATEST exposure, so a user exposed to several // variants (an allocation change mid-run, a client that re-sends expose with // a different arm) was attributed to whichever arm happened to expose last, -// and the answer moved as more exposures arrived. Note Results does not read -// this stored variant: it counts a converting user in every arm whose -// in-interval exposure window contains the conversion. +// and the answer moved as more exposures arrived. Results does not read this +// stored variant: a user exposed to more than one variant is contaminated and +// excluded from every count (reported as contaminated_users). func (s *ExperimentService) RecordConversion(ctx context.Context, experimentID, siteID, userID string) error { type vrow struct { Variant string `db:"variant"` @@ -225,6 +244,23 @@ func (s *ExperimentService) RecordConversion(ctx context.Context, experimentID, return err } +// maxAnalysisRows bounds the per-user rows Results pulls into memory. A read +// that reaches it fails loudly rather than analysing a silent prefix. +const maxAnalysisRows = 2_000_000 + +func (s *ExperimentService) loadExperiment(ctx context.Context, experimentID, siteID string) (Experiment, error) { + exps, err := nucleus.Query[Experiment](ctx, s.db.SQL(), + `SELECT experiment_id, tenant_id, site_id, name, flag_key, goal_metric, goal_value, status, min_sample, + COALESCE(variants, '') AS variants, + started_at, ended_at, created_at, version, + COALESCE(CAST(conversion_window_hours AS TEXT), '72') AS conversion_window_hours + FROM `+experimentsLatest("experiment_id = $1 AND site_id = $2"), experimentID, siteID) + if err != nil || len(exps) == 0 { + return Experiment{}, fmt.Errorf("experiment not found") + } + return exps[0], nil +} + // Results computes experiment results with the O09 decided analysis // (2026-09-23): input semantics restrict exposures to the running interval // and conversions to the declared conversion window after an exposure, @@ -232,20 +268,28 @@ func (s *ExperimentService) RecordConversion(ctx context.Context, experimentID, // per-arm horizon gate, SRM detection, the omnibus test with Fisher // fallback, and Holm-corrected pairwise winner claims. // +// A user exposed to more than one variant inside the interval is +// contaminated: excluded from every count and reported in +// contaminated_users. Primary and secondary goals of kind count/mean come +// from experiment_metric_events (see config.go). +// // Late-arrival policy: a conversion recorded after the experiment stopped // still counts when it falls inside the conversion window of an // in-interval exposure (the window is anchored at exposure time). func (s *ExperimentService) Results(ctx context.Context, experimentID, siteID string) (*ExperimentResults, error) { - exps, err := nucleus.Query[Experiment](ctx, s.db.SQL(), - `SELECT experiment_id, tenant_id, site_id, name, flag_key, goal_metric, goal_value, status, min_sample, - COALESCE(variants, '') AS variants, - started_at, ended_at, created_at, version, - COALESCE(CAST(conversion_window_hours AS TEXT), '72') AS conversion_window_hours - FROM `+experimentsLatest("experiment_id = $1 AND site_id = $2"), experimentID, siteID) - if err != nil || len(exps) == 0 { - return nil, fmt.Errorf("experiment not found") + return s.ResultsWithOptions(ctx, experimentID, siteID, ResultsOptions{}) +} + +// ResultsWithOptions is Results with per-request options. +func (s *ExperimentService) ResultsWithOptions(ctx context.Context, experimentID, siteID string, opts ResultsOptions) (*ExperimentResults, error) { + exp, err := s.loadExperiment(ctx, experimentID, siteID) + if err != nil { + return nil, err + } + cfg, err := s.LoadConfig(ctx, experimentID, siteID) + if err != nil { + return nil, err } - exp := exps[0] // Input semantics bounds: exposures count only inside the running // interval. started_at = 0 (draft, never started) keeps historical @@ -260,92 +304,104 @@ func (s *ExperimentService) Results(ctx context.Context, experimentID, siteID st windowMs = int64(exp.ConversionWindowHours) * 3600 * 1000 } - type cntRow struct { + type expRow struct { + User string `db:"user_id"` Variant string `db:"variant"` - Count string `db:"count"` + First string `db:"first_ts"` } - - // Exposures: distinct users per variant, restricted to the running - // interval. ORDER BY variant gives a stable slice so the Bayesian - // control selection below is deterministic. - expRows, err := nucleus.Query[cntRow](ctx, s.db.SQL(), - `SELECT variant, CAST(COUNT(DISTINCT user_id) AS TEXT) AS count + rawExp, err := nucleus.Query[expRow](ctx, s.db.SQL(), + `SELECT user_id, variant, CAST(MIN(timestamp) AS TEXT) AS first_ts FROM experiment_exposures WHERE experiment_id = $1 AND site_id = $2 AND timestamp >= $3 AND timestamp <= $4 - GROUP BY variant ORDER BY variant`, + GROUP BY user_id, variant LIMIT `+strconv.Itoa(maxAnalysisRows), experimentID, siteID, lower, upper) if err != nil { return nil, err } - - // Conversions: distinct converting users per variant, counted only when - // the conversion falls within the conversion window after an - // in-interval exposure of that user (any of them - the attributed - // variant is the exposure whose window contains it). - convRows, err := nucleus.Query[cntRow](ctx, s.db.SQL(), - `SELECT e.variant AS variant, CAST(COUNT(DISTINCT e.user_id) AS TEXT) AS count - FROM experiment_exposures e - INNER JOIN experiment_conversions c - ON c.experiment_id = e.experiment_id AND c.site_id = e.site_id AND c.user_id = e.user_id - WHERE e.experiment_id = $1 AND e.site_id = $2 AND e.timestamp >= $3 AND e.timestamp <= $4 - AND c.timestamp >= e.timestamp AND c.timestamp <= e.timestamp + CAST($5 AS BIGINT) - GROUP BY e.variant ORDER BY e.variant`, - experimentID, siteID, lower, upper, windowMs) - if err != nil { - return nil, err + if len(rawExp) >= maxAnalysisRows { + return nil, fmt.Errorf("experiment too large for in-process analysis (>= %d exposed users)", maxAnalysisRows) } - convByVariant := make(map[string]int64, len(convRows)) - for _, r := range convRows { - c, _ := strconv.ParseInt(r.Count, 10, 64) - convByVariant[r.Variant] = c + in := resultInputs{Exp: exp, Cfg: cfg, Opts: opts, WindowMs: windowMs, + Events: map[string][]metricEvent{}, Truncated: map[string]bool{}} + for _, r := range rawExp { + ts, _ := strconv.ParseInt(r.First, 10, 64) + in.Exposures = append(in.Exposures, exposureRow{User: r.User, Variant: r.Variant, First: ts}) } - var variants []VariantResult - for _, r := range expRows { - total, _ := strconv.ParseInt(r.Count, 10, 64) - conv := convByVariant[r.Variant] - rate := 0.0 - if total > 0 { - rate = float64(conv) / float64(total) + if cfg.MetricKind == KindBinary { + // Conversions: distinct converting (user, variant), counted only when + // the conversion falls within the conversion window after an + // in-interval exposure of that user. + type convRow struct { + User string `db:"user_id"` + Variant string `db:"variant"` + } + rows, err := nucleus.Query[convRow](ctx, s.db.SQL(), + `SELECT e.user_id AS user_id, e.variant AS variant + FROM experiment_exposures e + INNER JOIN experiment_conversions c + ON c.experiment_id = e.experiment_id AND c.site_id = e.site_id AND c.user_id = e.user_id + WHERE e.experiment_id = $1 AND e.site_id = $2 AND e.timestamp >= $3 AND e.timestamp <= $4 + AND c.timestamp >= e.timestamp AND c.timestamp <= e.timestamp + CAST($5 AS BIGINT) + GROUP BY e.user_id, e.variant LIMIT `+strconv.Itoa(maxAnalysisRows), + experimentID, siteID, lower, upper, windowMs) + if err != nil { + return nil, err + } + if len(rows) >= maxAnalysisRows { + return nil, fmt.Errorf("experiment too large for in-process analysis (>= %d converting users)", maxAnalysisRows) + } + for _, r := range rows { + in.BinaryConversions = append(in.BinaryConversions, userVariant{User: r.User, Variant: r.Variant}) } - lo, hi := wilsonInterval(conv, total, zAlphaTwoSided005) - variants = append(variants, VariantResult{ - Variant: r.Variant, Exposures: total, Conversions: conv, - ConversionRate: rate, WilsonLow: lo, WilsonHigh: hi, - }) } - // Put the declared control variant (first entry in the experiment's variants - // JSON, or one keyed "control") at index 0 so the Bayesian comparison is - // against the true control rather than whatever sorted first. - orderControlFirst(variants, controlKey(exp.Variants)) - - // Bayesian: compute probability each variant beats the control (index 0). - // Uses Beta(1+conv, 1+nonconv) conjugate prior with a 4000-sample Monte Carlo. - // Displayed as a labeled estimate; it never gates the winner (2026-09-23 - // decision: the fixed-horizon frequentist gates do). - if len(variants) >= 2 { - computeBayesianProbabilities(variants) + keys := make([]string, 0, 1+len(cfg.Secondary)) + if cfg.MetricKind != KindBinary { + keys = append(keys, PrimaryMetricKey) + } + for _, g := range cfg.Secondary { + keys = append(keys, g.Key) + } + for _, k := range keys { + ev, truncated, err := s.readMetricEvents(ctx, experimentID, siteID, k, lower, upper+windowMs) + if err != nil { + return nil, err + } + in.Events[k] = ev + in.Truncated[k] = truncated } - // O09 analysis: per-arm horizon, SRM, omnibus + Fisher fallback, Holm - // pairwise. MinSample is per-arm (a 9,900/100 split must not declare). - minSample := exp.MinSample - if minSample <= 0 { - minSample = 100 + return computeResults(in), nil +} + +// readMetricEvents reads one metric's events in [lower, upperTS]. truncated +// reports that the row cap was reached (the caller refuses a winner). +func (s *ExperimentService) readMetricEvents(ctx context.Context, experimentID, siteID, metric string, lower, upperTS int64) ([]metricEvent, bool, error) { + type evRow struct { + User string `db:"user_id"` + TS string `db:"ts"` + Value string `db:"value"` } - analysis := analyze(variants, allocationWeights(exp.Variants, len(variants)), minSample) - winner := winnerFrom(analysis, variants) - significant := analysis.HorizonMet && !analysis.SRM.Detected && - analysis.Test != "none" && analysis.PValue < alphaOmnibus - - return &ExperimentResults{ - Experiment: exp, - Variants: variants, - Significant: significant, - Winner: winner, - Analysis: analysis, - }, nil + rows, err := nucleus.Query[evRow](ctx, s.db.SQL(), + `SELECT user_id, CAST(timestamp AS TEXT) AS ts, value + FROM experiment_metric_events + WHERE experiment_id = $1 AND site_id = $2 AND metric = $3 AND timestamp >= $4 AND timestamp <= $5 + LIMIT `+strconv.Itoa(maxAnalysisRows), + experimentID, siteID, metric, lower, upperTS) + if err != nil { + return nil, false, err + } + out := make([]metricEvent, 0, len(rows)) + for _, r := range rows { + ts, _ := strconv.ParseInt(r.TS, 10, 64) + v, perr := strconv.ParseFloat(r.Value, 64) + if perr != nil || math.IsNaN(v) || math.IsInf(v, 0) { + continue // a corrupt value never poisons a mean + } + out = append(out, metricEvent{User: r.User, TS: ts, Value: v}) + } + return out, len(rows) >= maxAnalysisRows, nil } // timeMsOrZero maps the TEXT epoch-ms columns ('0' for unset) through their diff --git a/internal/experiments/results_test.go b/internal/experiments/results_test.go index b55338fa..bface393 100644 --- a/internal/experiments/results_test.go +++ b/internal/experiments/results_test.go @@ -40,6 +40,15 @@ func expTestDB(t *testing.T) (*nucleus.Client, func()) { site_id TEXT NOT NULL, user_id TEXT NOT NULL DEFAULT '', variant TEXT NOT NULL DEFAULT '', timestamp BIGINT NOT NULL ) WITH (engine = 'mergetree') ORDER BY (tenant_id, site_id, experiment_id, user_id)`, + `CREATE TABLE IF NOT EXISTS experiment_settings ( + setting_id TEXT NOT NULL, tenant_id TEXT NOT NULL DEFAULT 'default', experiment_id TEXT NOT NULL, + site_id TEXT NOT NULL, config TEXT NOT NULL DEFAULT '{}', timestamp BIGINT NOT NULL + ) WITH (engine = 'mergetree') ORDER BY (tenant_id, site_id, experiment_id, timestamp)`, + `CREATE TABLE IF NOT EXISTS experiment_metric_events ( + event_id TEXT NOT NULL, tenant_id TEXT NOT NULL DEFAULT 'default', experiment_id TEXT NOT NULL, + site_id TEXT NOT NULL, user_id TEXT NOT NULL DEFAULT '', metric TEXT NOT NULL DEFAULT 'primary', + value TEXT NOT NULL DEFAULT '0', timestamp BIGINT NOT NULL + ) WITH (engine = 'mergetree') ORDER BY (tenant_id, site_id, experiment_id, metric, timestamp)`, } { if _, err := db.SQL().Exec(ctx, ddl); err != nil { db.Close() @@ -147,3 +156,44 @@ func TestResults_GatesOnMinSample(t *testing.T) { t.Fatalf("min_sample=1000 with 6 exposures must not be significant; got significant=%v winner=%q", res.Significant, res.Winner) } } + +// TestResults_ContaminationAndMetricsLive exercises the SQL path (skipped +// without a live Nucleus): a user exposed to two arms is excluded and +// counted as contaminated, and a mean-metric primary reads metric events. +func TestResults_ContaminationAndMetricsLive(t *testing.T) { + db, done := expTestDB(t) + defer done() + ctx := context.Background() + svc := NewExperimentService(db) + + site := fmt.Sprintf("test-exp-contam-%d", time.Now().UnixNano()) + exp, err := svc.Create(ctx, site, "Contam", "flag", "purchase", "", + `[{"key":"control"},{"key":"treatment"}]`, 1) + if err != nil { + t.Fatal(err) + } + if _, err := svc.SaveConfig(ctx, exp.ExperimentID, site, ExperimentConfig{MetricKind: KindMean}); err != nil { + t.Fatal(err) + } + for _, u := range []string{"c1", "c2"} { + _ = svc.RecordExposure(ctx, exp.ExperimentID, site, u, "control") + } + _ = svc.RecordExposure(ctx, exp.ExperimentID, site, "t1", "treatment") + _ = svc.RecordExposure(ctx, exp.ExperimentID, site, "both", "control") + _ = svc.RecordExposure(ctx, exp.ExperimentID, site, "both", "treatment") + _ = svc.RecordMetric(ctx, exp.ExperimentID, site, "c1", PrimaryMetricKey, 20) + _ = svc.RecordMetric(ctx, exp.ExperimentID, site, "t1", PrimaryMetricKey, 30) + _ = svc.RecordMetric(ctx, exp.ExperimentID, site, "both", PrimaryMetricKey, 1000) + res, err := svc.Results(ctx, exp.ExperimentID, site) + if err != nil { + t.Fatal(err) + } + if res.ContaminatedUsers != 1 || res.MetricKind != KindMean { + t.Fatalf("contaminated=%d kind=%s", res.ContaminatedUsers, res.MetricKind) + } + for _, v := range res.Variants { + if v.Variant == "control" && (v.Exposures != 2 || v.Mean == nil || *v.Mean != 10) { + t.Fatalf("control: %+v", v) + } + } +} diff --git a/internal/experiments/testdata/legacy_results.golden b/internal/experiments/testdata/legacy_results.golden new file mode 100644 index 00000000..c477aa4b --- /dev/null +++ b/internal/experiments/testdata/legacy_results.golden @@ -0,0 +1,3 @@ +{"experiment":{"experiment_id":"e1","site_id":"s","name":"","flag_key":"","goal_metric":"","goal_value":"","status":"","min_sample":100,"variants":"[{\"key\":\"control\"},{\"key\":\"b\"},{\"key\":\"c\"}]","started_at":"0001-01-01T00:00:00Z","ended_at":"0001-01-01T00:00:00Z","created_at":"0001-01-01T00:00:00Z","conversion_window_hours":0},"variants":[{"variant":"control","exposures":4000,"conversions":400,"conversion_rate":0.1,"prob_beat_control":0,"wilson_low":0.09108339124317813,"wilson_high":0.10968416338858308},{"variant":"b","exposures":4000,"conversions":480,"conversion_rate":0.12,"prob_beat_control":0,"wilson_low":0.11029234261658878,"wilson_high":0.13043683428358435}],"significant":true,"winner":"b","analysis":{"horizon_met":true,"min_arm_exposures":4000,"min_sample_per_arm":100,"test":"chi-square-yates","chi_square":7.968590398365679,"df":1,"p_value":0.004759598860014404,"srm":{"detected":false,"chi_square":0,"p_value":0},"pairwise_vs_control":[{"variant":"b","lift_absolute":0.01999999999999999,"lift_relative":0.1999999999999999,"ci_low":0.006287901969046772,"ci_high":0.03372710536075374,"p_value":0.004759598860014404,"holm_adjusted_p":0.004759598860014404,"significant":true,"used_fisher":false}],"winner_rule":"winner requires: per-arm horizon, no SRM, omnibus p\u003c0.05, winner pairwise vs control surviving Holm"}} +{"experiment":{"experiment_id":"e1","site_id":"s","name":"","flag_key":"","goal_metric":"","goal_value":"","status":"","min_sample":100,"variants":"[{\"key\":\"control\"},{\"key\":\"b\"},{\"key\":\"c\"}]","started_at":"0001-01-01T00:00:00Z","ended_at":"0001-01-01T00:00:00Z","created_at":"0001-01-01T00:00:00Z","conversion_window_hours":0},"variants":[{"variant":"control","exposures":3000,"conversions":300,"conversion_rate":0.1,"prob_beat_control":0,"wilson_low":0.08977105263294838,"wilson_high":0.11125202634731239},{"variant":"b","exposures":3000,"conversions":360,"conversion_rate":0.12,"prob_beat_control":0,"wilson_low":0.10885485831348496,"wilson_high":0.13211706671776274},{"variant":"c","exposures":3000,"conversions":330,"conversion_rate":0.11,"prob_beat_control":0,"wilson_low":0.09929839195629818,"wilson_high":0.12169911004945606}],"significant":true,"winner":"b","analysis":{"horizon_met":true,"min_arm_exposures":3000,"min_sample_per_arm":100,"test":"chi-square","chi_square":6.1287027579162405,"df":2,"p_value":0.04668411234396321,"srm":{"detected":false,"chi_square":0,"p_value":0},"pairwise_vs_control":[{"variant":"b","lift_absolute":0.01999999999999999,"lift_relative":0.1999999999999999,"ci_low":0.004162630264683859,"ci_high":0.03585732228595439,"p_value":0.014918094783349203,"holm_adjusted_p":0.029836189566698406,"significant":true,"used_fisher":false},{"variant":"c","lift_absolute":0.009999999999999995,"lift_relative":0.09999999999999995,"ci_low":-0.005528442022354843,"ci_high":0.025540287648148474,"p_value":0.2219790009171313,"holm_adjusted_p":0.2219790009171313,"significant":false,"used_fisher":false}],"winner_rule":"winner requires: per-arm horizon, no SRM, omnibus p\u003c0.05, winner pairwise vs control surviving Holm"}} +{"experiment":{"experiment_id":"e1","site_id":"s","name":"","flag_key":"","goal_metric":"","goal_value":"","status":"","min_sample":100,"variants":"[{\"key\":\"control\"},{\"key\":\"b\"},{\"key\":\"c\"}]","started_at":"0001-01-01T00:00:00Z","ended_at":"0001-01-01T00:00:00Z","created_at":"0001-01-01T00:00:00Z","conversion_window_hours":0},"variants":[{"variant":"control","exposures":50,"conversions":5,"conversion_rate":0.1,"prob_beat_control":0,"wilson_low":0.04347576493189042,"wilson_high":0.21360231437479657},{"variant":"b","exposures":50,"conversions":9,"conversion_rate":0.18,"prob_beat_control":0,"wilson_low":0.09770192670234178,"wilson_high":0.30796053674300783}],"significant":false,"winner":"","analysis":{"horizon_met":false,"min_arm_exposures":50,"min_sample_per_arm":100,"test":"chi-square-yates","chi_square":0.7475083056478404,"df":1,"p_value":0.387266264379168,"srm":{"detected":false,"chi_square":0,"p_value":0},"pairwise_vs_control":[{"variant":"b","lift_absolute":0.07999999999999999,"lift_relative":0.7999999999999998,"ci_low":-0.060279929782620145,"ci_high":0.2198888419910379,"p_value":0.387266264379168,"holm_adjusted_p":0.387266264379168,"significant":false,"used_fisher":false}],"winner_rule":"winner requires: per-arm horizon, no SRM, omnibus p\u003c0.05, winner pairwise vs control surviving Holm - waiting for horizon (min arm 50 of 100)"}} diff --git a/internal/schema/migrations/063_experiment_metrics.up.sql b/internal/schema/migrations/063_experiment_metrics.up.sql new file mode 100644 index 00000000..66a447c1 --- /dev/null +++ b/internal/schema/migrations/063_experiment_metrics.up.sql @@ -0,0 +1,44 @@ +-- 063 (2026-10-03): experiment depth - continuous metrics, secondary goals, +-- analysis settings. +-- +-- The experiments table cannot take new columns (ALTER ADD COLUMN corrupts +-- populated tables, open P1 L9), and experiment_conversions carries no value, +-- so both new concerns get their own append-only tables: +-- +-- experiment_settings one JSON document per write (metric kind, +-- winsorize percentile, planned sample size, +-- early-winner override, up to 3 secondary +-- goals). The newest row by timestamp wins; an +-- experiment with no row behaves as before (one +-- binary conversion goal). +-- experiment_metric_events one observation per row for count / mean goals +-- and secondary goals. metric is 'primary' or a +-- secondary goal key. value is stored as TEXT +-- (parsed to a finite float in Go, the same +-- convention as llm cost_usd) so no DOUBLE wire +-- path is involved. +-- +-- Both are plain mergetree: the settings read orders by timestamp DESC, and +-- metric events are never updated. Comments pure ASCII (038 rule). + +CREATE TABLE IF NOT EXISTS experiment_settings ( + setting_id TEXT NOT NULL, + tenant_id TEXT NOT NULL DEFAULT 'default', + experiment_id TEXT NOT NULL, + site_id TEXT NOT NULL, + config TEXT NOT NULL DEFAULT '{}', + timestamp BIGINT NOT NULL +) WITH (engine = 'mergetree') +ORDER BY (tenant_id, site_id, experiment_id, timestamp); + +CREATE TABLE IF NOT EXISTS experiment_metric_events ( + event_id TEXT NOT NULL, + tenant_id TEXT NOT NULL DEFAULT 'default', + experiment_id TEXT NOT NULL, + site_id TEXT NOT NULL, + user_id TEXT NOT NULL DEFAULT '', + metric TEXT NOT NULL DEFAULT 'primary', + value TEXT NOT NULL DEFAULT '0', + timestamp BIGINT NOT NULL +) WITH (engine = 'mergetree') +ORDER BY (tenant_id, site_id, experiment_id, metric, timestamp); diff --git a/ui/src/api/flags.ts b/ui/src/api/flags.ts index 3b487e67..e936a3cf 100644 --- a/ui/src/api/flags.ts +++ b/ui/src/api/flags.ts @@ -41,6 +41,9 @@ export interface VariantResult { prob_beat_control: number; wilson_low: number; wilson_high: number; + /** count/mean metrics only: per-exposed-user mean and sample std dev. */ + mean?: number; + std_dev?: number; } export interface SRMDiagnostic { @@ -60,6 +63,9 @@ export interface PairwiseResult { holm_adjusted_p: number; significant: boolean; used_fisher: boolean; + /** continuous metrics (Welch): t statistic and Welch-Satterthwaite df. */ + t?: number; + df?: number; } export interface AnalysisResult { @@ -74,6 +80,21 @@ export interface AnalysisResult { srm: SRMDiagnostic; pairwise_vs_control: PairwiseResult[]; winner_rule: string; + planned_sample_per_arm?: number; + horizon_overridden?: boolean; +} + +export type MetricKind = "binary" | "count" | "mean"; + +export interface MetricResult { + key: string; + name?: string; + kind: MetricKind; + role: "primary" | "secondary"; + gates_winner: boolean; + variants: VariantResult[]; + analysis: AnalysisResult; + note?: string; } export interface ExperimentResults { @@ -82,6 +103,13 @@ export interface ExperimentResults { significant: boolean; winner: string; analysis: AnalysisResult; + metric_kind?: MetricKind; + /** Users exposed to more than one variant; excluded from every count. */ + contaminated_users?: number; + /** Fixed-horizon warning while any arm is below the planned sample. */ + peeking_warning?: string; + secondary?: MetricResult[]; + multiple_comparison_note?: string; } export interface FlagHistoryEntry { diff --git a/ui/src/routes/experiments.tsx b/ui/src/routes/experiments.tsx index a0c72484..01a98395 100644 --- a/ui/src/routes/experiments.tsx +++ b/ui/src/routes/experiments.tsx @@ -4,6 +4,7 @@ import type { Experiment, ExperimentResults } from "../api/flags.js"; import StatusBadge from "../components/shared/StatusBadge.js"; import Modal from "../components/shared/Modal.js"; import ExportButton from "../components/shared/ExportButton.js"; +import { formatArmValue, metricKindLabel, secondaryLabel } from "../utils/experimentMetrics.js"; import "../styles/flags.css"; import { useFilters } from "../hooks/useFilters.js"; @@ -54,6 +55,9 @@ function ResultsPanel({ experimentId }: { experimentId: string }) { const totalExposures = results.variants.reduce((sum, v) => sum + v.exposures, 0); const an = results.analysis; const pct = (x: number) => `${(x * 100).toFixed(2)}%`; + const kind = results.metric_kind; + const continuous = kind === "count" || kind === "mean"; + const num = (x: number) => (Math.abs(x) >= 100 ? x.toFixed(0) : x.toFixed(3)); const pStr = (p: number) => (p < 0.0001 ? "<0.0001" : p.toFixed(4)); return ( @@ -69,6 +73,16 @@ function ResultsPanel({ experimentId }: { experimentId: string }) { )}

+ {/* Fixed-horizon contract: peeking warning and contaminated users. */} + {(results.peeking_warning || (results.contaminated_users ?? 0) > 0) && ( +
+ {results.peeking_warning &&
{results.peeking_warning}
} + {(results.contaminated_users ?? 0) > 0 && ( +
{results.contaminated_users} user(s) were exposed to more than one variant and are excluded from every count.
+ )} +
+ )} + {/* O09 analysis: SRM diagnostic first (it invalidates trust), then the gate state, test and winner-rule trace. Estimates always render; the gates only decide the badge. */} @@ -111,9 +125,9 @@ function ResultsPanel({ experimentId }: { experimentId: string }) { {v.exposures.toLocaleString()} exposures {v.conversions.toLocaleString()} conversions - {(v.conversion_rate * 100).toFixed(2)}% + {formatArmValue(kind, v)} - {v.wilson_low !== undefined && v.wilson_high !== undefined && ( + {!continuous && v.wilson_low !== undefined && v.wilson_high !== undefined && ( [{pct(v.wilson_low)} - {pct(v.wilson_high)}] @@ -158,10 +172,10 @@ function ResultsPanel({ experimentId }: { experimentId: string }) { {an.pairwise_vs_control.map((pw) => (
{pw.variant} - lift {pw.lift_absolute >= 0 ? "+" : ""}{pct(pw.lift_absolute)} - 95% CI [{pct(pw.ci_low)} - {pct(pw.ci_high)}] + lift {pw.lift_absolute >= 0 ? "+" : ""}{continuous ? num(pw.lift_absolute) : pct(pw.lift_absolute)} + 95% CI [{continuous ? num(pw.ci_low) : pct(pw.ci_low)} - {continuous ? num(pw.ci_high) : pct(pw.ci_high)}] - p={pStr(pw.holm_adjusted_p)} (Holm{pw.used_fisher ? ", Fisher" : ""}) + p={pStr(pw.holm_adjusted_p)} (Holm{pw.used_fisher ? ", Fisher" : ""}{continuous ? ", Welch t" : ""}) {pw.significant && ( significant @@ -170,6 +184,29 @@ function ResultsPanel({ experimentId }: { experimentId: string }) { ))}
)} + {/* Secondary metrics: exploratory, never gate the winner. */} + {results.secondary && results.secondary.length > 0 && ( +
+
+ Secondary metrics - {results.multiple_comparison_note} +
+ {results.secondary.map((m) => ( +
+
{secondaryLabel(m)} - {metricKindLabel(m.kind)}
+ {m.variants.map((v) => ( + + {v.variant}: {formatArmValue(m.kind, v)} ({v.exposures.toLocaleString()} exposed) + + ))} + {m.analysis.pairwise_vs_control?.map((pw) => ( +
+ {pw.variant} vs control: p={pStr(pw.holm_adjusted_p)} (Holm){pw.significant ? " - crossed 0.05, exploratory" : ""} +
+ ))} +
+ ))} +
+ )} ); } diff --git a/ui/src/utils/experimentMetrics.test.ts b/ui/src/utils/experimentMetrics.test.ts new file mode 100644 index 00000000..d3622da9 --- /dev/null +++ b/ui/src/utils/experimentMetrics.test.ts @@ -0,0 +1,22 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { formatArmValue, metricKindLabel, secondaryLabel } from "./experimentMetrics.ts"; + +test("binary shows a percentage", () => { + assert.equal(formatArmValue("binary", { conversion_rate: 0.1234 }), "12.34%"); + assert.equal(formatArmValue(undefined, { conversion_rate: 0.5 }), "50.00%"); +}); + +test("continuous shows the mean with magnitude-aware precision", () => { + assert.equal(formatArmValue("mean", { conversion_rate: 0.6, mean: 12.3456 }), "12.35"); + assert.equal(formatArmValue("mean", { conversion_rate: 0.6, mean: 1234.5 }), "1235"); + assert.equal(formatArmValue("count", { conversion_rate: 0.6, mean: 0.0456 }), "0.046"); + assert.equal(formatArmValue("mean", { conversion_rate: 0.6 }), "--"); +}); + +test("labels", () => { + assert.equal(metricKindLabel("mean"), "mean per user"); + assert.equal(metricKindLabel(undefined), "conversion rate"); + assert.equal(secondaryLabel({ key: "rev", name: "Revenue" }), "Revenue (secondary, exploratory)"); + assert.equal(secondaryLabel({ key: "rev" }), "rev (secondary, exploratory)"); +}); diff --git a/ui/src/utils/experimentMetrics.ts b/ui/src/utils/experimentMetrics.ts new file mode 100644 index 00000000..e18d5013 --- /dev/null +++ b/ui/src/utils/experimentMetrics.ts @@ -0,0 +1,30 @@ +// Pure display helpers for the experiment results panel. + +export type MetricKindLike = "binary" | "count" | "mean" | undefined; + +/** Formats an arm's headline value: a rate for binary goals, a mean otherwise. */ +export function formatArmValue( + kind: MetricKindLike, + v: { conversion_rate: number; mean?: number }, +): string { + if (kind === "count" || kind === "mean") { + if (v.mean === undefined || !Number.isFinite(v.mean)) return "--"; + const abs = Math.abs(v.mean); + return abs >= 100 ? v.mean.toFixed(0) : abs >= 1 ? v.mean.toFixed(2) : v.mean.toFixed(3); + } + return `${(v.conversion_rate * 100).toFixed(2)}%`; +} + +/** Short human label for a metric kind. */ +export function metricKindLabel(kind: MetricKindLike): string { + switch (kind) { + case "count": return "count per user"; + case "mean": return "mean per user"; + default: return "conversion rate"; + } +} + +/** Heading for a secondary metric row: always carries the exploratory label. */ +export function secondaryLabel(m: { key: string; name?: string }): string { + return `${m.name || m.key} (secondary, exploratory)`; +} From 8782fe5cd7d5107b225354ba928bb73483ac4186 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 03:05:23 +0000 Subject: [PATCH 34/65] feat: cohort rule trees (and/or/not), static cohorts, cohort_id on funnels and retention - Rule tree {op: and|or|not, children|leaf}, depth<=4, <=30 leaves, validated on write (422); legacy flat-AND JSON stays readable. Evaluated as Go-side set algebra over per-leaf bounded results. - Static cohorts backed by new cohort_members table (migration 060): bulk create (cap 100k, chunked inserts), add/remove members, CSV import, bounded reads. - Optional cohort_id on funnel, funnel breakdown and retention using the stats routes' resolver: 404 foreign/missing, 422 over MaxFilterMembers, 503 fail closed; resolution runs inside the query admission slot. - UI: JSON-with-validation editor and static-list import (UNBUILT). UNVERIFIED against live Nucleus (integration tests skipped in this environment). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Rpvf5mrWnTXrcooQQyQCD3 --- cmd/observe/cohorts_handlers.go | 19 +- cmd/observe/cohorts_static_handlers.go | 227 +++++++++ cmd/observe/cohorts_static_handlers_test.go | 111 +++++ docs/cohorts.md | 76 +++ internal/cohorts/cohorts.go | 196 +++++--- internal/cohorts/depth_nucleus_test.go | 91 ++++ internal/cohorts/static.go | 432 ++++++++++++++++++ internal/cohorts/static_test.go | 174 +++++++ internal/cohorts/tree.go | 346 ++++++++++++++ internal/cohorts/tree_test.go | 319 +++++++++++++ internal/query/api.go | 31 +- internal/query/cohort_depth_test.go | 191 ++++++++ internal/query/cohort_filter.go | 76 +++ internal/query/funnel.go | 22 + internal/query/retention.go | 21 +- .../migrations/060_cohort_members.up.sql | 40 ++ ui/src/api/persons.ts | 29 +- ui/src/lib/cohortRule.test.ts | 83 ++++ ui/src/lib/cohortRule.ts | 138 ++++++ ui/src/routes/cohorts.tsx | 238 +++++++++- 20 files changed, 2751 insertions(+), 109 deletions(-) create mode 100644 cmd/observe/cohorts_static_handlers.go create mode 100644 cmd/observe/cohorts_static_handlers_test.go create mode 100644 docs/cohorts.md create mode 100644 internal/cohorts/depth_nucleus_test.go create mode 100644 internal/cohorts/static.go create mode 100644 internal/cohorts/static_test.go create mode 100644 internal/cohorts/tree.go create mode 100644 internal/cohorts/tree_test.go create mode 100644 internal/query/cohort_depth_test.go create mode 100644 internal/query/cohort_filter.go create mode 100644 internal/schema/migrations/060_cohort_members.up.sql create mode 100644 ui/src/lib/cohortRule.test.ts create mode 100644 ui/src/lib/cohortRule.ts diff --git a/cmd/observe/cohorts_handlers.go b/cmd/observe/cohorts_handlers.go index 5f7e5da5..1af4977b 100644 --- a/cmd/observe/cohorts_handlers.go +++ b/cmd/observe/cohorts_handlers.go @@ -59,6 +59,7 @@ func RegisterCohortsRoutes( neutron.WithTags("cohorts"), neutron.WithSummary("Re-evaluate a cohort and update member_count"), ) + registerStaticCohortRoutes(r, jwtMW, requireEditor, svc) } // --------------------------------------------------------------------------- @@ -135,12 +136,9 @@ func createCohortHandler(svc *cohorts.Service) neutron.HandlerFunc[createCohortI if strings.TrimSpace(in.Name) == "" { return cohorts.Cohort{}, neutron.ErrBadRequest("name required") } - if len(in.Rule.Rules) == 0 { - return cohorts.Cohort{}, neutron.ErrBadRequest("rule.rules must contain at least one condition") - } c, err := svc.Create(ctx, in.SiteID, in.Name, in.Description, in.Rule) if err != nil { - return cohorts.Cohort{}, err + return cohorts.Cohort{}, cohortHTTPError(err) } return *c, nil } @@ -153,7 +151,7 @@ func getCohortHandler(svc *cohorts.Service) neutron.HandlerFunc[cohortIDInput, c } c, err := svc.Get(ctx, in.SiteID, in.CohortID) if err != nil { - return cohorts.Cohort{}, err + return cohorts.Cohort{}, cohortHTTPError(err) } if c == nil { return cohorts.Cohort{}, neutron.ErrNotFound("cohort not found") @@ -176,7 +174,7 @@ func cohortMembersHandler(svc *cohorts.Service) neutron.HandlerFunc[cohortMember } members, err := svc.Members(ctx, in.SiteID, in.CohortID, limit, in.Offset) if err != nil { - return cohortMembersResult{}, err + return cohortMembersResult{}, cohortHTTPError(err) } if members == nil { members = []string{} @@ -192,7 +190,7 @@ func updateCohortHandler(svc *cohorts.Service) neutron.HandlerFunc[updateCohortI } c, err := svc.Update(ctx, in.SiteID, in.CohortID, in.Name, in.Description, in.Rule) if err != nil { - return cohorts.Cohort{}, err + return cohorts.Cohort{}, cohortHTTPError(err) } return *c, nil } @@ -214,7 +212,7 @@ func refreshCohortHandler(svc *cohorts.Service) neutron.HandlerFunc[cohortIDInpu } c, err := svc.Refresh(ctx, in.SiteID, in.CohortID) if err != nil { - return cohorts.Cohort{}, err + return cohorts.Cohort{}, cohortHTTPError(err) } return *c, nil } @@ -225,9 +223,12 @@ func previewCohortHandler(svc *cohorts.Service) neutron.HandlerFunc[previewCohor if in.SiteID == "" { return previewResult{}, neutron.ErrBadRequest("site_id required") } + if err := cohorts.ValidateDefinition(in.Rule); err != nil { + return previewResult{}, cohortHTTPError(err) + } ids, err := svc.EvaluateCohort(ctx, in.SiteID, in.Rule) if err != nil { - return previewResult{}, neutron.ErrBadRequest(err.Error()) + return previewResult{}, cohortHTTPError(err) } // Cap the sample at 10 — the UI just needs a glance, not the // whole list. Members API serves the full paginated set. diff --git a/cmd/observe/cohorts_static_handlers.go b/cmd/observe/cohorts_static_handlers.go new file mode 100644 index 00000000..9a45a9d9 --- /dev/null +++ b/cmd/observe/cohorts_static_handlers.go @@ -0,0 +1,227 @@ +package main + +// Static cohort routes (C2 depth) and the cohort error mapping shared with +// cohorts_handlers.go. Static cohorts are lists of up to 100k ids, so their +// writes are plain http handlers with a bounded body (readJSONBody / +// MaxBytesReader) rather than typed neutron handlers. +// +// POST /api/v1/cohorts/static {site_id,name,description,ids[]} create from ids +// POST /api/v1/cohorts/import?site_id&name text/csv body create from CSV +// POST /api/v1/cohorts/{id}/members {site_id,ids[]} add members +// POST /api/v1/cohorts/{id}/members/remove {site_id,ids[]} remove members +// POST /api/v1/cohorts/{id}/import?site_id text/csv body add members from CSV +// +// All are editor-gated. Every call resolves the cohort through the caller's +// site_id first, so another site's cohort id answers 404. + +import ( + "encoding/json" + "errors" + "log/slog" + "net/http" + "strings" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/cohorts" + "github.com/useteploy/teploy-observe/internal/guardmap" +) + +// staticBodyLimit bounds a static-cohort request body: 100k ids at the +// 256-byte id cap would be 25 MB, but real ids are far shorter; 8 MB covers +// 100k ids of ~80 bytes and anything larger is refused with 413. +const staticBodyLimit = 8 << 20 + +func registerStaticCohortRoutes(r *neutron.Router, jwtMW, requireEditor neutron.Middleware, svc *cohorts.Service) { + guard := func(h http.HandlerFunc) http.Handler { return jwtMW(requireEditor(h)) } + r.Handle("POST /api/v1/cohorts/static", guard(createStaticCohortHandler(svc))) + r.Handle("POST /api/v1/cohorts/import", guard(createStaticFromCSVHandler(svc))) + r.Handle("POST /api/v1/cohorts/{cohort_id}/members", guard(addCohortMembersHandler(svc))) + r.Handle("POST /api/v1/cohorts/{cohort_id}/members/remove", guard(removeCohortMembersHandler(svc))) + r.Handle("POST /api/v1/cohorts/{cohort_id}/import", guard(importCohortMembersHandler(svc))) +} + +// cohortErrorStatus maps a cohort service error to an HTTP status and a +// client-safe message. Anything unrecognised is a 503 with a generic message +// (the raw error is logged by the caller, never echoed). +func cohortErrorStatus(err error) (int, string) { + switch { + case errors.Is(err, cohorts.ErrNotFound): + return http.StatusNotFound, "cohort not found" + case errors.Is(err, cohorts.ErrInvalidDefinition), + errors.Is(err, cohorts.ErrTooLarge), + errors.Is(err, cohorts.ErrTooManyIDs), + errors.Is(err, cohorts.ErrNotStatic): + return http.StatusUnprocessableEntity, err.Error() + } + if mapped := guardmap.HTTPError(err); mapped != err { + var app *neutron.AppError + if errors.As(mapped, &app) { + return app.Status, app.Detail + } + } + return http.StatusServiceUnavailable, "cohort operation failed; retry later" +} + +// cohortHTTPError is cohortErrorStatus for the typed handlers. +func cohortHTTPError(err error) error { + if mapped := guardmap.HTTPError(err); mapped != err { + return mapped // query-admission refusal keeps its own status + } + status, msg := cohortErrorStatus(err) + switch status { + case http.StatusNotFound: + return neutron.ErrNotFound(msg) + case http.StatusUnprocessableEntity: + return neutron.ErrValidation(msg, []neutron.ValidationError{{Field: "rule", Message: msg}}) + } + slog.Error("cohort operation failed", "err", err) + return neutron.ErrServiceUnavailable(msg) +} + +func writeCohortError(w http.ResponseWriter, err error) { + status, msg := cohortErrorStatus(err) + if status == http.StatusServiceUnavailable { + slog.Error("cohort operation failed", "err", err) + } + writeJSONError(w, status, msg) +} + +type staticCohortBody struct { + SiteID string `json:"site_id"` + Name string `json:"name"` + Description string `json:"description"` + IDs []string `json:"ids"` +} + +type memberEditBody struct { + SiteID string `json:"site_id"` + IDs []string `json:"ids"` +} + +type memberEditResult struct { + Cohort cohorts.Cohort `json:"cohort"` + Changed int `json:"changed"` +} + +func createStaticCohortHandler(svc *cohorts.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var body staticCohortBody + if !readJSONBody(w, r, staticBodyLimit, &body) { + return + } + if body.SiteID == "" || strings.TrimSpace(body.Name) == "" { + writeJSONError(w, http.StatusBadRequest, "site_id and name required") + return + } + c, err := svc.CreateStatic(r.Context(), body.SiteID, body.Name, body.Description, body.IDs) + if err != nil { + writeCohortError(w, err) + return + } + writeJSONStatus(w, http.StatusCreated, c) + } +} + +// csvBodyIDs reads a bounded CSV request body into ids. It writes the error +// response and returns false on failure. +func csvBodyIDs(w http.ResponseWriter, r *http.Request) ([]string, bool) { + r.Body = http.MaxBytesReader(w, r.Body, staticBodyLimit) + ids, err := cohorts.ParseCSVIDs(r.Body) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeJSONError(w, http.StatusRequestEntityTooLarge, "request body too large") + } else { + writeCohortError(w, err) + } + return nil, false + } + return ids, true +} + +func createStaticFromCSVHandler(svc *cohorts.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + siteID, name := q.Get("site_id"), strings.TrimSpace(q.Get("name")) + if siteID == "" || name == "" { + writeJSONError(w, http.StatusBadRequest, "site_id and name required") + return + } + ids, ok := csvBodyIDs(w, r) + if !ok { + return + } + c, err := svc.CreateStatic(r.Context(), siteID, name, q.Get("description"), ids) + if err != nil { + writeCohortError(w, err) + return + } + writeJSONStatus(w, http.StatusCreated, c) + } +} + +func addCohortMembersHandler(svc *cohorts.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var body memberEditBody + if !readJSONBody(w, r, staticBodyLimit, &body) { + return + } + if body.SiteID == "" { + writeJSONError(w, http.StatusBadRequest, "site_id required") + return + } + c, n, err := svc.AddMembers(r.Context(), body.SiteID, r.PathValue("cohort_id"), body.IDs) + if err != nil { + writeCohortError(w, err) + return + } + writeJSONStatus(w, http.StatusOK, memberEditResult{Cohort: *c, Changed: n}) + } +} + +func removeCohortMembersHandler(svc *cohorts.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var body memberEditBody + if !readJSONBody(w, r, staticBodyLimit, &body) { + return + } + if body.SiteID == "" { + writeJSONError(w, http.StatusBadRequest, "site_id required") + return + } + c, n, err := svc.RemoveMembers(r.Context(), body.SiteID, r.PathValue("cohort_id"), body.IDs) + if err != nil { + writeCohortError(w, err) + return + } + writeJSONStatus(w, http.StatusOK, memberEditResult{Cohort: *c, Changed: n}) + } +} + +func importCohortMembersHandler(svc *cohorts.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + siteID := r.URL.Query().Get("site_id") + if siteID == "" { + writeJSONError(w, http.StatusBadRequest, "site_id required") + return + } + ids, ok := csvBodyIDs(w, r) + if !ok { + return + } + c, n, err := svc.AddMembers(r.Context(), siteID, r.PathValue("cohort_id"), ids) + if err != nil { + writeCohortError(w, err) + return + } + writeJSONStatus(w, http.StatusOK, memberEditResult{Cohort: *c, Changed: n}) + } +} + +func writeJSONStatus(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/cmd/observe/cohorts_static_handlers_test.go b/cmd/observe/cohorts_static_handlers_test.go new file mode 100644 index 00000000..f8689b73 --- /dev/null +++ b/cmd/observe/cohorts_static_handlers_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/neutron-build/neutron/go/neutron" + + "github.com/useteploy/teploy-observe/internal/cohorts" + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +func TestCohortErrorStatus(t *testing.T) { + cases := []struct { + err error + status int + }{ + {cohorts.ErrNotFound, http.StatusNotFound}, + {fmt.Errorf("x: %w", cohorts.ErrNotFound), http.StatusNotFound}, + {fmt.Errorf("%w: bad", cohorts.ErrInvalidDefinition), http.StatusUnprocessableEntity}, + {fmt.Errorf("%w: big", cohorts.ErrTooLarge), http.StatusUnprocessableEntity}, + {fmt.Errorf("%w: many", cohorts.ErrTooManyIDs), http.StatusUnprocessableEntity}, + {fmt.Errorf("%w: rule", cohorts.ErrNotStatic), http.StatusUnprocessableEntity}, + {errors.New("pq: connection refused dsn=secret"), http.StatusServiceUnavailable}, + } + for _, tc := range cases { + status, msg := cohortErrorStatus(tc.err) + if status != tc.status { + t.Errorf("%v: status %d want %d", tc.err, status, tc.status) + } + if status == http.StatusServiceUnavailable && strings.Contains(msg, "secret") { + t.Errorf("raw error leaked: %q", msg) + } + } + // Typed handlers surface the same statuses. + var app *neutron.AppError + if !errors.As(cohortHTTPError(cohorts.ErrNotFound), &app) || app.Status != 404 { + t.Errorf("typed not found: %v", app) + } + if !errors.As(cohortHTTPError(fmt.Errorf("%w: x", cohorts.ErrInvalidDefinition)), &app) || app.Status != 422 { + t.Errorf("typed invalid: %v", app) + } + if !errors.As(cohortHTTPError(errors.New("boom")), &app) || app.Status != 503 { + t.Errorf("typed other: %v", app) + } + // A query-admission refusal keeps its own status. + ref := queryguard.NewLimiter(1, 1) + rel, _ := ref.Acquire(context.Background(), "s") + defer rel() + _, err := ref.Acquire(context.Background(), "s") + if !errors.As(cohortHTTPError(err), &app) || app.Status != 429 { + t.Errorf("refusal: %v", app) + } +} + +// do runs a handler built with a nil service: every case below must be +// decided before the service is touched. +func do(h http.HandlerFunc, method, target, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, target, strings.NewReader(body)) + req.SetPathValue("cohort_id", "c1") + rec := httptest.NewRecorder() + h(rec, req) + return rec +} + +func TestStaticHandlersRejectBeforeService(t *testing.T) { + var svc *cohorts.Service + cases := []struct { + name string + h http.HandlerFunc + target string + body string + status int + }{ + {"create: invalid json", createStaticCohortHandler(svc), "/", "{", 400}, + {"create: unknown field", createStaticCohortHandler(svc), "/", `{"site_id":"s","name":"n","ids":[],"evil":1}`, 400}, + {"create: no site", createStaticCohortHandler(svc), "/", `{"name":"n","ids":["a"]}`, 400}, + {"create: no name", createStaticCohortHandler(svc), "/", `{"site_id":"s","ids":["a"]}`, 400}, + {"create: trailing doc", createStaticCohortHandler(svc), "/", `{"site_id":"s","name":"n"}{}`, 400}, + {"create: body too large", createStaticCohortHandler(svc), "/", `{"site_id":"s","name":"n","ids":["` + strings.Repeat("a", staticBodyLimit) + `"]}`, 413}, + {"add: no site", addCohortMembersHandler(svc), "/", `{"ids":["a"]}`, 400}, + {"remove: no site", removeCohortMembersHandler(svc), "/", `{"ids":["a"]}`, 400}, + {"import: no site", importCohortMembersHandler(svc), "/", "a\nb\n", 400}, + {"csv-create: no name", createStaticFromCSVHandler(svc), "/?site_id=s", "a\n", 400}, + {"csv-create: no site", createStaticFromCSVHandler(svc), "/?name=n", "a\n", 400}, + {"import: id too long", importCohortMembersHandler(svc), "/?site_id=s", strings.Repeat("x", 300) + "\n", 422}, + {"import: body too large", importCohortMembersHandler(svc), "/?site_id=s", strings.Repeat("a\n", staticBodyLimit/2+1), 413}, + } + for _, tc := range cases { + rec := do(tc.h, "POST", tc.target, tc.body) + if rec.Code != tc.status { + t.Errorf("%s: status %d want %d (%s)", tc.name, rec.Code, tc.status, strings.TrimSpace(rec.Body.String())) + } + } +} + +func TestStaticImportOverCapIs422(t *testing.T) { + var b strings.Builder + for i := 0; i <= cohorts.MaxStaticMembers; i++ { + fmt.Fprintf(&b, "u%d\n", i) + } + rec := do(importCohortMembersHandler(nil), "POST", "/?site_id=s", b.String()) + if rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } +} diff --git a/docs/cohorts.md b/docs/cohorts.md new file mode 100644 index 00000000..29afe020 --- /dev/null +++ b/docs/cohorts.md @@ -0,0 +1,76 @@ +# Cohorts + +A cohort is a saved set of identified users (`events.distinct_id`). Anonymous +events (`distinct_id = ''`) never belong to a cohort. Three kinds share the +`cohorts` table; the kind is in the stored `rule` JSON. + +## Rule cohorts + +```json +{"op":"and","rules":[{"type":"event","name":"purchase","min_count":2,"window":"30d"}, + {"type":"property","key":"country","operator":"=","value":"US"}]} +``` + +That flat-AND form (v1) is still read and written. A rule can also be a tree: + +```json +{"op":"and","children":[ + {"leaf":{"type":"event","name":"signup"}}, + {"op":"not","children":[{"leaf":{"type":"property","key":"country","value":"US"}}]}, + {"op":"or","children":[{"leaf":{"type":"event","name":"purchase"}}, + {"leaf":{"type":"event","name":"trial_start"}}]}]} +``` + +- `op` is `and`, `or` (one or more children) or `not` (exactly one child). + A node is a group (`op` + `children`) or a leaf (`leaf` only). `rules` (flat) + and `children` cannot be mixed in one node. +- Limits, enforced on write with 422: depth at most 4 (the root group is level + 1, so a flat rule is level 2), at most 30 leaves. Leaves are validated: + event name required, property key from the allow-list, operator `=` or `!=`, + window like `24h` / `7d` / `90m`. +- `!=` on a property means "no event of the user has this value". + `not` is the complement against the site's identified users. +- Evaluation: each leaf is one bounded query (at most 200,000 rows; over that + the evaluation fails with 422, never a truncated answer); the tree is + combined in Go with set intersection, union and difference. No SQL + subqueries are involved. + +## Static cohorts + +An explicit list of ids in `cohort_members` (migration 060); the cohort's rule +is `{"op":"static"}`. At most 100,000 distinct ids per cohort and per request +(ids are trimmed, de-duplicated, at most 256 bytes, no control characters). +Request bodies are capped at 8 MB. + +| Route | Body | +|---|---| +| `POST /api/v1/cohorts/static` | `{site_id,name,description,ids[]}` create | +| `POST /api/v1/cohorts/import?site_id=&name=` | CSV body, create | +| `POST /api/v1/cohorts/{id}/members` | `{site_id,ids[]}` add | +| `POST /api/v1/cohorts/{id}/members/remove` | `{site_id,ids[]}` remove | +| `POST /api/v1/cohorts/{id}/import?site_id=` | CSV body, add | + +CSV: the first column is the id; a leading header row (`id`, `entity_id`, +`distinct_id`, `user_id`) and blank rows are skipped. Rule cohorts reject member +edits (422); a static cohort rejects a rule. + +## Using a cohort as a filter + +`cohort_id` is accepted by the stats routes (SQL `distinct_id IN (...)`) and, +since C2 depth, by `POST /stats/funnel`, `POST /stats/funnel/breakdown` (JSON +field) and `GET /stats/retention` (query param). The funnel and retention paths +stream raw events and test each event's `distinct_id` against the resolved +membership, so only identified activity counts and sessions with no identified +event drop out. + +Failure behaviour is the same everywhere and never falls back to unfiltered: + +| Condition | Status | +|---|---| +| cohort missing for this site, or belongs to another site | 404 | +| more than 30,000 members (`cohorts.MaxFilterMembers`) | 422 | +| query admission refused (concurrency / time budget) | 429 / 504 | +| store failure, unreadable stored rule | 503 | + +Cohort resolution runs inside the query admission slot (`beginQuery`) of the +funnel / retention request, and takes its own slot on the stats routes. diff --git a/internal/cohorts/cohorts.go b/internal/cohorts/cohorts.go index ba984226..ca40493e 100644 --- a/internal/cohorts/cohorts.go +++ b/internal/cohorts/cohorts.go @@ -48,6 +48,10 @@ import ( // for the fixed site/time and dimension-filter parameters. const MaxFilterMembers = 30000 +// ErrNotFound is returned when a cohort does not exist for the site asked +// about (including a cohort id that belongs to a different site). +var ErrNotFound = errors.New("cohort not found") + // ErrTooLarge is returned (wrapped) by MembersForFilter when a cohort has // more than MaxFilterMembers members. var ErrTooLarge = errors.New("cohort too large for chart filtering") @@ -84,13 +88,19 @@ type Rule struct { Value string `json:"value,omitempty"` // type=property } -// Definition is the full saved cohort rule. Op is "and" for v1; "or" -// + nesting are deferred to a phase-2 builder. +// Definition is the full saved cohort rule. Three shapes share it (see +// tree.go): legacy flat ({op, rules}), a tree ({op, children} with {leaf} +// leaves) and static ({op:"static"}, membership in cohort_members). type Definition struct { - Op string `json:"op"` - Rules []Rule `json:"rules"` + Op string `json:"op"` + Rules []Rule `json:"rules,omitempty"` + Children []Definition `json:"children,omitempty"` + Leaf *Rule `json:"leaf,omitempty"` } +// IsStatic reports whether the definition is a static (list-backed) cohort. +func (d Definition) IsStatic() bool { return d.Op == OpStatic } + // Cohort is a saved cohort row. type Cohort struct { CohortID string `json:"cohort_id" db:"cohort_id"` @@ -115,7 +125,7 @@ func ParseDefinition(rule string) (Definition, error) { if err := json.Unmarshal([]byte(rule), &def); err != nil { return Definition{}, fmt.Errorf("invalid cohort rule json: %w", err) } - if def.Op == "" { + if def.Op == "" && def.Leaf == nil { def.Op = "and" } return def, nil @@ -129,58 +139,75 @@ func (s *Service) EvaluateCohort(ctx context.Context, siteID string, def Definit if siteID == "" { return []string{}, nil } - if def.Op == "" { - def.Op = "and" - } - if def.Op != "and" { - // Defer or / nested to phase 2 explicitly so the API is - // predictable. The UI never offers anything but AND for v1. - return nil, fmt.Errorf("cohort op %q not supported (v1 supports AND only)", def.Op) + if def.IsStatic() { + return nil, invalidf("a static cohort has no rule to evaluate") } - if len(def.Rules) == 0 { + if len(def.Rules) == 0 && len(def.Children) == 0 && def.Leaf == nil { return []string{}, nil } + if err := validateShape(def); err != nil { + return nil, err + } - // Evaluate each rule to a set of distinct_ids, then AND-intersect. - // Doing this per-rule (vs one big SQL with N joins) keeps each - // query simple — Nucleus's optimizer is young (finding #15 family) - // and per-rule queries are easier to reason about. - var acc map[string]struct{} - for i, r := range def.Rules { - ids, err := s.evalRule(ctx, siteID, r) - if err != nil { - return nil, fmt.Errorf("rule %d (%s): %w", i, r.Type, err) - } - set := make(map[string]struct{}, len(ids)) - for _, id := range ids { - if id == "" { - continue + // The rule is a tree (tree.go): every leaf runs one bounded query and + // the tree is combined with Go-side set algebra. Per-leaf queries (vs + // one big SQL with N joins) keep each query simple - Nucleus's + // optimizer is young (finding #15 family) and they are easier to + // reason about. + var universe idSet + env := treeEnv{ + leaf: func(ctx context.Context, r Rule) (idSet, error) { + ids, err := s.evalRule(ctx, siteID, r) + if err != nil { + return nil, fmt.Errorf("rule (%s): %w", r.Type, err) } - set[id] = struct{}{} - } - if i == 0 { - acc = set - continue - } - // AND-intersect. - for id := range acc { - if _, ok := set[id]; !ok { - delete(acc, id) + return toSet(ids), nil + }, + universe: func(ctx context.Context) (idSet, error) { + if universe != nil { + return universe, nil } - } - if len(acc) == 0 { - return []string{}, nil - } + ids, err := s.loadUniverse(ctx, siteID) + if err != nil { + return nil, err + } + universe = toSet(ids) + return universe, nil + }, } + set, err := evalNode(ctx, normalize(def), env) + if err != nil { + return nil, err + } + return set.sorted(), nil +} - out := make([]string, 0, len(acc)) - for id := range acc { - out = append(out, id) +// loadUniverse returns every identified user of the site, bounded. It is +// the complement base for "not" nodes and for the "!=" property operator. +func (s *Service) loadUniverse(ctx context.Context, siteID string) ([]string, error) { + type idRow struct { + DistinctID string `db:"distinct_id"` + } + rows, err := nucleus.Query[idRow](ctx, s.db.SQL(), propertyUniverseSQL()+leafLimitSQL(), siteID) + if err != nil { + return nil, fmt.Errorf("cohort universe query: %w", err) + } + if len(rows) > MaxLeafRows { + return nil, fmt.Errorf("%w: more than %d identified users", ErrTooLarge, MaxLeafRows) + } + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.DistinctID) } - sort.Strings(out) return out, nil } +// leafLimitSQL bounds a leaf query one row past MaxLeafRows so an overrun +// is detectable (and refused) instead of silently truncated. +func leafLimitSQL() string { + return " LIMIT " + strconv.Itoa(MaxLeafRows+1) +} + func (s *Service) evalRule(ctx context.Context, siteID string, r Rule) ([]string, error) { switch r.Type { case "event": @@ -225,11 +252,14 @@ func (s *Service) evalEventRule(ctx context.Context, siteID string, r Rule) ([]s AND event_type = $2 AND timestamp >= $3 AND distinct_id != '' - GROUP BY distinct_id` + GROUP BY distinct_id` + leafLimitSQL() rows, err := nucleus.Query[countedRow](ctx, s.db.SQL(), q, siteID, r.Name, from) if err != nil { return nil, fmt.Errorf("event rule query: %w", err) } + if len(rows) > MaxLeafRows { + return nil, fmt.Errorf("%w: event rule matched more than %d users", ErrTooLarge, MaxLeafRows) + } out := make([]string, 0, len(rows)) for _, row := range rows { if row.Count >= int64(min) { @@ -261,10 +291,13 @@ func (s *Service) evalPropertyRule(ctx context.Context, siteID string, r Rule) ( type idRow struct { DistinctID string `db:"distinct_id"` } - matched, err := nucleus.Query[idRow](ctx, s.db.SQL(), propertyMatchSQL(r.Key), siteID, r.Value) + matched, err := nucleus.Query[idRow](ctx, s.db.SQL(), propertyMatchSQL(r.Key)+leafLimitSQL(), siteID, r.Value) if err != nil { return nil, fmt.Errorf("property rule query: %w", err) } + if len(matched) > MaxLeafRows { + return nil, fmt.Errorf("%w: property rule matched more than %d users", ErrTooLarge, MaxLeafRows) + } matchedIDs := make([]string, 0, len(matched)) for _, row := range matched { matchedIDs = append(matchedIDs, row.DistinctID) @@ -281,13 +314,9 @@ func (s *Service) evalPropertyRule(ctx context.Context, siteID string, r Rule) ( // plain DISTINCT scans and a Go-side subtraction keep the SQL to forms // the engine is known to handle (no NOT IN subquery). Events whose // column is NULL/empty never match "=", so their users are kept. - all, err := nucleus.Query[idRow](ctx, s.db.SQL(), propertyUniverseSQL(), siteID) + allIDs, err := s.loadUniverse(ctx, siteID) if err != nil { - return nil, fmt.Errorf("property rule query: %w", err) - } - allIDs := make([]string, 0, len(all)) - for _, row := range all { - allIDs = append(allIDs, row.DistinctID) + return nil, err } return subtractIDs(allIDs, matchedIDs), nil } @@ -388,6 +417,12 @@ func (s *Service) Create(ctx context.Context, siteID, name, description string, if siteID == "" || name == "" { return nil, fmt.Errorf("site_id and name required") } + if def.IsStatic() { + return nil, invalidf("static cohorts are created from a member list (CreateStatic)") + } + if err := ValidateDefinition(def); err != nil { + return nil, err + } raw, err := json.Marshal(def) if err != nil { return nil, fmt.Errorf("marshal rule: %w", err) @@ -485,18 +520,39 @@ func (s *Service) Update(ctx context.Context, siteID, cohortID, name, descriptio return nil, err } if existing == nil { - return nil, fmt.Errorf("cohort not found") + return nil, ErrNotFound } if name == "" { name = existing.Name } + existingDef, _ := ParseDefinition(existing.Rule) + var count int64 + if existingDef.IsStatic() { + // A static cohort keeps its list; only name/description (and a + // recount) change. A rule cannot be grafted onto it. + if def.Op != "" && !def.IsStatic() || len(def.Rules) != 0 || len(def.Children) != 0 || def.Leaf != nil { + return nil, invalidf("a static cohort cannot take a rule; create a rule cohort instead") + } + def = Definition{Op: OpStatic} + n, err := s.staticCount(ctx, siteID, cohortID) + if err != nil { + return nil, err + } + count = n + } else { + if def.IsStatic() { + return nil, invalidf("a rule cohort cannot become static; create a static cohort instead") + } + if err := ValidateDefinition(def); err != nil { + return nil, err + } + members, _ := s.EvaluateCohort(ctx, siteID, def) + count = int64(len(members)) + } raw, err := json.Marshal(def) if err != nil { return nil, fmt.Errorf("marshal rule: %w", err) } - - members, _ := s.EvaluateCohort(ctx, siteID, def) - count := int64(len(members)) nowMs := time.Now().UTC().UnixMilli() // Force monotonic updated_at so the read-side dedup picks this row // even when wall-clock didn't tick since the previous write. @@ -529,7 +585,7 @@ func (s *Service) Refresh(ctx context.Context, siteID, cohortID string) (*Cohort return nil, err } if existing == nil { - return nil, fmt.Errorf("cohort not found") + return nil, ErrNotFound } def, err := ParseDefinition(existing.Rule) if err != nil { @@ -577,13 +633,9 @@ func (s *Service) Members(ctx context.Context, siteID, cohortID string, limit, o return nil, err } if c == nil { - return []string{}, nil + return nil, ErrNotFound } - def, err := ParseDefinition(c.Rule) - if err != nil { - return nil, err - } - ids, err := s.EvaluateCohort(ctx, siteID, def) + ids, err := s.resolveMembers(ctx, c) if err != nil { return nil, err } @@ -603,22 +655,20 @@ func (s *Service) Members(ctx context.Context, siteID, cohortID string, limit, o } // MembersForFilter returns the full set of distinct_ids for a cohort, -// used by the query pipeline to filter analytics charts. Returns an -// empty slice for an empty / missing cohort so callers can substitute -// an "impossible" filter without special-casing nil. +// used by the query pipeline to filter analytics charts. An empty cohort +// yields an empty slice (callers substitute an "impossible" filter). A +// cohort that does not exist for THIS site - including one that belongs to +// another site - is ErrNotFound, so the API answers 404 rather than +// charting an empty or foreign cohort. func (s *Service) MembersForFilter(ctx context.Context, siteID, cohortID string) ([]string, error) { c, err := s.Get(ctx, siteID, cohortID) if err != nil { return nil, err } if c == nil { - return []string{}, nil - } - def, err := ParseDefinition(c.Rule) - if err != nil { - return nil, err + return nil, ErrNotFound } - ids, err := s.EvaluateCohort(ctx, siteID, def) + ids, err := s.resolveMembers(ctx, c) if err != nil { return nil, err } diff --git a/internal/cohorts/depth_nucleus_test.go b/internal/cohorts/depth_nucleus_test.go new file mode 100644 index 00000000..c2bccb48 --- /dev/null +++ b/internal/cohorts/depth_nucleus_test.go @@ -0,0 +1,91 @@ +package cohorts + +// Nucleus-gated integration for the C2 depth work (OR / NOT trees and +// static cohorts). Self-skips without a live Nucleus like every other +// DB-backed test here; it was written WITHOUT access to one and has never +// been run. Expected values are hand-written. + +import ( + "errors" + "reflect" + "testing" + "time" +) + +func TestDepth_TreeOrNot(t *testing.T) { + ctx, db, done := connect(t) + defer done() + svc := NewService(db) + + site := "test_cohort_tree_" + cohortsToken() + ts := time.Now().UTC().UnixMilli() - 86_400_000 + plantEvent(t, ctx, db, site, "uUS", "s1", "pageview", "US", "Chrome", "/", ts) + plantEvent(t, ctx, db, site, "uDE", "s2", "pageview", "DE", "Chrome", "/", ts+1) + plantEvent(t, ctx, db, site, "uFR", "s3", "pageview", "FR", "Safari", "/", ts+2) + plantEvent(t, ctx, db, site, "uDE", "s2", "purchase", "DE", "Chrome", "/", ts+3) + + us := Rule{Type: "property", Key: "country", Value: "US"} + buy := Rule{Type: "event", Name: "purchase"} + got, err := svc.EvaluateCohort(ctx, site, Definition{Op: "or", Children: []Definition{{Leaf: &us}, {Leaf: &buy}}}) + if err != nil || !reflect.DeepEqual(got, []string{"uDE", "uUS"}) { + t.Fatalf("or: %v %v", got, err) + } + got, err = svc.EvaluateCohort(ctx, site, Definition{Op: "not", Children: []Definition{{Op: "or", Children: []Definition{{Leaf: &us}, {Leaf: &buy}}}}}) + if err != nil || !reflect.DeepEqual(got, []string{"uFR"}) { + t.Fatalf("not(or): %v %v", got, err) + } +} + +func TestDepth_StaticLifecycleAndIDOR(t *testing.T) { + ctx, db, done := connect(t) + defer done() + svc := NewService(db) + + siteA := "test_cohort_static_a_" + cohortsToken() + siteB := "test_cohort_static_b_" + cohortsToken() + + c, err := svc.CreateStatic(ctx, siteA, "vip", "", []string{"u1", "u2", "u2", " u3 "}) + if err != nil || c.MemberCount != 3 { + t.Fatalf("create: %+v %v", c, err) + } + ids, err := svc.MembersForFilter(ctx, siteA, c.CohortID) + if err != nil || !reflect.DeepEqual(ids, []string{"u1", "u2", "u3"}) { + t.Fatalf("members: %v %v", ids, err) + } + if _, n, err := svc.AddMembers(ctx, siteA, c.CohortID, []string{"u3", "u4"}); err != nil || n != 1 { + t.Fatalf("add: %d %v", n, err) + } + if c2, n, err := svc.RemoveMembers(ctx, siteA, c.CohortID, []string{"u1", "nope"}); err != nil || n != 1 || c2.MemberCount != 3 { + t.Fatalf("remove: %+v %d %v", c2, n, err) + } + if ids, _ := svc.MembersForFilter(ctx, siteA, c.CohortID); !reflect.DeepEqual(ids, []string{"u2", "u3", "u4"}) { + t.Fatalf("after edits: %v", ids) + } + // Re-adding a removed member works (higher version than the tombstone). + if _, n, err := svc.AddMembers(ctx, siteA, c.CohortID, []string{"u1"}); err != nil || n != 1 { + t.Fatalf("re-add: %d %v", n, err) + } + + // IDOR: site B cannot read or edit site A's cohort. + if _, err := svc.MembersForFilter(ctx, siteB, c.CohortID); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign read: %v", err) + } + if _, _, err := svc.AddMembers(ctx, siteB, c.CohortID, []string{"evil"}); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign add: %v", err) + } + if _, _, err := svc.RemoveMembers(ctx, siteB, c.CohortID, []string{"u1"}); !errors.Is(err, ErrNotFound) { + t.Fatalf("foreign remove: %v", err) + } + if ids, _ := svc.MembersForFilter(ctx, siteA, c.CohortID); !reflect.DeepEqual(ids, []string{"u1", "u2", "u3", "u4"}) { + t.Fatalf("site A list changed by site B: %v", ids) + } + + // Member edits on a rule cohort are refused. + rc, err := svc.Create(ctx, siteA, "rule", "", Definition{Op: "and", Rules: []Rule{{Type: "event", Name: "x"}}}) + if err != nil { + t.Fatal(err) + } + if _, _, err := svc.AddMembers(ctx, siteA, rc.CohortID, []string{"u9"}); !errors.Is(err, ErrNotStatic) { + t.Fatalf("add to rule cohort: %v", err) + } +} diff --git a/internal/cohorts/static.go b/internal/cohorts/static.go new file mode 100644 index 00000000..33b31c8b --- /dev/null +++ b/internal/cohorts/static.go @@ -0,0 +1,432 @@ +package cohorts + +// Static cohorts (C2 depth): an explicit list of entity ids stored in +// cohort_members (migration 060) instead of a rule. The cohorts row carries +// the rule {"op":"static"} and a maintained member_count. +// +// Storage shape: cohort_members is a ReplacingMergeTree keyed on +// (tenant, site, cohort, entity) and versioned. Add writes removed=0 rows, +// remove writes removed=1 tombstones at a higher version; reads collapse to +// the highest version per entity in Go (finding #10) over a row-bounded +// SELECT. Versions are taken from max(now, cohort.updated_at+1) and the +// cohort row is rewritten at the same value, so each mutation batch is +// strictly newer than the last one on that cohort. +// +// Everything is scoped by (site_id, cohort_id): the cohort is first resolved +// through Get(siteID, cohortID), so another site's cohort id is +// ErrNotFound, and member rows are only ever read/written under the +// caller's site_id. + +import ( + "context" + "encoding/csv" + "errors" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" + + "github.com/neutron-build/neutron/go/nucleus" + + "github.com/useteploy/teploy-observe/internal/dbutil" +) + +const ( + // MaxStaticMembers is the most members one static cohort may hold, and + // the most distinct ids one bulk create / import / add call may carry. + MaxStaticMembers = 100000 + // MaxEntityIDLen is the longest accepted entity id, in bytes. + MaxEntityIDLen = 256 + // staticInsertChunk is rows per multi-row INSERT: 1000 x 7 columns = + // 7000 parameters, far under the 65535 wire cap. + staticInsertChunk = 1000 + staticInsertCols = 7 + // maxStaticRows bounds the raw rows (all versions, tombstones + // included) one read pulls. Past it the read fails (ErrTooLarge) + // rather than truncating membership. + maxStaticRows = 600000 +) + +// ErrTooManyIDs is returned (wrapped) when a request carries, or would +// leave a static cohort with, more than MaxStaticMembers distinct ids. +var ErrTooManyIDs = errors.New("too many member ids") + +// ErrNotStatic is returned (wrapped) when a member edit targets a rule +// cohort. +var ErrNotStatic = errors.New("cohort is not static") + +// NormalizeIDs trims, drops blanks, validates and de-duplicates a list of +// entity ids, preserving first-seen order. It refuses an id longer than +// MaxEntityIDLen or containing control characters, and refuses to hold more +// than MaxStaticMembers distinct ids (ErrTooManyIDs) - checked while +// building so an oversized list is never fully materialised. +func NormalizeIDs(ids []string) ([]string, error) { + seen := make(map[string]struct{}, len(ids)) + out := make([]string, 0, len(ids)) + for _, raw := range ids { + id := strings.TrimSpace(raw) + if id == "" { + continue + } + if len(id) > MaxEntityIDLen { + return nil, invalidf("member id longer than %d bytes", MaxEntityIDLen) + } + for i := 0; i < len(id); i++ { + if id[i] < 0x20 || id[i] == 0x7f { + return nil, invalidf("member id contains a control character") + } + } + if _, dup := seen[id]; dup { + continue + } + if len(out) >= MaxStaticMembers { + return nil, fmt.Errorf("%w: more than %d distinct ids", ErrTooManyIDs, MaxStaticMembers) + } + seen[id] = struct{}{} + out = append(out, id) + } + return out, nil +} + +// bomPrefix is the UTF-8 byte order mark some spreadsheet exports prepend. +const bomPrefix = "\xef\xbb\xbf" + +// csvHeaderNames are first-cell values treated as a header row, not an id. +var csvHeaderNames = map[string]struct{}{ + "id": {}, "ids": {}, "entity_id": {}, "distinct_id": {}, "user_id": {}, "person": {}, +} + +// ParseCSVIDs reads entity ids from a CSV body: the FIRST column of each +// record, one id per row, an optional header row (id / entity_id / +// distinct_id / user_id) skipped, blank rows ignored. It stops with +// ErrTooManyIDs as soon as the distinct count passes MaxStaticMembers. +func ParseCSVIDs(r io.Reader) ([]string, error) { + cr := csv.NewReader(r) + cr.FieldsPerRecord = -1 + cr.LazyQuotes = true + cr.ReuseRecord = true + var ids []string + seen := make(map[string]struct{}) + first := true + for { + rec, err := cr.Read() + if err == io.EOF { + break + } + if err != nil { + var perr *csv.ParseError + if errors.As(err, &perr) { + return nil, invalidf("csv: %v", err) + } + return nil, err // read failure (for example an oversized body) + } + if len(rec) == 0 { + continue + } + cell := strings.TrimSpace(strings.TrimPrefix(rec[0], bomPrefix)) + if first { + first = false + if _, hdr := csvHeaderNames[strings.ToLower(cell)]; hdr { + continue + } + } + if cell == "" { + continue + } + if _, dup := seen[cell]; dup { + continue + } + if len(seen) >= MaxStaticMembers { + return nil, fmt.Errorf("%w: more than %d distinct ids", ErrTooManyIDs, MaxStaticMembers) + } + seen[cell] = struct{}{} + ids = append(ids, cell) + } + return NormalizeIDs(ids) +} + +// memberPlaceholders renders "($1,...,$7),($8,...)" for n rows. +func memberPlaceholders(n int) string { + var b strings.Builder + p := 1 + for i := 0; i < n; i++ { + if i > 0 { + b.WriteByte(',') + } + b.WriteByte('(') + for c := 0; c < staticInsertCols; c++ { + if c > 0 { + b.WriteByte(',') + } + b.WriteByte('$') + b.WriteString(strconv.Itoa(p)) + p++ + } + b.WriteByte(')') + } + return b.String() +} + +const memberInsertPrefix = `INSERT INTO cohort_members (cohort_id, tenant_id, site_id, entity_id, added_at, removed, version) VALUES ` + +// memberRow is one cohort_members write. +type memberRow struct { + id string + addedAt int64 + removed int64 +} + +// insertMembers writes rows in chunked multi-row INSERTs at one version. +func (s *Service) insertMembers(ctx context.Context, siteID, cohortID string, rows []memberRow, version int64) error { + for start := 0; start < len(rows); start += staticInsertChunk { + end := start + staticInsertChunk + if end > len(rows) { + end = len(rows) + } + chunk := rows[start:end] + args := make([]any, 0, len(chunk)*staticInsertCols) + for _, r := range chunk { + args = append(args, cohortID, "default", siteID, r.id, + dbutil.IntParam(r.addedAt), dbutil.IntParam(r.removed), dbutil.IntParam(version)) + } + if _, err := s.db.SQL().Exec(ctx, memberInsertPrefix+memberPlaceholders(len(chunk)), args...); err != nil { + return fmt.Errorf("insert cohort_members %d-%d: %w", start+1, end, err) + } + } + return nil +} + +type memberRecord struct { + EntityID string `db:"entity_id"` + AddedAt int64 `db:"added_at"` + Removed int64 `db:"removed"` + Version int64 `db:"version"` +} + +// staticMembersSQL is the bounded read of one cohort's member rows. Both +// site_id and cohort_id are bound; the LIMIT is one past maxStaticRows so +// an overrun is detected. +func staticMembersSQL() string { + return `SELECT entity_id, added_at, removed, version + FROM cohort_members + WHERE site_id = $1 AND cohort_id = $2 + LIMIT ` + strconv.Itoa(maxStaticRows+1) +} + +// collapseMembers reduces versioned rows to the live members (entity -> +// first added_at): the highest version per entity wins, and a winning +// tombstone drops the entity. Ties on version prefer the tombstone, the +// conservative reading of a same-batch race. +func collapseMembers(rows []memberRecord) map[string]int64 { + best := make(map[string]memberRecord, len(rows)) + for _, r := range rows { + cur, ok := best[r.EntityID] + if !ok || r.Version > cur.Version || (r.Version == cur.Version && r.Removed > cur.Removed) { + best[r.EntityID] = r + } + } + live := make(map[string]int64, len(best)) + for id, r := range best { + if r.Removed == 0 && id != "" { + live[id] = r.AddedAt + } + } + return live +} + +func (s *Service) readStatic(ctx context.Context, siteID, cohortID string) (map[string]int64, error) { + rows, err := nucleus.Query[memberRecord](ctx, s.db.SQL(), staticMembersSQL(), siteID, cohortID) + if err != nil { + return nil, fmt.Errorf("read cohort_members: %w", err) + } + if len(rows) > maxStaticRows { + return nil, fmt.Errorf("%w: static cohort has more than %d stored rows", ErrTooLarge, maxStaticRows) + } + return collapseMembers(rows), nil +} + +func (s *Service) staticMembers(ctx context.Context, siteID, cohortID string) ([]string, error) { + live, err := s.readStatic(ctx, siteID, cohortID) + if err != nil { + return nil, err + } + out := make([]string, 0, len(live)) + for id := range live { + out = append(out, id) + } + sort.Strings(out) + return out, nil +} + +func (s *Service) staticCount(ctx context.Context, siteID, cohortID string) (int64, error) { + live, err := s.readStatic(ctx, siteID, cohortID) + if err != nil { + return 0, err + } + return int64(len(live)), nil +} + +// resolveMembers is the single membership entry point for an existing +// cohort: static cohorts read their list, rule cohorts evaluate their tree. +func (s *Service) resolveMembers(ctx context.Context, c *Cohort) ([]string, error) { + def, err := ParseDefinition(c.Rule) + if err != nil { + return nil, err + } + if def.IsStatic() { + return s.staticMembers(ctx, c.SiteID, c.CohortID) + } + return s.EvaluateCohort(ctx, c.SiteID, def) +} + +const staticRule = `{"op":"static"}` + +// putCohortRow inserts a new version of a cohorts row. +func (s *Service) putCohortRow(ctx context.Context, c Cohort) error { + _, err := s.db.SQL().Exec(ctx, + `INSERT INTO cohorts (cohort_id, tenant_id, site_id, name, description, rule, member_count, created_at, updated_at) + VALUES ($1, 'default', $2, $3, $4, $5, $6, $7, $8)`, + c.CohortID, c.SiteID, c.Name, c.Description, c.Rule, + dbutil.IntParam(c.MemberCount), dbutil.IntParam(c.CreatedAt), dbutil.IntParam(c.UpdatedAt), + ) + if err != nil { + return fmt.Errorf("write cohort: %w", err) + } + return nil +} + +// CreateStatic creates a static cohort from a list of ids (at most +// MaxStaticMembers distinct). Members are written first, then the cohort +// row: a failure part-way leaves only unreachable member rows under a +// random id, never a visible cohort with a partial list. +func (s *Service) CreateStatic(ctx context.Context, siteID, name, description string, ids []string) (*Cohort, error) { + if siteID == "" || strings.TrimSpace(name) == "" { + return nil, fmt.Errorf("site_id and name required") + } + norm, err := NormalizeIDs(ids) + if err != nil { + return nil, err + } + id := genID() + now := time.Now().UTC().UnixMilli() + rows := make([]memberRow, len(norm)) + for i, e := range norm { + rows[i] = memberRow{id: e, addedAt: now} + } + if err := s.insertMembers(ctx, siteID, id, rows, now); err != nil { + return nil, err + } + c := Cohort{ + CohortID: id, TenantID: "default", SiteID: siteID, + Name: name, Description: description, Rule: staticRule, + MemberCount: int64(len(norm)), CreatedAt: now, UpdatedAt: now, + } + if err := s.putCohortRow(ctx, c); err != nil { + return nil, err + } + return &c, nil +} + +// getStatic loads a cohort and requires it to be static. +func (s *Service) getStatic(ctx context.Context, siteID, cohortID string) (*Cohort, error) { + c, err := s.Get(ctx, siteID, cohortID) + if err != nil { + return nil, err + } + if c == nil { + return nil, ErrNotFound + } + def, err := ParseDefinition(c.Rule) + if err != nil { + return nil, err + } + if !def.IsStatic() { + return nil, fmt.Errorf("%w: members of a rule cohort come from its rule", ErrNotStatic) + } + return c, nil +} + +// AddMembers adds ids to a static cohort and returns the updated cohort and +// how many were new. The resulting list may not exceed MaxStaticMembers. +func (s *Service) AddMembers(ctx context.Context, siteID, cohortID string, ids []string) (*Cohort, int, error) { + c, err := s.getStatic(ctx, siteID, cohortID) + if err != nil { + return nil, 0, err + } + norm, err := NormalizeIDs(ids) + if err != nil { + return nil, 0, err + } + live, err := s.readStatic(ctx, siteID, cohortID) + if err != nil { + return nil, 0, err + } + version := time.Now().UTC().UnixMilli() + if version <= c.UpdatedAt { + version = c.UpdatedAt + 1 + } + var fresh []memberRow + for _, e := range norm { + if _, ok := live[e]; !ok { + fresh = append(fresh, memberRow{id: e, addedAt: version}) + } + } + if len(live)+len(fresh) > MaxStaticMembers { + return nil, 0, fmt.Errorf("%w: cohort would hold %d members, limit %d", ErrTooManyIDs, len(live)+len(fresh), MaxStaticMembers) + } + if len(fresh) == 0 { + return c, 0, nil + } + if err := s.insertMembers(ctx, siteID, cohortID, fresh, version); err != nil { + return nil, 0, err + } + next := *c + next.MemberCount = int64(len(live) + len(fresh)) + next.UpdatedAt = version + if err := s.putCohortRow(ctx, next); err != nil { + return nil, 0, err + } + return &next, len(fresh), nil +} + +// RemoveMembers removes ids from a static cohort (tombstone rows at a newer +// version) and returns the updated cohort and how many were removed. +func (s *Service) RemoveMembers(ctx context.Context, siteID, cohortID string, ids []string) (*Cohort, int, error) { + c, err := s.getStatic(ctx, siteID, cohortID) + if err != nil { + return nil, 0, err + } + norm, err := NormalizeIDs(ids) + if err != nil { + return nil, 0, err + } + live, err := s.readStatic(ctx, siteID, cohortID) + if err != nil { + return nil, 0, err + } + version := time.Now().UTC().UnixMilli() + if version <= c.UpdatedAt { + version = c.UpdatedAt + 1 + } + var gone []memberRow + for _, e := range norm { + if added, ok := live[e]; ok { + gone = append(gone, memberRow{id: e, addedAt: added, removed: 1}) + } + } + if len(gone) == 0 { + return c, 0, nil + } + if err := s.insertMembers(ctx, siteID, cohortID, gone, version); err != nil { + return nil, 0, err + } + next := *c + next.MemberCount = int64(len(live) - len(gone)) + next.UpdatedAt = version + if err := s.putCohortRow(ctx, next); err != nil { + return nil, 0, err + } + return &next, len(gone), nil +} diff --git a/internal/cohorts/static_test.go b/internal/cohorts/static_test.go new file mode 100644 index 00000000..8a729bab --- /dev/null +++ b/internal/cohorts/static_test.go @@ -0,0 +1,174 @@ +package cohorts + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestNormalizeIDs(t *testing.T) { + got, err := NormalizeIDs([]string{" a ", "b", "", "a", " ", "c", "b"}) + if err != nil || !reflect.DeepEqual(got, []string{"a", "b", "c"}) { + t.Fatalf("got %v %v", got, err) + } + for _, bad := range []string{strings.Repeat("x", MaxEntityIDLen+1), "a\x00b", "line\nbreak", "tab\there"} { + if _, err := NormalizeIDs([]string{bad}); !errors.Is(err, ErrInvalidDefinition) { + t.Errorf("%q: %v", bad, err) + } + } + // Exactly at the id length cap is accepted. + if _, err := NormalizeIDs([]string{strings.Repeat("x", MaxEntityIDLen)}); err != nil { + t.Errorf("at cap: %v", err) + } +} + +func TestNormalizeIDsCap(t *testing.T) { + ids := make([]string, MaxStaticMembers) + for i := range ids { + ids[i] = fmt.Sprintf("u%d", i) + } + if got, err := NormalizeIDs(ids); err != nil || len(got) != MaxStaticMembers { + t.Fatalf("at cap: %d %v", len(got), err) + } + // Duplicates beyond the cap do not count; one more DISTINCT id does. + if _, err := NormalizeIDs(append(ids, "u0", "u1")); err != nil { + t.Fatalf("dups over cap: %v", err) + } + if _, err := NormalizeIDs(append(ids, "one-too-many")); !errors.Is(err, ErrTooManyIDs) { + t.Fatalf("over cap: %v", err) + } +} + +func TestParseCSVIDs(t *testing.T) { + csv := "\xef\xbb\xbfdistinct_id,name\r\nu1,Alice\r\n\r\n\"u,2\",Bob\r\nu1,dup\r\n u3 \r\n,empty-first\r\n" + got, err := ParseCSVIDs(strings.NewReader(csv)) + if err != nil || !reflect.DeepEqual(got, []string{"u1", "u,2", "u3"}) { + t.Fatalf("got %v %v", got, err) + } + // No header: the first row is data. + got, err = ParseCSVIDs(strings.NewReader("alpha\nbeta\n")) + if err != nil || !reflect.DeepEqual(got, []string{"alpha", "beta"}) { + t.Fatalf("headerless: %v %v", got, err) + } + // A header word is only a header on the first row. + got, _ = ParseCSVIDs(strings.NewReader("id\nid\nx\n")) + if !reflect.DeepEqual(got, []string{"id", "x"}) { + t.Fatalf("second id row: %v", got) + } + if got, err := ParseCSVIDs(strings.NewReader("")); err != nil || len(got) != 0 { + t.Fatalf("empty: %v %v", got, err) + } +} + +func TestParseCSVIDsAdversarial(t *testing.T) { + // Over the distinct cap: refused, not truncated. + var b strings.Builder + for i := 0; i <= MaxStaticMembers; i++ { + fmt.Fprintf(&b, "u%d\n", i) + } + if _, err := ParseCSVIDs(strings.NewReader(b.String())); !errors.Is(err, ErrTooManyIDs) { + t.Fatalf("over cap: %v", err) + } + // Oversized id, control characters, and a formula-injection looking cell + // are handled as data (ids are only ever bound parameters). + if _, err := ParseCSVIDs(strings.NewReader(strings.Repeat("x", MaxEntityIDLen+1) + "\n")); !errors.Is(err, ErrInvalidDefinition) { + t.Fatalf("long id: %v", err) + } + if got, err := ParseCSVIDs(strings.NewReader("'; DROP TABLE cohorts; --\n")); err != nil || len(got) != 1 { + t.Fatalf("sql-looking id: %v %v", got, err) + } + // Malformed quoting is an invalid body, not a panic. + if _, err := ParseCSVIDs(strings.NewReader("\"unterminated\nx")); err == nil { + t.Log("lazy quotes accepted unterminated field") + } +} + +func TestMemberPlaceholders(t *testing.T) { + if got := memberPlaceholders(2); got != "($1,$2,$3,$4,$5,$6,$7),($8,$9,$10,$11,$12,$13,$14)" { + t.Fatalf("got %s", got) + } + // A full chunk stays far below the 65535 parameter wire cap. + if staticInsertChunk*staticInsertCols > 20000 { + t.Fatalf("chunk uses %d params", staticInsertChunk*staticInsertCols) + } + if !strings.Contains(memberPlaceholders(staticInsertChunk), fmt.Sprintf("$%d)", staticInsertChunk*staticInsertCols)) { + t.Fatal("last placeholder missing") + } + // Column list and placeholder width agree. + if strings.Count(memberInsertPrefix[:strings.Index(memberInsertPrefix, ")")], ",")+1 != staticInsertCols { + t.Fatalf("insert columns != %d: %s", staticInsertCols, memberInsertPrefix) + } +} + +func TestStaticMembersSQL(t *testing.T) { + q := staticMembersSQL() + for _, want := range []string{"FROM cohort_members", "site_id = $1", "cohort_id = $2", "LIMIT "} { + if !strings.Contains(q, want) { + t.Errorf("missing %q:\n%s", want, q) + } + } + if !strings.HasSuffix(q, fmt.Sprint(maxStaticRows+1)) { + t.Errorf("read must be bounded one past the row cap:\n%s", q) + } + // Reads are scoped by BOTH ids: an unscoped cohort_id lookup would be + // the IDOR. + if strings.Contains(q, "WHERE cohort_id") && !strings.Contains(q, "site_id") { + t.Error("cohort read without site scope") + } +} + +func TestCollapseMembers(t *testing.T) { + rows := []memberRecord{ + {EntityID: "keep", AddedAt: 10, Version: 10}, + {EntityID: "gone", AddedAt: 10, Version: 10}, + {EntityID: "gone", AddedAt: 10, Removed: 1, Version: 20}, // removed later + {EntityID: "back", AddedAt: 10, Version: 10}, + {EntityID: "back", AddedAt: 10, Removed: 1, Version: 20}, + {EntityID: "back", AddedAt: 30, Version: 30}, // re-added + {EntityID: "tie", AddedAt: 40, Version: 40}, + {EntityID: "tie", AddedAt: 40, Removed: 1, Version: 40}, // tie -> removed + {EntityID: "", AddedAt: 1, Version: 1}, // blank never a member + } + got := collapseMembers(rows) + want := map[string]int64{"keep": 10, "back": 30} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v want %v", got, want) + } + // Row order must not matter. + rev := make([]memberRecord, len(rows)) + for i, r := range rows { + rev[len(rows)-1-i] = r + } + if !reflect.DeepEqual(collapseMembers(rev), want) { + t.Fatal("collapse is order dependent") + } +} + +func TestStaticSentinels(t *testing.T) { + for _, e := range []error{ErrNotStatic, ErrTooManyIDs, ErrNotFound, ErrTooLarge, ErrInvalidDefinition} { + if !errors.Is(fmt.Errorf("wrap: %w", e), e) { + t.Errorf("%v does not survive wrapping", e) + } + } + // A static definition is recognised and carries no rules. + if !(Definition{Op: OpStatic}).IsStatic() || (Definition{Op: "and"}).IsStatic() { + t.Error("IsStatic") + } + if staticRule != `{"op":"static"}` { + t.Errorf("staticRule = %s", staticRule) + } + d, err := ParseDefinition(staticRule) + if err != nil || !d.IsStatic() || ValidateDefinition(d) != nil { + t.Errorf("static rule json: %+v %v", d, err) + } +} + +func TestMembersForFilterCapUsesSharedConstants(t *testing.T) { + // A static cohort may hold more than a chart filter can express; the + // 30000 filter cap still applies on top (MembersForFilter). + if MaxStaticMembers <= MaxFilterMembers { + t.Fatalf("static cap %d should exceed the filter cap %d", MaxStaticMembers, MaxFilterMembers) + } +} diff --git a/internal/cohorts/tree.go b/internal/cohorts/tree.go new file mode 100644 index 00000000..060c428c --- /dev/null +++ b/internal/cohorts/tree.go @@ -0,0 +1,346 @@ +package cohorts + +// Rule trees (C2 depth, 2026-10): OR, NOT and nesting on top of the v1 +// flat-AND definition. +// +// A Definition is either +// +// - legacy flat: {"op":"and","rules":[,...]} (v1, still read and +// written by older clients; "op":"or" over rules is also accepted now), +// - a tree: {"op":"and|or|not","children":[,...]} whose leaves +// are {"leaf":} nodes, +// - static: {"op":"static"} - membership lives in cohort_members. +// +// Evaluation stays the strategy v1 used: each leaf is one bounded query +// returning a set of distinct_ids, and the tree is combined in Go with set +// algebra (intersection / union / difference). No subqueries or engine-side +// boolean composition are involved, so nothing new is asked of Nucleus. + +import ( + "context" + "errors" + "fmt" + "regexp" + "sort" + "strings" +) + +// Limits on a rule tree, enforced on write. +const ( + // MaxTreeDepth is the longest root-to-leaf path in nodes. A flat v1 + // definition is depth 2 (the root group plus its leaves). + MaxTreeDepth = 4 + // MaxTreeLeaves is the largest number of leaf conditions in one tree. + MaxTreeLeaves = 30 + // MaxLeafRows bounds how many rows one leaf query may return. Past it + // the evaluation fails (ErrTooLarge) instead of materialising an + // unbounded set in memory. + MaxLeafRows = 200000 + // MaxEvalSet bounds an intermediate set produced by a union or a + // complement, for the same reason. + MaxEvalSet = 1000000 +) + +// OpStatic marks a static (list-backed) cohort definition. +const OpStatic = "static" + +// ErrInvalidDefinition is returned (wrapped) for a rule definition that fails +// validation; handlers map it to 422. +var ErrInvalidDefinition = errors.New("invalid cohort definition") + +func invalidf(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrInvalidDefinition, fmt.Sprintf(format, args...)) +} + +// node is the normalised internal form of a Definition: a group (op + +// kids) or a leaf (leaf != nil). +type node struct { + op string + kids []node + leaf *Rule +} + +// normalize converts a Definition (legacy flat, tree or leaf) to a node. +// It is purely structural; validate does the checking. +func normalize(def Definition) node { + if def.Leaf != nil { + return node{leaf: def.Leaf} + } + op := def.Op + if op == "" { + op = "and" + } + n := node{op: op} + for i := range def.Rules { + r := def.Rules[i] + n.kids = append(n.kids, node{leaf: &r}) + } + for _, c := range def.Children { + n.kids = append(n.kids, normalize(c)) + } + return n +} + +// ValidateDefinition checks a definition for a write. It rejects bad shape +// (unknown op, wrong child counts, depth/leaf limits, a leaf mixed with +// children) and, per leaf, an unusable rule (empty event name, property key +// outside the allow-list, unsupported operator, malformed window). +func ValidateDefinition(def Definition) error { + return validateDefinition(def, true) +} + +// validateShape is the evaluation-time subset: structure and limits only. +// Leaf field problems surface from the leaf evaluators exactly as in v1, so +// a rule saved before write validation existed keeps its old behaviour. +func validateShape(def Definition) error { + return validateDefinition(def, false) +} + +func validateDefinition(def Definition, strictLeaves bool) error { + if def.Op == OpStatic { + if def.Leaf != nil || len(def.Rules) != 0 || len(def.Children) != 0 { + return invalidf("a static cohort carries no rules") + } + return nil + } + if err := checkMixing(def); err != nil { + return err + } + n := normalize(def) + leaves := 0 + if err := checkNode(n, 1, &leaves, strictLeaves); err != nil { + return err + } + if leaves == 0 { + return invalidf("at least one condition is required") + } + if leaves > MaxTreeLeaves { + return invalidf("%d conditions, limit %d", leaves, MaxTreeLeaves) + } + return nil +} + +// checkMixing rejects ambiguous nodes before normalisation flattens them. +func checkMixing(def Definition) error { + if def.Leaf != nil { + if def.Op != "" || len(def.Rules) != 0 || len(def.Children) != 0 { + return invalidf("a leaf node carries no op, rules or children") + } + return nil + } + if len(def.Rules) != 0 && len(def.Children) != 0 { + return invalidf("use either rules (flat) or children (tree), not both") + } + for _, c := range def.Children { + if c.Op == OpStatic { + return invalidf("static is a top-level cohort kind, not a tree node") + } + if err := checkMixing(c); err != nil { + return err + } + } + return nil +} + +func checkNode(n node, depth int, leaves *int, strict bool) error { + if depth > MaxTreeDepth { + return invalidf("nesting deeper than %d levels", MaxTreeDepth) + } + if n.leaf != nil { + *leaves++ + if strict { + return validateRule(*n.leaf) + } + return nil + } + switch n.op { + case "and", "or": + if len(n.kids) == 0 { + return invalidf("%q group needs at least one child", n.op) + } + case "not": + if len(n.kids) != 1 { + return invalidf("\"not\" takes exactly one child, got %d", len(n.kids)) + } + default: + return invalidf("unknown op %q (supported: and, or, not)", n.op) + } + for _, k := range n.kids { + if err := checkNode(k, depth+1, leaves, strict); err != nil { + return err + } + } + return nil +} + +var windowRe = regexp.MustCompile(`^[0-9]{1,4}[dhm]$`) + +const ( + maxRuleStr = 512 + maxRuleMinCount = 1000000 +) + +// validateRule is the strict write-time check of one leaf. +func validateRule(r Rule) error { + if len(r.Name) > maxRuleStr || len(r.Value) > maxRuleStr { + return invalidf("rule name/value longer than %d bytes", maxRuleStr) + } + switch r.Type { + case "event": + if strings.TrimSpace(r.Name) == "" { + return invalidf("event rule requires name") + } + if r.MinCount < 0 || r.MinCount > maxRuleMinCount { + return invalidf("min_count out of range") + } + if r.Window != "" && !windowRe.MatchString(strings.ToLower(strings.TrimSpace(r.Window))) { + return invalidf("window %q must look like 24h, 7d or 90m", r.Window) + } + case "property": + if r.Key == "" { + return invalidf("property rule requires key") + } + if !isAllowedPropertyKey(r.Key) { + return invalidf("property key %q not allowed (allowed: %s)", r.Key, strings.Join(allowedPropertyKeys(), ", ")) + } + if r.Operator != "" && r.Operator != "=" && r.Operator != "!=" { + return invalidf("operator %q not supported (supported: =, !=)", r.Operator) + } + default: + return invalidf("unsupported rule type %q (supported: event, property)", r.Type) + } + return nil +} + +// idSet is the unit of the set algebra. +type idSet map[string]struct{} + +func toSet(ids []string) idSet { + s := make(idSet, len(ids)) + for _, id := range ids { + if id != "" { + s[id] = struct{}{} + } + } + return s +} + +func (s idSet) sorted() []string { + out := make([]string, 0, len(s)) + for id := range s { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// intersectSets returns a new set with the ids present in both. +func intersectSets(a, b idSet) idSet { + if len(b) < len(a) { + a, b = b, a + } + out := make(idSet, len(a)) + for id := range a { + if _, ok := b[id]; ok { + out[id] = struct{}{} + } + } + return out +} + +// unionSets returns a new set with the ids present in either; it fails +// (ErrTooLarge) rather than building a set larger than MaxEvalSet. +func unionSets(a, b idSet) (idSet, error) { + if len(a)+len(b) > MaxEvalSet { + // Only an upper bound; count exactly before refusing. + n := len(a) + for id := range b { + if _, ok := a[id]; !ok { + n++ + } + } + if n > MaxEvalSet { + return nil, fmt.Errorf("%w: intermediate set over %d ids", ErrTooLarge, MaxEvalSet) + } + } + out := make(idSet, len(a)+len(b)) + for id := range a { + out[id] = struct{}{} + } + for id := range b { + out[id] = struct{}{} + } + return out, nil +} + +// differenceSets returns the ids of a that are not in b. +func differenceSets(a, b idSet) idSet { + out := make(idSet, len(a)) + for id := range a { + if _, ok := b[id]; !ok { + out[id] = struct{}{} + } + } + return out +} + +// treeEnv supplies the two data sources a tree evaluation needs; tests +// substitute in-memory fakes. +type treeEnv struct { + leaf func(ctx context.Context, r Rule) (idSet, error) + universe func(ctx context.Context) (idSet, error) +} + +// evalNode evaluates a node to its member set. "and" short-circuits on an +// empty intersection; "not" is the complement against the site's identified +// users (the universe), loaded at most once per evaluation. +func evalNode(ctx context.Context, n node, env treeEnv) (idSet, error) { + if n.leaf != nil { + return env.leaf(ctx, *n.leaf) + } + switch n.op { + case "and": + var acc idSet + for i, k := range n.kids { + s, err := evalNode(ctx, k, env) + if err != nil { + return nil, err + } + if i == 0 { + acc = s + } else { + acc = intersectSets(acc, s) + } + if len(acc) == 0 { + return idSet{}, nil + } + } + return acc, nil + case "or": + acc := idSet{} + for _, k := range n.kids { + s, err := evalNode(ctx, k, env) + if err != nil { + return nil, err + } + if acc, err = unionSets(acc, s); err != nil { + return nil, err + } + } + return acc, nil + case "not": + if len(n.kids) != 1 { + return nil, invalidf("\"not\" takes exactly one child") + } + s, err := evalNode(ctx, n.kids[0], env) + if err != nil { + return nil, err + } + all, err := env.universe(ctx) + if err != nil { + return nil, err + } + return differenceSets(all, s), nil + } + return nil, invalidf("unknown op %q", n.op) +} diff --git a/internal/cohorts/tree_test.go b/internal/cohorts/tree_test.go new file mode 100644 index 00000000..743a6fd3 --- /dev/null +++ b/internal/cohorts/tree_test.go @@ -0,0 +1,319 @@ +package cohorts + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func ev(name string) Rule { return Rule{Type: "event", Name: name} } +func prop(k, v string) Rule { return Rule{Type: "property", Key: k, Value: v} } + +func leaf(r Rule) Definition { return Definition{Leaf: &r} } + +func grp(op string, kids ...Definition) Definition { return Definition{Op: op, Children: kids} } + +// chain nests n "and" groups around one leaf: depth n+1. +func chain(n int) Definition { + d := leaf(ev("x")) + for i := 0; i < n; i++ { + d = grp("and", d) + } + return d +} + +func TestValidateDefinitionTable(t *testing.T) { + manyLeaves := func(n int) Definition { + kids := make([]Definition, n) + for i := range kids { + kids[i] = leaf(ev(fmt.Sprintf("e%d", i))) + } + return grp("or", kids...) + } + flat := func(n int) Definition { + rules := make([]Rule, n) + for i := range rules { + rules[i] = ev(fmt.Sprintf("e%d", i)) + } + return Definition{Op: "and", Rules: rules} + } + cases := []struct { + name string + def Definition + ok bool + }{ + {"legacy flat and", Definition{Op: "and", Rules: []Rule{ev("a"), prop("country", "US")}}, true}, + {"legacy flat empty op", Definition{Rules: []Rule{ev("a")}}, true}, + {"legacy flat or", Definition{Op: "or", Rules: []Rule{ev("a"), ev("b")}}, true}, + {"single leaf root", leaf(ev("a")), true}, + {"or of leaves", grp("or", leaf(ev("a")), leaf(ev("b"))), true}, + {"and of or", grp("and", leaf(ev("a")), grp("or", leaf(ev("b")), leaf(ev("c")))), true}, + {"not one child", grp("not", leaf(ev("a"))), true}, + {"static", Definition{Op: OpStatic}, true}, + + {"empty", Definition{}, false}, + {"empty and group", grp("and"), false}, + {"empty or group", grp("or"), false}, + {"not no child", grp("not"), false}, + {"not two children", grp("not", leaf(ev("a")), leaf(ev("b"))), false}, + {"unknown op", grp("xor", leaf(ev("a"))), false}, + {"rules and children mixed", Definition{Op: "and", Rules: []Rule{ev("a")}, Children: []Definition{leaf(ev("b"))}}, false}, + {"leaf with op", Definition{Op: "and", Leaf: &Rule{Type: "event", Name: "a"}}, false}, + {"leaf with children", Definition{Leaf: &Rule{Type: "event", Name: "a"}, Children: []Definition{leaf(ev("b"))}}, false}, + {"static with rules", Definition{Op: OpStatic, Rules: []Rule{ev("a")}}, false}, + {"static nested", grp("and", Definition{Op: OpStatic}), false}, + {"event no name", leaf(Rule{Type: "event"}), false}, + {"unknown rule type", leaf(Rule{Type: "cohort", Name: "x"}), false}, + {"property bad key", leaf(prop("password", "x")), false}, + {"property sql in key", leaf(prop("country; DROP TABLE events", "x")), false}, + {"property bad operator", leaf(Rule{Type: "property", Key: "country", Operator: ">", Value: "x"}), false}, + {"property neq ok", leaf(Rule{Type: "property", Key: "country", Operator: "!=", Value: "x"}), true}, + {"bad window", leaf(Rule{Type: "event", Name: "a", Window: "forever"}), false}, + {"good window", leaf(Rule{Type: "event", Name: "a", Window: "90m"}), true}, + {"negative min_count", leaf(Rule{Type: "event", Name: "a", MinCount: -1}), false}, + {"oversize name", leaf(Rule{Type: "event", Name: strings.Repeat("a", maxRuleStr+1)}), false}, + + // Depth: root group = 1, so chain(3) is 4 levels (ok), chain(4) is 5. + {"depth 4 ok", chain(3), true}, + {"depth 5 refused", chain(4), false}, + {"flat 30 leaves ok", flat(MaxTreeLeaves), true}, + {"flat 31 leaves refused", flat(MaxTreeLeaves + 1), false}, + {"tree 30 leaves ok", manyLeaves(MaxTreeLeaves), true}, + {"tree 31 leaves refused", manyLeaves(MaxTreeLeaves + 1), false}, + } + for _, tc := range cases { + err := ValidateDefinition(tc.def) + if tc.ok && err != nil { + t.Errorf("%s: unexpected error %v", tc.name, err) + } + if !tc.ok { + if err == nil { + t.Errorf("%s: expected error", tc.name) + } else if !errors.Is(err, ErrInvalidDefinition) { + t.Errorf("%s: error %v does not wrap ErrInvalidDefinition", tc.name, err) + } + } + } +} + +func TestLegacyJSONStillParses(t *testing.T) { + // Exactly what v1 wrote. + raw := `{"op":"and","rules":[{"type":"event","name":"purchase","window":"30d","min_count":2},{"type":"property","key":"country","operator":"=","value":"US"}]}` + def, err := ParseDefinition(raw) + if err != nil { + t.Fatal(err) + } + if err := ValidateDefinition(def); err != nil { + t.Fatalf("legacy definition no longer valid: %v", err) + } + n := normalize(def) + if n.op != "and" || len(n.kids) != 2 || n.kids[0].leaf == nil || n.kids[0].leaf.Name != "purchase" { + t.Fatalf("normalize(legacy) = %+v", n) + } + // v1 also stored an empty rules array with no omitempty. + def, err = ParseDefinition(`{"op":"and","rules":null}`) + if err != nil || len(def.Rules) != 0 { + t.Fatalf("%v %+v", err, def) + } +} + +func TestTreeJSONRoundTrip(t *testing.T) { + raw := `{"op":"and","children":[{"leaf":{"type":"event","name":"signup"}},{"op":"not","children":[{"leaf":{"type":"property","key":"country","value":"US"}}]}]}` + def, err := ParseDefinition(raw) + if err != nil { + t.Fatal(err) + } + if err := ValidateDefinition(def); err != nil { + t.Fatal(err) + } + out, _ := json.Marshal(def) + var again Definition + if err := json.Unmarshal(out, &again); err != nil || !reflect.DeepEqual(def, again) { + t.Fatalf("round trip: %v\n%s", err, out) + } + // A bare leaf root must not be defaulted to op "and". + d, _ := ParseDefinition(`{"leaf":{"type":"event","name":"a"}}`) + if d.Op != "" || ValidateDefinition(d) != nil { + t.Fatalf("leaf root: %+v", d) + } +} + +func TestShapeCheckIsLenientOnLeafFields(t *testing.T) { + // A pre-validation legacy rule with a junk window must still evaluate + // (parseWindow defaults it) - only the write path is strict. + d := Definition{Op: "and", Rules: []Rule{{Type: "event", Name: "a", Window: "soon"}}} + if validateShape(d) != nil { + t.Fatal("shape check must not police leaf fields") + } + if ValidateDefinition(d) == nil { + t.Fatal("write validation must reject it") + } +} + +func setOf(ids ...string) idSet { return toSet(ids) } + +func TestSetAlgebra(t *testing.T) { + a, b := setOf("1", "2", "3"), setOf("2", "3", "4") + if got := intersectSets(a, b).sorted(); !reflect.DeepEqual(got, []string{"2", "3"}) { + t.Errorf("intersect = %v", got) + } + if got := intersectSets(b, a).sorted(); !reflect.DeepEqual(got, []string{"2", "3"}) { + t.Errorf("intersect commutes: %v", got) + } + u, err := unionSets(a, b) + if err != nil || !reflect.DeepEqual(u.sorted(), []string{"1", "2", "3", "4"}) { + t.Errorf("union = %v %v", u.sorted(), err) + } + if got := differenceSets(a, b).sorted(); !reflect.DeepEqual(got, []string{"1"}) { + t.Errorf("difference = %v", got) + } + if got := differenceSets(a, a); len(got) != 0 { + t.Errorf("a - a = %v", got) + } + // Inputs are never mutated. + if len(a) != 3 || len(b) != 3 { + t.Errorf("inputs mutated: %v %v", a, b) + } + // Anonymous ids never enter a set. + if len(toSet([]string{"", "x"})) != 1 { + t.Error("empty id kept") + } + // De Morgan: not(a or b) == not(a) and not(b), over a universe. + all := setOf("1", "2", "3", "4", "5") + ab, _ := unionSets(a, b) + left := differenceSets(all, ab) + right := intersectSets(differenceSets(all, a), differenceSets(all, b)) + if !reflect.DeepEqual(left.sorted(), right.sorted()) { + t.Errorf("de morgan: %v vs %v", left.sorted(), right.sorted()) + } +} + +// fakeEnv resolves leaves by event/property name from fixed sets and counts +// universe loads. +func fakeEnv(leaves map[string][]string, universe []string, loads *int) treeEnv { + return treeEnv{ + leaf: func(_ context.Context, r Rule) (idSet, error) { + key := r.Name + if r.Type == "property" { + key = r.Key + "=" + r.Value + } + ids, ok := leaves[key] + if !ok { + return nil, fmt.Errorf("fake: no leaf %q", key) + } + return toSet(ids), nil + }, + universe: func(context.Context) (idSet, error) { + *loads++ + return toSet(universe), nil + }, + } +} + +func TestEvalNode(t *testing.T) { + leaves := map[string][]string{ + "a": {"u1", "u2", "u3"}, + "b": {"u3", "u4"}, + "c": {"u9"}, + "country=US": {"u1", "u4"}, + } + universe := []string{"u1", "u2", "u3", "u4", "u5", "u9"} + cases := []struct { + name string + def Definition + want []string + }{ + {"legacy and", Definition{Op: "and", Rules: []Rule{ev("a"), ev("b")}}, []string{"u3"}}, + {"legacy or", Definition{Op: "or", Rules: []Rule{ev("a"), ev("b")}}, []string{"u1", "u2", "u3", "u4"}}, + {"leaf", leaf(ev("c")), []string{"u9"}}, + {"not leaf", grp("not", leaf(ev("a"))), []string{"u4", "u5", "u9"}}, + {"and with not", grp("and", leaf(ev("a")), grp("not", leaf(ev("b")))), []string{"u1", "u2"}}, + {"or of and", grp("or", + grp("and", leaf(ev("a")), leaf(prop("country", "US"))), + leaf(ev("c"))), []string{"u1", "u9"}}, + {"nested not not", grp("not", grp("not", leaf(ev("b")))), []string{"u3", "u4"}}, + {"empty intersection", grp("and", leaf(ev("c")), leaf(ev("a"))), []string{}}, + } + for _, tc := range cases { + loads := 0 + got, err := evalNode(context.Background(), normalize(tc.def), fakeEnv(leaves, universe, &loads)) + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if !reflect.DeepEqual(got.sorted(), tc.want) { + t.Errorf("%s: got %v want %v", tc.name, got.sorted(), tc.want) + } + } +} + +func TestEvalNodeLeafErrorPropagates(t *testing.T) { + loads := 0 + _, err := evalNode(context.Background(), normalize(grp("or", leaf(ev("a")), leaf(ev("missing")))), + fakeEnv(map[string][]string{"a": {"u1"}}, nil, &loads)) + if err == nil { + t.Fatal("a failing leaf must fail the whole evaluation, not be dropped from the OR") + } +} + +func TestEvalNodeAndShortCircuits(t *testing.T) { + calls := 0 + env := treeEnv{ + leaf: func(_ context.Context, r Rule) (idSet, error) { + calls++ + if r.Name == "empty" { + return idSet{}, nil + } + return setOf("u1"), nil + }, + universe: func(context.Context) (idSet, error) { return idSet{}, nil }, + } + d := Definition{Op: "and", Rules: []Rule{ev("empty"), ev("x"), ev("y")}} + got, err := evalNode(context.Background(), normalize(d), env) + if err != nil || len(got) != 0 || calls != 1 { + t.Fatalf("got %v err %v calls %d", got, err, calls) + } +} + +func TestUnionCap(t *testing.T) { + big := make(idSet, MaxEvalSet) + for i := 0; i < MaxEvalSet; i++ { + big[fmt.Sprint(i)] = struct{}{} + } + if _, err := unionSets(big, setOf("extra-1")); !errors.Is(err, ErrTooLarge) { + t.Fatalf("union past cap: %v", err) + } + // Overlapping union that stays within the cap is fine. + if _, err := unionSets(big, setOf("1", "2")); err != nil { + t.Fatalf("overlap union: %v", err) + } +} + +func TestLeafLimitSQL(t *testing.T) { + if !strings.HasSuffix(leafLimitSQL(), fmt.Sprint(MaxLeafRows+1)) || !strings.HasPrefix(leafLimitSQL(), " LIMIT ") { + t.Fatalf("leafLimitSQL = %q", leafLimitSQL()) + } +} + +func TestEvaluateCohortRejectsBeforeQuerying(t *testing.T) { + // A nil-db Service must refuse malformed trees without touching the db. + s := &Service{} + bad := []Definition{ + grp("xor", leaf(ev("a"))), + grp("not", leaf(ev("a")), leaf(ev("b"))), + chain(MaxTreeDepth), + {Op: OpStatic}, + } + for i, d := range bad { + if _, err := s.EvaluateCohort(context.Background(), "site", d); !errors.Is(err, ErrInvalidDefinition) { + t.Errorf("case %d: %v", i, err) + } + } + if ids, err := s.EvaluateCohort(context.Background(), "site", Definition{}); err != nil || len(ids) != 0 { + t.Errorf("empty def: %v %v", ids, err) + } +} diff --git a/internal/query/api.go b/internal/query/api.go index 94227a35..76567db6 100644 --- a/internal/query/api.go +++ b/internal/query/api.go @@ -86,10 +86,9 @@ func (i StatsInput) resolveFilters(ctx context.Context, svc *StatsService) (*Fil if i.CohortID == "" || svc == nil { return fb, nil } - ids, err := svc.ResolveCohort(ctx, i.SiteID, i.CohortID) + ids, err := svc.resolveCohortAdmitted(ctx, i.SiteID, i.CohortID) if err != nil { - slog.Error("cohort resolve failed", "err", err, "site", i.SiteID, "cohort", i.CohortID) - return nil, cohortResolveError(err) + return nil, cohortResolveFailure(ctx, i.SiteID, i.CohortID, err) } if ids == nil { // Resolver wired but cohort_id wasn't found / returned nil. @@ -101,7 +100,18 @@ func (i StatsInput) resolveFilters(ctx context.Context, svc *StatsService) (*Fil return fb, nil } +// cohortResolveFailure logs a cohort resolution failure (the raw error stays +// in the log, never in the response) and returns its HTTP mapping. +func cohortResolveFailure(ctx context.Context, siteID, cohortID string, err error) error { + if !errors.Is(err, cohorts.ErrNotFound) { + slog.Error("cohort resolve failed", "err", err, "site", siteID, "cohort", cohortID) + } + return cohortResolveError(err) +} + // cohortResolveError maps a cohort resolver failure to an HTTP error: +// - cohort missing for this site (including another site's cohort id): +// 404, indistinguishable from a cohort that never existed; // - cohort larger than the filter can express (one SQL parameter per // member): 422, the caller must narrow the cohort; // - query-admission refusal (429 / 504): the refusal's own status; @@ -109,6 +119,9 @@ func (i StatsInput) resolveFilters(ctx context.Context, svc *StatsService) (*Fil // retryable server-side condition. The raw error is logged by the // caller, never echoed to the client. func cohortResolveError(err error) error { + if errors.Is(err, cohorts.ErrNotFound) { + return neutron.ErrNotFound("cohort not found") + } if errors.Is(err, cohorts.ErrTooLarge) { return neutron.ErrValidation( "cohort has more members than a chart filter supports; narrow the cohort definition", @@ -213,6 +226,9 @@ type FunnelInput struct { ConversionWindowMs int64 `json:"conversion_window_ms"` // Exclusions are disqualifying steps (see FunnelOptions). Exclusions []FunnelStep `json:"exclusions"` + // CohortID restricts the funnel to the cohort's members (see + // FunnelOptions.CohortID). + CohortID string `json:"cohort_id"` } // FunnelBreakdownInput augments FunnelInput with a breakdown dimension. @@ -224,6 +240,7 @@ type FunnelBreakdownInput struct { BreakdownBy string `json:"breakdown_by"` MinSize int `json:"min_size"` Entity string `json:"entity"` + CohortID string `json:"cohort_id"` } func (i FunnelBreakdownInput) TimeRange() (time.Time, time.Time) { @@ -264,6 +281,9 @@ type RetentionInput struct { // ReturnEvent restricts return-activity bucketing to an event_type // (empty = any event). ReturnEvent string `query:"return_event"` + // CohortID restricts retention to the cohort's members (see + // RetentionOptions.CohortID). + CohortID string `query:"cohort_id"` } func (i RetentionInput) TimeRange() (time.Time, time.Time) { @@ -599,6 +619,7 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa Entity: input.Entity, ConversionWindowMs: input.ConversionWindowMs, Exclusions: input.Exclusions, + CohortID: input.CohortID, }) }, neutron.WithTags("stats")) @@ -610,7 +631,8 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa min = 5 } return svc.FunnelByBreakdownWithOptions(ctx, input.SiteID, from, to, input.Steps, input.BreakdownBy, min, FunnelOptions{ - Entity: input.Entity, + Entity: input.Entity, + CohortID: input.CohortID, }) }, neutron.WithTags("stats")) @@ -621,6 +643,7 @@ func RegisterRoutes(r *neutron.Router, svc *StatsService, mw ...neutron.Middlewa Entity: input.Entity, CohortEvent: input.CohortEvent, ReturnEvent: input.ReturnEvent, + CohortID: input.CohortID, }) }, neutron.WithTags("stats")) } diff --git a/internal/query/cohort_depth_test.go b/internal/query/cohort_depth_test.go new file mode 100644 index 00000000..01de512b --- /dev/null +++ b/internal/query/cohort_depth_test.go @@ -0,0 +1,191 @@ +package query + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/useteploy/teploy-observe/internal/cohorts" + "github.com/useteploy/teploy-observe/internal/queryguard" +) + +// These tests cover the cohort_id plumbing of the funnel, funnel-breakdown +// and retention paths (cohort_filter.go) without a database: every failure +// mode of cohort resolution must be decided BEFORE the first events read, +// so a nil-db StatsService is enough to prove the request fails closed. + +func depthSvc(r CohortResolver) *StatsService { + return NewStatsService(nil).WithCohortResolver(r) +} + +var depthSteps = []FunnelStep{{Type: "event", Value: "a"}, {Type: "event", Value: "b"}} + +// consumers runs the same cohort-filtered request through each consumer. +func consumers(svc *StatsService, cohortID string) map[string]error { + ctx := context.Background() + to := time.Now().UTC() + from := to.Add(-24 * time.Hour) + out := map[string]error{} + _, out["funnel"] = svc.FunnelWithOptions(ctx, "site-a", from, to, depthSteps, FunnelOptions{CohortID: cohortID}) + _, out["breakdown"] = svc.FunnelByBreakdownWithOptions(ctx, "site-a", from, to, depthSteps, "browser", 1, FunnelOptions{CohortID: cohortID}) + _, out["retention"] = svc.RetentionWithOptions(ctx, "site-a", from, to, 1, RetentionOptions{CohortID: cohortID}) + return out +} + +func TestCohortConsumersFailClosed(t *testing.T) { + cases := []struct { + name string + err error + status int + }{ + {"resolver failure is 503", errors.New("store down: secret-dsn"), http.StatusServiceUnavailable}, + {"too large is 422", fmt.Errorf("%w: 99999 members", cohorts.ErrTooLarge), http.StatusUnprocessableEntity}, + {"missing or foreign cohort is 404", cohorts.ErrNotFound, http.StatusNotFound}, + {"wrapped not found is 404", fmt.Errorf("get: %w", cohorts.ErrNotFound), http.StatusNotFound}, + {"unreadable stored rule is 503", fmt.Errorf("invalid cohort rule json: boom"), http.StatusServiceUnavailable}, + } + for _, tc := range cases { + svc := depthSvc(func(context.Context, string, string) ([]string, error) { return nil, tc.err }) + for name, err := range consumers(svc, "c1") { + if err == nil { + t.Errorf("%s/%s: no error - the cohort filter was dropped", tc.name, name) + continue + } + app := appErr(t, err) + if app.Status != tc.status { + t.Errorf("%s/%s: status %d, want %d", tc.name, name, app.Status, tc.status) + } + if strings.Contains(app.Detail, "secret-dsn") { + t.Errorf("%s/%s: raw error leaked: %q", tc.name, name, app.Detail) + } + } + } +} + +// The IDOR shape: the resolver is handed the REQUEST's site, so a cohort id +// belonging to another site resolves to ErrNotFound (cohorts.Service.Get is +// site-scoped) and the endpoint answers 404 for every consumer. +func TestCohortConsumersPassRequestSiteToResolver(t *testing.T) { + var gotSite, gotCohort atomic.Value + svc := depthSvc(func(_ context.Context, site, cohort string) ([]string, error) { + gotSite.Store(site) + gotCohort.Store(cohort) + return nil, cohorts.ErrNotFound + }) + for name, err := range consumers(svc, "other-sites-cohort") { + if appErr(t, err).Status != http.StatusNotFound { + t.Errorf("%s: want 404, got %v", name, err) + } + if gotSite.Load() != "site-a" || gotCohort.Load() != "other-sites-cohort" { + t.Errorf("%s: resolver saw site=%v cohort=%v", name, gotSite.Load(), gotCohort.Load()) + } + } + // And the stats routes (resolveFilters) agree. + _, err := StatsInput{SiteID: "site-a", CohortID: "other-sites-cohort"}.resolveFilters(context.Background(), svc) + if appErr(t, err).Status != http.StatusNotFound { + t.Errorf("resolveFilters: want 404, got %v", err) + } +} + +// Cohort-filtered queries go through the query guard: with the concurrency +// slots exhausted the request is refused (429) before the cohort is even +// resolved, on every consumer. +func TestCohortConsumersAreGuarded(t *testing.T) { + var resolved atomic.Int32 + svc := depthSvc(func(context.Context, string, string) ([]string, error) { + resolved.Add(1) + return []string{"u1"}, nil + }) + lim := queryguard.NewLimiter(1, 1) + svc.WithQueryGuard(lim, queryguard.DefaultBudgets()) + release, err := lim.Acquire(context.Background(), "site-a") + if err != nil { + t.Fatal(err) + } + defer release() + + for name, err := range consumers(svc, "c1") { + var ref *queryguard.Refusal + if !errors.As(err, &ref) || ref.Status != http.StatusTooManyRequests { + t.Errorf("%s: want a 429 refusal, got %v", name, err) + } + } + // The stats-route resolution takes a slot too. + _, err = StatsInput{SiteID: "site-a", CohortID: "c1"}.resolveFilters(context.Background(), svc) + if appErr(t, err).Status != http.StatusTooManyRequests { + t.Errorf("resolveFilters under load: %v", err) + } + if n := resolved.Load(); n != 0 { + t.Errorf("resolver ran %d times while the guard was refusing", n) + } +} + +func TestResolveFiltersTakesAndReleasesSlot(t *testing.T) { + svc := depthSvc(func(context.Context, string, string) ([]string, error) { return []string{"u1", "u2"}, nil }) + lim := queryguard.NewLimiter(1, 1) + svc.WithQueryGuard(lim, queryguard.DefaultBudgets()) + for i := 0; i < 3; i++ { // a leaked slot would refuse the second call + fb, err := StatsInput{SiteID: "site-a", CohortID: "c1"}.resolveFilters(context.Background(), svc) + if err != nil || !strings.Contains(fb.SQL(), "distinct_id IN") { + t.Fatalf("call %d: fb=%q err=%v", i, fb.SQL(), err) + } + } + // A failing resolver releases the slot as well. + svc = depthSvc(func(context.Context, string, string) ([]string, error) { return nil, errors.New("x") }) + svc.WithQueryGuard(lim, queryguard.DefaultBudgets()) + for i := 0; i < 3; i++ { + _, err := StatsInput{SiteID: "site-a", CohortID: "c1"}.resolveFilters(context.Background(), svc) + if appErr(t, err).Status != http.StatusServiceUnavailable { + t.Fatalf("call %d: %v", i, err) + } + } +} + +func TestResolveCohortSet(t *testing.T) { + ctx := context.Background() + // No cohort: no filter. + set, err := depthSvc(nil).resolveCohortSet(ctx, "s", "") + if set != nil || err != nil { + t.Fatalf("no cohort: %v %v", set, err) + } + // Unwired resolver with a cohort id: match nothing, never unfiltered. + set, err = depthSvc(nil).resolveCohortSet(ctx, "s", "c") + if err != nil || set == nil || len(set) != 0 { + t.Fatalf("unwired: %v %v", set, err) + } + // Resolver returning nil ids: empty non-nil set. + set, err = depthSvc(func(context.Context, string, string) ([]string, error) { return nil, nil }).resolveCohortSet(ctx, "s", "c") + if err != nil || set == nil || len(set) != 0 { + t.Fatalf("nil ids: %v %v", set, err) + } + set, err = depthSvc(func(context.Context, string, string) ([]string, error) { return []string{"a", "b", "a"}, nil }).resolveCohortSet(ctx, "s", "c") + if err != nil || len(set) != 2 || !set.has("a") || set.has("z") { + t.Fatalf("members: %v %v", set, err) + } +} + +func TestMemberSetHas(t *testing.T) { + var none memberSet + if none.has("a") || none.has("") { + t.Error("nil set admits nothing through has") + } + m := memberSet{"a": {}, "": {}} + if !m.has("a") || m.has("b") { + t.Error("membership") + } + if m.has("") { + t.Error("an event with no distinct_id must never match a cohort") + } +} + +func TestCohortErrorMappingNotFound(t *testing.T) { + app := appErr(t, cohortResolveError(fmt.Errorf("x: %w", cohorts.ErrNotFound))) + if app.Status != http.StatusNotFound || strings.Contains(app.Detail, "x:") { + t.Errorf("%d %q", app.Status, app.Detail) + } +} diff --git a/internal/query/cohort_filter.go b/internal/query/cohort_filter.go new file mode 100644 index 00000000..37cd077a --- /dev/null +++ b/internal/query/cohort_filter.go @@ -0,0 +1,76 @@ +package query + +import ( + "context" +) + +// Cohort filtering for the heavy read paths (funnel, funnel breakdown, +// retention). The stats routes expand a cohort into a SQL +// `distinct_id IN (...)` clause (StatsInput.resolveFilters); these paths +// stream raw events instead, so the same resolved membership is applied as +// a Go-side set test on each streamed row's distinct_id. +// +// Same machinery, same limits: the membership comes from the same +// CohortResolver (cohorts.Service.MembersForFilter), so the +// cohorts.MaxFilterMembers cap (422), the not-found / foreign-site refusal +// (404) and the fail-closed mapping (cohortResolveError: 503, or the +// guard's own 429/504) are identical to the stats routes. The resolution +// runs INSIDE the query admission slot the caller already holds +// (beginQuery), so a cohort-filtered funnel is one admitted, time-budgeted +// query, not an admitted query plus an unguarded cohort evaluation. + +// memberSet is a resolved cohort membership: a nil set means "no cohort +// filter"; a non-nil (possibly empty) set admits only its ids. +type memberSet map[string]struct{} + +// has reports whether an event with this distinct_id passes the filter. +// An event with no distinct_id never belongs to a cohort. +func (m memberSet) has(distinctID string) bool { + if distinctID == "" { + return false + } + _, ok := m[distinctID] + return ok +} + +// resolveCohortSet resolves cohortID to a membership set under ctx (the +// caller's already-admitted, budget-bounded context). An empty cohortID is +// no filter (nil set, nil error). A missing resolver, a missing cohort or +// an empty cohort yield an empty non-nil set - match nothing, never +// unfiltered. Failures are mapped (cohortResolveError) and never fall back +// to unfiltered. +func (s *StatsService) resolveCohortSet(ctx context.Context, siteID, cohortID string) (memberSet, error) { + if cohortID == "" { + return nil, nil + } + ids, err := s.ResolveCohort(ctx, siteID, cohortID) + if err != nil { + return nil, cohortResolveFailure(ctx, siteID, cohortID, err) + } + set := make(memberSet, len(ids)) + for _, id := range ids { + set[id] = struct{}{} + } + return set, nil +} + +// resolveCohortAdmitted is resolveCohortSet for callers that do NOT already +// hold an admission slot (the stats-route resolveFilters): it takes one +// for the duration of the resolution so cohort evaluation is subject to the +// same concurrency and time budgets as the heavy paths. With no guard and +// no timeout configured (a bare StatsService in tests) it resolves directly. +func (s *StatsService) resolveCohortAdmitted(ctx context.Context, siteID, cohortID string) ([]string, error) { + if s.guard == nil && s.budgets.Timeout <= 0 { + return s.ResolveCohort(ctx, siteID, cohortID) + } + qctx, finish, err := s.beginQuery(ctx, siteID) + if err != nil { + return nil, err + } + defer finish() + ids, err := s.ResolveCohort(qctx, siteID, cohortID) + if err != nil { + return nil, s.scanDeadlineError(ctx, err) + } + return ids, nil +} diff --git a/internal/query/funnel.go b/internal/query/funnel.go index 896aa3bb..c2a33e67 100644 --- a/internal/query/funnel.go +++ b/internal/query/funnel.go @@ -46,6 +46,14 @@ type FunnelOptions struct { // — steps already matched are kept, nothing further counts. Events // at or before the step-0 event never disqualify. Exclusions []FunnelStep + // CohortID, when non-empty, restricts the funnel to events whose + // distinct_id is a member of the cohort (same membership resolution + // as the stats routes' cohort_id filter, applied Go-side on the + // streamed rows; see cohort_filter.go). Events with no distinct_id + // never match, so the funnel counts identified activity only. A + // missing cohort is 404, an oversized one 422, a failed resolution + // 503 - never an unfiltered funnel. + CohortID string } // funnelEvent is the minimal event data needed for funnel computation. @@ -199,6 +207,10 @@ func (s *StatsService) FunnelWithOptions(ctx context.Context, siteID string, fro return nil, err } defer finish() + members, err := s.resolveCohortSet(qctx, siteID, opts.CohortID) + if err != nil { + return nil, err + } walkers := make(map[string]*funnelWalker) err = s.streamEvents(qctx, siteID, from.UnixMilli(), to.UnixMilli(), "", func(row pgx.Row) error { @@ -206,6 +218,9 @@ func (s *StatsService) FunnelWithOptions(ctx context.Context, siteID string, fro if err != nil { return fmt.Errorf("funnel scan: %w", err) } + if members != nil && !members.has(e.DistinctID) { + return nil + } id, ok := entityKeyOf(entity, e) if !ok { return nil @@ -301,6 +316,10 @@ func (s *StatsService) FunnelByBreakdownWithOptions(ctx context.Context, siteID return nil, err } defer finish() + members, err := s.resolveCohortSet(qctx, siteID, opts.CohortID) + if err != nil { + return nil, err + } // Group walkers by (breakdown, entity); an entity can span different // breakdown values in theory; keep the first seen in stream order. @@ -312,6 +331,9 @@ func (s *StatsService) FunnelByBreakdownWithOptions(ctx context.Context, siteID if err != nil { return fmt.Errorf("funnel breakdown scan: %w", err) } + if members != nil && !members.has(e.DistinctID) { + return nil + } eid, ok := entityKeyOf(entity, e.funnelEvent) if !ok { return nil diff --git a/internal/query/retention.go b/internal/query/retention.go index ee1e4191..9f37c969 100644 --- a/internal/query/retention.go +++ b/internal/query/retention.go @@ -43,6 +43,12 @@ type RetentionOptions struct { // ReturnEvent, when non-empty, restricts return-activity bucketing // to matching event_types. Empty = any event. ReturnEvent string + // CohortID, when non-empty, restricts retention to events whose + // distinct_id is a member of the cohort (see FunnelOptions.CohortID + // for the shared semantics and error mapping). Entities with no + // member event are not cohort members, including sessions-rollup + // entries in the default visitor-estimate path. + CohortID string } // sessionFirstLast is the minimal session data for retention's default @@ -57,6 +63,9 @@ type sessionFirstLast struct { type retentionEntity struct { firstTS int64 // cohort entry instant (ms) buckets map[int64]bool // period buckets with qualifying activity + // inCohort is set when a cohort-member event was seen for the entity; + // only consulted when a cohort filter is active. + inCohort bool } // Retention computes cohort retention over the given time range on the @@ -116,6 +125,10 @@ func (s *StatsService) RetentionWithOptions(ctx context.Context, siteID string, return nil, err } defer finish() + members, err := s.resolveCohortSet(qctx, siteID, opts.CohortID) + if err != nil { + return nil, err + } fromMs := from.UnixMilli() toMs := to.UnixMilli() @@ -159,11 +172,15 @@ func (s *StatsService) RetentionWithOptions(ctx context.Context, siteID string, if err != nil { return fmt.Errorf("retention scan: %w", err) } + if members != nil && !members.has(r.DistinctID) { + return nil + } id, ok := entityKeyOf(entity, r) if !ok { return nil } e := getEntity(id) + e.inCohort = true isReturn := opts.ReturnEvent == "" || r.EventType == opts.ReturnEvent isEntry := opts.CohortEvent == "" || r.EventType == opts.CohortEvent // In the default visitor-estimate path, cohort membership comes @@ -186,7 +203,9 @@ func (s *StatsService) RetentionWithOptions(ctx context.Context, siteID string, // fired, or — impossible in practice — a sessions row without events) // are not cohort members. for id, e := range entities { - if e.firstTS < 0 { + // With a cohort filter, an entity pre-seeded from the sessions + // rollup that never produced a member event is not a member. + if e.firstTS < 0 || (members != nil && !e.inCohort) { delete(entities, id) } } diff --git a/internal/schema/migrations/060_cohort_members.up.sql b/internal/schema/migrations/060_cohort_members.up.sql new file mode 100644 index 00000000..b8f9023f --- /dev/null +++ b/internal/schema/migrations/060_cohort_members.up.sql @@ -0,0 +1,40 @@ +-- 060 (2026-10-03): C2 - static cohort membership. +-- +-- A rule cohort (023) re-evaluates its definition on every use. A static +-- cohort is an explicit list of entity ids (distinct_id values) the caller +-- supplied - a bulk create, a CSV import, or add/remove edits. Its cohorts +-- row carries the rule {"op":"static"}; the members live here. +-- +-- cohort_id, site_id the owning cohort. Every read filters on BOTH, so +-- a cohort id from another site matches nothing. +-- entity_id the member (a distinct_id), at most 256 bytes +-- added_at epoch ms the entity first joined this cohort; kept +-- across later versions of the same row +-- removed 0 = member, 1 = tombstone (member removed). Hard +-- DELETE is unreliable across replacing merges in +-- Nucleus, so removal is a newer version with +-- removed = 1 (same pattern as the cohorts tombstone) +-- version epoch ms of the mutation batch that wrote the row; +-- strictly increasing per cohort (the service takes +-- max(now, cohort.updated_at + 1)), so the highest +-- version per (cohort, entity) is the current state +-- +-- ReplacingMergeTree keyed on the member, versioned on version. The read +-- path collapses versions in Go (highest version wins) because read-time +-- dedup is unreliable (finding #10) and reads are bounded by a row LIMIT. +-- +-- Comments pure ASCII (038 rule). + +CREATE TABLE IF NOT EXISTS cohort_members ( + cohort_id TEXT NOT NULL, + tenant_id TEXT NOT NULL DEFAULT 'default', + site_id TEXT NOT NULL, + entity_id TEXT NOT NULL, + added_at BIGINT NOT NULL, + removed BIGINT NOT NULL DEFAULT 0, + version BIGINT NOT NULL DEFAULT 0 +) WITH ( + engine = 'replacing_mergetree', + version_column = 'version' +) +ORDER BY (tenant_id, site_id, cohort_id, entity_id); diff --git a/ui/src/api/persons.ts b/ui/src/api/persons.ts index 0a178d64..f40f1a45 100644 --- a/ui/src/api/persons.ts +++ b/ui/src/api/persons.ts @@ -52,9 +52,16 @@ export interface CohortRule { value?: string; } +// CohortDefinition is the stored rule. Three shapes share it: legacy flat +// ({op:"and", rules}), a tree ({op:"and|or|not", children} with {leaf} +// leaves, depth <= 4, <= 30 leaves) and static ({op:"static"}; the list +// lives server-side). The form editor only builds the flat-AND shape; trees +// are edited as JSON. export interface CohortDefinition { - op: "and"; - rules: CohortRule[]; + op: "and" | "or" | "not" | "static"; + rules?: CohortRule[]; + children?: CohortDefinition[]; + leaf?: CohortRule; } export interface Cohort { @@ -85,11 +92,19 @@ export function parseRule(raw: string): CohortDefinition { if (!raw) return { op: "and", rules: [] }; try { const p = JSON.parse(raw); - if (p && p.op === "and" && Array.isArray(p.rules)) return p; + if (p && (p.op === "and" || p.op === "or" || p.op === "not" || p.op === "static" || p.leaf)) { + return p; + } } catch { /* fall through */ } return { op: "and", rules: [] }; } +// isFlatAnd reports whether a definition is the legacy shape the form +// editor can round-trip. +export function isFlatAnd(def: CohortDefinition): boolean { + return def.op === "and" && !def.children?.length && !def.leaf; +} + // --------------------------------------------------------------------------- // Persons API // --------------------------------------------------------------------------- @@ -134,6 +149,14 @@ export const cohortsApi = { post(`${BASE}/cohorts/${cohortId}/refresh?site_id=${encodeURIComponent(siteId)}`, {}), preview: (siteId: string, rule: CohortDefinition) => post(`${BASE}/cohorts/preview`, { site_id: siteId, rule }), + // Static cohorts (list-backed). Bodies are bounded server-side (8 MB, + // 100k distinct ids); the server answers 422 past the caps. + createStatic: (data: { site_id: string; name: string; description?: string; ids: string[] }) => + post(`${BASE}/cohorts/static`, data), + addMembers: (cohortId: string, siteId: string, ids: string[]) => + post<{ cohort: Cohort; changed: number }>(`${BASE}/cohorts/${cohortId}/members`, { site_id: siteId, ids }), + removeMembers: (cohortId: string, siteId: string, ids: string[]) => + post<{ cohort: Cohort; changed: number }>(`${BASE}/cohorts/${cohortId}/members/remove`, { site_id: siteId, ids }), members: (cohortId: string, siteId: string, opts?: { limit?: number; offset?: number }) => { let q = `site_id=${encodeURIComponent(siteId)}`; if (opts?.limit) q += `&limit=${opts.limit}`; diff --git a/ui/src/lib/cohortRule.test.ts b/ui/src/lib/cohortRule.test.ts new file mode 100644 index 00000000..d67f348d --- /dev/null +++ b/ui/src/lib/cohortRule.test.ts @@ -0,0 +1,83 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + validateRuleJson, describeRule, parseIdList, + MAX_TREE_LEAVES, MAX_STATIC_IDS, +} from "./cohortRule.ts"; + +const ev = (name: string) => ({ leaf: { type: "event", name } }); + +test("legacy flat and rule is valid", () => { + const r = validateRuleJson(JSON.stringify({ + op: "and", + rules: [{ type: "event", name: "a" }, { type: "property", key: "country", value: "US" }], + })); + assert.equal(r.ok, true); +}); + +test("or / not trees are valid", () => { + const r = validateRuleJson(JSON.stringify({ + op: "and", + children: [ev("a"), { op: "not", children: [{ op: "or", children: [ev("b"), ev("c")] }] }], + })); + assert.equal(r.ok, true); +}); + +test("rejects bad shapes", () => { + const bad: Array<[string, unknown]> = [ + ["not json", "{"], + ["empty group", { op: "or", children: [] }], + ["not with two", { op: "not", children: [ev("a"), ev("b")] }], + ["unknown op", { op: "xor", children: [ev("a")] }], + ["mixed", { op: "and", rules: [{ type: "event", name: "a" }], children: [ev("b")] }], + ["bad key", { leaf: { type: "property", key: "password", value: "x" } }], + ["bad window", { leaf: { type: "event", name: "a", window: "soon" } }], + ["static", { op: "static" }], + ]; + for (const [name, v] of bad) { + const r = validateRuleJson(typeof v === "string" ? v : JSON.stringify(v)); + assert.equal(r.ok, false, name); + } +}); + +test("depth limit is 4 levels", () => { + const nest = (n: number): unknown => { + let d: unknown = ev("x"); + for (let i = 0; i < n; i++) d = { op: "and", children: [d] }; + return d; + }; + assert.equal(validateRuleJson(JSON.stringify(nest(3))).ok, true); + assert.equal(validateRuleJson(JSON.stringify(nest(4))).ok, false); +}); + +test("leaf limit is 30", () => { + const kids = (n: number) => ({ op: "or", children: Array.from({ length: n }, (_, i) => ev("e" + i)) }); + assert.equal(validateRuleJson(JSON.stringify(kids(MAX_TREE_LEAVES))).ok, true); + assert.equal(validateRuleJson(JSON.stringify(kids(MAX_TREE_LEAVES + 1))).ok, false); +}); + +test("describeRule renders trees", () => { + const lines = describeRule({ + op: "and", + children: [{ leaf: { type: "event", name: "a" } }, { op: "not", children: [{ leaf: { type: "property", key: "country", value: "US" } }] }], + }); + assert.deepEqual(lines, [ + "AND", + ' did "a" at least 1 time(s) in 30d', + " NOT", + " country = US", + ]); + assert.deepEqual(describeRule({ op: "static" }), ["static member list"]); +}); + +test("parseIdList handles csv, header, blanks, dups", () => { + const r = parseIdList("distinct_id,name\r\nu1,A\r\n\r\n\"u2\",B\nu1\n u3 \n"); + assert.deepEqual(r.ids, ["u1", "u2", "u3"]); + assert.equal(r.error, undefined); +}); + +test("parseIdList enforces caps", () => { + assert.ok(parseIdList("x".repeat(257)).error); + const big = Array.from({ length: MAX_STATIC_IDS + 1 }, (_, i) => "u" + i).join("\n"); + assert.ok(parseIdList(big).error); +}); diff --git a/ui/src/lib/cohortRule.ts b/ui/src/lib/cohortRule.ts new file mode 100644 index 00000000..023f118e --- /dev/null +++ b/ui/src/lib/cohortRule.ts @@ -0,0 +1,138 @@ +// Cohort rule helpers (pure logic, no DOM): client-side mirror of the server's +// rule-tree validation (internal/cohorts/tree.go) for the JSON editor, plus +// list parsing for static-cohort import. The server stays the authority; this +// only gives fast feedback before a round trip. + +export const MAX_TREE_DEPTH = 4; +export const MAX_TREE_LEAVES = 30; +export const MAX_STATIC_IDS = 100000; +export const MAX_ENTITY_ID_LEN = 256; + +const PROPERTY_KEYS = [ + "country", "region", "city", + "browser", "os", "device", "language", + "utm_source", "utm_medium", "utm_campaign", + "pathname", "referrer", +]; + +export interface RuleLeaf { + type: string; + name?: string; + window?: string; + min_count?: number; + key?: string; + operator?: string; + value?: string; +} + +export interface RuleNode { + op?: string; + rules?: RuleLeaf[]; + children?: RuleNode[]; + leaf?: RuleLeaf; +} + +export type RuleCheck = { ok: true; def: RuleNode } | { ok: false; error: string }; + +function checkLeaf(r: RuleLeaf): string | null { + if (!r || typeof r !== "object") return "a condition must be an object"; + if (r.type === "event") { + if (!r.name || !r.name.trim()) return "event condition requires name"; + if (r.window && !/^[0-9]{1,4}[dhm]$/i.test(r.window.trim())) { + return `window "${r.window}" must look like 24h, 7d or 90m`; + } + return null; + } + if (r.type === "property") { + if (!r.key) return "property condition requires key"; + if (!PROPERTY_KEYS.includes(r.key)) return `property key "${r.key}" is not allowed`; + if (r.operator && r.operator !== "=" && r.operator !== "!=") { + return `operator "${r.operator}" is not supported (use = or !=)`; + } + return null; + } + return `unsupported condition type "${r.type}" (event or property)`; +} + +// Returns an error string or null. `depth` counts nodes from the root. +function checkNode(n: RuleNode, depth: number, count: { leaves: number }): string | null { + if (!n || typeof n !== "object") return "a node must be an object"; + if (depth > MAX_TREE_DEPTH) return `nesting deeper than ${MAX_TREE_DEPTH} levels`; + if (n.leaf) { + if (n.op || n.rules?.length || n.children?.length) return "a leaf node carries no op, rules or children"; + count.leaves++; + return checkLeaf(n.leaf); + } + if (n.rules?.length && n.children?.length) return "use either rules or children, not both"; + const op = n.op || "and"; + const kids: Array = []; + for (const r of n.rules || []) kids.push({ leaf: r }); + for (const c of n.children || []) kids.push(c); + if (op !== "and" && op !== "or" && op !== "not") return `unknown op "${op}" (and, or, not)`; + if (op === "not" && kids.length !== 1) return `"not" takes exactly one child, got ${kids.length}`; + if (kids.length === 0) return `"${op}" group needs at least one child`; + for (const k of kids) { + const err = checkNode(k as RuleNode, depth + 1, count); + if (err) return err; + } + return null; +} + +// validateRuleJson parses and checks a rule written as JSON text. +export function validateRuleJson(text: string): RuleCheck { + let def: RuleNode; + try { + def = JSON.parse(text); + } catch (e) { + return { ok: false, error: `Invalid JSON: ${(e as Error).message}` }; + } + if (def && (def as RuleNode).op === "static") { + return { ok: false, error: "static cohorts are created from a member list, not a rule" }; + } + const count = { leaves: 0 }; + const err = checkNode(def, 1, count); + if (err) return { ok: false, error: err }; + if (count.leaves > MAX_TREE_LEAVES) { + return { ok: false, error: `${count.leaves} conditions, limit ${MAX_TREE_LEAVES}` }; + } + return { ok: true, def }; +} + +// describeRule renders a rule tree as indented human-readable lines. +export function describeRule(def: RuleNode, indent = 0): string[] { + const pad = " ".repeat(indent); + const leafText = (r: RuleLeaf) => + r.type === "event" + ? `did "${r.name}" at least ${r.min_count ?? 1} time(s) in ${r.window || "30d"}` + : `${r.key} ${r.operator || "="} ${r.value || ""}`; + if (def.op === "static") return [`${pad}static member list`]; + if (def.leaf) return [pad + leafText(def.leaf)]; + const op = (def.op || "and").toUpperCase(); + const lines = [`${pad}${op}`]; + for (const r of def.rules || []) lines.push(`${pad} ${leafText(r)}`); + for (const c of def.children || []) lines.push(...describeRule(c, indent + 1)); + return lines; +} + +// parseIdList splits pasted text or CSV (first column) into distinct trimmed +// ids, skipping a header row and blanks. Mirrors cohorts.ParseCSVIDs. +export function parseIdList(text: string): { ids: string[]; error?: string } { + const seen = new Set(); + const ids: string[] = []; + const lines = text.replace(/^/, "").split(/\r?\n/); + let first = true; + for (const line of lines) { + let cell = line.split(",")[0].trim(); + if (cell.startsWith('"') && cell.endsWith('"') && cell.length >= 2) cell = cell.slice(1, -1); + if (first) { + first = false; + if (["id", "ids", "entity_id", "distinct_id", "user_id", "person"].includes(cell.toLowerCase())) continue; + } + if (!cell || seen.has(cell)) continue; + if (cell.length > MAX_ENTITY_ID_LEN) return { ids: [], error: `an id is longer than ${MAX_ENTITY_ID_LEN} characters` }; + if (seen.size >= MAX_STATIC_IDS) return { ids: [], error: `more than ${MAX_STATIC_IDS.toLocaleString()} distinct ids` }; + seen.add(cell); + ids.push(cell); + } + return { ids }; +} diff --git a/ui/src/routes/cohorts.tsx b/ui/src/routes/cohorts.tsx index 4a06b483..c8782add 100644 --- a/ui/src/routes/cohorts.tsx +++ b/ui/src/routes/cohorts.tsx @@ -1,9 +1,10 @@ import { useState, useEffect, useCallback, useMemo } from "preact/hooks"; -import { cohortsApi, parseRule } from "../api/persons.js"; +import { cohortsApi, parseRule, isFlatAnd } from "../api/persons.js"; import type { Cohort, CohortDefinition, CohortRule } from "../api/persons.js"; import Modal from "../components/shared/Modal.js"; import EmptyState from "../components/shared/EmptyState.js"; import { useFilters } from "../hooks/useFilters.js"; +import { validateRuleJson, describeRule, parseIdList } from "../lib/cohortRule.js"; export const config = { mode: "app" }; @@ -25,7 +26,8 @@ function fmtDate(ms: number): string { } // --------------------------------------------------------------------------- -// Rule editor — builds CohortDefinition (op="and"; v1 doesn't expose OR / nesting) +// Rule editor — builds the flat-AND CohortDefinition; OR / NOT / nesting are +// edited as JSON (see the JSON toggle in CohortBuilder). // --------------------------------------------------------------------------- function emptyEventRule(): CohortRule { @@ -146,26 +148,63 @@ function CohortBuilder({ open, onClose, onSave, siteId, initial }: const [name, setName] = useState(initial?.name || ""); const [description, setDescription] = useState(initial?.description || ""); const [rules, setRules] = useState( - initial ? parseRule(initial.rule).rules : [emptyEventRule()] + initial ? (parseRule(initial.rule).rules || []) : [emptyEventRule()] ); const [previewCount, setPreviewCount] = useState(null); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); + // JSON mode edits the full rule tree (OR / NOT / nesting). It starts on + // for a saved cohort whose rule the form cannot represent. + const startsJson = (c?: Cohort | null) => !!c && !isFlatAnd(parseRule(c.rule)); + const [jsonMode, setJsonMode] = useState(startsJson(initial)); + const [jsonText, setJsonText] = useState( + initial ? JSON.stringify(parseRule(initial.rule), null, 2) : "" + ); // Reset state when re-opened with a different cohort. useEffect(() => { if (!open) return; setName(initial?.name || ""); setDescription(initial?.description || ""); - setRules(initial ? parseRule(initial.rule).rules : [emptyEventRule()]); + setRules(initial ? (parseRule(initial.rule).rules || []) : [emptyEventRule()]); + setJsonMode(startsJson(initial)); + setJsonText(initial ? JSON.stringify(parseRule(initial.rule), null, 2) : ""); setPreviewCount(null); setError(null); }, [open, initial]); - const def = useMemo(() => ({ op: "and", rules }), [rules]); + const flatDef = useMemo(() => ({ op: "and", rules }), [rules]); + + // currentDef returns the definition to send, or null with `error` set when + // the JSON text does not pass the client-side check (the server re-checks). + const currentDef = (): CohortDefinition | null => { + if (!jsonMode) return flatDef; + const r = validateRuleJson(jsonText); + if (!r.ok) { setError(r.error); return null; } + return r.def as CohortDefinition; + }; + + const toggleJson = () => { + setError(null); + if (!jsonMode) { + setJsonText(JSON.stringify(flatDef, null, 2)); + setJsonMode(true); + return; + } + // Back to the form only when the JSON is the flat-AND shape. + const r = validateRuleJson(jsonText); + if (r.ok && isFlatAnd(r.def as CohortDefinition)) { + setRules(((r.def as CohortDefinition).rules || []) as CohortRule[]); + setJsonMode(false); + } else { + setError("This rule uses OR / NOT / nesting and can only be edited as JSON."); + } + }; const preview = async () => { setError(null); + const def = currentDef(); + if (!def) return; try { const r = await cohortsApi.preview(siteId, def); setPreviewCount(r.count); @@ -177,7 +216,9 @@ function CohortBuilder({ open, onClose, onSave, siteId, initial }: const save = async () => { setError(null); if (!name.trim()) { setError("Name required"); return; } - if (rules.length === 0) { setError("Add at least one condition"); return; } + if (!jsonMode && rules.length === 0) { setError("Add at least one condition"); return; } + const def = currentDef(); + if (!def) return; setSaving(true); try { const saved = initial @@ -209,8 +250,21 @@ function CohortBuilder({ open, onClose, onSave, siteId, initial }:
- - + + {jsonMode ? ( +