diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28513cb0..b4872bd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -441,6 +441,70 @@ jobs: - name: Validate observability/rpc-alerts.yml run: ./promtool check rules observability/rpc-alerts.yml + - name: Download amtool + run: | + AM_VERSION="0.27.0" + curl -sSfL \ + "https://github.com/prometheus/alertmanager/releases/download/v${AM_VERSION}/alertmanager-${AM_VERSION}.linux-amd64.tar.gz" \ + | tar xz --strip-components=1 "alertmanager-${AM_VERSION}.linux-amd64/amtool" + chmod +x amtool + ./amtool --version + + # Issue #526: the routing tree is only half the requirement — an alert + # must actually reach a person. These checks fail the build on the two + # ways that silently breaks: a receiver that resolves nowhere, and a + # receiver defined with no delivery integration at all (which + # Alertmanager accepts and then discards every notification through). + - name: Validate monitoring/alertmanager.yml + env: + # Deploy-time values. Alertmanager only requires a well-formed URL + # and a non-empty routing key to validate, so these are deliberately + # not shaped like real credentials: a hooks.slack.com placeholder + # trips GitHub push protection, which is the correct behaviour for + # anything matching that pattern. + PAGERDUTY_ROUTING_KEY: example-routing-key-not-a-real-credential + SLACK_WEBHOOK_URL: https://alertmanager-config-check.invalid/webhook-placeholder + run: | + set -euo pipefail + envsubst < monitoring/alertmanager.yml > /tmp/alertmanager.rendered.yml + ./amtool check-config /tmp/alertmanager.rendered.yml + + # Every alert severity/service combination alerts.yml can emit must + # resolve to the on-call receiver, not the null sink. + for args in \ + "service=indexer severity=critical on-call-critical" \ + "service=indexer severity=warning on-call-warning" \ + "service=api severity=critical on-call-critical"; do + set -- $args + actual=$(./amtool config routes test \ + --config.file=/tmp/alertmanager.rendered.yml "$1" "$2") + if [ "$actual" != "$3" ]; then + echo "::error::$1 $2 routed to '$actual', expected '$3'" + exit 1 + fi + echo "OK: $1 $2 -> $actual" + done + + # A receiver with no *_configs is valid YAML and a silent black + # hole. Assert the two on-call receivers each define at least one. + python3 - <<'PY' + import sys, yaml + cfg = yaml.safe_load(open("/tmp/alertmanager.rendered.yml")) + failed = False + for r in cfg.get("receivers", []): + name = r.get("name") + if name == "default-null": + continue + integrations = [k for k in r if k.endswith("_configs") and r[k]] + if not integrations: + print(f"::error::receiver '{name}' has no delivery integration — " + "it would accept and discard every alert") + failed = True + else: + print(f"OK: receiver '{name}' delivers via {', '.join(integrations)}") + sys.exit(1 if failed else 0) + PY + # --------------------------------------------------------------------------- # Rust integration tests — real Postgres + Redis # Runs integration tests that are silently skipped when TEST_DATABASE_URL @@ -1068,6 +1132,72 @@ jobs: # Environment reference — fails if code reads an env var undocumented in # docs/ENVIRONMENT.md (issue #312). # --------------------------------------------------------------------------- + # --------------------------------------------------------------------------- + # Launch gate (issue #503) — keeps the go/no-go check mechanical. + # + # This deliberately does NOT fail the build on a NO-GO verdict. The gate is + # unexecuted (every Pass/Fail cell blank) and correctly reports NO-GO today, + # so enforcing the verdict here would red every PR until launch day and get + # the job disabled. What IS enforced is that the checker keeps working: the + # table stays parseable, and the script's own GO/NO-GO branches still behave. + # A gate that silently stopped parsing its own table would otherwise report + # a clean NO-GO for the wrong reason. + # + # Run `bash scripts/check-launch-gate.sh` for the real verdict at launch. + # --------------------------------------------------------------------------- + launch-gate: + name: Launch gate checker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Shell syntax + run: bash -n scripts/check-launch-gate.sh + + - name: Checklist table is still parseable + run: | + set -euo pipefail + # 2>&1 matters: the NO-GO verdict is written to stderr (it is the + # failure path), so capturing stdout alone loses the RESULT line the + # assertions below look for. + out=$(bash scripts/check-launch-gate.sh 2>&1 || true) + echo "$out" + if echo "$out" | grep -q "no gate rows found"; then + echo "::error::the launch checklist table is no longer parseable by check-launch-gate.sh" + exit 1 + fi + if echo "$out" | grep -q "no gate row matching 'Rollback rehearsed'"; then + echo "::error::the rollback rehearsal row was renamed; update the match in check-launch-gate.sh" + exit 1 + fi + echo "$out" | grep -qE '^RESULT: (GO|NO-GO)' \ + || { echo "::error::script produced no GO/NO-GO verdict"; exit 1; } + + # Exercises both verdicts against synthetic checklists so a regression in + # the parser cannot pass by always answering NO-GO. + - name: Verdict branches behave + run: | + set -euo pipefail + tmp=$(mktemp -d) + hdr=$'| # | Gate | Pass/Fail | Evidence | Signed off by |\n|---|---|---|---|---|' + today=$(date +%Y-%m-%d) + stale=$(date -d '60 days ago' +%Y-%m-%d) + + printf '%s\n| 1 | Alerts | Pass | ev | carol |\n| 2 | Rollback rehearsed | Pass | drill %s | dave |\n' \ + "$hdr" "$today" > "$tmp/pass.md" + printf '%s\n| 1 | Alerts | Fail | ev | carol |\n| 2 | Rollback rehearsed | Pass | drill %s | dave |\n' \ + "$hdr" "$today" > "$tmp/fail.md" + printf '%s\n| 1 | Alerts | Pass | ev | carol |\n| 2 | Rollback rehearsed | Pass | drill %s | dave |\n' \ + "$hdr" "$stale" > "$tmp/stale.md" + + bash scripts/check-launch-gate.sh "$tmp/pass.md" \ + || { echo "::error::a fully-passing checklist must exit 0"; exit 1; } + ! bash scripts/check-launch-gate.sh "$tmp/fail.md" \ + || { echo "::error::an explicit Fail row must exit non-zero"; exit 1; } + ! bash scripts/check-launch-gate.sh "$tmp/stale.md" \ + || { echo "::error::a >30-day-old rollback rehearsal must exit non-zero"; exit 1; } + echo "All verdict branches behaved as expected." + env-reference: name: Env reference runs-on: ubuntu-latest @@ -1303,8 +1433,20 @@ jobs: go-version: "1.25" cache-dependency-path: services/api/go.sum + # Issue #516: the API-key lifecycle tests (issue/rotate/revoke against a + # real Redis-cached auth path) skip themselves unless these are set — + # this job is the only one with Postgres and Redis attached, so without + # them the lifecycle suite silently never runs anywhere in CI. + # REQUIRE_TEST_SERVICES turns a missing service into a hard failure + # rather than a green run that tested nothing. - name: Go coverage working-directory: services/api + env: + DATABASE_URL: postgres://postgres:trident@localhost:5432/trident_test + TEST_DATABASE_URL: postgres://postgres:trident@localhost:5432/trident_test + TEST_REDIS_URL: redis://localhost:6379 + REQUIRE_TEST_SERVICES: "1" + CI: "true" run: | go test ./... -coverprofile=coverage.out go tool cover -func=coverage.out | tee go-coverage.txt @@ -1315,6 +1457,12 @@ jobs: # variation does not flap the build. - name: Enforce Go critical-package floors working-directory: services/api + env: + DATABASE_URL: postgres://postgres:trident@localhost:5432/trident_test + TEST_DATABASE_URL: postgres://postgres:trident@localhost:5432/trident_test + TEST_REDIS_URL: redis://localhost:6379 + REQUIRE_TEST_SERVICES: "1" + CI: "true" run: | set -euo pipefail check() { diff --git a/.github/workflows/explorer-build.yml b/.github/workflows/explorer-build.yml new file mode 100644 index 00000000..9527a612 --- /dev/null +++ b/.github/workflows/explorer-build.yml @@ -0,0 +1,96 @@ +name: Explorer Build + +# Issue #520: "wire it into CI so it breaks loudly when the API changes." +# explorer-perf.yml already exercises explorer/ against a real testnet API, +# but only on a weekly schedule/manual dispatch, and it never actually runs +# `npm run build` — it discovered the explorer's build had been broken +# outright (a duplicated template block in one page, since fixed) only +# because someone ran `npm run build` by hand. This job runs the real +# production build on every push/PR that touches explorer/, so a build +# break (from an API/type change, a bad merge, or anything else) is a red +# CI check within minutes, not something waiting to be found by hand. + +on: + push: + branches: [main, dev] + paths: + - "explorer/**" + - "sdk/typescript/**" + - ".github/workflows/explorer-build.yml" + pull_request: + paths: + - "explorer/**" + - "sdk/typescript/**" + - ".github/workflows/explorer-build.yml" + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + # Astro 7 refuses to run on anything below 22.12 ("Node.js v20.x is + # not supported by Astro"). Pinned at 20, this job failed before it + # compiled a line — the workflow added to catch a broken explorer + # build could not itself run. + node-version: "22.12" + + - name: Build the TypeScript SDK + working-directory: sdk/typescript + run: | + npm install + npm run build + + # explorer/ depends on the SDK via a local `file:../sdk/typescript` + # dependency (the SDK is not yet published — see #517/#429, blocked on + # #512), so the SDK must be built before `npm install` in explorer/ + # resolves and links it. + - name: Install explorer dependencies + working-directory: explorer + run: npm install --legacy-peer-deps + + - name: Type-check + working-directory: explorer + # astro check has pre-existing, unrelated failures in + # scripts/a11y-test.ts and scripts/perf-test.ts (tracked + # separately) — this job checks the build, which is the specific + # "breaks loudly" signal #520 asks for; a full green astro check + # across the whole package is a separate concern. + run: npm run build + + - name: Verify the built server actually starts + working-directory: explorer + env: + TRIDENT_TESTNET_API_URL: https://api.testnet.trident.dev + TRIDENT_MAINNET_API_URL: https://api.mainnet.trident.dev + EXPLORER_API_KEY: ci-smoke-test-key + PORT: 4321 + run: | + node dist/server/entry.mjs & + SERVER_PID=$! + for i in $(seq 1 20); do + # localhost, not 127.0.0.1: @astrojs/node binds the hostname and + # on a dual-stack runner that can be ::1 only, so the v4 literal + # never connects. + # + # No --fail, and any HTTP status counts. This step verifies the + # built server boots and serves — not that the page renders + # successfully. Rendering "/" calls the real testnet API with a + # dummy key, so a non-2xx here is the expected, correct outcome; + # requiring 2xx made the check depend on a live external service. + code=$(curl --silent --show-error --max-time 2 -o /dev/null -w '%{http_code}' \ + "http://localhost:4321/" 2>/dev/null || echo "000") + if [ "$code" != "000" ]; then + echo "Server responded with HTTP $code" + kill "$SERVER_PID" + exit 0 + fi + sleep 0.5 + done + echo "Server did not respond within the timeout" >&2 + kill "$SERVER_PID" 2>/dev/null || true + exit 1 diff --git a/docs/LAUNCH_CHECKLIST.md b/docs/LAUNCH_CHECKLIST.md index 468b42d5..e155fa8f 100644 --- a/docs/LAUNCH_CHECKLIST.md +++ b/docs/LAUNCH_CHECKLIST.md @@ -1,4 +1,4 @@ -# Pre-launch verification checklist +# Pre-launch verification checklist (MVP go/no-go gate) **Status: template — not yet executed.** This is the checklist structure issue #459 asks for; running it against production configuration with @@ -7,6 +7,27 @@ infrastructure access and team availability, which this pass doesn't have. Filling in "Evidence" and "Signed off by" for each row, against production config, is what turns this from a template into a completed launch gate. +## Enforcement + +This table is the blocking set for testnet launch (issue #503) — every row +below is a launch blocker, distinct from the ~50 other open launch issues that +are not. Each row's Pass/Fail column is meant to be an objective, checkable +fact rather than an opinion, and `scripts/check-launch-gate.sh` enforces that +mechanically instead of relying on someone reading the table carefully: + +```bash +scripts/check-launch-gate.sh # checks docs/LAUNCH_CHECKLIST.md +scripts/check-launch-gate.sh path/to.md # or an explicit path +``` + +It fails (exit 1) if any row's Pass/Fail column is blank, unrecognized, or +`Fail`, if a row marked `Pass` is missing Evidence or a Signed-off-by name, or +if the rollback rehearsal (row 9) has no dated evidence or is older than the +30-day limit below. It exits 0 only when the table itself says every gate is +satisfied. Run it locally before any go/no-go call; see the script's header +comment for what it deliberately does not check (truthfulness of the Evidence +text, and open P1/P2 incidents — both still require a human). + | # | Gate | Pass/Fail | Evidence | Signed off by | |---|------|-----------|----------|----------------| | 1 | Alerts verified firing (trigger each alert deliberately, confirm on-call receives it) | | | | diff --git a/docs/runbooks/alerts.md b/docs/runbooks/alerts.md index e63c621f..662dbf9a 100644 --- a/docs/runbooks/alerts.md +++ b/docs/runbooks/alerts.md @@ -4,7 +4,10 @@ One section per alert in [`monitoring/alerts.yml`](../../monitoring/alerts.yml). Each section covers what the alert means, why its threshold was picked, and the first steps to take when it fires. See [`docs/metrics-catalog.md`](../metrics-catalog.md) for what every metric -referenced here actually measures. +referenced here actually measures. Routing (which severity/service pages +whom) is configured in [`monitoring/alertmanager.yml`](../../monitoring/alertmanager.yml) — +"page on-call" below means whatever's wired into that file's +`on-call-critical`/`on-call-warning` receivers. **Related runbooks:** - [`incident-response.md`](incident-response.md) — severity classification (SEV-1/2/3), on-call owner, escalation path, and user communication channel. diff --git a/docs/runbooks/chaos-drill-findings.md b/docs/runbooks/chaos-drill-findings.md new file mode 100644 index 00000000..2cb27e9e --- /dev/null +++ b/docs/runbooks/chaos-drill-findings.md @@ -0,0 +1,150 @@ +# Chaos drill findings (issue #499) + +## Status: harness verified, staging run not performed + +Issue #499 asks for `load-tests/chaos-launch.sh` (built for #439) to be run +against staging for Postgres, Redis, and RPC faults, with real observed +degradation behavior recorded. That could not be done from this environment: + +- Staging is a Kubernetes deployment reached only through CI secrets + (`STAGING_KUBECONFIG`, `STAGING_URL`, `STAGING_DATABASE_URL` — see + `.github/workflows/staging-deploy.yml`), none of which are available here. +- The harness itself is compose-native (`docker compose stop/pause/exec` + against `docker/docker-compose.yml`'s service names), so it cannot target a + Kubernetes staging deployment as written without a rewrite to `kubectl` + equivalents (see "Running against Kubernetes staging" below) — it is built + and documented for a compose-backed environment. +- A local compose dry run was also not possible: this environment has the + Docker CLI but no reachable Docker daemon (`docker info` fails to connect + to the daemon socket), so even `docker compose up` against the existing + local `docker/docker-compose.yml` could not be exercised here. + +Rather than claim a run that didn't happen, this documents what was actually +done: a full static verification of the harness's logic against the real +`/v1/ready` contract and the production compose topology, which surfaced two +genuine gaps worth filing as follow-ups, plus the exact procedure for someone +with staging or local Docker access to execute the real drill. + +## What was verified + +1. **Shell correctness.** `bash -n load-tests/chaos-launch.sh` and + `shellcheck load-tests/chaos-launch.sh` both pass clean — no syntax errors, + no shellcheck warnings. + +2. **Readiness contract cross-check.** Read + `services/api/handlers/health.go` (`Ready` handler) directly against the + harness's assertions: + - `GET /v1/ready` returns `503` if any of Postgres, Redis, or the gRPC + backend check fails, `200` only if all three pass. `expect_degraded` + (harness line 62) and `expect_healthy` (line 72) match this exactly: + 503 during a fault is a pass, 200 during a fault is a failure, and a + hang (curl's `000` from `--max-time 10`) is a failure either way rather + than being mistaken for either state. + - Redis is checked with a plain `Ping` (`checkRedis`, health.go:186) with + no dependency on cached keys existing, so `FLUSHDB` correctly leaves + `/v1/ready` at 200 — the harness's `run_redis_evicting` scenario + (chaos-launch.sh:124) asserting `expect_healthy` "during" the flush, + not `expect_degraded`, matches the handler's actual behavior rather than + the more intuitive-but-wrong assumption that clearing the cache should + look unhealthy. + - No RPC/gRPC dependency is reachable from a stubbed local run, so the + gRPC-down path (`checkGRPC`, health.go:193, a real `ListEvents` call) + was reasoned through code rather than exercised. + +3. **Compose topology cross-check** (`docker/docker-compose.yml`). This is + where the two findings below came from — see "Findings to file as + follow-up issues". + +## Findings to file as follow-up issues + +### Finding 1 — the harness never faults PgBouncer itself + +Every service (`indexer`, `api`) connects to `pgbouncer:6432`, never to +`postgres:5432` directly (`docker-compose.yml:91,129`, both comments say +"Route through PgBouncer, never postgres directly"). `chaos-launch.sh`'s +`POSTGRES_SERVICE` defaults to `postgres` and every Postgres scenario +(`postgres-down`, `postgres-slow`) stops or pauses the `postgres` container, +which is one hop upstream of what the application actually talks to. + +This is a real gap, not just a naming nit: PgBouncer sits in front of +Postgres and is responsible for detecting a dead backend and either failing +fast or queuing, depending on its pool state — behavior the current +scenarios never exercise. `/v1/ready` failing when `postgres` is stopped only +proves PgBouncer eventually surfaces the outage to `checkPostgres`'s `Ping`; +it says nothing about how long PgBouncer takes to notice, whether it queues +client connections against a dead backend (which would show up as `/v1/ready` +hanging past `--max-time 10` rather than failing fast, indistinguishable in +the harness's output from a genuine timeout), or what happens if `pgbouncer` +itself is stopped/paused instead of the Postgres behind it — a distinct +failure mode (`api`/`indexer` lose their pooler, not their database) that has +no scenario at all today. + +**Suggested follow-up**: add a `pgbouncer-down`/`pgbouncer-slow` scenario +pair (stop/pause the `pgbouncer` service directly, same before/during/after +probe shape as the existing scenarios), and consider adding a +`PGBOUNCER_SERVICE` variable alongside `POSTGRES_SERVICE` so the existing +Postgres scenarios can optionally be run against the pooler layer too. + +### Finding 2 — recovery timing assumption doesn't account for PgBouncer's reconnect behavior + +`RECOVERY_SECONDS` defaults to 45s, and Postgres's own compose healthcheck +needs up to `start_period(15s) + retries(10) × interval(10s)` ≈ 115s in the +worst case to report `healthy` again after a restart +(`docker-compose.yml:13-18`). The harness's 45s default assumes the +application-level recovery (PgBouncer reconnecting to Postgres, `checkPostgres` +succeeding again) happens well before Postgres's own healthcheck would +declare it healthy — which is plausible (PgBouncer doesn't wait on compose's +healthcheck; it retries its own backend connection independently) but was +never actually measured, only assumed by the person who wrote #439's harness +and reused here. + +**Suggested follow-up**: when the harness is actually run (staging or local +compose with a working daemon), record the wall-clock time from +`postgres-down-during` to the first `postgres-down-after` probe that returns +200, and if it's ever close to or past `RECOVERY_SECONDS`, either raise the +default or make it configurable per-scenario rather than one shared value for +Postgres, Redis, and RPC recovery. + +## Running the real drill (for whoever has staging or local Docker access) + +### Against local compose (once a Docker daemon is reachable) + +```bash +docker compose -f docker/docker-compose.yml up -d --build +BASE_URL=http://localhost:3000 \ +COMPOSE_FILE=docker/docker-compose.yml \ +FAULT_SECONDS=30 \ +RECOVERY_SECONDS=45 \ +./load-tests/chaos-launch.sh +``` + +There is no local RPC container in `docker/docker-compose.yml`, so +`RPC_SERVICE` will be unset and the rpc-down/rpc-slow scenarios will be +skipped with the harness's own printed instructions for exercising them at +the network/provider layer instead (chaos-launch.sh:149-158). + +### Against staging + +The harness as written targets `docker compose` service names and cannot run +against a Kubernetes deployment unmodified. Two options, in order of +preference: + +1. **Point `BASE_URL` at staging, induce faults at the Kubernetes layer.** + Keep the harness's probe/assert logic (`probe`, `expect_degraded`, + `expect_healthy`) but replace the `compose stop/pause/exec` calls with + `kubectl scale --replicas=0` (Postgres/Redis down), + `kubectl exec ... -- redis-cli FLUSHDB` (Redis eviction), or a + network-policy/`kubectl exec ... -- tc` byte-delay for the "slow" variants. + This preserves the exact readiness assertions this script already encodes + against `/v1/ready`'s real contract, which is the part worth reusing. +2. **Run `docker compose` directly against staging's Postgres/Redis + connection strings** if staging genuinely runs the same compose stack + behind `STAGING_KUBECONFIG` (unclear from this repo alone — the deploy + workflow uses Helm, which implies Kubernetes, not compose, actually runs + in staging, making option 1 the realistic path). + +Either way: run it, record the produced `load-tests/chaos-results//` +directory's `summary.txt` and `probes.csv` (both gitignored — copy the +relevant numbers into this file or a dated section here rather than the raw +directory), and file every unexpected result as its own issue per the +harness's own printed review checklist and issue #499's "Done when". diff --git a/explorer/src/pages/contract/[address]/index.astro b/explorer/src/pages/contract/[address]/index.astro index 17770ce4..8499067e 100644 --- a/explorer/src/pages/contract/[address]/index.astro +++ b/explorer/src/pages/contract/[address]/index.astro @@ -93,6 +93,40 @@ if (!isValidContractId(address)) { address, }); } +} catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + + // The SDK (issue #520) throws TridentApiError with a real, structured + // .status field on any non-2xx response — the previous hand-rolled + // fetchWithTimeout instead encoded the status INTO the message + // ("API 404"), which is what the regex below used to parse back out. + // TridentApiError's message comes from the API's own JSON error + // envelope and is arbitrary server text, so it won't reliably contain + // "API " (or "not found"/"timeout" substrings classifyError also + // checks) — reading .status directly is both simpler and correct for + // both error shapes, since a plain Error just has it as undefined. + const statusCode = err instanceof TridentApiError ? String(err.status) : undefined; + + // Classify the error + let errorType = classifyError(error, statusCode ? Number(statusCode) : undefined); + + // Timeouts surface as TridentError{code:"TIMEOUT"} from the SDK's own + // per-attempt AbortController. The old hand-rolled client raised a bare + // DOMException named "AbortError" instead, so that shape is still checked + // for the client-side