From 55d1262921c0b2bf1da50dd05a401750299be3ec Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Thu, 30 Jul 2026 15:28:44 -0700 Subject: [PATCH 1/5] [COVAL-4319] Automate weekly CLI parity releases --- .github/workflows/api-parity-audit.yml | 107 ++++++++++ .github/workflows/ci.yml | 30 ++- .github/workflows/release-on-version-bump.yml | 97 +++++++++ .github/workflows/release.yml | 202 +++++++++++++----- .gitignore | 2 + README.md | 47 +++- api-coverage.toml | 10 +- scripts/audit_api_coverage.py | 189 ++++++++++++---- scripts/bump_version.py | 74 +++++++ scripts/release_version.py | 76 +++++++ scripts/render_homebrew_formula.py | 91 ++++++++ scripts/requirements-audit.txt | 2 + scripts/test_audit_api_coverage.py | 116 +++++++++- scripts/test_release_automation.py | 79 +++++++ 14 files changed, 1014 insertions(+), 108 deletions(-) create mode 100644 .github/workflows/api-parity-audit.yml create mode 100644 .github/workflows/release-on-version-bump.yml create mode 100644 scripts/bump_version.py create mode 100644 scripts/release_version.py create mode 100644 scripts/render_homebrew_formula.py create mode 100644 scripts/requirements-audit.txt create mode 100644 scripts/test_release_automation.py diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml new file mode 100644 index 0000000..b9585d6 --- /dev/null +++ b/.github/workflows/api-parity-audit.yml @@ -0,0 +1,107 @@ +name: Weekly API parity audit + +# The scheduled Codex job does the judgment-heavy command implementation and +# opens a reviewed PR. This workflow is the deterministic backstop: it runs the +# same live audit without credentials and keeps one GitHub issue open whenever +# the public API and first-class CLI command surface drift. + +on: + workflow_dispatch: + schedule: + # Monday 10:00 UTC, after the SDK regeneration job. + - cron: "0 10 * * 1" + +permissions: + contents: read + issues: write + +concurrency: + group: weekly-api-parity-audit + cancel-in-progress: false + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/requirements-audit.txt + + - name: Install pinned dependencies + run: python -m pip install --requirement scripts/requirements-audit.txt + + - name: Test the audit + run: python -m unittest scripts/test_audit_api_coverage.py + + - name: Compare first-class commands with the live public API + run: python scripts/audit_api_coverage.py + + - name: Open or update the parity failure issue + if: failure() + uses: actions/github-script@v7 + with: + script: | + const title = 'Weekly CLI API parity audit is failing'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + 'The scheduled CLI API parity audit failed.', + '', + `Run: ${runUrl}`, + '', + 'Until this is reconciled, the first-class CLI command coverage may have drifted from the published API.', + ].join('\n'); + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + }); + } + + - name: Close a recovered parity failure issue + if: success() + uses: actions/github-script@v7 + with: + script: | + const title = 'Weekly CLI API parity audit is failing'; + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: `Recovered in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`, + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed', + state_reason: 'completed', + }); + } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df0fa13..9b31ce7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,38 @@ on: env: CARGO_TERM_COLOR: always +permissions: + contents: read + jobs: check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/requirements-audit.txt + + - name: Install API audit dependencies + run: python -m pip install --requirement scripts/requirements-audit.txt + + - name: Test automation scripts + run: python -m unittest discover --start-directory scripts --pattern 'test_*.py' + + - name: Lint automation scripts + run: | + python -m ruff check scripts + python -m ruff format --check scripts + + - name: Audit live API command coverage + run: python scripts/audit_api_coverage.py + + - name: Validate release manifests + run: python scripts/release_version.py + - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -25,10 +51,10 @@ jobs: run: cargo fmt --check - name: Clippy - run: cargo clippy -- -D warnings + run: cargo clippy --all-targets --all-features -- -D warnings - name: Test - run: cargo test + run: cargo test --all-targets --all-features build: needs: check diff --git a/.github/workflows/release-on-version-bump.yml b/.github/workflows/release-on-version-bump.yml new file mode 100644 index 0000000..016dee0 --- /dev/null +++ b/.github/workflows/release-on-version-bump.yml @@ -0,0 +1,97 @@ +name: Release on version bump + +# Wait for the exact main-branch CI run to pass, then release an untagged Cargo +# version. The release workflow is called directly because a tag pushed with +# GITHUB_TOKEN does not trigger another workflow. + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + workflow_dispatch: + +permissions: + contents: write + issues: write + +concurrency: + group: release-on-version-bump + cancel-in-progress: false + +jobs: + prepare: + if: >- + github.event_name == 'workflow_dispatch' || + ( + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' + ) + runs-on: ubuntu-latest + outputs: + release: ${{ steps.tag.outputs.release }} + release_ref: ${{ steps.tag.outputs.release_ref }} + tag: ${{ steps.version.outputs.tag }} + steps: + - name: Resolve the CI-tested commit + id: ref + env: + WORKFLOW_RUN_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + else + echo "sha=$WORKFLOW_RUN_SHA" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@v6 + with: + ref: ${{ steps.ref.outputs.sha }} + fetch-depth: 0 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Read and validate the release version + id: version + run: python scripts/release_version.py --github-output "$GITHUB_OUTPUT" + + - name: Tag an unreleased version + id: tag + env: + RELEASE_SHA: ${{ steps.ref.outputs.sha }} + RELEASE_TAG: ${{ steps.version.outputs.tag }} + run: | + set -euo pipefail + git fetch --tags --force + + if git rev-parse -q --verify "refs/tags/$RELEASE_TAG" >/dev/null; then + existing_sha="$(git rev-list -n 1 "$RELEASE_TAG")" + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Manual recovery will rerun the existing ${RELEASE_TAG} release." + echo "release=true" >> "$GITHUB_OUTPUT" + echo "release_ref=$existing_sha" >> "$GITHUB_OUTPUT" + else + echo "${RELEASE_TAG} is already released; nothing to do." + echo "release=false" >> "$GITHUB_OUTPUT" + echo "release_ref=$existing_sha" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$RELEASE_TAG" "$RELEASE_SHA" -m "$RELEASE_TAG" + git push origin "$RELEASE_TAG" + echo "release=true" >> "$GITHUB_OUTPUT" + echo "release_ref=$RELEASE_SHA" >> "$GITHUB_OUTPUT" + + release: + needs: prepare + if: needs.prepare.outputs.release == 'true' + uses: ./.github/workflows/release.yml + with: + release_ref: ${{ needs.prepare.outputs.release_ref }} + tag_name: ${{ needs.prepare.outputs.tag }} + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b994217..33a8c11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,50 @@ name: Release +# A human-pushed tag can still invoke this workflow directly. The gated +# version-bump bridge calls it as a reusable workflow because tags pushed with +# GITHUB_TOKEN do not trigger a second workflow. + on: push: tags: - - 'v*' + - "v*" + workflow_call: + inputs: + release_ref: + description: Commit or tag to build + required: true + type: string + tag_name: + description: Existing v-prefixed release tag + required: true + type: string env: CARGO_TERM_COLOR: always + RELEASE_REF: ${{ inputs.release_ref || github.ref }} + RELEASE_TAG: ${{ inputs.tag_name || github.ref_name }} + +concurrency: + group: release-${{ inputs.tag_name || github.ref_name }} + cancel-in-progress: false jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ env.RELEASE_REF }} + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Match Cargo manifests and release tag + run: python scripts/release_version.py --expected-tag "$RELEASE_TAG" + build: + needs: validate strategy: matrix: include: @@ -38,6 +73,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + ref: ${{ env.RELEASE_REF }} - uses: dtolnay/rust-toolchain@stable with: @@ -77,8 +114,6 @@ jobs: contents: write steps: - - uses: actions/checkout@v6 - - name: Download artifacts uses: actions/download-artifact@v8 with: @@ -92,9 +127,11 @@ jobs: done cat -- */SHA256SUMS > ../SHA256SUMS - - name: Create release + - name: Create or update release uses: softprops/action-gh-release@v3 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.RELEASE_REF }} files: | artifacts/*/*.tar.gz SHA256SUMS @@ -105,15 +142,19 @@ jobs: runs-on: ubuntu-latest steps: - - name: Extract version from tag - id: version - run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@v6 + with: + ref: ${{ env.RELEASE_REF }} + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" - name: Download SHA256SUMS from release env: GH_TOKEN: ${{ github.token }} run: | - gh release download "$GITHUB_REF_NAME" \ + gh release download "$RELEASE_TAG" \ --repo "$GITHUB_REPOSITORY" \ --pattern SHA256SUMS @@ -130,54 +171,107 @@ jobs: - name: Update Homebrew tap env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} - VERSION: ${{ steps.version.outputs.version }} + VERSION: ${{ env.RELEASE_TAG }} SHA_MACOS_ARM64: ${{ steps.sha.outputs.macos_arm64 }} SHA_MACOS_X64: ${{ steps.sha.outputs.macos_x64 }} SHA_LINUX_ARM64: ${{ steps.sha.outputs.linux_arm64 }} SHA_LINUX_X64: ${{ steps.sha.outputs.linux_x64 }} run: | - git clone "https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/coval-ai/homebrew-tap.git" tap - cd tap - - cat > Formula/coval.rb << FORMULA - class Coval < Formula - desc "CLI for Coval AI agent evaluation platform" - homepage "https://coval.dev" - version "${VERSION}" - license "MIT" - - on_macos do - if Hardware::CPU.arm? - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-macos-arm64.tar.gz" - sha256 "${SHA_MACOS_ARM64}" - else - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-macos-x64.tar.gz" - sha256 "${SHA_MACOS_X64}" - end - end - - on_linux do - if Hardware::CPU.arm? - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-linux-arm64.tar.gz" - sha256 "${SHA_LINUX_ARM64}" - else - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-linux-x64.tar.gz" - sha256 "${SHA_LINUX_X64}" - end - end - - def install - bin.install "coval" - end - - test do - system "#{bin}/coval", "--version" - end - end - FORMULA - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add Formula/coval.rb - git commit -m "chore: update coval to ${VERSION}" - git push origin main + set -euo pipefail + if [ -z "$HOMEBREW_TAP_TOKEN" ]; then + echo "::error::HOMEBREW_TAP_TOKEN is not configured." + exit 1 + fi + + tap_dir="${RUNNER_TEMP}/homebrew-tap" + git clone "https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/coval-ai/homebrew-tap.git" "$tap_dir" + python scripts/render_homebrew_formula.py \ + --version "${VERSION#v}" \ + --macos-arm64 "$SHA_MACOS_ARM64" \ + --macos-x64 "$SHA_MACOS_X64" \ + --linux-arm64 "$SHA_LINUX_ARM64" \ + --linux-x64 "$SHA_LINUX_X64" \ + --output "$tap_dir/Formula/coval.rb" + + if git -C "$tap_dir" diff --quiet -- Formula/coval.rb; then + echo "Homebrew formula already matches ${VERSION}; nothing to push." + exit 0 + fi + + git -C "$tap_dir" config user.name "github-actions[bot]" + git -C "$tap_dir" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$tap_dir" add Formula/coval.rb + git -C "$tap_dir" commit -m "chore: update coval to ${VERSION#v}" + git -C "$tap_dir" push origin main + + report-failure: + needs: [validate, build, release, update-homebrew] + if: ${{ always() && contains(needs.*.result, 'failure') }} + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open or update the release failure issue + uses: actions/github-script@v7 + with: + script: | + const title = 'CLI release workflow is failing'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `Release \`${process.env.RELEASE_TAG}\` failed: ${runUrl}`; + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + }); + } + + close-recovered-failure: + needs: [validate, build, release, update-homebrew] + if: ${{ always() && needs.update-homebrew.result == 'success' }} + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Close a recovered release failure issue + uses: actions/github-script@v7 + with: + script: | + const title = 'CLI release workflow is failing'; + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: `Recovered in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`, + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed', + state_reason: 'completed', + }); + } diff --git a/.gitignore b/.gitignore index 13ac30d..db5c4b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /target +__pycache__/ +*.py[cod] .env .DS_Store *.log diff --git a/README.md b/README.md index b1ee343..866f154 100644 --- a/README.md +++ b/README.md @@ -186,16 +186,53 @@ coval traces search --input-json @trace-search.json --format json ## API Coverage Audit The checked-in coverage manifest records every published API operation that the -CLI does not yet expose. Run the live audit after API or client changes: +CLI does not yet expose as a first-class command. The audit traces each literal +client route back to a resource-client method referenced by `src/commands/`, so +an unused HTTP helper does not count as command coverage. + +Run the deterministic tests and live audit after API, client, or command changes: ```bash -python3 -m pip install PyYAML +python3 -m pip install --requirement scripts/requirements-audit.txt +python3 -m unittest scripts/test_audit_api_coverage.py python3 scripts/audit_api_coverage.py ``` -The audit fails for new or stale gaps and for CLI routes absent from the public -OpenAPI catalog unless they are explicitly marked as planned or documented extras -in `api-coverage.toml`. +The audit fails for new or stale gaps, a stale checked-in snapshot, or command +routes absent from the public OpenAPI catalog unless they are explicitly marked +as planned or documented extras in `api-coverage.toml`. A credential-free +GitHub workflow runs the same audit every Monday and reuses one failure issue +until parity recovers. A separate Monday 3:00 AM Pacific Codex automation takes +one bounded resource family through implementation, tests, a version bump, and +a ready-for-review PR; it never merges or releases. + +## Release Automation + +CLI implementation PRs retain a human merge gate. When a merged PR changes the +Cargo version, the exact `main` CI run must pass before +`Release on version bump` creates the matching `v*` tag and calls the reusable +release workflow. A merge without a version bump does not release anything. + +Use the checked-in helper so `Cargo.toml` and `Cargo.lock` move together: + +```bash +# New first-class commands +python3 scripts/bump_version.py minor + +# Backward-compatible fixes +python3 scripts/bump_version.py patch +``` + +The release workflow validates tag/version consistency, builds all five target +binaries, creates or updates the GitHub release, and updates +`coval-ai/homebrew-tap`. A manual `Release on version bump` dispatch safely +retries the current version without creating another tag. + +Repository prerequisite: + +- `HOMEBREW_TAP_TOKEN`: a fine-grained token or GitHub App token with Contents + read/write access to `coval-ai/homebrew-tap`. The Homebrew update is + idempotent, so retrying an already-current formula succeeds without a commit. ## Configuration diff --git a/api-coverage.toml b/api-coverage.toml index 6316603..6296a61 100644 --- a/api-coverage.toml +++ b/api-coverage.toml @@ -1,20 +1,16 @@ # Reviewed against the live public OpenAPI catalog by scripts/audit_api_coverage.py. -# Every published operation must either appear in the CLI client or have a reason here. +# Every published operation must either reach a first-class CLI command or have a reason here. [snapshot] catalog_url = "https://api.coval.dev/v1/openapi" reviewed_at = "2026-07-30" -published_operations = 173 -cli_supported_operations = 123 +published_operations = 174 +cli_supported_operations = 124 [[allowed_extra]] operation = "POST /test-cases/{test_case_id}/media:upload-url" reason = "Implemented by the public API but currently omitted from its published OpenAPI specs." -[[planned_operation]] -operation = "POST /traces/search" -reason = "Implemented with COVAL-4307 and expected to appear after the backend change is deployed." - [[known_gap]] operation = "GET /agents/{agent_id}/versions" reason = "Agent version history remains to be modeled under COVAL-2079." diff --git a/scripts/audit_api_coverage.py b/scripts/audit_api_coverage.py index 5faba2f..59d7e5c 100644 --- a/scripts/audit_api_coverage.py +++ b/scripts/audit_api_coverage.py @@ -1,4 +1,4 @@ -"""Compare CLI HTTP operations with Coval's published OpenAPI catalog. +"""Compare first-class CLI operations with Coval's published OpenAPI catalog. The audit is intentionally live: the public catalog is the source of truth, while ``api-coverage.toml`` records reviewed gaps and temporary pre-deploy operations. @@ -17,7 +17,9 @@ import json import re import sys +import time from pathlib import Path +from urllib.error import HTTPError, URLError from urllib.parse import urlsplit from urllib.request import HTTPRedirectHandler, Request, build_opener @@ -27,8 +29,10 @@ CATALOG_URL = "https://api.coval.dev/v1/openapi" DEFAULT_ALLOWED_ORIGINS = frozenset({"https://api.coval.dev"}) HTTP_METHODS = frozenset({"delete", "get", "patch", "post", "put"}) +FETCH_ATTEMPTS = 3 ROOT = Path(__file__).resolve().parents[1] CLIENT_PATH = ROOT / "src" / "client" / "mod.rs" +COMMANDS_PATH = ROOT / "src" / "commands" MANIFEST_PATH = ROOT / "api-coverage.toml" @@ -83,9 +87,19 @@ def _fetch(url: str, allowed_origins: frozenset[str]) -> bytes: url, headers={"User-Agent": "coval-cli-api-coverage-audit"} ) opener = build_opener(_AllowedOriginRedirectHandler(allowed_origins)) - with opener.open(request, timeout=30) as response: - _validate_fetch_url(response.geturl(), allowed_origins) - return response.read() + for attempt in range(FETCH_ATTEMPTS): + try: + with opener.open(request, timeout=30) as response: + _validate_fetch_url(response.geturl(), allowed_origins) + return response.read() + except HTTPError as error: + if error.code < 500 or attempt == FETCH_ATTEMPTS - 1: + raise + except (TimeoutError, URLError): + if attempt == FETCH_ATTEMPTS - 1: + raise + time.sleep(2**attempt) + raise AssertionError("fetch retry loop exited unexpectedly") def _canonical_operation(method: str, path: str) -> str: @@ -108,46 +122,105 @@ def _published_operations( return operations -def _rust_function_blocks(source: str) -> list[str]: - starts = list(re.finditer(r"(?m)^ pub async fn ", source)) - blocks: list[str] = [] +def _rust_function_blocks(source: str) -> list[tuple[str, str]]: + starts = list(re.finditer(r"(?m)^ pub async fn (?P[A-Za-z0-9_]+)", source)) + blocks: list[tuple[str, str]] = [] for index, match in enumerate(starts): end = starts[index + 1].start() if index + 1 < len(starts) else len(source) - blocks.append(source[match.start() : end]) + blocks.append((match.group("name"), source[match.start() : end])) return blocks -def _client_operations() -> dict[str, str]: +def _snake_case_client_name(name: str) -> str: + return re.sub(r"(? dict[str, dict]: source = CLIENT_PATH.read_text() - operations: dict[str, str] = {} - for block in _rust_function_blocks(source): - paths = re.findall(r'"(/v1/[^"]+)"', block) - if not paths: - continue - - methods = [] - if re.search(r"\.(?:post|post_empty)\(", block): - methods.append("POST") - if re.search(r"\.patch\(", block): - methods.append("PATCH") - if re.search(r"\.delete\(", block): - methods.append("DELETE") - if re.search(r"\.get\(", block): - methods.append("GET") - - if len(methods) != 1: - function_name = re.search(r"pub async fn ([a-zA-Z0-9_]+)", block) - name = function_name.group(1) if function_name else "" - raise RuntimeError( - f"could not infer exactly one HTTP method for client function {name}: {methods}" - ) + implementations = list( + re.finditer(r"(?m)^impl (?P[A-Za-z0-9_]+Client)<'_> \{", source) + ) + operations: dict[str, dict] = {} - for path in paths: - canonical = _canonical_operation(methods[0], path) - operations[canonical] = f"{methods[0]} {path.removeprefix('/v1')}" + for index, implementation in enumerate(implementations): + end = ( + implementations[index + 1].start() + if index + 1 < len(implementations) + else len(source) + ) + accessor = _snake_case_client_name(implementation.group("name")) + implementation_source = source[implementation.end() : end] + + for function_name, block in _rust_function_blocks(implementation_source): + paths = re.findall(r'"(/v1/[^"]+)"', block) + if not paths: + continue + + method_calls = set( + re.findall(r"\.(get|post|post_empty|patch|delete)\(", block) + ) + methods = { + "POST" if method in {"post", "post_empty"} else method.upper() + for method in method_calls + } + if len(methods) != 1: + raise RuntimeError( + "could not infer exactly one HTTP method for client function " + f"{accessor}.{function_name}: {sorted(methods)}" + ) + + method = methods.pop() + client_method = f"{accessor}.{function_name}" + for path in paths: + canonical = _canonical_operation(method, path) + operation = operations.setdefault( + canonical, + { + "operation": f"{method} {path.removeprefix('/v1')}", + "client_methods": set(), + }, + ) + operation["client_methods"].add(client_method) return operations +def _command_client_methods(allowed_accessors: set[str]) -> dict[str, set[str]]: + """Return resource-client methods referenced by first-class command modules.""" + + pattern = re.compile( + r"\bclient\s*\.\s*(?P[A-Za-z0-9_]+)\s*" + r"\([^)]*\)\s*\.\s*(?P[A-Za-z0-9_]+)\s*\(", + re.MULTILINE, + ) + methods: dict[str, set[str]] = {} + for path in sorted(COMMANDS_PATH.glob("*.rs")): + for match in pattern.finditer(path.read_text()): + if match.group("accessor") not in allowed_accessors: + continue + method = f"{match.group('accessor')}.{match.group('method')}" + methods.setdefault(method, set()).add(path.name) + return methods + + +def _command_operations( + client_operations: dict[str, dict], +) -> tuple[dict[str, dict], list[str]]: + known_client_methods = { + method + for operation in client_operations.values() + for method in operation["client_methods"] + } + allowed_accessors = {method.partition(".")[0] for method in known_client_methods} + command_methods = _command_client_methods(allowed_accessors) + operations = { + canonical: operation + for canonical, operation in client_operations.items() + if operation["client_methods"] & set(command_methods) + } + unmapped_methods = sorted(set(command_methods) - known_client_methods) + return operations, unmapped_methods + + def _manifest_operations(entries: list[dict], section: str) -> dict[str, dict]: operations: dict[str, dict] = {} for entry in entries: @@ -171,6 +244,25 @@ def _manifest_operations(entries: list[dict], section: str) -> dict[str, dict]: return operations +def _snapshot_mismatches( + snapshot: dict, + *, + catalog_url: str, + published_operation_count: int, + supported_operation_count: int, +) -> list[str]: + expected = { + "catalog_url": catalog_url, + "published_operations": published_operation_count, + "cli_supported_operations": supported_operation_count, + } + return [ + f"{field}: recorded {snapshot.get(field)!r}, current {value!r}" + for field, value in expected.items() + if snapshot.get(field) != value + ] + + def audit( catalog_url: str, allowed_origins: frozenset[str] = DEFAULT_ALLOWED_ORIGINS, @@ -178,6 +270,7 @@ def audit( manifest = tomllib.loads(MANIFEST_PATH.read_text()) published = _published_operations(catalog_url, allowed_origins) client = _client_operations() + commands, unmapped_command_methods = _command_operations(client) known_gaps = _manifest_operations(manifest.get("known_gap", []), "known_gap") allowed_extras = _manifest_operations( manifest.get("allowed_extra", []), "allowed_extra" @@ -198,20 +291,29 @@ def audit( published_keys = set(published) client_keys = set(client) - actual_gaps = published_keys - client_keys - actual_extras = client_keys - published_keys + command_keys = set(commands) + actual_gaps = published_keys - command_keys + actual_extras = command_keys - published_keys new_gaps = actual_gaps - set(known_gaps) stale_gaps = set(known_gaps) - actual_gaps unexpected_extras = actual_extras - set(allowed_extras) - set(planned) stale_allowed_extras = set(allowed_extras) - actual_extras stale_planned = set(planned) - actual_extras + client_only = client_keys - command_keys + snapshot_mismatches = _snapshot_mismatches( + manifest.get("snapshot", {}), + catalog_url=catalog_url, + published_operation_count=len(published), + supported_operation_count=len(published_keys & command_keys), + ) report = { "catalog_url": catalog_url, "published_operation_count": len(published), - "cli_operation_count": len(client), - "supported_operation_count": len(published_keys & client_keys), + "client_operation_count": len(client), + "command_operation_count": len(commands), + "supported_operation_count": len(published_keys & command_keys), "known_gap_count": len(actual_gaps & set(known_gaps)), "new_gaps": [published[item] for item in sorted(new_gaps)], "stale_gaps": [known_gaps[item]["operation"] for item in sorted(stale_gaps)], @@ -224,6 +326,11 @@ def audit( "stale_planned_operations": [ planned[item]["operation"] for item in sorted(stale_planned) ], + "client_only_operations": [ + client[item]["operation"] for item in sorted(client_only) + ], + "unmapped_command_client_methods": unmapped_command_methods, + "snapshot_mismatches": snapshot_mismatches, "all_current_gaps": [published[item] for item in sorted(actual_gaps)], } passed = not ( @@ -232,6 +339,8 @@ def audit( or unexpected_extras or stale_allowed_extras or stale_planned + or unmapped_command_methods + or snapshot_mismatches ) return report, passed @@ -260,7 +369,7 @@ def main() -> int: status = "PASS" if passed else "FAIL" print( f"{status}: {report['supported_operation_count']}/{report['published_operation_count']} " - f"published operations have CLI HTTP coverage; " + f"published operations have first-class CLI command coverage; " f"{report['known_gap_count']} reviewed gaps remain." ) for key in ( @@ -269,6 +378,8 @@ def main() -> int: "unexpected_cli_operations", "stale_allowed_extras", "stale_planned_operations", + "unmapped_command_client_methods", + "snapshot_mismatches", ): values = report[key] if values: diff --git a/scripts/bump_version.py b/scripts/bump_version.py new file mode 100644 index 0000000..639ff3b --- /dev/null +++ b/scripts/bump_version.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Bump the CLI's stable Cargo version in both release manifests.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +if __package__: + from .release_version import ROOT + from .release_version import release_version +else: + from release_version import ROOT + from release_version import release_version + + +def next_version(version: str, part: str) -> str: + major, minor, patch = (int(value) for value in version.split(".")) + if part == "minor": + return f"{major}.{minor + 1}.0" + if part == "patch": + return f"{major}.{minor}.{patch + 1}" + raise ValueError(f"unsupported version part: {part!r}") + + +def _replace_once(contents: str, pattern: str, replacement: str, path: Path) -> str: + updated, count = re.subn( + pattern, replacement, contents, count=1, flags=re.MULTILINE + ) + if count != 1: + raise ValueError(f"could not find one release version in {path}") + return updated + + +def bump_version(part: str, root: Path = ROOT) -> str: + current = release_version(root) + updated = next_version(current, part) + cargo_path = root / "Cargo.toml" + lock_path = root / "Cargo.lock" + + cargo = _replace_once( + cargo_path.read_text(), + rf'^version = "{re.escape(current)}"$', + f'version = "{updated}"', + cargo_path, + ) + lock = _replace_once( + lock_path.read_text(), + ( + rf'(^\[\[package\]\]\nname = "coval"\nversion = ")' + rf"{re.escape(current)}" + r'("$)' + ), + rf"\g<1>{updated}\g<2>", + lock_path, + ) + + cargo_path.write_text(cargo) + lock_path.write_text(lock) + return updated + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("part", choices=("minor", "patch")) + parser.add_argument("--root", type=Path, default=ROOT) + args = parser.parse_args() + + print(bump_version(args.part, args.root)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 0000000..a3f3912 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Validate CLI release manifests and emit the matching tag.""" + +from __future__ import annotations + +import argparse +import json +import re +import tomllib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +VERSION_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") + + +def release_version(root: Path = ROOT) -> str: + cargo = tomllib.loads((root / "Cargo.toml").read_text()) + name = cargo["package"]["name"] + version = cargo["package"]["version"] + if VERSION_RE.fullmatch(version) is None: + raise ValueError(f"Cargo.toml has a non-release version: {version!r}") + + lock = tomllib.loads((root / "Cargo.lock").read_text()) + workspace_packages = [ + package + for package in lock["package"] + if package["name"] == name and "source" not in package + ] + if len(workspace_packages) != 1: + raise ValueError( + f"expected one workspace package named {name!r} in Cargo.lock, " + f"found {len(workspace_packages)}" + ) + locked_version = workspace_packages[0]["version"] + if locked_version != version: + raise ValueError( + f"Cargo.lock has {name} {locked_version}, but Cargo.toml has {version}" + ) + return version + + +def release_metadata( + root: Path = ROOT, expected_tag: str | None = None +) -> dict[str, str]: + version = release_version(root) + tag = f"v{version}" + if expected_tag is not None and expected_tag != tag: + raise ValueError( + f"release tag {expected_tag!r} does not match Cargo version tag {tag!r}" + ) + return {"version": version, "tag": tag} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--expected-tag") + parser.add_argument("--github-output", type=Path) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + metadata = release_metadata(args.root, args.expected_tag) + if args.github_output is not None: + with args.github_output.open("a") as output: + for key, value in metadata.items(): + output.write(f"{key}={value}\n") + if args.json: + print(json.dumps(metadata, sort_keys=True)) + else: + print(metadata["tag"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_homebrew_formula.py b/scripts/render_homebrew_formula.py new file mode 100644 index 0000000..7304cdb --- /dev/null +++ b/scripts/render_homebrew_formula.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Render the Homebrew formula for one validated Coval CLI release.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +VERSION_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +def render_formula(version: str, checksums: dict[str, str]) -> str: + if VERSION_RE.fullmatch(version) is None: + raise ValueError(f"invalid release version: {version!r}") + + expected_platforms = {"macos_arm64", "macos_x64", "linux_arm64", "linux_x64"} + if set(checksums) != expected_platforms: + missing = sorted(expected_platforms - set(checksums)) + unexpected = sorted(set(checksums) - expected_platforms) + raise ValueError( + f"invalid checksum platforms; missing={missing}, unexpected={unexpected}" + ) + for platform, checksum in checksums.items(): + if SHA256_RE.fullmatch(checksum) is None: + raise ValueError(f"invalid SHA-256 for {platform}: {checksum!r}") + + return f'''class Coval < Formula + desc "CLI for Coval AI agent evaluation platform" + homepage "https://coval.dev" + version "{version}" + license "MIT" + + on_macos do + if Hardware::CPU.arm? + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-macos-arm64.tar.gz" + sha256 "{checksums["macos_arm64"]}" + else + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-macos-x64.tar.gz" + sha256 "{checksums["macos_x64"]}" + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-linux-arm64.tar.gz" + sha256 "{checksums["linux_arm64"]}" + else + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-linux-x64.tar.gz" + sha256 "{checksums["linux_x64"]}" + end + end + + def install + bin.install "coval" + end + + test do + system "#{{bin}}/coval", "--version" + end +end +''' + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--macos-arm64", required=True) + parser.add_argument("--macos-x64", required=True) + parser.add_argument("--linux-arm64", required=True) + parser.add_argument("--linux-x64", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + formula = render_formula( + args.version, + { + "macos_arm64": args.macos_arm64, + "macos_x64": args.macos_x64, + "linux_arm64": args.linux_arm64, + "linux_x64": args.linux_x64, + }, + ) + args.output.write_text(formula) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/requirements-audit.txt b/scripts/requirements-audit.txt new file mode 100644 index 0000000..9db3f9a --- /dev/null +++ b/scripts/requirements-audit.txt @@ -0,0 +1,2 @@ +PyYAML==6.0.3 +ruff==0.15.9 diff --git a/scripts/test_audit_api_coverage.py b/scripts/test_audit_api_coverage.py index 499c286..c680067 100644 --- a/scripts/test_audit_api_coverage.py +++ b/scripts/test_audit_api_coverage.py @@ -1,6 +1,9 @@ -"""Tests for the live API-coverage audit's network boundary.""" +"""Tests for the live API-coverage audit.""" +import tempfile import unittest +from pathlib import Path +from urllib.error import URLError from unittest.mock import Mock from unittest.mock import patch @@ -50,6 +53,117 @@ def test_validates_final_response_url(self, build_opener): audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, ) + @patch("scripts.audit_api_coverage.time.sleep") + @patch("scripts.audit_api_coverage.build_opener") + def test_retries_transient_network_failure(self, build_opener, sleep): + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.geturl.return_value = audit_api_coverage.CATALOG_URL + response.read.return_value = b"ok" + build_opener.return_value.open.side_effect = [ + URLError("temporary"), + response, + ] + + result = audit_api_coverage._fetch( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + + self.assertEqual(b"ok", result) + sleep.assert_called_once_with(1) + + +class CommandCoverageTests(unittest.TestCase): + def test_only_counts_client_operations_referenced_by_commands(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + client_path = root / "client.rs" + commands_path = root / "commands" + commands_path.mkdir() + client_path.write_text( + """ +impl AgentsClient<'_> { + pub async fn list(&self) { + let url = self.0.url("/v1/agents"); + self.0.get(url).await + } + + pub async fn get(&self, id: &str) { + let url = self.0.url(&format!("/v1/agents/{id}")); + self.0.get(url).await + } +} +""" + ) + (commands_path / "agents.rs").write_text( + """ +let agents = client + .agents() + .list() + .await?; +let response = client.get(url).send().await?; +""" + ) + + with ( + patch.object(audit_api_coverage, "CLIENT_PATH", client_path), + patch.object(audit_api_coverage, "COMMANDS_PATH", commands_path), + ): + client_operations = audit_api_coverage._client_operations() + command_operations, unmapped = audit_api_coverage._command_operations( + client_operations + ) + + self.assertEqual(2, len(client_operations)) + self.assertEqual( + ["GET /agents"], + [operation["operation"] for operation in command_operations.values()], + ) + self.assertEqual([], unmapped) + + def test_any_exposed_client_method_covers_a_shared_operation(self): + client_operations = { + "GET /conversations": { + "operation": "GET /conversations", + "client_methods": { + "conversations.list", + "conversations.list_with_metric_outputs", + }, + } + } + with patch.object( + audit_api_coverage, + "_command_client_methods", + return_value={"conversations.list": {"conversations.rs"}}, + ): + command_operations, unmapped = audit_api_coverage._command_operations( + client_operations + ) + + self.assertEqual(client_operations, command_operations) + self.assertEqual([], unmapped) + + +class SnapshotTests(unittest.TestCase): + def test_reports_stale_snapshot_fields(self): + mismatches = audit_api_coverage._snapshot_mismatches( + { + "catalog_url": audit_api_coverage.CATALOG_URL, + "published_operations": 10, + "cli_supported_operations": 8, + }, + catalog_url=audit_api_coverage.CATALOG_URL, + published_operation_count=11, + supported_operation_count=8, + ) + + self.assertEqual( + ["published_operations: recorded 10, current 11"], + mismatches, + ) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_release_automation.py b/scripts/test_release_automation.py new file mode 100644 index 0000000..5d551d2 --- /dev/null +++ b/scripts/test_release_automation.py @@ -0,0 +1,79 @@ +"""Tests for release metadata and Homebrew formula generation.""" + +import tempfile +import unittest +from pathlib import Path + +from scripts import bump_version +from scripts import release_version +from scripts import render_homebrew_formula + + +class ReleaseVersionTests(unittest.TestCase): + def _root( + self, cargo_version: str, lock_version: str + ) -> tempfile.TemporaryDirectory: + directory = tempfile.TemporaryDirectory() + root = Path(directory.name) + (root / "Cargo.toml").write_text( + f'[package]\nname = "coval"\nversion = "{cargo_version}"\n' + ) + (root / "Cargo.lock").write_text( + f'[[package]]\nname = "coval"\nversion = "{lock_version}"\n' + ) + return directory + + def test_returns_matching_release_metadata(self): + with self._root("0.6.0", "0.6.0") as directory: + metadata = release_version.release_metadata(Path(directory), "v0.6.0") + + self.assertEqual({"version": "0.6.0", "tag": "v0.6.0"}, metadata) + + def test_rejects_manifest_mismatch(self): + with self._root("0.6.0", "0.5.0") as directory: + with self.assertRaisesRegex(ValueError, "Cargo.lock"): + release_version.release_metadata(Path(directory)) + + def test_rejects_tag_mismatch(self): + with self._root("0.6.0", "0.6.0") as directory: + with self.assertRaisesRegex(ValueError, "does not match"): + release_version.release_metadata(Path(directory), "v0.5.0") + + +class BumpVersionTests(unittest.TestCase): + def test_bumps_minor_version_in_both_manifests(self): + with ReleaseVersionTests()._root("0.5.0", "0.5.0") as directory: + root = Path(directory) + updated = bump_version.bump_version("minor", root) + + self.assertEqual("0.6.0", updated) + self.assertEqual("0.6.0", release_version.release_version(root)) + + def test_bumps_patch_version(self): + self.assertEqual("0.5.1", bump_version.next_version("0.5.0", "patch")) + + +class HomebrewFormulaTests(unittest.TestCase): + def setUp(self): + self.checksums = { + "macos_arm64": "a" * 64, + "macos_x64": "b" * 64, + "linux_arm64": "c" * 64, + "linux_x64": "d" * 64, + } + + def test_renders_release_urls_and_ruby_interpolation(self): + formula = render_homebrew_formula.render_formula("0.6.0", self.checksums) + + self.assertIn("releases/download/v0.6.0/coval-macos-arm64.tar.gz", formula) + self.assertIn('system "#{bin}/coval", "--version"', formula) + + def test_rejects_invalid_checksum(self): + self.checksums["linux_x64"] = "not-a-checksum" + + with self.assertRaisesRegex(ValueError, "invalid SHA-256"): + render_homebrew_formula.render_formula("0.6.0", self.checksums) + + +if __name__ == "__main__": + unittest.main() From b3a1e541ca199f8d83ede635fb3a8161ef79ccfb Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Thu, 30 Jul 2026 15:47:54 -0700 Subject: [PATCH 2/5] [COVAL-4319] Address release automation review --- .github/workflows/api-parity-audit.yml | 2 + .github/workflows/ci.yml | 4 + .github/workflows/release-on-version-bump.yml | 16 ++- .github/workflows/release.yml | 37 ++++- scripts/audit_api_coverage.py | 9 +- scripts/bump_version.py | 30 +++- scripts/release_version.py | 29 +++- scripts/render_homebrew_formula.py | 5 +- scripts/test_audit_api_coverage.py | 128 ++++++++++++++++++ scripts/test_release_automation.py | 80 ++++++++--- src/commands/traces.rs | 9 ++ tests/cli_tests.rs | 18 +++ 12 files changed, 336 insertions(+), 31 deletions(-) diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml index b9585d6..5f09c09 100644 --- a/.github/workflows/api-parity-audit.yml +++ b/.github/workflows/api-parity-audit.yml @@ -24,6 +24,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b31ce7..158ac2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: actions/setup-python@v6 with: @@ -81,6 +83,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/release-on-version-bump.yml b/.github/workflows/release-on-version-bump.yml index 016dee0..95fcb23 100644 --- a/.github/workflows/release-on-version-bump.yml +++ b/.github/workflows/release-on-version-bump.yml @@ -55,15 +55,29 @@ jobs: - name: Read and validate the release version id: version - run: python scripts/release_version.py --github-output "$GITHUB_OUTPUT" + run: | + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + python scripts/release_version.py --github-output "$GITHUB_OUTPUT" + else + python scripts/release_version.py \ + --allow-non-release \ + --github-output "$GITHUB_OUTPUT" + fi - name: Tag an unreleased version id: tag env: + RELEASE_CANDIDATE: ${{ steps.version.outputs.release_candidate }} RELEASE_SHA: ${{ steps.ref.outputs.sha }} RELEASE_TAG: ${{ steps.version.outputs.tag }} run: | set -euo pipefail + if [ "$RELEASE_CANDIDATE" != "true" ]; then + echo "Cargo.toml does not contain a stable release candidate; nothing to do." + echo "release=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + git fetch --tags --force if git rev-parse -q --verify "refs/tags/$RELEASE_TAG" >/dev/null; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33a8c11..47035ef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,10 +31,13 @@ concurrency: jobs: validate: runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v6 with: ref: ${{ env.RELEASE_REF }} + persist-credentials: false - uses: actions/setup-python@v6 with: @@ -45,6 +48,8 @@ jobs: build: needs: validate + permissions: + contents: read strategy: matrix: include: @@ -75,6 +80,7 @@ jobs: - uses: actions/checkout@v6 with: ref: ${{ env.RELEASE_REF }} + persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: @@ -140,11 +146,14 @@ jobs: update-homebrew: needs: release runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v6 with: ref: ${{ env.RELEASE_REF }} + persist-credentials: false - uses: actions/setup-python@v6 with: @@ -184,7 +193,16 @@ jobs: fi tap_dir="${RUNNER_TEMP}/homebrew-tap" - git clone "https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/coval-ai/homebrew-tap.git" "$tap_dir" + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "$HOMEBREW_TAP_TOKEN" | base64 | tr -d '\n')" + git_auth() { + GIT_CONFIG_COUNT=1 \ + GIT_CONFIG_KEY_0=http.https://github.com/.extraheader \ + GIT_CONFIG_VALUE_0="$auth_header" \ + git "$@" + } + + git_auth clone "https://github.com/coval-ai/homebrew-tap.git" "$tap_dir" + mkdir -p "$tap_dir/Formula" python scripts/render_homebrew_formula.py \ --version "${VERSION#v}" \ --macos-arm64 "$SHA_MACOS_ARM64" \ @@ -193,7 +211,7 @@ jobs: --linux-x64 "$SHA_LINUX_X64" \ --output "$tap_dir/Formula/coval.rb" - if git -C "$tap_dir" diff --quiet -- Formula/coval.rb; then + if [ -z "$(git -C "$tap_dir" status --porcelain -- Formula/coval.rb)" ]; then echo "Homebrew formula already matches ${VERSION}; nothing to push." exit 0 fi @@ -202,7 +220,20 @@ jobs: git -C "$tap_dir" config user.email "41898282+github-actions[bot]@users.noreply.github.com" git -C "$tap_dir" add Formula/coval.rb git -C "$tap_dir" commit -m "chore: update coval to ${VERSION#v}" - git -C "$tap_dir" push origin main + for attempt in 1 2 3; do + if git_auth -C "$tap_dir" push origin HEAD:main; then + exit 0 + fi + if [ "$attempt" -eq 3 ]; then + break + fi + echo "Push attempt ${attempt} failed; rebasing on origin/main." + git_auth -C "$tap_dir" fetch origin main + git -C "$tap_dir" rebase origin/main + sleep $((attempt * 5)) + done + echo "::error::Could not push the Homebrew formula update after 3 attempts." + exit 1 report-failure: needs: [validate, build, release, update-homebrew] diff --git a/scripts/audit_api_coverage.py b/scripts/audit_api_coverage.py index 59d7e5c..420938d 100644 --- a/scripts/audit_api_coverage.py +++ b/scripts/audit_api_coverage.py @@ -226,7 +226,12 @@ def _manifest_operations(entries: list[dict], section: str) -> dict[str, dict]: for entry in entries: operation = entry.get("operation", "") reason = entry.get("reason", "") - if not operation or not reason.strip(): + if ( + not isinstance(operation, str) + or not operation + or not isinstance(reason, str) + or not reason.strip() + ): raise ValueError( f"{section} entries require non-empty operation and reason fields" ) @@ -318,7 +323,7 @@ def audit( "new_gaps": [published[item] for item in sorted(new_gaps)], "stale_gaps": [known_gaps[item]["operation"] for item in sorted(stale_gaps)], "unexpected_cli_operations": [ - client[item] for item in sorted(unexpected_extras) + client[item]["operation"] for item in sorted(unexpected_extras) ], "stale_allowed_extras": [ allowed_extras[item]["operation"] for item in sorted(stale_allowed_extras) diff --git a/scripts/bump_version.py b/scripts/bump_version.py index 639ff3b..8df2b8d 100644 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -5,6 +5,7 @@ import argparse import re +import tomllib from pathlib import Path if __package__: @@ -33,22 +34,41 @@ def _replace_once(contents: str, pattern: str, replacement: str, path: Path) -> return updated +def _replace_package_version( + contents: str, current: str, updated: str, path: Path +) -> str: + sections = list(re.finditer(r"(?ms)^\[package\][ \t]*\n.*?(?=^\[|\Z)", contents)) + if len(sections) != 1: + raise ValueError(f"expected one [package] section in {path}") + + section = sections[0] + updated_section = _replace_once( + section.group(), + rf'^(version[ \t]*=[ \t]*"){re.escape(current)}("[ \t]*)$', + rf"\g<1>{updated}\g<2>", + path, + ) + return contents[: section.start()] + updated_section + contents[section.end() :] + + def bump_version(part: str, root: Path = ROOT) -> str: current = release_version(root) updated = next_version(current, part) cargo_path = root / "Cargo.toml" lock_path = root / "Cargo.lock" + cargo_contents = cargo_path.read_text() + package_name = tomllib.loads(cargo_contents)["package"]["name"] - cargo = _replace_once( - cargo_path.read_text(), - rf'^version = "{re.escape(current)}"$', - f'version = "{updated}"', + cargo = _replace_package_version( + cargo_contents, + current, + updated, cargo_path, ) lock = _replace_once( lock_path.read_text(), ( - rf'(^\[\[package\]\]\nname = "coval"\nversion = ")' + rf'(^\[\[package\]\]\nname = "{re.escape(package_name)}"\nversion = ")' rf"{re.escape(current)}" + r'("$)' ), rf"\g<1>{updated}\g<2>", diff --git a/scripts/release_version.py b/scripts/release_version.py index a3f3912..1aef8e7 100644 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -14,12 +14,18 @@ VERSION_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") +class NonReleaseVersionError(ValueError): + """Raised when Cargo.toml intentionally contains a non-stable version.""" + + def release_version(root: Path = ROOT) -> str: cargo = tomllib.loads((root / "Cargo.toml").read_text()) name = cargo["package"]["name"] version = cargo["package"]["version"] if VERSION_RE.fullmatch(version) is None: - raise ValueError(f"Cargo.toml has a non-release version: {version!r}") + raise NonReleaseVersionError( + f"Cargo.toml has a non-release version: {version!r}" + ) lock = tomllib.loads((root / "Cargo.lock").read_text()) workspace_packages = [ @@ -57,12 +63,31 @@ def main() -> int: parser.add_argument("--root", type=Path, default=ROOT) parser.add_argument("--expected-tag") parser.add_argument("--github-output", type=Path) + parser.add_argument( + "--allow-non-release", + action="store_true", + help="Exit successfully without a tag when Cargo.toml is not stable semver", + ) parser.add_argument("--json", action="store_true") args = parser.parse_args() - metadata = release_metadata(args.root, args.expected_tag) + try: + metadata = release_metadata(args.root, args.expected_tag) + except NonReleaseVersionError as error: + if not args.allow_non_release: + raise + if args.github_output is not None: + with args.github_output.open("a") as output: + output.write("release_candidate=false\n") + if args.json: + print(json.dumps({"release_candidate": False})) + else: + print(f"No stable release candidate: {error}") + return 0 + if args.github_output is not None: with args.github_output.open("a") as output: + output.write("release_candidate=true\n") for key, value in metadata.items(): output.write(f"{key}={value}\n") if args.json: diff --git a/scripts/render_homebrew_formula.py b/scripts/render_homebrew_formula.py index 7304cdb..0c012f4 100644 --- a/scripts/render_homebrew_formula.py +++ b/scripts/render_homebrew_formula.py @@ -7,8 +7,11 @@ import re from pathlib import Path +if __package__: + from .release_version import VERSION_RE +else: + from release_version import VERSION_RE -VERSION_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") SHA256_RE = re.compile(r"^[0-9a-f]{64}$") diff --git a/scripts/test_audit_api_coverage.py b/scripts/test_audit_api_coverage.py index c680067..f23311e 100644 --- a/scripts/test_audit_api_coverage.py +++ b/scripts/test_audit_api_coverage.py @@ -146,6 +146,134 @@ def test_any_exposed_client_method_covers_a_shared_operation(self): self.assertEqual([], unmapped) +class ManifestTests(unittest.TestCase): + def test_rejects_missing_reason(self): + with self.assertRaisesRegex(ValueError, "non-empty"): + audit_api_coverage._manifest_operations( + [{"operation": "GET /agents"}], + "known_gap", + ) + + def test_rejects_invalid_http_method(self): + with self.assertRaisesRegex(ValueError, "invalid operation"): + audit_api_coverage._manifest_operations( + [{"operation": "HEAD /agents", "reason": "Not supported"}], + "known_gap", + ) + + def test_rejects_duplicate_operation(self): + with self.assertRaisesRegex(ValueError, "duplicate operation"): + audit_api_coverage._manifest_operations( + [ + {"operation": "GET /agents", "reason": "First"}, + {"operation": "GET /agents", "reason": "Second"}, + ], + "known_gap", + ) + + +class AuditAggregationTests(unittest.TestCase): + def _audit( + self, + manifest: str, + published: dict[str, str], + commands: dict[str, dict], + ) -> tuple[dict, bool]: + with tempfile.TemporaryDirectory() as directory: + manifest_path = Path(directory) / "api-coverage.toml" + manifest_path.write_text(manifest) + with ( + patch.object(audit_api_coverage, "MANIFEST_PATH", manifest_path), + patch.object( + audit_api_coverage, + "_published_operations", + return_value=published, + ), + patch.object( + audit_api_coverage, + "_client_operations", + return_value=commands, + ), + patch.object( + audit_api_coverage, + "_command_operations", + return_value=(commands, []), + ), + ): + return audit_api_coverage.audit(audit_api_coverage.CATALOG_URL) + + def test_accepts_reviewed_gap_and_allowed_extra(self): + manifest = f""" +[snapshot] +catalog_url = "{audit_api_coverage.CATALOG_URL}" +published_operations = 2 +cli_supported_operations = 1 + +[[known_gap]] +operation = "GET /beta" +reason = "Reviewed" + +[[allowed_extra]] +operation = "POST /gamma" +reason = "Pre-deploy" +""" + published = { + "GET /alpha": "GET /v1/alpha", + "GET /beta": "GET /v1/beta", + } + commands = { + "GET /alpha": {"operation": "GET /alpha"}, + "POST /gamma": {"operation": "POST /gamma"}, + } + + report, passed = self._audit(manifest, published, commands) + + self.assertTrue(passed) + self.assertEqual(1, report["known_gap_count"]) + self.assertEqual([], report["new_gaps"]) + self.assertEqual([], report["unexpected_cli_operations"]) + + def test_classifies_new_gap_and_unexpected_extra_as_failure(self): + manifest = f""" +[snapshot] +catalog_url = "{audit_api_coverage.CATALOG_URL}" +published_operations = 2 +cli_supported_operations = 1 +""" + published = { + "GET /alpha": "GET /v1/alpha", + "GET /beta": "GET /v1/beta", + } + commands = { + "GET /alpha": {"operation": "GET /alpha"}, + "POST /gamma": {"operation": "POST /gamma"}, + } + + report, passed = self._audit(manifest, published, commands) + + self.assertFalse(passed) + self.assertEqual(["GET /v1/beta"], report["new_gaps"]) + self.assertEqual(["POST /gamma"], report["unexpected_cli_operations"]) + + def test_rejects_operation_in_multiple_manifest_sections(self): + manifest = """ +[snapshot] +catalog_url = "https://api.coval.dev/v1/openapi" +published_operations = 0 +cli_supported_operations = 0 + +[[known_gap]] +operation = "GET /agents" +reason = "Reviewed" + +[[allowed_extra]] +operation = "GET /agents" +reason = "Pre-deploy" +""" + with self.assertRaisesRegex(ValueError, "only one section"): + self._audit(manifest, {}, {}) + + class SnapshotTests(unittest.TestCase): def test_reports_stale_snapshot_fields(self): mismatches = audit_api_coverage._snapshot_mismatches( diff --git a/scripts/test_release_automation.py b/scripts/test_release_automation.py index 5d551d2..b21c91f 100644 --- a/scripts/test_release_automation.py +++ b/scripts/test_release_automation.py @@ -1,48 +1,79 @@ """Tests for release metadata and Homebrew formula generation.""" +import io import tempfile import unittest +from contextlib import redirect_stdout from pathlib import Path +from unittest.mock import patch from scripts import bump_version from scripts import release_version from scripts import render_homebrew_formula -class ReleaseVersionTests(unittest.TestCase): - def _root( - self, cargo_version: str, lock_version: str - ) -> tempfile.TemporaryDirectory: - directory = tempfile.TemporaryDirectory() - root = Path(directory.name) - (root / "Cargo.toml").write_text( - f'[package]\nname = "coval"\nversion = "{cargo_version}"\n' - ) - (root / "Cargo.lock").write_text( - f'[[package]]\nname = "coval"\nversion = "{lock_version}"\n' - ) - return directory +def temporary_release_root( + cargo_version: str, + lock_version: str, + *, + package_name: str = "coval", + cargo_prefix: str = "", +) -> tempfile.TemporaryDirectory: + directory = tempfile.TemporaryDirectory() + root = Path(directory.name) + (root / "Cargo.toml").write_text( + f'{cargo_prefix}[package]\nname = "{package_name}"\n' + f'version = "{cargo_version}"\n' + ) + (root / "Cargo.lock").write_text( + f'[[package]]\nname = "{package_name}"\nversion = "{lock_version}"\n' + ) + return directory + +class ReleaseVersionTests(unittest.TestCase): def test_returns_matching_release_metadata(self): - with self._root("0.6.0", "0.6.0") as directory: + with temporary_release_root("0.6.0", "0.6.0") as directory: metadata = release_version.release_metadata(Path(directory), "v0.6.0") self.assertEqual({"version": "0.6.0", "tag": "v0.6.0"}, metadata) def test_rejects_manifest_mismatch(self): - with self._root("0.6.0", "0.5.0") as directory: + with temporary_release_root("0.6.0", "0.5.0") as directory: with self.assertRaisesRegex(ValueError, "Cargo.lock"): release_version.release_metadata(Path(directory)) def test_rejects_tag_mismatch(self): - with self._root("0.6.0", "0.6.0") as directory: + with temporary_release_root("0.6.0", "0.6.0") as directory: with self.assertRaisesRegex(ValueError, "does not match"): release_version.release_metadata(Path(directory), "v0.5.0") + def test_identifies_non_release_version(self): + with temporary_release_root("0.6.0-rc.1", "0.6.0-rc.1") as directory: + with self.assertRaises(release_version.NonReleaseVersionError): + release_version.release_metadata(Path(directory)) + + def test_automatic_release_can_skip_non_release_version(self): + with temporary_release_root("0.6.0-rc.1", "0.6.0-rc.1") as directory: + output_path = Path(directory) / "github-output" + argv = [ + "release_version.py", + "--root", + directory, + "--allow-non-release", + "--github-output", + str(output_path), + ] + with patch("sys.argv", argv), redirect_stdout(io.StringIO()): + result = release_version.main() + + self.assertEqual(0, result) + self.assertEqual("release_candidate=false\n", output_path.read_text()) + class BumpVersionTests(unittest.TestCase): def test_bumps_minor_version_in_both_manifests(self): - with ReleaseVersionTests()._root("0.5.0", "0.5.0") as directory: + with temporary_release_root("0.5.0", "0.5.0") as directory: root = Path(directory) updated = bump_version.bump_version("minor", root) @@ -52,6 +83,21 @@ def test_bumps_minor_version_in_both_manifests(self): def test_bumps_patch_version(self): self.assertEqual("0.5.1", bump_version.next_version("0.5.0", "patch")) + def test_only_bumps_renamed_workspace_package(self): + prefix = '[dependencies]\nhelper = "0.5.0"\n\n' + with temporary_release_root( + "0.5.0", + "0.5.0", + package_name="renamed-cli", + cargo_prefix=prefix, + ) as directory: + root = Path(directory) + bump_version.bump_version("minor", root) + + cargo = (root / "Cargo.toml").read_text() + self.assertIn('helper = "0.5.0"', cargo) + self.assertEqual("0.6.0", release_version.release_version(root)) + class HomebrewFormulaTests(unittest.TestCase): def setUp(self): diff --git a/src/commands/traces.rs b/src/commands/traces.rs index 2baa325..3fd0042 100644 --- a/src/commands/traces.rs +++ b/src/commands/traces.rs @@ -245,6 +245,15 @@ fn build_search_request(args: SearchArgs) -> Result { input.insert("filters".to_string(), serde_json::Value::Object(filters)); } let request: TraceSearchRequest = input_json::finish(input)?; + if let (Some(minimum), Some(maximum)) = ( + request.filters.duration_ms_min, + request.filters.duration_ms_max, + ) { + anyhow::ensure!( + minimum <= maximum, + "trace search duration minimum ({minimum}) must not exceed maximum ({maximum})" + ); + } if let Some(attribute_filters) = &request.filters.attribute_filters { anyhow::ensure!( attribute_filters.len() <= 10, diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index f3cf01c..370fe06 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -3992,6 +3992,24 @@ fn test_traces_search_rejects_ambiguous_attribute_filter() { )); } +#[test] +fn test_traces_search_rejects_inverted_duration_range() { + coval() + .arg("--api-key") + .arg("test_key") + .arg("traces") + .arg("search") + .arg("--duration-ms-min") + .arg("10") + .arg("--duration-ms-max") + .arg("5") + .assert() + .failure() + .stderr(predicate::str::contains( + "trace search duration minimum (10) must not exceed maximum (5)", + )); +} + #[test] fn test_traces_search_rejects_more_than_ten_attribute_filter_flags() { let mut command = coval(); From a382d3cd919ff47a888e93809ee3ca9ed1646810 Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Thu, 30 Jul 2026 16:22:49 -0700 Subject: [PATCH 3/5] [COVAL-4319] Keep CLI automation repository-owned --- .github/workflows/api-parity-audit.yml | 8 ++++---- README.md | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml index 5f09c09..fe01d91 100644 --- a/.github/workflows/api-parity-audit.yml +++ b/.github/workflows/api-parity-audit.yml @@ -1,9 +1,9 @@ name: Weekly API parity audit -# The scheduled Codex job does the judgment-heavy command implementation and -# opens a reviewed PR. This workflow is the deterministic backstop: it runs the -# same live audit without credentials and keeps one GitHub issue open whenever -# the public API and first-class CLI command surface drift. +# Repository-owned drift detection for the hand-written CLI. Unlike the SDK +# workflow, this does not claim to regenerate command UX from OpenAPI. It runs +# the deterministic audit without credentials and keeps one GitHub issue open +# whenever the public API and first-class CLI command surface drift. on: workflow_dispatch: diff --git a/README.md b/README.md index 866f154..2796b41 100644 --- a/README.md +++ b/README.md @@ -202,9 +202,14 @@ The audit fails for new or stale gaps, a stale checked-in snapshot, or command routes absent from the public OpenAPI catalog unless they are explicitly marked as planned or documented extras in `api-coverage.toml`. A credential-free GitHub workflow runs the same audit every Monday and reuses one failure issue -until parity recovers. A separate Monday 3:00 AM Pacific Codex automation takes -one bounded resource family through implementation, tests, a version bump, and -a ready-for-review PR; it never merges or releases. +until parity recovers. + +The SDK regeneration workflow can open deterministic codegen PRs because its +published clients are generated from OpenAPI. The CLI command surface is still +hand-written, so this repository does not present an automated audit as command +generation. Repository-owned generated-model PRs are tracked separately under +COVAL-2079; they require the CLI's OpenAPI type-codegen migration to be +completed first. ## Release Automation From cfb0eda8ace71f39e113d4e957f86a63c85b4438 Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Thu, 30 Jul 2026 16:47:55 -0700 Subject: [PATCH 4/5] [COVAL-4319] Open parity drift PRs --- .github/workflows/api-parity-audit.yml | 59 ++++++++++---- .github/workflows/ci.yml | 5 +- README.md | 17 +++- api-coverage-report.md | 104 +++++++++++++++++++++++++ scripts/audit_api_coverage.py | 67 +++++++++++++++- scripts/test_audit_api_coverage.py | 65 ++++++++++++++++ 6 files changed, 295 insertions(+), 22 deletions(-) create mode 100644 api-coverage-report.md diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml index fe01d91..96993e3 100644 --- a/.github/workflows/api-parity-audit.yml +++ b/.github/workflows/api-parity-audit.yml @@ -1,9 +1,13 @@ -name: Weekly API parity audit +name: Weekly API parity PR -# Repository-owned drift detection for the hand-written CLI. Unlike the SDK -# workflow, this does not claim to regenerate command UX from OpenAPI. It runs -# the deterministic audit without credentials and keeps one GitHub issue open -# whenever the public API and first-class CLI command surface drift. +# Repository-owned drift detection for the hand-written CLI. The workflow +# refreshes a deterministic coverage report and opens or updates one rolling PR +# when the public API and first-class CLI command surface change. It does not +# claim to regenerate hand-written command UX from OpenAPI. +# +# Prerequisite: +# REGEN_PR_TOKEN: a token with Contents and Pull requests write access to this +# repository. The organization does not allow GITHUB_TOKEN to create PRs. on: workflow_dispatch: @@ -12,19 +16,21 @@ on: - cron: "0 10 * * 1" permissions: - contents: read + contents: write + pull-requests: write issues: write concurrency: - group: weekly-api-parity-audit + group: weekly-api-parity-pr cancel-in-progress: false jobs: - audit: + refresh: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 with: + ref: main persist-credentials: false - uses: actions/setup-python@v6 @@ -39,22 +45,43 @@ jobs: - name: Test the audit run: python -m unittest scripts/test_audit_api_coverage.py - - name: Compare first-class commands with the live public API - run: python scripts/audit_api_coverage.py + - name: Refresh the deterministic parity report + run: >- + python scripts/audit_api_coverage.py + --write-markdown api-coverage-report.md + --allow-drift - - name: Open or update the parity failure issue + - name: Open or update the parity PR + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.REGEN_PR_TOKEN }} + commit-message: "chore(cli): refresh API parity report" + title: "chore(cli): reconcile weekly API parity" + body: | + Automated comparison of the live public OpenAPI catalog with first-class CLI commands. + + - `api-coverage-report.md` is deterministic and changes only when API or CLI coverage changes. + - This PR does not claim to generate the CLI's hand-written command UX. + - If the report says `ACTION REQUIRED`, implement the missing commands or explicitly review the manifest exception, regenerate the report, and get CI green before merging. + + The branch is stable, so future weekly runs update this PR instead of opening duplicates. + branch: chore/weekly-api-parity + delete-branch: false + add-paths: api-coverage-report.md + + - name: Open or update the automation failure issue if: failure() uses: actions/github-script@v7 with: script: | - const title = 'Weekly CLI API parity audit is failing'; + const title = 'Weekly CLI API parity PR automation is failing'; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const body = [ - 'The scheduled CLI API parity audit failed.', + 'The scheduled CLI API parity workflow failed before it could produce or update its rolling PR.', '', `Run: ${runUrl}`, '', - 'Until this is reconciled, the first-class CLI command coverage may have drifted from the published API.', + 'This issue reports broken automation, not ordinary API drift. Ordinary drift is reported through the rolling PR.', ].join('\n'); const issues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, @@ -79,12 +106,12 @@ jobs: }); } - - name: Close a recovered parity failure issue + - name: Close a recovered automation failure issue if: success() uses: actions/github-script@v7 with: script: | - const title = 'Weekly CLI API parity audit is failing'; + const title = 'Weekly CLI API parity PR automation is failing'; const issues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 158ac2f..e00c574 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,10 @@ jobs: python -m ruff format --check scripts - name: Audit live API command coverage - run: python scripts/audit_api_coverage.py + run: | + python scripts/audit_api_coverage.py \ + --write-markdown api-coverage-report.md + git diff --exit-code -- api-coverage-report.md - name: Validate release manifests run: python scripts/release_version.py diff --git a/README.md b/README.md index 2796b41..17a6c11 100644 --- a/README.md +++ b/README.md @@ -195,14 +195,20 @@ Run the deterministic tests and live audit after API, client, or command changes ```bash python3 -m pip install --requirement scripts/requirements-audit.txt python3 -m unittest scripts/test_audit_api_coverage.py -python3 scripts/audit_api_coverage.py +python3 scripts/audit_api_coverage.py \ + --write-markdown api-coverage-report.md ``` The audit fails for new or stale gaps, a stale checked-in snapshot, or command routes absent from the public OpenAPI catalog unless they are explicitly marked -as planned or documented extras in `api-coverage.toml`. A credential-free -GitHub workflow runs the same audit every Monday and reuses one failure issue -until parity recovers. +as planned or documented extras in `api-coverage.toml`. + +A repository-owned GitHub workflow runs every Monday and refreshes the +deterministic `api-coverage-report.md`. When coverage changes, it opens or +updates one rolling PR on `chore/weekly-api-parity`; the PR's CI remains blocked +until the command implementation or an explicitly reviewed manifest exception +reconciles the drift. A GitHub issue is used only if the automation itself +fails before it can create or update that PR. The SDK regeneration workflow can open deterministic codegen PRs because its published clients are generated from OpenAPI. The CLI command surface is still @@ -235,6 +241,9 @@ retries the current version without creating another tag. Repository prerequisite: +- `REGEN_PR_TOKEN`: a fine-grained token with Contents and Pull requests + read/write access to `coval-ai/cli`. The organization does not allow + `GITHUB_TOKEN` to create pull requests. - `HOMEBREW_TAP_TOKEN`: a fine-grained token or GitHub App token with Contents read/write access to `coval-ai/homebrew-tap`. The Homebrew update is idempotent, so retrying an already-current formula succeeds without a commit. diff --git a/api-coverage-report.md b/api-coverage-report.md new file mode 100644 index 0000000..f24e498 --- /dev/null +++ b/api-coverage-report.md @@ -0,0 +1,104 @@ +# CLI API Coverage Report + + + +This report compares the published Coval OpenAPI catalog with first-class +CLI commands. It is intentionally timestamp-free so the weekly workflow +opens or updates a PR only when coverage actually changes. + +## Summary + +| Metric | Value | +| --- | ---: | +| Reconciliation status | PASS | +| Published operations | 174 | +| First-class CLI operations | 124 | +| Reviewed gaps | 50 | +| Client operations | 125 | + +Catalog: https://api.coval.dev/v1/openapi + +## New published operations without CLI commands + +- None. + +## Reviewed gaps no longer present + +- None. + +## CLI operations absent from published OpenAPI + +- None. + +## Allowed extras no longer present + +- None. + +## Planned operations no longer present + +- None. + +## Client methods not mapped to operations + +- None. + +## Coverage snapshot mismatches + +- None. + +## Client-only operations + +- None. + +## All current published gaps + +- `DELETE /integrations/slack` +- `DELETE /metrics/flows/{flow_id}` +- `DELETE /test-sets/{test_set_id}/agents/{agent_id}` +- `DELETE /webhooks/{webhook_id}` +- `GET /agents/{agent_id}/versions` +- `GET /integrations/slack` +- `GET /metrics/flows` +- `GET /metrics/recently-deleted` +- `GET /metrics/sql-schema` +- `GET /metrics/tags` +- `GET /metrics/template-variables` +- `GET /metrics/{metric_id}/baselines/{baseline_id}/history` +- `GET /organization/conversation-metrics` +- `GET /organization/monitoring-metrics` +- `GET /personas/tags` +- `GET /personas/{persona_id}/versions` +- `GET /reports/{report_id}/rows` +- `GET /review-annotations/metric-health-stats` +- `GET /review-projects/{project_id}/insights` +- `GET /review-projects/{project_id}/metric-agreement` +- `GET /review-projects/{project_id}/progress` +- `GET /runs/tags` +- `GET /scheduled-runs/{scheduled_run_id}/runs` +- `GET /test-sets/{test_set_id}/agents` +- `GET /test-sets/{test_set_id}/records` +- `GET /test-sets/{test_set_id}/versions` +- `GET /webhooks` +- `PATCH /metrics/flows/{flow_id}` +- `PATCH /organization/conversation-metrics` +- `PATCH /organization/monitoring-metrics` +- `PATCH /webhooks/{webhook_id}` +- `POST /agents/{agent_id}/duplicate` +- `POST /agents/{agent_id}/versions/{version_id}/revert` +- `POST /audio:upload` +- `POST /integrations/slack/connect` +- `POST /metrics/flows` +- `POST /metrics/outputs:batchGet` +- `POST /metrics/sql:test` +- `POST /metrics/{metric_id}/restore` +- `POST /metrics/{metric_id}/versions/{version_id}/revert` +- `POST /personas/{persona_id}/duplicate` +- `POST /personas/{persona_id}/versions/{version_id}/revert` +- `POST /review-annotations:withMetricOutputs` +- `POST /review-projects/disagreement-state` +- `POST /simulations:rerunMetrics` +- `POST /test-sets/{test_set_id}/agents:add` +- `POST /test-sets/{test_set_id}/duplicate` +- `POST /test-sets/{test_set_id}/versions/{version_id}/revert` +- `POST /traces` +- `POST /webhooks` diff --git a/scripts/audit_api_coverage.py b/scripts/audit_api_coverage.py index 420938d..50fddb3 100644 --- a/scripts/audit_api_coverage.py +++ b/scripts/audit_api_coverage.py @@ -350,6 +350,54 @@ def audit( return report, passed +def render_markdown_report(report: dict, passed: bool) -> str: + """Render a deterministic, reviewable API-parity report.""" + + lines = [ + "# CLI API Coverage Report", + "", + "", + "", + "This report compares the published Coval OpenAPI catalog with first-class", + "CLI commands. It is intentionally timestamp-free so the weekly workflow", + "opens or updates a PR only when coverage actually changes.", + "", + "## Summary", + "", + "| Metric | Value |", + "| --- | ---: |", + f"| Reconciliation status | {'PASS' if passed else 'ACTION REQUIRED'} |", + f"| Published operations | {report['published_operation_count']} |", + f"| First-class CLI operations | {report['supported_operation_count']} |", + f"| Reviewed gaps | {report['known_gap_count']} |", + f"| Client operations | {report['client_operation_count']} |", + "", + f"Catalog: {report['catalog_url']}", + "", + ] + + sections = ( + ("New published operations without CLI commands", "new_gaps"), + ("Reviewed gaps no longer present", "stale_gaps"), + ("CLI operations absent from published OpenAPI", "unexpected_cli_operations"), + ("Allowed extras no longer present", "stale_allowed_extras"), + ("Planned operations no longer present", "stale_planned_operations"), + ("Client methods not mapped to operations", "unmapped_command_client_methods"), + ("Coverage snapshot mismatches", "snapshot_mismatches"), + ("Client-only operations", "client_only_operations"), + ("All current published gaps", "all_current_gaps"), + ) + for title, key in sections: + lines.extend((f"## {title}", "")) + values = report[key] + lines.extend(f"- `{value}`" for value in values) + if not values: + lines.append("- None.") + lines.append("") + + return "\n".join(lines) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--catalog-url", default=CATALOG_URL) @@ -361,13 +409,30 @@ def main() -> int: parser.add_argument( "--json", action="store_true", help="Emit the full machine-readable report" ) + parser.add_argument( + "--write-markdown", + type=Path, + help="Write a deterministic Markdown report to this path", + ) + parser.add_argument( + "--allow-drift", + action="store_true", + help=( + "Exit successfully after writing a report even when parity needs " + "reconciliation; requires --write-markdown" + ), + ) args = parser.parse_args() + if args.allow_drift and args.write_markdown is None: + parser.error("--allow-drift requires --write-markdown") configured_origins = args.allowed_origin or sorted(DEFAULT_ALLOWED_ORIGINS) allowed_origins = frozenset( _normalize_allowed_origin(origin) for origin in configured_origins ) report, passed = audit(args.catalog_url, allowed_origins) + if args.write_markdown is not None: + args.write_markdown.write_text(render_markdown_report(report, passed)) if args.json: print(json.dumps(report, indent=2)) else: @@ -391,7 +456,7 @@ def main() -> int: print(f"{key}:") for value in values: print(f" - {value}") - return 0 if passed else 1 + return 0 if passed or args.allow_drift else 1 if __name__ == "__main__": diff --git a/scripts/test_audit_api_coverage.py b/scripts/test_audit_api_coverage.py index f23311e..c8040f0 100644 --- a/scripts/test_audit_api_coverage.py +++ b/scripts/test_audit_api_coverage.py @@ -1,7 +1,10 @@ """Tests for the live API-coverage audit.""" +import io import tempfile import unittest +from contextlib import redirect_stderr +from contextlib import redirect_stdout from pathlib import Path from urllib.error import URLError from unittest.mock import Mock @@ -293,5 +296,67 @@ def test_reports_stale_snapshot_fields(self): ) +class MarkdownReportTests(unittest.TestCase): + def setUp(self): + self.report = { + "catalog_url": audit_api_coverage.CATALOG_URL, + "published_operation_count": 3, + "client_operation_count": 2, + "command_operation_count": 2, + "supported_operation_count": 2, + "known_gap_count": 0, + "new_gaps": ["GET /v1/beta"], + "stale_gaps": [], + "unexpected_cli_operations": [], + "stale_allowed_extras": [], + "stale_planned_operations": [], + "client_only_operations": [], + "unmapped_command_client_methods": [], + "snapshot_mismatches": ["published_operations: recorded 2, current 3"], + "all_current_gaps": ["GET /v1/beta"], + } + + def test_renders_deterministic_actionable_report(self): + rendered = audit_api_coverage.render_markdown_report(self.report, False) + + self.assertIn("| Reconciliation status | ACTION REQUIRED |", rendered) + self.assertIn("- `GET /v1/beta`", rendered) + self.assertIn( + "- `published_operations: recorded 2, current 3`", + rendered, + ) + self.assertNotIn("Generated at", rendered) + + @patch("scripts.audit_api_coverage.audit") + def test_allow_drift_writes_report_and_returns_success(self, audit): + audit.return_value = (self.report, False) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "coverage.md" + argv = [ + "audit_api_coverage.py", + "--write-markdown", + str(output), + "--allow-drift", + ] + with ( + patch("sys.argv", argv), + redirect_stdout(io.StringIO()), + ): + result = audit_api_coverage.main() + + self.assertEqual(0, result) + self.assertIn("ACTION REQUIRED", output.read_text()) + + def test_allow_drift_requires_markdown_output(self): + with ( + patch("sys.argv", ["audit_api_coverage.py", "--allow-drift"]), + redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit) as exception, + ): + audit_api_coverage.main() + + self.assertEqual(2, exception.exception.code) + + if __name__ == "__main__": unittest.main() From 7d3464243118d78a59325794a5805543195cf2eb Mon Sep 17 00:00:00 2001 From: Callum Reid Date: Thu, 30 Jul 2026 16:59:23 -0700 Subject: [PATCH 5/5] [COVAL-4319] Address parity workflow review --- .github/workflows/api-parity-audit.yml | 4 +- .github/workflows/ci.yml | 3 + .github/workflows/release-on-version-bump.yml | 5 + scripts/audit_api_coverage.py | 9 +- scripts/test_audit_api_coverage.py | 92 +++++++++++++++++++ 5 files changed, 110 insertions(+), 3 deletions(-) diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml index 96993e3..af6014c 100644 --- a/.github/workflows/api-parity-audit.yml +++ b/.github/workflows/api-parity-audit.yml @@ -16,8 +16,7 @@ on: - cron: "0 10 * * 1" permissions: - contents: write - pull-requests: write + contents: read issues: write concurrency: @@ -27,6 +26,7 @@ concurrency: jobs: refresh: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e00c574..f2be4b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,9 @@ jobs: python -m ruff format --check scripts - name: Audit live API command coverage + # Live API/network failures are advisory on ordinary CI runs. The + # repository-owned weekly parity PR is the strict reconciliation gate. + continue-on-error: ${{ github.event_name != 'pull_request' || github.head_ref != 'chore/weekly-api-parity' }} run: | python scripts/audit_api_coverage.py \ --write-markdown api-coverage-report.md diff --git a/.github/workflows/release-on-version-bump.yml b/.github/workflows/release-on-version-bump.yml index 95fcb23..cda2627 100644 --- a/.github/workflows/release-on-version-bump.yml +++ b/.github/workflows/release-on-version-bump.yml @@ -38,7 +38,12 @@ jobs: env: WORKFLOW_RUN_SHA: ${{ github.event.workflow_run.head_sha }} run: | + set -euo pipefail if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + if [ "$GITHUB_REF_NAME" != "main" ]; then + echo "::error::Manual releases must be dispatched from main (got $GITHUB_REF_NAME)." + exit 1 + fi echo "sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT" else echo "sha=$WORKFLOW_RUN_SHA" >> "$GITHUB_OUTPUT" diff --git a/scripts/audit_api_coverage.py b/scripts/audit_api_coverage.py index 50fddb3..4275f3d 100644 --- a/scripts/audit_api_coverage.py +++ b/scripts/audit_api_coverage.py @@ -118,7 +118,14 @@ def _published_operations( for path, path_item in (spec.get("paths") or {}).items(): for method in HTTP_METHODS & set(path_item): canonical = _canonical_operation(method, path) - operations[canonical] = f"{method.upper()} {path}" + display = f"{method.upper()} {path}" + previous = operations.get(canonical) + if previous is not None and previous != display: + raise RuntimeError( + f"published operations {previous!r} and {display!r} " + f"share canonical key {canonical!r}" + ) + operations[canonical] = display return operations diff --git a/scripts/test_audit_api_coverage.py b/scripts/test_audit_api_coverage.py index c8040f0..08a3b0c 100644 --- a/scripts/test_audit_api_coverage.py +++ b/scripts/test_audit_api_coverage.py @@ -6,6 +6,7 @@ from contextlib import redirect_stderr from contextlib import redirect_stdout from pathlib import Path +from urllib.error import HTTPError from urllib.error import URLError from unittest.mock import Mock from unittest.mock import patch @@ -77,6 +78,97 @@ def test_retries_transient_network_failure(self, build_opener, sleep): self.assertEqual(b"ok", result) sleep.assert_called_once_with(1) + @patch("scripts.audit_api_coverage.time.sleep") + @patch("scripts.audit_api_coverage.build_opener") + def test_retries_server_http_error(self, build_opener, sleep): + server_error = HTTPError( + audit_api_coverage.CATALOG_URL, + 503, + "Service Unavailable", + {}, + None, + ) + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.geturl.return_value = audit_api_coverage.CATALOG_URL + response.read.return_value = b"ok" + build_opener.return_value.open.side_effect = [ + server_error, + response, + ] + + try: + result = audit_api_coverage._fetch( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + finally: + server_error.close() + + self.assertEqual(b"ok", result) + sleep.assert_called_once_with(1) + + @patch("scripts.audit_api_coverage.time.sleep") + @patch("scripts.audit_api_coverage.build_opener") + def test_does_not_retry_client_http_error(self, build_opener, sleep): + client_error = HTTPError( + audit_api_coverage.CATALOG_URL, + 404, + "Not Found", + {}, + None, + ) + build_opener.return_value.open.side_effect = client_error + + try: + with self.assertRaises(HTTPError): + audit_api_coverage._fetch( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + finally: + client_error.close() + + build_opener.return_value.open.assert_called_once() + sleep.assert_not_called() + + +class PublishedOperationsTests(unittest.TestCase): + @patch("scripts.audit_api_coverage._fetch") + def test_rejects_distinct_operations_with_same_canonical_key(self, fetch): + fetch.side_effect = [ + b'{"specs":[{"url":"https://api.coval.dev/a"},' + b'{"url":"https://api.coval.dev/b"}]}', + b"paths:\n /v1/agents/{agent_id}:\n get: {}\n", + b"paths:\n /v1/agents/{id}:\n get: {}\n", + ] + + with self.assertRaisesRegex(RuntimeError, "share canonical key"): + audit_api_coverage._published_operations( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + + @patch("scripts.audit_api_coverage._fetch") + def test_deduplicates_identical_operation_across_specs(self, fetch): + fetch.side_effect = [ + b'{"specs":[{"url":"https://api.coval.dev/a"},' + b'{"url":"https://api.coval.dev/b"}]}', + b"paths:\n /v1/agents/{agent_id}:\n get: {}\n", + b"paths:\n /v1/agents/{agent_id}:\n get: {}\n", + ] + + operations = audit_api_coverage._published_operations( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + + self.assertEqual( + {"GET /agents/{id}": "GET /v1/agents/{agent_id}"}, + operations, + ) + class CommandCoverageTests(unittest.TestCase): def test_only_counts_client_operations_referenced_by_commands(self):