diff --git a/.github/workflows/api-parity-audit.yml b/.github/workflows/api-parity-audit.yml new file mode 100644 index 0000000..af6014c --- /dev/null +++ b/.github/workflows/api-parity-audit.yml @@ -0,0 +1,136 @@ +name: Weekly API parity PR + +# Repository-owned drift detection for the hand-written CLI. The workflow +# refreshes a deterministic coverage report and opens or updates one rolling PR +# when the public API and first-class CLI command surface change. It does not +# claim to regenerate hand-written command UX from OpenAPI. +# +# Prerequisite: +# REGEN_PR_TOKEN: a token with Contents and Pull requests write access to this +# repository. The organization does not allow GITHUB_TOKEN to create PRs. + +on: + workflow_dispatch: + schedule: + # Monday 10:00 UTC, after the SDK regeneration job. + - cron: "0 10 * * 1" + +permissions: + contents: read + issues: write + +concurrency: + group: weekly-api-parity-pr + cancel-in-progress: false + +jobs: + refresh: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + ref: main + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/requirements-audit.txt + + - name: Install pinned dependencies + run: python -m pip install --requirement scripts/requirements-audit.txt + + - name: Test the audit + run: python -m unittest scripts/test_audit_api_coverage.py + + - name: Refresh the deterministic parity report + run: >- + python scripts/audit_api_coverage.py + --write-markdown api-coverage-report.md + --allow-drift + + - name: Open or update the parity PR + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.REGEN_PR_TOKEN }} + commit-message: "chore(cli): refresh API parity report" + title: "chore(cli): reconcile weekly API parity" + body: | + Automated comparison of the live public OpenAPI catalog with first-class CLI commands. + + - `api-coverage-report.md` is deterministic and changes only when API or CLI coverage changes. + - This PR does not claim to generate the CLI's hand-written command UX. + - If the report says `ACTION REQUIRED`, implement the missing commands or explicitly review the manifest exception, regenerate the report, and get CI green before merging. + + The branch is stable, so future weekly runs update this PR instead of opening duplicates. + branch: chore/weekly-api-parity + delete-branch: false + add-paths: api-coverage-report.md + + - name: Open or update the automation failure issue + if: failure() + uses: actions/github-script@v7 + with: + script: | + const title = 'Weekly CLI API parity PR automation is failing'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + 'The scheduled CLI API parity workflow failed before it could produce or update its rolling PR.', + '', + `Run: ${runUrl}`, + '', + 'This issue reports broken automation, not ordinary API drift. Ordinary drift is reported through the rolling PR.', + ].join('\n'); + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + }); + } + + - name: Close a recovered automation failure issue + if: success() + uses: actions/github-script@v7 + with: + script: | + const title = 'Weekly CLI API parity PR automation is failing'; + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: `Recovered in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`, + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed', + state_reason: 'completed', + }); + } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df0fa13..f2be4b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,11 +9,45 @@ on: env: CARGO_TERM_COLOR: always +permissions: + contents: read + jobs: check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/requirements-audit.txt + + - name: Install API audit dependencies + run: python -m pip install --requirement scripts/requirements-audit.txt + + - name: Test automation scripts + run: python -m unittest discover --start-directory scripts --pattern 'test_*.py' + + - name: Lint automation scripts + run: | + python -m ruff check scripts + python -m ruff format --check scripts + + - name: Audit live API command coverage + # Live API/network failures are advisory on ordinary CI runs. The + # repository-owned weekly parity PR is the strict reconciliation gate. + continue-on-error: ${{ github.event_name != 'pull_request' || github.head_ref != 'chore/weekly-api-parity' }} + run: | + python scripts/audit_api_coverage.py \ + --write-markdown api-coverage-report.md + git diff --exit-code -- api-coverage-report.md + + - name: Validate release manifests + run: python scripts/release_version.py - uses: dtolnay/rust-toolchain@stable with: @@ -25,10 +59,10 @@ jobs: run: cargo fmt --check - name: Clippy - run: cargo clippy -- -D warnings + run: cargo clippy --all-targets --all-features -- -D warnings - name: Test - run: cargo test + run: cargo test --all-targets --all-features build: needs: check @@ -55,6 +89,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/release-on-version-bump.yml b/.github/workflows/release-on-version-bump.yml new file mode 100644 index 0000000..cda2627 --- /dev/null +++ b/.github/workflows/release-on-version-bump.yml @@ -0,0 +1,116 @@ +name: Release on version bump + +# Wait for the exact main-branch CI run to pass, then release an untagged Cargo +# version. The release workflow is called directly because a tag pushed with +# GITHUB_TOKEN does not trigger another workflow. + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + workflow_dispatch: + +permissions: + contents: write + issues: write + +concurrency: + group: release-on-version-bump + cancel-in-progress: false + +jobs: + prepare: + if: >- + github.event_name == 'workflow_dispatch' || + ( + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' + ) + runs-on: ubuntu-latest + outputs: + release: ${{ steps.tag.outputs.release }} + release_ref: ${{ steps.tag.outputs.release_ref }} + tag: ${{ steps.version.outputs.tag }} + steps: + - name: Resolve the CI-tested commit + id: ref + env: + WORKFLOW_RUN_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + if [ "$GITHUB_REF_NAME" != "main" ]; then + echo "::error::Manual releases must be dispatched from main (got $GITHUB_REF_NAME)." + exit 1 + fi + echo "sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + else + echo "sha=$WORKFLOW_RUN_SHA" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@v6 + with: + ref: ${{ steps.ref.outputs.sha }} + fetch-depth: 0 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Read and validate the release version + id: version + run: | + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + python scripts/release_version.py --github-output "$GITHUB_OUTPUT" + else + python scripts/release_version.py \ + --allow-non-release \ + --github-output "$GITHUB_OUTPUT" + fi + + - name: Tag an unreleased version + id: tag + env: + RELEASE_CANDIDATE: ${{ steps.version.outputs.release_candidate }} + RELEASE_SHA: ${{ steps.ref.outputs.sha }} + RELEASE_TAG: ${{ steps.version.outputs.tag }} + run: | + set -euo pipefail + if [ "$RELEASE_CANDIDATE" != "true" ]; then + echo "Cargo.toml does not contain a stable release candidate; nothing to do." + echo "release=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git fetch --tags --force + + if git rev-parse -q --verify "refs/tags/$RELEASE_TAG" >/dev/null; then + existing_sha="$(git rev-list -n 1 "$RELEASE_TAG")" + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Manual recovery will rerun the existing ${RELEASE_TAG} release." + echo "release=true" >> "$GITHUB_OUTPUT" + echo "release_ref=$existing_sha" >> "$GITHUB_OUTPUT" + else + echo "${RELEASE_TAG} is already released; nothing to do." + echo "release=false" >> "$GITHUB_OUTPUT" + echo "release_ref=$existing_sha" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$RELEASE_TAG" "$RELEASE_SHA" -m "$RELEASE_TAG" + git push origin "$RELEASE_TAG" + echo "release=true" >> "$GITHUB_OUTPUT" + echo "release_ref=$RELEASE_SHA" >> "$GITHUB_OUTPUT" + + release: + needs: prepare + if: needs.prepare.outputs.release == 'true' + uses: ./.github/workflows/release.yml + with: + release_ref: ${{ needs.prepare.outputs.release_ref }} + tag_name: ${{ needs.prepare.outputs.tag }} + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b994217..47035ef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,55 @@ name: Release +# A human-pushed tag can still invoke this workflow directly. The gated +# version-bump bridge calls it as a reusable workflow because tags pushed with +# GITHUB_TOKEN do not trigger a second workflow. + on: push: tags: - - 'v*' + - "v*" + workflow_call: + inputs: + release_ref: + description: Commit or tag to build + required: true + type: string + tag_name: + description: Existing v-prefixed release tag + required: true + type: string env: CARGO_TERM_COLOR: always + RELEASE_REF: ${{ inputs.release_ref || github.ref }} + RELEASE_TAG: ${{ inputs.tag_name || github.ref_name }} + +concurrency: + group: release-${{ inputs.tag_name || github.ref_name }} + cancel-in-progress: false jobs: + validate: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ env.RELEASE_REF }} + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Match Cargo manifests and release tag + run: python scripts/release_version.py --expected-tag "$RELEASE_TAG" + build: + needs: validate + permissions: + contents: read strategy: matrix: include: @@ -38,6 +78,9 @@ jobs: steps: - uses: actions/checkout@v6 + with: + ref: ${{ env.RELEASE_REF }} + persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: @@ -77,8 +120,6 @@ jobs: contents: write steps: - - uses: actions/checkout@v6 - - name: Download artifacts uses: actions/download-artifact@v8 with: @@ -92,9 +133,11 @@ jobs: done cat -- */SHA256SUMS > ../SHA256SUMS - - name: Create release + - name: Create or update release uses: softprops/action-gh-release@v3 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.RELEASE_REF }} files: | artifacts/*/*.tar.gz SHA256SUMS @@ -103,17 +146,24 @@ jobs: update-homebrew: needs: release runs-on: ubuntu-latest + permissions: + contents: read steps: - - name: Extract version from tag - id: version - run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@v6 + with: + ref: ${{ env.RELEASE_REF }} + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" - name: Download SHA256SUMS from release env: GH_TOKEN: ${{ github.token }} run: | - gh release download "$GITHUB_REF_NAME" \ + gh release download "$RELEASE_TAG" \ --repo "$GITHUB_REPOSITORY" \ --pattern SHA256SUMS @@ -130,54 +180,129 @@ jobs: - name: Update Homebrew tap env: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} - VERSION: ${{ steps.version.outputs.version }} + VERSION: ${{ env.RELEASE_TAG }} SHA_MACOS_ARM64: ${{ steps.sha.outputs.macos_arm64 }} SHA_MACOS_X64: ${{ steps.sha.outputs.macos_x64 }} SHA_LINUX_ARM64: ${{ steps.sha.outputs.linux_arm64 }} SHA_LINUX_X64: ${{ steps.sha.outputs.linux_x64 }} run: | - git clone "https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/coval-ai/homebrew-tap.git" tap - cd tap - - cat > Formula/coval.rb << FORMULA - class Coval < Formula - desc "CLI for Coval AI agent evaluation platform" - homepage "https://coval.dev" - version "${VERSION}" - license "MIT" - - on_macos do - if Hardware::CPU.arm? - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-macos-arm64.tar.gz" - sha256 "${SHA_MACOS_ARM64}" - else - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-macos-x64.tar.gz" - sha256 "${SHA_MACOS_X64}" - end - end - - on_linux do - if Hardware::CPU.arm? - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-linux-arm64.tar.gz" - sha256 "${SHA_LINUX_ARM64}" - else - url "https://github.com/coval-ai/cli/releases/download/v${VERSION}/coval-linux-x64.tar.gz" - sha256 "${SHA_LINUX_X64}" - end - end - - def install - bin.install "coval" - end - - test do - system "#{bin}/coval", "--version" - end - end - FORMULA - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add Formula/coval.rb - git commit -m "chore: update coval to ${VERSION}" - git push origin main + set -euo pipefail + if [ -z "$HOMEBREW_TAP_TOKEN" ]; then + echo "::error::HOMEBREW_TAP_TOKEN is not configured." + exit 1 + fi + + tap_dir="${RUNNER_TEMP}/homebrew-tap" + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "$HOMEBREW_TAP_TOKEN" | base64 | tr -d '\n')" + git_auth() { + GIT_CONFIG_COUNT=1 \ + GIT_CONFIG_KEY_0=http.https://github.com/.extraheader \ + GIT_CONFIG_VALUE_0="$auth_header" \ + git "$@" + } + + git_auth clone "https://github.com/coval-ai/homebrew-tap.git" "$tap_dir" + mkdir -p "$tap_dir/Formula" + python scripts/render_homebrew_formula.py \ + --version "${VERSION#v}" \ + --macos-arm64 "$SHA_MACOS_ARM64" \ + --macos-x64 "$SHA_MACOS_X64" \ + --linux-arm64 "$SHA_LINUX_ARM64" \ + --linux-x64 "$SHA_LINUX_X64" \ + --output "$tap_dir/Formula/coval.rb" + + if [ -z "$(git -C "$tap_dir" status --porcelain -- Formula/coval.rb)" ]; then + echo "Homebrew formula already matches ${VERSION}; nothing to push." + exit 0 + fi + + git -C "$tap_dir" config user.name "github-actions[bot]" + git -C "$tap_dir" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$tap_dir" add Formula/coval.rb + git -C "$tap_dir" commit -m "chore: update coval to ${VERSION#v}" + for attempt in 1 2 3; do + if git_auth -C "$tap_dir" push origin HEAD:main; then + exit 0 + fi + if [ "$attempt" -eq 3 ]; then + break + fi + echo "Push attempt ${attempt} failed; rebasing on origin/main." + git_auth -C "$tap_dir" fetch origin main + git -C "$tap_dir" rebase origin/main + sleep $((attempt * 5)) + done + echo "::error::Could not push the Homebrew formula update after 3 attempts." + exit 1 + + report-failure: + needs: [validate, build, release, update-homebrew] + if: ${{ always() && contains(needs.*.result, 'failure') }} + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open or update the release failure issue + uses: actions/github-script@v7 + with: + script: | + const title = 'CLI release workflow is failing'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `Release \`${process.env.RELEASE_TAG}\` failed: ${runUrl}`; + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + }); + } + + close-recovered-failure: + needs: [validate, build, release, update-homebrew] + if: ${{ always() && needs.update-homebrew.result == 'success' }} + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Close a recovered release failure issue + uses: actions/github-script@v7 + with: + script: | + const title = 'CLI release workflow is failing'; + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + }); + const existing = issues.find((issue) => !issue.pull_request && issue.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: `Recovered in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`, + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed', + state_reason: 'completed', + }); + } diff --git a/.gitignore b/.gitignore index 13ac30d..db5c4b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /target +__pycache__/ +*.py[cod] .env .DS_Store *.log diff --git a/README.md b/README.md index b1ee343..17a6c11 100644 --- a/README.md +++ b/README.md @@ -186,16 +186,67 @@ coval traces search --input-json @trace-search.json --format json ## API Coverage Audit The checked-in coverage manifest records every published API operation that the -CLI does not yet expose. Run the live audit after API or client changes: +CLI does not yet expose as a first-class command. The audit traces each literal +client route back to a resource-client method referenced by `src/commands/`, so +an unused HTTP helper does not count as command coverage. + +Run the deterministic tests and live audit after API, client, or command changes: ```bash -python3 -m pip install PyYAML -python3 scripts/audit_api_coverage.py +python3 -m pip install --requirement scripts/requirements-audit.txt +python3 -m unittest scripts/test_audit_api_coverage.py +python3 scripts/audit_api_coverage.py \ + --write-markdown api-coverage-report.md ``` -The audit fails for new or stale gaps and for CLI routes absent from the public -OpenAPI catalog unless they are explicitly marked as planned or documented extras -in `api-coverage.toml`. +The audit fails for new or stale gaps, a stale checked-in snapshot, or command +routes absent from the public OpenAPI catalog unless they are explicitly marked +as planned or documented extras in `api-coverage.toml`. + +A repository-owned GitHub workflow runs every Monday and refreshes the +deterministic `api-coverage-report.md`. When coverage changes, it opens or +updates one rolling PR on `chore/weekly-api-parity`; the PR's CI remains blocked +until the command implementation or an explicitly reviewed manifest exception +reconciles the drift. A GitHub issue is used only if the automation itself +fails before it can create or update that PR. + +The SDK regeneration workflow can open deterministic codegen PRs because its +published clients are generated from OpenAPI. The CLI command surface is still +hand-written, so this repository does not present an automated audit as command +generation. Repository-owned generated-model PRs are tracked separately under +COVAL-2079; they require the CLI's OpenAPI type-codegen migration to be +completed first. + +## Release Automation + +CLI implementation PRs retain a human merge gate. When a merged PR changes the +Cargo version, the exact `main` CI run must pass before +`Release on version bump` creates the matching `v*` tag and calls the reusable +release workflow. A merge without a version bump does not release anything. + +Use the checked-in helper so `Cargo.toml` and `Cargo.lock` move together: + +```bash +# New first-class commands +python3 scripts/bump_version.py minor + +# Backward-compatible fixes +python3 scripts/bump_version.py patch +``` + +The release workflow validates tag/version consistency, builds all five target +binaries, creates or updates the GitHub release, and updates +`coval-ai/homebrew-tap`. A manual `Release on version bump` dispatch safely +retries the current version without creating another tag. + +Repository prerequisite: + +- `REGEN_PR_TOKEN`: a fine-grained token with Contents and Pull requests + read/write access to `coval-ai/cli`. The organization does not allow + `GITHUB_TOKEN` to create pull requests. +- `HOMEBREW_TAP_TOKEN`: a fine-grained token or GitHub App token with Contents + read/write access to `coval-ai/homebrew-tap`. The Homebrew update is + idempotent, so retrying an already-current formula succeeds without a commit. ## Configuration diff --git a/api-coverage-report.md b/api-coverage-report.md new file mode 100644 index 0000000..f24e498 --- /dev/null +++ b/api-coverage-report.md @@ -0,0 +1,104 @@ +# CLI API Coverage Report + + + +This report compares the published Coval OpenAPI catalog with first-class +CLI commands. It is intentionally timestamp-free so the weekly workflow +opens or updates a PR only when coverage actually changes. + +## Summary + +| Metric | Value | +| --- | ---: | +| Reconciliation status | PASS | +| Published operations | 174 | +| First-class CLI operations | 124 | +| Reviewed gaps | 50 | +| Client operations | 125 | + +Catalog: https://api.coval.dev/v1/openapi + +## New published operations without CLI commands + +- None. + +## Reviewed gaps no longer present + +- None. + +## CLI operations absent from published OpenAPI + +- None. + +## Allowed extras no longer present + +- None. + +## Planned operations no longer present + +- None. + +## Client methods not mapped to operations + +- None. + +## Coverage snapshot mismatches + +- None. + +## Client-only operations + +- None. + +## All current published gaps + +- `DELETE /integrations/slack` +- `DELETE /metrics/flows/{flow_id}` +- `DELETE /test-sets/{test_set_id}/agents/{agent_id}` +- `DELETE /webhooks/{webhook_id}` +- `GET /agents/{agent_id}/versions` +- `GET /integrations/slack` +- `GET /metrics/flows` +- `GET /metrics/recently-deleted` +- `GET /metrics/sql-schema` +- `GET /metrics/tags` +- `GET /metrics/template-variables` +- `GET /metrics/{metric_id}/baselines/{baseline_id}/history` +- `GET /organization/conversation-metrics` +- `GET /organization/monitoring-metrics` +- `GET /personas/tags` +- `GET /personas/{persona_id}/versions` +- `GET /reports/{report_id}/rows` +- `GET /review-annotations/metric-health-stats` +- `GET /review-projects/{project_id}/insights` +- `GET /review-projects/{project_id}/metric-agreement` +- `GET /review-projects/{project_id}/progress` +- `GET /runs/tags` +- `GET /scheduled-runs/{scheduled_run_id}/runs` +- `GET /test-sets/{test_set_id}/agents` +- `GET /test-sets/{test_set_id}/records` +- `GET /test-sets/{test_set_id}/versions` +- `GET /webhooks` +- `PATCH /metrics/flows/{flow_id}` +- `PATCH /organization/conversation-metrics` +- `PATCH /organization/monitoring-metrics` +- `PATCH /webhooks/{webhook_id}` +- `POST /agents/{agent_id}/duplicate` +- `POST /agents/{agent_id}/versions/{version_id}/revert` +- `POST /audio:upload` +- `POST /integrations/slack/connect` +- `POST /metrics/flows` +- `POST /metrics/outputs:batchGet` +- `POST /metrics/sql:test` +- `POST /metrics/{metric_id}/restore` +- `POST /metrics/{metric_id}/versions/{version_id}/revert` +- `POST /personas/{persona_id}/duplicate` +- `POST /personas/{persona_id}/versions/{version_id}/revert` +- `POST /review-annotations:withMetricOutputs` +- `POST /review-projects/disagreement-state` +- `POST /simulations:rerunMetrics` +- `POST /test-sets/{test_set_id}/agents:add` +- `POST /test-sets/{test_set_id}/duplicate` +- `POST /test-sets/{test_set_id}/versions/{version_id}/revert` +- `POST /traces` +- `POST /webhooks` diff --git a/api-coverage.toml b/api-coverage.toml index 6316603..6296a61 100644 --- a/api-coverage.toml +++ b/api-coverage.toml @@ -1,20 +1,16 @@ # Reviewed against the live public OpenAPI catalog by scripts/audit_api_coverage.py. -# Every published operation must either appear in the CLI client or have a reason here. +# Every published operation must either reach a first-class CLI command or have a reason here. [snapshot] catalog_url = "https://api.coval.dev/v1/openapi" reviewed_at = "2026-07-30" -published_operations = 173 -cli_supported_operations = 123 +published_operations = 174 +cli_supported_operations = 124 [[allowed_extra]] operation = "POST /test-cases/{test_case_id}/media:upload-url" reason = "Implemented by the public API but currently omitted from its published OpenAPI specs." -[[planned_operation]] -operation = "POST /traces/search" -reason = "Implemented with COVAL-4307 and expected to appear after the backend change is deployed." - [[known_gap]] operation = "GET /agents/{agent_id}/versions" reason = "Agent version history remains to be modeled under COVAL-2079." diff --git a/scripts/audit_api_coverage.py b/scripts/audit_api_coverage.py index 5faba2f..4275f3d 100644 --- a/scripts/audit_api_coverage.py +++ b/scripts/audit_api_coverage.py @@ -1,4 +1,4 @@ -"""Compare CLI HTTP operations with Coval's published OpenAPI catalog. +"""Compare first-class CLI operations with Coval's published OpenAPI catalog. The audit is intentionally live: the public catalog is the source of truth, while ``api-coverage.toml`` records reviewed gaps and temporary pre-deploy operations. @@ -17,7 +17,9 @@ import json import re import sys +import time from pathlib import Path +from urllib.error import HTTPError, URLError from urllib.parse import urlsplit from urllib.request import HTTPRedirectHandler, Request, build_opener @@ -27,8 +29,10 @@ CATALOG_URL = "https://api.coval.dev/v1/openapi" DEFAULT_ALLOWED_ORIGINS = frozenset({"https://api.coval.dev"}) HTTP_METHODS = frozenset({"delete", "get", "patch", "post", "put"}) +FETCH_ATTEMPTS = 3 ROOT = Path(__file__).resolve().parents[1] CLIENT_PATH = ROOT / "src" / "client" / "mod.rs" +COMMANDS_PATH = ROOT / "src" / "commands" MANIFEST_PATH = ROOT / "api-coverage.toml" @@ -83,9 +87,19 @@ def _fetch(url: str, allowed_origins: frozenset[str]) -> bytes: url, headers={"User-Agent": "coval-cli-api-coverage-audit"} ) opener = build_opener(_AllowedOriginRedirectHandler(allowed_origins)) - with opener.open(request, timeout=30) as response: - _validate_fetch_url(response.geturl(), allowed_origins) - return response.read() + for attempt in range(FETCH_ATTEMPTS): + try: + with opener.open(request, timeout=30) as response: + _validate_fetch_url(response.geturl(), allowed_origins) + return response.read() + except HTTPError as error: + if error.code < 500 or attempt == FETCH_ATTEMPTS - 1: + raise + except (TimeoutError, URLError): + if attempt == FETCH_ATTEMPTS - 1: + raise + time.sleep(2**attempt) + raise AssertionError("fetch retry loop exited unexpectedly") def _canonical_operation(method: str, path: str) -> str: @@ -104,56 +118,127 @@ def _published_operations( for path, path_item in (spec.get("paths") or {}).items(): for method in HTTP_METHODS & set(path_item): canonical = _canonical_operation(method, path) - operations[canonical] = f"{method.upper()} {path}" + display = f"{method.upper()} {path}" + previous = operations.get(canonical) + if previous is not None and previous != display: + raise RuntimeError( + f"published operations {previous!r} and {display!r} " + f"share canonical key {canonical!r}" + ) + operations[canonical] = display return operations -def _rust_function_blocks(source: str) -> list[str]: - starts = list(re.finditer(r"(?m)^ pub async fn ", source)) - blocks: list[str] = [] +def _rust_function_blocks(source: str) -> list[tuple[str, str]]: + starts = list(re.finditer(r"(?m)^ pub async fn (?P[A-Za-z0-9_]+)", source)) + blocks: list[tuple[str, str]] = [] for index, match in enumerate(starts): end = starts[index + 1].start() if index + 1 < len(starts) else len(source) - blocks.append(source[match.start() : end]) + blocks.append((match.group("name"), source[match.start() : end])) return blocks -def _client_operations() -> dict[str, str]: +def _snake_case_client_name(name: str) -> str: + return re.sub(r"(? dict[str, dict]: source = CLIENT_PATH.read_text() - operations: dict[str, str] = {} - for block in _rust_function_blocks(source): - paths = re.findall(r'"(/v1/[^"]+)"', block) - if not paths: - continue - - methods = [] - if re.search(r"\.(?:post|post_empty)\(", block): - methods.append("POST") - if re.search(r"\.patch\(", block): - methods.append("PATCH") - if re.search(r"\.delete\(", block): - methods.append("DELETE") - if re.search(r"\.get\(", block): - methods.append("GET") - - if len(methods) != 1: - function_name = re.search(r"pub async fn ([a-zA-Z0-9_]+)", block) - name = function_name.group(1) if function_name else "" - raise RuntimeError( - f"could not infer exactly one HTTP method for client function {name}: {methods}" - ) + implementations = list( + re.finditer(r"(?m)^impl (?P[A-Za-z0-9_]+Client)<'_> \{", source) + ) + operations: dict[str, dict] = {} + + for index, implementation in enumerate(implementations): + end = ( + implementations[index + 1].start() + if index + 1 < len(implementations) + else len(source) + ) + accessor = _snake_case_client_name(implementation.group("name")) + implementation_source = source[implementation.end() : end] + + for function_name, block in _rust_function_blocks(implementation_source): + paths = re.findall(r'"(/v1/[^"]+)"', block) + if not paths: + continue - for path in paths: - canonical = _canonical_operation(methods[0], path) - operations[canonical] = f"{methods[0]} {path.removeprefix('/v1')}" + method_calls = set( + re.findall(r"\.(get|post|post_empty|patch|delete)\(", block) + ) + methods = { + "POST" if method in {"post", "post_empty"} else method.upper() + for method in method_calls + } + if len(methods) != 1: + raise RuntimeError( + "could not infer exactly one HTTP method for client function " + f"{accessor}.{function_name}: {sorted(methods)}" + ) + + method = methods.pop() + client_method = f"{accessor}.{function_name}" + for path in paths: + canonical = _canonical_operation(method, path) + operation = operations.setdefault( + canonical, + { + "operation": f"{method} {path.removeprefix('/v1')}", + "client_methods": set(), + }, + ) + operation["client_methods"].add(client_method) return operations +def _command_client_methods(allowed_accessors: set[str]) -> dict[str, set[str]]: + """Return resource-client methods referenced by first-class command modules.""" + + pattern = re.compile( + r"\bclient\s*\.\s*(?P[A-Za-z0-9_]+)\s*" + r"\([^)]*\)\s*\.\s*(?P[A-Za-z0-9_]+)\s*\(", + re.MULTILINE, + ) + methods: dict[str, set[str]] = {} + for path in sorted(COMMANDS_PATH.glob("*.rs")): + for match in pattern.finditer(path.read_text()): + if match.group("accessor") not in allowed_accessors: + continue + method = f"{match.group('accessor')}.{match.group('method')}" + methods.setdefault(method, set()).add(path.name) + return methods + + +def _command_operations( + client_operations: dict[str, dict], +) -> tuple[dict[str, dict], list[str]]: + known_client_methods = { + method + for operation in client_operations.values() + for method in operation["client_methods"] + } + allowed_accessors = {method.partition(".")[0] for method in known_client_methods} + command_methods = _command_client_methods(allowed_accessors) + operations = { + canonical: operation + for canonical, operation in client_operations.items() + if operation["client_methods"] & set(command_methods) + } + unmapped_methods = sorted(set(command_methods) - known_client_methods) + return operations, unmapped_methods + + def _manifest_operations(entries: list[dict], section: str) -> dict[str, dict]: operations: dict[str, dict] = {} for entry in entries: operation = entry.get("operation", "") reason = entry.get("reason", "") - if not operation or not reason.strip(): + if ( + not isinstance(operation, str) + or not operation + or not isinstance(reason, str) + or not reason.strip() + ): raise ValueError( f"{section} entries require non-empty operation and reason fields" ) @@ -171,6 +256,25 @@ def _manifest_operations(entries: list[dict], section: str) -> dict[str, dict]: return operations +def _snapshot_mismatches( + snapshot: dict, + *, + catalog_url: str, + published_operation_count: int, + supported_operation_count: int, +) -> list[str]: + expected = { + "catalog_url": catalog_url, + "published_operations": published_operation_count, + "cli_supported_operations": supported_operation_count, + } + return [ + f"{field}: recorded {snapshot.get(field)!r}, current {value!r}" + for field, value in expected.items() + if snapshot.get(field) != value + ] + + def audit( catalog_url: str, allowed_origins: frozenset[str] = DEFAULT_ALLOWED_ORIGINS, @@ -178,6 +282,7 @@ def audit( manifest = tomllib.loads(MANIFEST_PATH.read_text()) published = _published_operations(catalog_url, allowed_origins) client = _client_operations() + commands, unmapped_command_methods = _command_operations(client) known_gaps = _manifest_operations(manifest.get("known_gap", []), "known_gap") allowed_extras = _manifest_operations( manifest.get("allowed_extra", []), "allowed_extra" @@ -198,25 +303,34 @@ def audit( published_keys = set(published) client_keys = set(client) - actual_gaps = published_keys - client_keys - actual_extras = client_keys - published_keys + command_keys = set(commands) + actual_gaps = published_keys - command_keys + actual_extras = command_keys - published_keys new_gaps = actual_gaps - set(known_gaps) stale_gaps = set(known_gaps) - actual_gaps unexpected_extras = actual_extras - set(allowed_extras) - set(planned) stale_allowed_extras = set(allowed_extras) - actual_extras stale_planned = set(planned) - actual_extras + client_only = client_keys - command_keys + snapshot_mismatches = _snapshot_mismatches( + manifest.get("snapshot", {}), + catalog_url=catalog_url, + published_operation_count=len(published), + supported_operation_count=len(published_keys & command_keys), + ) report = { "catalog_url": catalog_url, "published_operation_count": len(published), - "cli_operation_count": len(client), - "supported_operation_count": len(published_keys & client_keys), + "client_operation_count": len(client), + "command_operation_count": len(commands), + "supported_operation_count": len(published_keys & command_keys), "known_gap_count": len(actual_gaps & set(known_gaps)), "new_gaps": [published[item] for item in sorted(new_gaps)], "stale_gaps": [known_gaps[item]["operation"] for item in sorted(stale_gaps)], "unexpected_cli_operations": [ - client[item] for item in sorted(unexpected_extras) + client[item]["operation"] for item in sorted(unexpected_extras) ], "stale_allowed_extras": [ allowed_extras[item]["operation"] for item in sorted(stale_allowed_extras) @@ -224,6 +338,11 @@ def audit( "stale_planned_operations": [ planned[item]["operation"] for item in sorted(stale_planned) ], + "client_only_operations": [ + client[item]["operation"] for item in sorted(client_only) + ], + "unmapped_command_client_methods": unmapped_command_methods, + "snapshot_mismatches": snapshot_mismatches, "all_current_gaps": [published[item] for item in sorted(actual_gaps)], } passed = not ( @@ -232,10 +351,60 @@ def audit( or unexpected_extras or stale_allowed_extras or stale_planned + or unmapped_command_methods + or snapshot_mismatches ) return report, passed +def render_markdown_report(report: dict, passed: bool) -> str: + """Render a deterministic, reviewable API-parity report.""" + + lines = [ + "# CLI API Coverage Report", + "", + "", + "", + "This report compares the published Coval OpenAPI catalog with first-class", + "CLI commands. It is intentionally timestamp-free so the weekly workflow", + "opens or updates a PR only when coverage actually changes.", + "", + "## Summary", + "", + "| Metric | Value |", + "| --- | ---: |", + f"| Reconciliation status | {'PASS' if passed else 'ACTION REQUIRED'} |", + f"| Published operations | {report['published_operation_count']} |", + f"| First-class CLI operations | {report['supported_operation_count']} |", + f"| Reviewed gaps | {report['known_gap_count']} |", + f"| Client operations | {report['client_operation_count']} |", + "", + f"Catalog: {report['catalog_url']}", + "", + ] + + sections = ( + ("New published operations without CLI commands", "new_gaps"), + ("Reviewed gaps no longer present", "stale_gaps"), + ("CLI operations absent from published OpenAPI", "unexpected_cli_operations"), + ("Allowed extras no longer present", "stale_allowed_extras"), + ("Planned operations no longer present", "stale_planned_operations"), + ("Client methods not mapped to operations", "unmapped_command_client_methods"), + ("Coverage snapshot mismatches", "snapshot_mismatches"), + ("Client-only operations", "client_only_operations"), + ("All current published gaps", "all_current_gaps"), + ) + for title, key in sections: + lines.extend((f"## {title}", "")) + values = report[key] + lines.extend(f"- `{value}`" for value in values) + if not values: + lines.append("- None.") + lines.append("") + + return "\n".join(lines) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--catalog-url", default=CATALOG_URL) @@ -247,20 +416,37 @@ def main() -> int: parser.add_argument( "--json", action="store_true", help="Emit the full machine-readable report" ) + parser.add_argument( + "--write-markdown", + type=Path, + help="Write a deterministic Markdown report to this path", + ) + parser.add_argument( + "--allow-drift", + action="store_true", + help=( + "Exit successfully after writing a report even when parity needs " + "reconciliation; requires --write-markdown" + ), + ) args = parser.parse_args() + if args.allow_drift and args.write_markdown is None: + parser.error("--allow-drift requires --write-markdown") configured_origins = args.allowed_origin or sorted(DEFAULT_ALLOWED_ORIGINS) allowed_origins = frozenset( _normalize_allowed_origin(origin) for origin in configured_origins ) report, passed = audit(args.catalog_url, allowed_origins) + if args.write_markdown is not None: + args.write_markdown.write_text(render_markdown_report(report, passed)) if args.json: print(json.dumps(report, indent=2)) else: status = "PASS" if passed else "FAIL" print( f"{status}: {report['supported_operation_count']}/{report['published_operation_count']} " - f"published operations have CLI HTTP coverage; " + f"published operations have first-class CLI command coverage; " f"{report['known_gap_count']} reviewed gaps remain." ) for key in ( @@ -269,13 +455,15 @@ def main() -> int: "unexpected_cli_operations", "stale_allowed_extras", "stale_planned_operations", + "unmapped_command_client_methods", + "snapshot_mismatches", ): values = report[key] if values: print(f"{key}:") for value in values: print(f" - {value}") - return 0 if passed else 1 + return 0 if passed or args.allow_drift else 1 if __name__ == "__main__": diff --git a/scripts/bump_version.py b/scripts/bump_version.py new file mode 100644 index 0000000..8df2b8d --- /dev/null +++ b/scripts/bump_version.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Bump the CLI's stable Cargo version in both release manifests.""" + +from __future__ import annotations + +import argparse +import re +import tomllib +from pathlib import Path + +if __package__: + from .release_version import ROOT + from .release_version import release_version +else: + from release_version import ROOT + from release_version import release_version + + +def next_version(version: str, part: str) -> str: + major, minor, patch = (int(value) for value in version.split(".")) + if part == "minor": + return f"{major}.{minor + 1}.0" + if part == "patch": + return f"{major}.{minor}.{patch + 1}" + raise ValueError(f"unsupported version part: {part!r}") + + +def _replace_once(contents: str, pattern: str, replacement: str, path: Path) -> str: + updated, count = re.subn( + pattern, replacement, contents, count=1, flags=re.MULTILINE + ) + if count != 1: + raise ValueError(f"could not find one release version in {path}") + return updated + + +def _replace_package_version( + contents: str, current: str, updated: str, path: Path +) -> str: + sections = list(re.finditer(r"(?ms)^\[package\][ \t]*\n.*?(?=^\[|\Z)", contents)) + if len(sections) != 1: + raise ValueError(f"expected one [package] section in {path}") + + section = sections[0] + updated_section = _replace_once( + section.group(), + rf'^(version[ \t]*=[ \t]*"){re.escape(current)}("[ \t]*)$', + rf"\g<1>{updated}\g<2>", + path, + ) + return contents[: section.start()] + updated_section + contents[section.end() :] + + +def bump_version(part: str, root: Path = ROOT) -> str: + current = release_version(root) + updated = next_version(current, part) + cargo_path = root / "Cargo.toml" + lock_path = root / "Cargo.lock" + cargo_contents = cargo_path.read_text() + package_name = tomllib.loads(cargo_contents)["package"]["name"] + + cargo = _replace_package_version( + cargo_contents, + current, + updated, + cargo_path, + ) + lock = _replace_once( + lock_path.read_text(), + ( + rf'(^\[\[package\]\]\nname = "{re.escape(package_name)}"\nversion = ")' + rf"{re.escape(current)}" + r'("$)' + ), + rf"\g<1>{updated}\g<2>", + lock_path, + ) + + cargo_path.write_text(cargo) + lock_path.write_text(lock) + return updated + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("part", choices=("minor", "patch")) + parser.add_argument("--root", type=Path, default=ROOT) + args = parser.parse_args() + + print(bump_version(args.part, args.root)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 0000000..1aef8e7 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Validate CLI release manifests and emit the matching tag.""" + +from __future__ import annotations + +import argparse +import json +import re +import tomllib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +VERSION_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") + + +class NonReleaseVersionError(ValueError): + """Raised when Cargo.toml intentionally contains a non-stable version.""" + + +def release_version(root: Path = ROOT) -> str: + cargo = tomllib.loads((root / "Cargo.toml").read_text()) + name = cargo["package"]["name"] + version = cargo["package"]["version"] + if VERSION_RE.fullmatch(version) is None: + raise NonReleaseVersionError( + f"Cargo.toml has a non-release version: {version!r}" + ) + + lock = tomllib.loads((root / "Cargo.lock").read_text()) + workspace_packages = [ + package + for package in lock["package"] + if package["name"] == name and "source" not in package + ] + if len(workspace_packages) != 1: + raise ValueError( + f"expected one workspace package named {name!r} in Cargo.lock, " + f"found {len(workspace_packages)}" + ) + locked_version = workspace_packages[0]["version"] + if locked_version != version: + raise ValueError( + f"Cargo.lock has {name} {locked_version}, but Cargo.toml has {version}" + ) + return version + + +def release_metadata( + root: Path = ROOT, expected_tag: str | None = None +) -> dict[str, str]: + version = release_version(root) + tag = f"v{version}" + if expected_tag is not None and expected_tag != tag: + raise ValueError( + f"release tag {expected_tag!r} does not match Cargo version tag {tag!r}" + ) + return {"version": version, "tag": tag} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--expected-tag") + parser.add_argument("--github-output", type=Path) + parser.add_argument( + "--allow-non-release", + action="store_true", + help="Exit successfully without a tag when Cargo.toml is not stable semver", + ) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + try: + metadata = release_metadata(args.root, args.expected_tag) + except NonReleaseVersionError as error: + if not args.allow_non_release: + raise + if args.github_output is not None: + with args.github_output.open("a") as output: + output.write("release_candidate=false\n") + if args.json: + print(json.dumps({"release_candidate": False})) + else: + print(f"No stable release candidate: {error}") + return 0 + + if args.github_output is not None: + with args.github_output.open("a") as output: + output.write("release_candidate=true\n") + for key, value in metadata.items(): + output.write(f"{key}={value}\n") + if args.json: + print(json.dumps(metadata, sort_keys=True)) + else: + print(metadata["tag"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_homebrew_formula.py b/scripts/render_homebrew_formula.py new file mode 100644 index 0000000..0c012f4 --- /dev/null +++ b/scripts/render_homebrew_formula.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Render the Homebrew formula for one validated Coval CLI release.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +if __package__: + from .release_version import VERSION_RE +else: + from release_version import VERSION_RE + +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +def render_formula(version: str, checksums: dict[str, str]) -> str: + if VERSION_RE.fullmatch(version) is None: + raise ValueError(f"invalid release version: {version!r}") + + expected_platforms = {"macos_arm64", "macos_x64", "linux_arm64", "linux_x64"} + if set(checksums) != expected_platforms: + missing = sorted(expected_platforms - set(checksums)) + unexpected = sorted(set(checksums) - expected_platforms) + raise ValueError( + f"invalid checksum platforms; missing={missing}, unexpected={unexpected}" + ) + for platform, checksum in checksums.items(): + if SHA256_RE.fullmatch(checksum) is None: + raise ValueError(f"invalid SHA-256 for {platform}: {checksum!r}") + + return f'''class Coval < Formula + desc "CLI for Coval AI agent evaluation platform" + homepage "https://coval.dev" + version "{version}" + license "MIT" + + on_macos do + if Hardware::CPU.arm? + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-macos-arm64.tar.gz" + sha256 "{checksums["macos_arm64"]}" + else + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-macos-x64.tar.gz" + sha256 "{checksums["macos_x64"]}" + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-linux-arm64.tar.gz" + sha256 "{checksums["linux_arm64"]}" + else + url "https://github.com/coval-ai/cli/releases/download/v{version}/coval-linux-x64.tar.gz" + sha256 "{checksums["linux_x64"]}" + end + end + + def install + bin.install "coval" + end + + test do + system "#{{bin}}/coval", "--version" + end +end +''' + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--macos-arm64", required=True) + parser.add_argument("--macos-x64", required=True) + parser.add_argument("--linux-arm64", required=True) + parser.add_argument("--linux-x64", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + formula = render_formula( + args.version, + { + "macos_arm64": args.macos_arm64, + "macos_x64": args.macos_x64, + "linux_arm64": args.linux_arm64, + "linux_x64": args.linux_x64, + }, + ) + args.output.write_text(formula) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/requirements-audit.txt b/scripts/requirements-audit.txt new file mode 100644 index 0000000..9db3f9a --- /dev/null +++ b/scripts/requirements-audit.txt @@ -0,0 +1,2 @@ +PyYAML==6.0.3 +ruff==0.15.9 diff --git a/scripts/test_audit_api_coverage.py b/scripts/test_audit_api_coverage.py index 499c286..08a3b0c 100644 --- a/scripts/test_audit_api_coverage.py +++ b/scripts/test_audit_api_coverage.py @@ -1,6 +1,13 @@ -"""Tests for the live API-coverage audit's network boundary.""" +"""Tests for the live API-coverage audit.""" +import io +import tempfile import unittest +from contextlib import redirect_stderr +from contextlib import redirect_stdout +from pathlib import Path +from urllib.error import HTTPError +from urllib.error import URLError from unittest.mock import Mock from unittest.mock import patch @@ -50,6 +57,398 @@ def test_validates_final_response_url(self, build_opener): audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, ) + @patch("scripts.audit_api_coverage.time.sleep") + @patch("scripts.audit_api_coverage.build_opener") + def test_retries_transient_network_failure(self, build_opener, sleep): + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.geturl.return_value = audit_api_coverage.CATALOG_URL + response.read.return_value = b"ok" + build_opener.return_value.open.side_effect = [ + URLError("temporary"), + response, + ] + + result = audit_api_coverage._fetch( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + + self.assertEqual(b"ok", result) + sleep.assert_called_once_with(1) + + @patch("scripts.audit_api_coverage.time.sleep") + @patch("scripts.audit_api_coverage.build_opener") + def test_retries_server_http_error(self, build_opener, sleep): + server_error = HTTPError( + audit_api_coverage.CATALOG_URL, + 503, + "Service Unavailable", + {}, + None, + ) + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.geturl.return_value = audit_api_coverage.CATALOG_URL + response.read.return_value = b"ok" + build_opener.return_value.open.side_effect = [ + server_error, + response, + ] + + try: + result = audit_api_coverage._fetch( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + finally: + server_error.close() + + self.assertEqual(b"ok", result) + sleep.assert_called_once_with(1) + + @patch("scripts.audit_api_coverage.time.sleep") + @patch("scripts.audit_api_coverage.build_opener") + def test_does_not_retry_client_http_error(self, build_opener, sleep): + client_error = HTTPError( + audit_api_coverage.CATALOG_URL, + 404, + "Not Found", + {}, + None, + ) + build_opener.return_value.open.side_effect = client_error + + try: + with self.assertRaises(HTTPError): + audit_api_coverage._fetch( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + finally: + client_error.close() + + build_opener.return_value.open.assert_called_once() + sleep.assert_not_called() + + +class PublishedOperationsTests(unittest.TestCase): + @patch("scripts.audit_api_coverage._fetch") + def test_rejects_distinct_operations_with_same_canonical_key(self, fetch): + fetch.side_effect = [ + b'{"specs":[{"url":"https://api.coval.dev/a"},' + b'{"url":"https://api.coval.dev/b"}]}', + b"paths:\n /v1/agents/{agent_id}:\n get: {}\n", + b"paths:\n /v1/agents/{id}:\n get: {}\n", + ] + + with self.assertRaisesRegex(RuntimeError, "share canonical key"): + audit_api_coverage._published_operations( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + + @patch("scripts.audit_api_coverage._fetch") + def test_deduplicates_identical_operation_across_specs(self, fetch): + fetch.side_effect = [ + b'{"specs":[{"url":"https://api.coval.dev/a"},' + b'{"url":"https://api.coval.dev/b"}]}', + b"paths:\n /v1/agents/{agent_id}:\n get: {}\n", + b"paths:\n /v1/agents/{agent_id}:\n get: {}\n", + ] + + operations = audit_api_coverage._published_operations( + audit_api_coverage.CATALOG_URL, + audit_api_coverage.DEFAULT_ALLOWED_ORIGINS, + ) + + self.assertEqual( + {"GET /agents/{id}": "GET /v1/agents/{agent_id}"}, + operations, + ) + + +class CommandCoverageTests(unittest.TestCase): + def test_only_counts_client_operations_referenced_by_commands(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + client_path = root / "client.rs" + commands_path = root / "commands" + commands_path.mkdir() + client_path.write_text( + """ +impl AgentsClient<'_> { + pub async fn list(&self) { + let url = self.0.url("/v1/agents"); + self.0.get(url).await + } + + pub async fn get(&self, id: &str) { + let url = self.0.url(&format!("/v1/agents/{id}")); + self.0.get(url).await + } +} +""" + ) + (commands_path / "agents.rs").write_text( + """ +let agents = client + .agents() + .list() + .await?; +let response = client.get(url).send().await?; +""" + ) + + with ( + patch.object(audit_api_coverage, "CLIENT_PATH", client_path), + patch.object(audit_api_coverage, "COMMANDS_PATH", commands_path), + ): + client_operations = audit_api_coverage._client_operations() + command_operations, unmapped = audit_api_coverage._command_operations( + client_operations + ) + + self.assertEqual(2, len(client_operations)) + self.assertEqual( + ["GET /agents"], + [operation["operation"] for operation in command_operations.values()], + ) + self.assertEqual([], unmapped) + + def test_any_exposed_client_method_covers_a_shared_operation(self): + client_operations = { + "GET /conversations": { + "operation": "GET /conversations", + "client_methods": { + "conversations.list", + "conversations.list_with_metric_outputs", + }, + } + } + with patch.object( + audit_api_coverage, + "_command_client_methods", + return_value={"conversations.list": {"conversations.rs"}}, + ): + command_operations, unmapped = audit_api_coverage._command_operations( + client_operations + ) + + self.assertEqual(client_operations, command_operations) + self.assertEqual([], unmapped) + + +class ManifestTests(unittest.TestCase): + def test_rejects_missing_reason(self): + with self.assertRaisesRegex(ValueError, "non-empty"): + audit_api_coverage._manifest_operations( + [{"operation": "GET /agents"}], + "known_gap", + ) + + def test_rejects_invalid_http_method(self): + with self.assertRaisesRegex(ValueError, "invalid operation"): + audit_api_coverage._manifest_operations( + [{"operation": "HEAD /agents", "reason": "Not supported"}], + "known_gap", + ) + + def test_rejects_duplicate_operation(self): + with self.assertRaisesRegex(ValueError, "duplicate operation"): + audit_api_coverage._manifest_operations( + [ + {"operation": "GET /agents", "reason": "First"}, + {"operation": "GET /agents", "reason": "Second"}, + ], + "known_gap", + ) + + +class AuditAggregationTests(unittest.TestCase): + def _audit( + self, + manifest: str, + published: dict[str, str], + commands: dict[str, dict], + ) -> tuple[dict, bool]: + with tempfile.TemporaryDirectory() as directory: + manifest_path = Path(directory) / "api-coverage.toml" + manifest_path.write_text(manifest) + with ( + patch.object(audit_api_coverage, "MANIFEST_PATH", manifest_path), + patch.object( + audit_api_coverage, + "_published_operations", + return_value=published, + ), + patch.object( + audit_api_coverage, + "_client_operations", + return_value=commands, + ), + patch.object( + audit_api_coverage, + "_command_operations", + return_value=(commands, []), + ), + ): + return audit_api_coverage.audit(audit_api_coverage.CATALOG_URL) + + def test_accepts_reviewed_gap_and_allowed_extra(self): + manifest = f""" +[snapshot] +catalog_url = "{audit_api_coverage.CATALOG_URL}" +published_operations = 2 +cli_supported_operations = 1 + +[[known_gap]] +operation = "GET /beta" +reason = "Reviewed" + +[[allowed_extra]] +operation = "POST /gamma" +reason = "Pre-deploy" +""" + published = { + "GET /alpha": "GET /v1/alpha", + "GET /beta": "GET /v1/beta", + } + commands = { + "GET /alpha": {"operation": "GET /alpha"}, + "POST /gamma": {"operation": "POST /gamma"}, + } + + report, passed = self._audit(manifest, published, commands) + + self.assertTrue(passed) + self.assertEqual(1, report["known_gap_count"]) + self.assertEqual([], report["new_gaps"]) + self.assertEqual([], report["unexpected_cli_operations"]) + + def test_classifies_new_gap_and_unexpected_extra_as_failure(self): + manifest = f""" +[snapshot] +catalog_url = "{audit_api_coverage.CATALOG_URL}" +published_operations = 2 +cli_supported_operations = 1 +""" + published = { + "GET /alpha": "GET /v1/alpha", + "GET /beta": "GET /v1/beta", + } + commands = { + "GET /alpha": {"operation": "GET /alpha"}, + "POST /gamma": {"operation": "POST /gamma"}, + } + + report, passed = self._audit(manifest, published, commands) + + self.assertFalse(passed) + self.assertEqual(["GET /v1/beta"], report["new_gaps"]) + self.assertEqual(["POST /gamma"], report["unexpected_cli_operations"]) + + def test_rejects_operation_in_multiple_manifest_sections(self): + manifest = """ +[snapshot] +catalog_url = "https://api.coval.dev/v1/openapi" +published_operations = 0 +cli_supported_operations = 0 + +[[known_gap]] +operation = "GET /agents" +reason = "Reviewed" + +[[allowed_extra]] +operation = "GET /agents" +reason = "Pre-deploy" +""" + with self.assertRaisesRegex(ValueError, "only one section"): + self._audit(manifest, {}, {}) + + +class SnapshotTests(unittest.TestCase): + def test_reports_stale_snapshot_fields(self): + mismatches = audit_api_coverage._snapshot_mismatches( + { + "catalog_url": audit_api_coverage.CATALOG_URL, + "published_operations": 10, + "cli_supported_operations": 8, + }, + catalog_url=audit_api_coverage.CATALOG_URL, + published_operation_count=11, + supported_operation_count=8, + ) + + self.assertEqual( + ["published_operations: recorded 10, current 11"], + mismatches, + ) + + +class MarkdownReportTests(unittest.TestCase): + def setUp(self): + self.report = { + "catalog_url": audit_api_coverage.CATALOG_URL, + "published_operation_count": 3, + "client_operation_count": 2, + "command_operation_count": 2, + "supported_operation_count": 2, + "known_gap_count": 0, + "new_gaps": ["GET /v1/beta"], + "stale_gaps": [], + "unexpected_cli_operations": [], + "stale_allowed_extras": [], + "stale_planned_operations": [], + "client_only_operations": [], + "unmapped_command_client_methods": [], + "snapshot_mismatches": ["published_operations: recorded 2, current 3"], + "all_current_gaps": ["GET /v1/beta"], + } + + def test_renders_deterministic_actionable_report(self): + rendered = audit_api_coverage.render_markdown_report(self.report, False) + + self.assertIn("| Reconciliation status | ACTION REQUIRED |", rendered) + self.assertIn("- `GET /v1/beta`", rendered) + self.assertIn( + "- `published_operations: recorded 2, current 3`", + rendered, + ) + self.assertNotIn("Generated at", rendered) + + @patch("scripts.audit_api_coverage.audit") + def test_allow_drift_writes_report_and_returns_success(self, audit): + audit.return_value = (self.report, False) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "coverage.md" + argv = [ + "audit_api_coverage.py", + "--write-markdown", + str(output), + "--allow-drift", + ] + with ( + patch("sys.argv", argv), + redirect_stdout(io.StringIO()), + ): + result = audit_api_coverage.main() + + self.assertEqual(0, result) + self.assertIn("ACTION REQUIRED", output.read_text()) + + def test_allow_drift_requires_markdown_output(self): + with ( + patch("sys.argv", ["audit_api_coverage.py", "--allow-drift"]), + redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit) as exception, + ): + audit_api_coverage.main() + + self.assertEqual(2, exception.exception.code) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_release_automation.py b/scripts/test_release_automation.py new file mode 100644 index 0000000..b21c91f --- /dev/null +++ b/scripts/test_release_automation.py @@ -0,0 +1,125 @@ +"""Tests for release metadata and Homebrew formula generation.""" + +import io +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + +from scripts import bump_version +from scripts import release_version +from scripts import render_homebrew_formula + + +def temporary_release_root( + cargo_version: str, + lock_version: str, + *, + package_name: str = "coval", + cargo_prefix: str = "", +) -> tempfile.TemporaryDirectory: + directory = tempfile.TemporaryDirectory() + root = Path(directory.name) + (root / "Cargo.toml").write_text( + f'{cargo_prefix}[package]\nname = "{package_name}"\n' + f'version = "{cargo_version}"\n' + ) + (root / "Cargo.lock").write_text( + f'[[package]]\nname = "{package_name}"\nversion = "{lock_version}"\n' + ) + return directory + + +class ReleaseVersionTests(unittest.TestCase): + def test_returns_matching_release_metadata(self): + with temporary_release_root("0.6.0", "0.6.0") as directory: + metadata = release_version.release_metadata(Path(directory), "v0.6.0") + + self.assertEqual({"version": "0.6.0", "tag": "v0.6.0"}, metadata) + + def test_rejects_manifest_mismatch(self): + with temporary_release_root("0.6.0", "0.5.0") as directory: + with self.assertRaisesRegex(ValueError, "Cargo.lock"): + release_version.release_metadata(Path(directory)) + + def test_rejects_tag_mismatch(self): + with temporary_release_root("0.6.0", "0.6.0") as directory: + with self.assertRaisesRegex(ValueError, "does not match"): + release_version.release_metadata(Path(directory), "v0.5.0") + + def test_identifies_non_release_version(self): + with temporary_release_root("0.6.0-rc.1", "0.6.0-rc.1") as directory: + with self.assertRaises(release_version.NonReleaseVersionError): + release_version.release_metadata(Path(directory)) + + def test_automatic_release_can_skip_non_release_version(self): + with temporary_release_root("0.6.0-rc.1", "0.6.0-rc.1") as directory: + output_path = Path(directory) / "github-output" + argv = [ + "release_version.py", + "--root", + directory, + "--allow-non-release", + "--github-output", + str(output_path), + ] + with patch("sys.argv", argv), redirect_stdout(io.StringIO()): + result = release_version.main() + + self.assertEqual(0, result) + self.assertEqual("release_candidate=false\n", output_path.read_text()) + + +class BumpVersionTests(unittest.TestCase): + def test_bumps_minor_version_in_both_manifests(self): + with temporary_release_root("0.5.0", "0.5.0") as directory: + root = Path(directory) + updated = bump_version.bump_version("minor", root) + + self.assertEqual("0.6.0", updated) + self.assertEqual("0.6.0", release_version.release_version(root)) + + def test_bumps_patch_version(self): + self.assertEqual("0.5.1", bump_version.next_version("0.5.0", "patch")) + + def test_only_bumps_renamed_workspace_package(self): + prefix = '[dependencies]\nhelper = "0.5.0"\n\n' + with temporary_release_root( + "0.5.0", + "0.5.0", + package_name="renamed-cli", + cargo_prefix=prefix, + ) as directory: + root = Path(directory) + bump_version.bump_version("minor", root) + + cargo = (root / "Cargo.toml").read_text() + self.assertIn('helper = "0.5.0"', cargo) + self.assertEqual("0.6.0", release_version.release_version(root)) + + +class HomebrewFormulaTests(unittest.TestCase): + def setUp(self): + self.checksums = { + "macos_arm64": "a" * 64, + "macos_x64": "b" * 64, + "linux_arm64": "c" * 64, + "linux_x64": "d" * 64, + } + + def test_renders_release_urls_and_ruby_interpolation(self): + formula = render_homebrew_formula.render_formula("0.6.0", self.checksums) + + self.assertIn("releases/download/v0.6.0/coval-macos-arm64.tar.gz", formula) + self.assertIn('system "#{bin}/coval", "--version"', formula) + + def test_rejects_invalid_checksum(self): + self.checksums["linux_x64"] = "not-a-checksum" + + with self.assertRaisesRegex(ValueError, "invalid SHA-256"): + render_homebrew_formula.render_formula("0.6.0", self.checksums) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/commands/traces.rs b/src/commands/traces.rs index 2baa325..3fd0042 100644 --- a/src/commands/traces.rs +++ b/src/commands/traces.rs @@ -245,6 +245,15 @@ fn build_search_request(args: SearchArgs) -> Result { input.insert("filters".to_string(), serde_json::Value::Object(filters)); } let request: TraceSearchRequest = input_json::finish(input)?; + if let (Some(minimum), Some(maximum)) = ( + request.filters.duration_ms_min, + request.filters.duration_ms_max, + ) { + anyhow::ensure!( + minimum <= maximum, + "trace search duration minimum ({minimum}) must not exceed maximum ({maximum})" + ); + } if let Some(attribute_filters) = &request.filters.attribute_filters { anyhow::ensure!( attribute_filters.len() <= 10, diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index f3cf01c..370fe06 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -3992,6 +3992,24 @@ fn test_traces_search_rejects_ambiguous_attribute_filter() { )); } +#[test] +fn test_traces_search_rejects_inverted_duration_range() { + coval() + .arg("--api-key") + .arg("test_key") + .arg("traces") + .arg("search") + .arg("--duration-ms-min") + .arg("10") + .arg("--duration-ms-max") + .arg("5") + .assert() + .failure() + .stderr(predicate::str::contains( + "trace search duration minimum (10) must not exceed maximum (5)", + )); +} + #[test] fn test_traces_search_rejects_more_than_ten_attribute_filter_flags() { let mut command = coval();