fix(cloud-eval): lift grader execution errors into RowMetric.error (#… #296
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # AgentOps Toolkit — CI/CD Pipelines | |
| # | |
| # Workflows: | |
| # 1. ci.yml — Lint + test on every push/PR; publish dev builds to TestPyPI on develop; VSIX build validation | |
| # 2. _build.yml — Reusable build (test + package), called by staging and release | |
| # 3. staging.yml — Staging: release/* branch → TestPyPI → verify; VSIX pre-release → Marketplace | |
| # 4. release.yml — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace | |
| # 5. cut-release.yml — Manual dispatch: create release branch + PR from develop | |
| # | |
| # NOTE: VSIX publishing is handled exclusively by staging.yml (pre-release) | |
| # and release.yml (stable). CI only validates that the VSIX packages correctly. | |
| # | |
| # Trusted Publishing (OIDC): | |
| # The publish-dev job uses PyPI Trusted Publishing — no API tokens required. | |
| # Authentication is handled via OpenID Connect (OIDC) between GitHub Actions | |
| # and TestPyPI. | |
| # | |
| # Setup (Trusted Publishing): | |
| # 1. https://test.pypi.org/manage/project/agentops-accelerator/settings/publishing/ | |
| # → Add publisher: GitHub, owner=Azure, repo=agentops, workflow=ci.yml, environment=staging | |
| # (This is a SEPARATE entry from the workflow=staging.yml / workflow=release.yml | |
| # publishers — TestPyPI matches workflow filename exactly.) | |
| # 2. GitHub repo → Settings → Environments → "staging" → confirm "Deployment branches | |
| # and tags" allows `develop` (publish-dev is the only consumer triggered from develop). | |
| name: CI | |
| on: | |
| push: | |
| branches: [develop] | |
| pull_request: | |
| branches: [develop] | |
| permissions: | |
| contents: read | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: ">=0.9.0" | |
| enable-cache: false # test matrix saves this cache key | |
| - name: Set up Python | |
| run: uv python install 3.11 | |
| - name: Install dependencies | |
| run: uv sync --group dev | |
| - name: Lint with ruff | |
| run: uv run ruff check src/ tests/ | |
| - name: Type check with mypy | |
| run: uv run mypy src/agentops/ --ignore-missing-imports | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| python-version: ["3.11", "3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: ">=0.9.0" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: uv sync --group dev | |
| - name: Run tests | |
| run: uv run pytest tests/ -v --tb=short --junitxml=test-results.xml | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-${{ matrix.os }}-py${{ matrix.python-version }} | |
| path: test-results.xml | |
| coverage: | |
| runs-on: ubuntu-latest | |
| needs: test | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: ">=0.9.0" | |
| enable-cache: false # test matrix saves this cache key | |
| - name: Set up Python | |
| run: uv python install 3.13 | |
| - name: Install dependencies | |
| run: uv sync --group dev | |
| - name: Run tests with coverage | |
| run: uv run pytest tests/ --cov=agentops --cov-report=xml --cov-report=term-missing | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: coverage-report | |
| path: coverage.xml | |
| # Publish dev build to TestPyPI on every push to develop (not PRs) | |
| publish-dev: | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/develop' | |
| needs: [lint, test] | |
| runs-on: ubuntu-latest | |
| environment: staging | |
| permissions: | |
| id-token: write # Required for PyPI Trusted Publishing (OIDC) | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 # Full history for setuptools-scm | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: ">=0.9.0" | |
| enable-cache: false # test matrix saves this cache key | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --group dev | |
| - name: Build package | |
| run: uv build | |
| - name: Show version | |
| run: | | |
| ls -la dist/ | |
| uv run python -c "from importlib.metadata import version; print(f'Dev version: {version(\"agentops-accelerator\")}')" | |
| - name: Publish to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: https://test.pypi.org/legacy/ | |
| verbose: true | |
| skip-existing: true | |
| # Verify the dev build installs correctly | |
| verify-dev: | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/develop' | |
| needs: publish-dev | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Determine expected version | |
| id: version | |
| run: | | |
| pip install setuptools-scm | |
| VERSION=$(python -m setuptools_scm) | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Expected dev version: $VERSION" | |
| - name: Install from TestPyPI | |
| run: | | |
| for i in 1 2 3 4 5; do | |
| echo "Attempt $i: installing agentops-accelerator==${{ steps.version.outputs.version }}" | |
| if pip install \ | |
| "agentops-accelerator==${{ steps.version.outputs.version }}" \ | |
| --index-url https://test.pypi.org/simple/ \ | |
| --extra-index-url https://pypi.org/simple/; then | |
| exit 0 | |
| fi | |
| if [ "$i" -lt 5 ]; then | |
| echo "Not available yet, waiting 30s..." | |
| sleep 30 | |
| fi | |
| done | |
| echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after 5 attempts." | |
| exit 1 | |
| - name: Smoke test | |
| run: | | |
| agentops --version | |
| agentops --help | |
| - name: Summary | |
| run: | | |
| echo "## ✅ Dev build published and verified" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- Version: \`${{ steps.version.outputs.version }}\`" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- TestPyPI: https://test.pypi.org/project/agentops-accelerator/${{ steps.version.outputs.version }}/" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "Install: \`pip install agentops-accelerator==${{ steps.version.outputs.version }} --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/\`" >> "$GITHUB_STEP_SUMMARY" | |
| # Validate that the VSIX extension packages correctly | |
| build-vsix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| # CI uses the committed package.json version as-is (no publish, dry-run only). | |
| # The version in package.json is synced by cut-release.yml when a release branch is created. | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "22" | |
| - name: Install vsce | |
| run: npm install -g @vscode/vsce | |
| - name: Copy root assets for VSIX | |
| run: | | |
| cp CHANGELOG.md plugins/agentops/CHANGELOG.md | |
| cp icon.png plugins/agentops/icon.png | |
| - name: Package VSIX (dry run) | |
| working-directory: plugins/agentops | |
| run: vsce package -o agentops-skills.vsix | |
| - name: Show VSIX info | |
| run: | | |
| ls -la plugins/agentops/*.vsix | |
| echo "✅ VSIX packaging validated" | |
| - name: Upload VSIX artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: vsix | |
| path: plugins/agentops/*.vsix |