From 6849ea7007f589890e077631be3791ac93d87dab Mon Sep 17 00:00:00 2001 From: DB Lee Date: Mon, 23 Mar 2026 20:20:44 -0700 Subject: [PATCH] refactor: move agent skill plugins from .github/plugins to plugins/ Relocate agentops skill plugin folder to the repo root for better discoverability. Also includes updates to CI docs, release process docs, reporter, and foundry backend tests. --- .github/workflows/_build.yml | 2 +- docs/ci-github-actions.md | 90 ++++++------- docs/release-process.md | 126 +++++++++--------- .../agentops-investigate-regression/SKILL.md | 0 .../agentops-observability-triage/SKILL.md | 0 .../skills/agentops-run-evals/SKILL.md | 0 src/agentops/core/reporter.py | 12 +- tests/unit/test_foundry_backend.py | 53 ++++---- 8 files changed, 145 insertions(+), 138 deletions(-) rename {.github/plugins => plugins}/agentops/skills/agentops-investigate-regression/SKILL.md (100%) rename {.github/plugins => plugins}/agentops/skills/agentops-observability-triage/SKILL.md (100%) rename {.github/plugins => plugins}/agentops/skills/agentops-run-evals/SKILL.md (100%) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index 8e60abf9..2e87b552 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # Full history required for setuptools-scm + fetch-depth: 0 # Full history required for setuptools-scm - name: Install uv uses: astral-sh/setup-uv@v6 diff --git a/docs/ci-github-actions.md b/docs/ci-github-actions.md index bd22e0e9..48e5fc60 100644 --- a/docs/ci-github-actions.md +++ b/docs/ci-github-actions.md @@ -32,12 +32,12 @@ This guide explains how to add AgentOps evaluation to your CI pipeline using Git Your repository must contain these files for the workflow to succeed: -| File | Purpose | -| --- | --- | -| `.agentops/run.yaml` | Run specification — references the bundle, dataset, and backend | -| `.agentops/bundles/.yaml` | Evaluation bundle — evaluators + thresholds | -| `.agentops/datasets/.yaml` | Dataset metadata | -| `.agentops/datasets/.jsonl` | Dataset rows (JSONL format) | +| File | Purpose | +| --------------------------------- | --------------------------------------------------------------- | +| `.agentops/run.yaml` | Run specification — references the bundle, dataset, and backend | +| `.agentops/bundles/.yaml` | Evaluation bundle — evaluators + thresholds | +| `.agentops/datasets/.yaml` | Dataset metadata | +| `.agentops/datasets/.jsonl` | Dataset rows (JSONL format) | All paths in `run.yaml` are relative to the `.agentops/` directory. @@ -81,16 +81,16 @@ The workflow uses **Workload Identity Federation (OIDC)** — no client secrets Set these as **repository variables** (not secrets — they are not confidential): -| Variable | Value | -| --- | --- | -| `AZURE_CLIENT_ID` | Application (client) ID | -| `AZURE_TENANT_ID` | Directory (tenant) ID | -| `AZURE_SUBSCRIPTION_ID` | Azure subscription ID | +| Variable | Value | +| ----------------------- | ----------------------- | +| `AZURE_CLIENT_ID` | Application (client) ID | +| `AZURE_TENANT_ID` | Directory (tenant) ID | +| `AZURE_SUBSCRIPTION_ID` | Azure subscription ID | Set this as a **repository secret**: -| Secret | Value | -| --- | --- | +| Secret | Value | +| ----------------------------------- | ---------------------------- | | `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Foundry project endpoint URL | Go to **Settings** → **Secrets and variables** → **Actions** → **Variables** tab (for variables) or **Secrets** tab (for the endpoint). @@ -101,9 +101,9 @@ Go to **Settings** → **Secrets and variables** → **Actions** → **Variables The template workflow triggers on: -| Trigger | When | -| --- | --- | -| `pull_request` | Any PR targeting `main` or `develop` | +| Trigger | When | +| ------------------- | ---------------------------------------------------------------------------------- | +| `pull_request` | Any PR targeting `main` or `develop` | | `workflow_dispatch` | Manual run from the Actions tab (supports custom config path and output directory) | To change which branches trigger evaluations, edit the `on.pull_request.branches` array in the workflow file. @@ -114,11 +114,11 @@ To change which branches trigger evaluations, edit the `on.pull_request.branches AgentOps returns CI-friendly exit codes that GitHub Actions interprets directly: -| Exit Code | Meaning | CI Result | -| --- | --- | --- | -| `0` | Evaluation succeeded, all thresholds passed | ✅ Job passes | -| `2` | Evaluation succeeded, one or more thresholds failed | ❌ Job fails | -| `1` | Runtime or configuration error | ❌ Job fails | +| Exit Code | Meaning | CI Result | +| --------- | --------------------------------------------------- | ------------ | +| `0` | Evaluation succeeded, all thresholds passed | ✅ Job passes | +| `2` | Evaluation succeeded, one or more thresholds failed | ❌ Job fails | +| `1` | Runtime or configuration error | ❌ Job fails | No special handling is needed — GitHub Actions fails the job on any non-zero exit code. @@ -128,14 +128,14 @@ No special handling is needed — GitHub Actions fails the job on any non-zero e The workflow uploads the following files as a GitHub Actions artifact named `agentops-eval-results`: -| File | Description | -| --- | --- | -| `results.json` | Machine-readable evaluation results (versioned schema) | -| `report.md` | Human-readable Markdown summary | -| `backend_metrics.json` | Raw backend scores per row | +| File | Description | +| ----------------------- | -------------------------------------------------------------- | +| `results.json` | Machine-readable evaluation results (versioned schema) | +| `report.md` | Human-readable Markdown summary | +| `backend_metrics.json` | Raw backend scores per row | | `cloud_evaluation.json` | Cloud eval metadata with Foundry portal link (cloud mode only) | -| `backend.stdout.log` | Backend stdout capture | -| `backend.stderr.log` | Backend stderr capture | +| `backend.stdout.log` | Backend stdout capture | +| `backend.stderr.log` | Backend stderr capture | Artifacts are uploaded even when the evaluation fails (`if: always()`), so you can always inspect results. @@ -174,10 +174,10 @@ agentops config cicd Options: -| Flag | Description | Default | -| --- | --- | --- | +| Flag | Description | Default | +| ------------ | -------------------------------- | ----------------------- | | `--dir PATH` | Target repository root directory | `.` (current directory) | -| `--force` | Overwrite existing workflow file | `false` | +| `--force` | Overwrite existing workflow file | `false` | ### Regenerate (overwrite) @@ -243,13 +243,13 @@ Remove or comment out the "Post report as PR comment" step in the workflow. ## Troubleshooting -| Problem | Solution | -| --- | --- | -| `Error: evaluation failed: ...` (exit 1) | Check that `.agentops/run.yaml` exists, config is valid YAML, and secrets are set | -| `Threshold status: FAILED` (exit 2) | Review `report.md` — thresholds are too strict or model quality regressed | -| Missing artifacts | Ensure `.agentops/results/latest/` is not in `.gitignore` — the workflow reads this path | -| Authentication errors | Verify the federated credential entity matches your repo/branch; check that `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` are set as repository variables; confirm the app registration has access to the Foundry project | -| `agentops: command not found` | Ensure `pip install agentops-toolkit` runs before the eval step | +| Problem | Solution | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Error: evaluation failed: ...` (exit 1) | Check that `.agentops/run.yaml` exists, config is valid YAML, and secrets are set | +| `Threshold status: FAILED` (exit 2) | Review `report.md` — thresholds are too strict or model quality regressed | +| Missing artifacts | Ensure `.agentops/results/latest/` is not in `.gitignore` — the workflow reads this path | +| Authentication errors | Verify the federated credential entity matches your repo/branch; check that `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` are set as repository variables; confirm the app registration has access to the Foundry project | +| `agentops: command not found` | Ensure `pip install agentops-toolkit` runs before the eval step | --- @@ -257,13 +257,13 @@ Remove or comment out the "Post report as PR comment" step in the workflow. If you are contributing to the agentops-toolkit repository itself, the project has separate CI/CD workflows for building and releasing the package: -| Workflow | Trigger | Purpose | -| --- | --- | --- | -| `ci.yml` | Push to `develop`, PRs to `main`/`develop` | Lint (ruff) + test (matrix) + coverage | -| `_build.yml` | Called by staging/release | Reusable lint + test + build package | -| `staging.yml` | Push to `release/**` | Build → TestPyPI → verify install | -| `release.yml` | Push `v*` tag | TestPyPI → PyPI (with approval) → GitHub Release | -| `cut-release.yml` | Manual dispatch (Actions tab button) | Create release branch from `develop`, update CHANGELOG, open PR to `main` | +| Workflow | Trigger | Purpose | +| ----------------- | ------------------------------------------ | ------------------------------------------------------------------------- | +| `ci.yml` | Push to `develop`, PRs to `main`/`develop` | Lint (ruff) + test (matrix) + coverage | +| `_build.yml` | Called by staging/release | Reusable lint + test + build package | +| `staging.yml` | Push to `release/**` | Build → TestPyPI → verify install | +| `release.yml` | Push `v*` tag | TestPyPI → PyPI (with approval) → GitHub Release | +| `cut-release.yml` | Manual dispatch (Actions tab button) | Create release branch from `develop`, update CHANGELOG, open PR to `main` | The **Cut Release** workflow provides a one-click way to start a release: enter a version number in the Actions UI, and it creates the release branch, updates the changelog, and opens the PR automatically. diff --git a/docs/release-process.md b/docs/release-process.md index 5168ee74..3aaf17ef 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -50,12 +50,12 @@ develop ← integration branch, all feature PRs target here ### Branch Purposes -| Branch | Purpose | Who creates | Merges into | -| --- | --- | --- | --- | -| `main` | Production-ready code. Every commit here should be a tagged release. | Maintainers only | — | -| `develop` | Integration branch. All feature work flows through here. | — | `main` (via release branches) | -| `feature/*` | Individual features, bug fixes, or improvements. | Any contributor | `develop` | -| `release/v0.X.Y` | Release stabilization and staging. Triggers TestPyPI pipeline. | Maintainers | `main` | +| Branch | Purpose | Who creates | Merges into | +| ---------------- | -------------------------------------------------------------------- | ---------------- | ----------------------------- | +| `main` | Production-ready code. Every commit here should be a tagged release. | Maintainers only | — | +| `develop` | Integration branch. All feature work flows through here. | — | `main` (via release branches) | +| `feature/*` | Individual features, bug fixes, or improvements. | Any contributor | `develop` | +| `release/v0.X.Y` | Release stabilization and staging. Triggers TestPyPI pipeline. | Maintainers | `main` | ### Branch Lifecycle @@ -72,11 +72,11 @@ develop ← integration branch, all feature PRs target here Configure these in **Settings → Branches → Branch protection rules**: -| Branch | Rules | -| --- | --- | -| `main` | Require PR, require status checks (CI), require approvals, no force push | -| `develop` | Require PR, require status checks (CI), no force push | -| `release/*` | Require status checks (Staging pipeline), no force push | +| Branch | Rules | +| ----------- | ------------------------------------------------------------------------ | +| `main` | Require PR, require status checks (CI), require approvals, no force push | +| `develop` | Require PR, require status checks (CI), no force push | +| `release/*` | Require status checks (Staging pipeline), no force push | --- @@ -196,18 +196,18 @@ The CI pipeline runs on **every push and PR** to `main` or `develop`. ### Jobs -| Job | What it does | Runs on | -| --- | --- | --- | -| **lint** | `ruff check` (linting) + `mypy` (type checking, soft-fail) | Ubuntu, Python 3.11 | -| **test** | `pytest tests/` with JUnit XML output | Matrix: 3 OS × 3 Python versions | -| **coverage** | `pytest --cov` with XML coverage report | Ubuntu, Python 3.13 (after tests pass) | +| Job | What it does | Runs on | +| ------------ | ---------------------------------------------------------- | -------------------------------------- | +| **lint** | `ruff check` (linting) + `mypy` (type checking, soft-fail) | Ubuntu, Python 3.11 | +| **test** | `pytest tests/` with JUnit XML output | Matrix: 3 OS × 3 Python versions | +| **coverage** | `pytest --cov` with XML coverage report | Ubuntu, Python 3.13 (after tests pass) | ### Test Matrix -| OS | Python 3.11 | Python 3.12 | Python 3.13 | -| --- | --- | --- | --- | -| Ubuntu | ✅ | ✅ | ✅ | -| Windows | ✅ | ✅ | ✅ | +| OS | Python 3.11 | Python 3.12 | Python 3.13 | +| ------- | ----------- | ----------- | ----------- | +| Ubuntu | ✅ | ✅ | ✅ | +| Windows | ✅ | ✅ | ✅ | ### What CI Catches @@ -233,11 +233,11 @@ AgentOps uses [setuptools-scm](https://github.com/pypa/setuptools-scm) for **ful setuptools-scm reads your git history and computes the version: -| Git state | Example version | Explanation | -| --- | --- | --- | -| Exactly on tag `v0.2.0` | `0.2.0` | Clean release version | -| 3 commits after `v0.2.0` | `0.2.1.dev3` | Dev version, 3 commits ahead | -| 10 commits after `v0.1.2` on `release/v0.2.0` | `0.1.3.dev10` | Dev version on release branch | +| Git state | Example version | Explanation | +| --------------------------------------------- | --------------- | ----------------------------- | +| Exactly on tag `v0.2.0` | `0.2.0` | Clean release version | +| 3 commits after `v0.2.0` | `0.2.1.dev3` | Dev version, 3 commits ahead | +| 10 commits after `v0.1.2` on `release/v0.2.0` | `0.1.3.dev10` | Dev version on release branch | ### Configuration @@ -491,13 +491,13 @@ If you approved the PyPI publish, the test version (`0.0.0.test1`) will exist on ### 8.3 Quick E2E Test Summary -| What to test | Command | What to watch | -| --- | --- | --- | -| Staging only | `git push origin release/v0.0.0-test` | 3 jobs: build → TestPyPI → verify | -| Full release (safe) | `git push origin v0.0.0-test.1` then **reject** at approval | 4 jobs run, approval gate works | -| Full release (real) | `git push origin v0.0.0-test.1` then **approve** | All 5 jobs, package on PyPI | -| Cleanup (branch) | `git push origin --delete release/v0.0.0-test` | Branch removed | -| Cleanup (tag) | `git push origin --delete v0.0.0-test.1 && git tag -d v0.0.0-test.1` | Tag removed | +| What to test | Command | What to watch | +| ------------------- | -------------------------------------------------------------------- | --------------------------------- | +| Staging only | `git push origin release/v0.0.0-test` | 3 jobs: build → TestPyPI → verify | +| Full release (safe) | `git push origin v0.0.0-test.1` then **reject** at approval | 4 jobs run, approval gate works | +| Full release (real) | `git push origin v0.0.0-test.1` then **approve** | All 5 jobs, package on PyPI | +| Cleanup (branch) | `git push origin --delete release/v0.0.0-test` | Branch removed | +| Cleanup (tag) | `git push origin --delete v0.0.0-test.1 && git tag -d v0.0.0-test.1` | Tag removed | ### 8.4 Testing Workflow Changes on a Feature Branch @@ -680,8 +680,8 @@ Create two environments in **Settings → Environments → New environment**: - **Protection rules**: None required (auto-deploys), or add reviewers for extra safety - **Secrets**: - | Secret | Value | How to get it | - | --- | --- | --- | + | Secret | Value | How to get it | + | ----------------- | ------------------ | --------------------------------------------------------------------------------- | | `TEST_PYPI_TOKEN` | TestPyPI API token | [test.pypi.org/manage/account/token](https://test.pypi.org/manage/account/token/) | #### `release` Environment @@ -691,8 +691,8 @@ Create two environments in **Settings → Environments → New environment**: - **Deployment branches**: Optionally restrict to `main` branch and `v*` tags - **Secrets**: - | Secret | Value | How to get it | - | --- | --- | --- | + | Secret | Value | How to get it | + | ------------ | --------------------------------------------- | ----------------------------------------------------------------------- | | `PYPI_TOKEN` | PyPI API token (scoped to `agentops-toolkit`) | [pypi.org/manage/account/token](https://pypi.org/manage/account/token/) | ### 9.2 PyPI and TestPyPI Accounts @@ -826,45 +826,45 @@ Use this checklist when cutting a release: ### Build Failures -| Problem | Cause | Solution | -| --- | --- | --- | -| `setuptools_scm` can't determine version | Shallow clone (missing git history) | Ensure `fetch-depth: 0` in checkout step | -| Version shows `0.0.0` locally | Not in a git repo or no tags exist | Run `git tag v0.0.1` to create an initial tag | -| `ModuleNotFoundError` in tests | Dependencies not installed | Run `uv sync --group dev` | -| Tests fail on Windows but pass on Linux | Path separator issues | Use `pathlib.Path`, not string concatenation | +| Problem | Cause | Solution | +| ---------------------------------------- | ----------------------------------- | --------------------------------------------- | +| `setuptools_scm` can't determine version | Shallow clone (missing git history) | Ensure `fetch-depth: 0` in checkout step | +| Version shows `0.0.0` locally | Not in a git repo or no tags exist | Run `git tag v0.0.1` to create an initial tag | +| `ModuleNotFoundError` in tests | Dependencies not installed | Run `uv sync --group dev` | +| Tests fail on Windows but pass on Linux | Path separator issues | Use `pathlib.Path`, not string concatenation | ### TestPyPI Issues -| Problem | Cause | Solution | -| --- | --- | --- | -| Upload fails with 403 | Invalid or expired token | Regenerate `TEST_PYPI_TOKEN` and update the GitHub secret | -| Upload fails with "already exists" | Same version previously uploaded | Normal — `skip-existing: true` handles this. If you need a new upload, push another commit to increment the dev version | -| Install fails with "no matching distribution" | Package not yet indexed | The verify job retries automatically (5 attempts, 30s apart). If persistent, check TestPyPI status | -| Install fails with dependency errors | Dependency not on TestPyPI | Verify `--extra-index-url https://pypi.org/simple/` is present | +| Problem | Cause | Solution | +| --------------------------------------------- | -------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Upload fails with 403 | Invalid or expired token | Regenerate `TEST_PYPI_TOKEN` and update the GitHub secret | +| Upload fails with "already exists" | Same version previously uploaded | Normal — `skip-existing: true` handles this. If you need a new upload, push another commit to increment the dev version | +| Install fails with "no matching distribution" | Package not yet indexed | The verify job retries automatically (5 attempts, 30s apart). If persistent, check TestPyPI status | +| Install fails with dependency errors | Dependency not on TestPyPI | Verify `--extra-index-url https://pypi.org/simple/` is present | ### PyPI Issues -| Problem | Cause | Solution | -| --- | --- | --- | -| Publish step stuck on "Waiting for review" | Normal — requires approval | A designated reviewer must approve in the Actions UI | -| Upload fails with 403 | Invalid `PYPI_TOKEN` | Regenerate the token on pypi.org and update the GitHub secret | -| Version already exists on PyPI | Tag points to an already-released version | PyPI versions are immutable. You must use a new version number | +| Problem | Cause | Solution | +| ------------------------------------------ | ----------------------------------------- | -------------------------------------------------------------- | +| Publish step stuck on "Waiting for review" | Normal — requires approval | A designated reviewer must approve in the Actions UI | +| Upload fails with 403 | Invalid `PYPI_TOKEN` | Regenerate the token on pypi.org and update the GitHub secret | +| Version already exists on PyPI | Tag points to an already-released version | PyPI versions are immutable. You must use a new version number | ### Git and Version Issues -| Problem | Cause | Solution | -| --- | --- | --- | -| Wrong version in built package | Tag not on the expected commit | Verify with `git log --oneline --decorate` that the tag is where you expect | -| `pip install -e .` fails | `.git` directory missing | Editable installs need git history for setuptools-scm. Clone the repo, don't just download a zip | -| Merge conflicts between release and develop | Normal for concurrent work | Resolve conflicts on the release branch before merging to main | +| Problem | Cause | Solution | +| ------------------------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------ | +| Wrong version in built package | Tag not on the expected commit | Verify with `git log --oneline --decorate` that the tag is where you expect | +| `pip install -e .` fails | `.git` directory missing | Editable installs need git history for setuptools-scm. Clone the repo, don't just download a zip | +| Merge conflicts between release and develop | Normal for concurrent work | Resolve conflicts on the release branch before merging to main | ### Environment and Permissions -| Problem | Cause | Solution | -| --- | --- | --- | -| "Environment not found" error | GitHub Environment not created | Create `staging` and `release` environments in Settings → Environments | -| "Secret not found" error | Secret not added to the environment | Add secrets to the specific environment, not repository-level secrets | -| Reviewer can't approve deployment | Not listed as required reviewer | Update the environment's required reviewers list | +| Problem | Cause | Solution | +| --------------------------------- | ----------------------------------- | ---------------------------------------------------------------------- | +| "Environment not found" error | GitHub Environment not created | Create `staging` and `release` environments in Settings → Environments | +| "Secret not found" error | Secret not added to the environment | Add secrets to the specific environment, not repository-level secrets | +| Reviewer can't approve deployment | Not listed as required reviewer | Update the environment's required reviewers list | --- diff --git a/.github/plugins/agentops/skills/agentops-investigate-regression/SKILL.md b/plugins/agentops/skills/agentops-investigate-regression/SKILL.md similarity index 100% rename from .github/plugins/agentops/skills/agentops-investigate-regression/SKILL.md rename to plugins/agentops/skills/agentops-investigate-regression/SKILL.md diff --git a/.github/plugins/agentops/skills/agentops-observability-triage/SKILL.md b/plugins/agentops/skills/agentops-observability-triage/SKILL.md similarity index 100% rename from .github/plugins/agentops/skills/agentops-observability-triage/SKILL.md rename to plugins/agentops/skills/agentops-observability-triage/SKILL.md diff --git a/.github/plugins/agentops/skills/agentops-run-evals/SKILL.md b/plugins/agentops/skills/agentops-run-evals/SKILL.md similarity index 100% rename from .github/plugins/agentops/skills/agentops-run-evals/SKILL.md rename to plugins/agentops/skills/agentops-run-evals/SKILL.md diff --git a/src/agentops/core/reporter.py b/src/agentops/core/reporter.py index 95e6d9cd..82080751 100644 --- a/src/agentops/core/reporter.py +++ b/src/agentops/core/reporter.py @@ -381,11 +381,13 @@ def generate_comparison_markdown(result: ComparisonResult) -> str: ( "Status", None, - lambda r: "PASS" - if r.overall_passed - else "FAIL" - if r.overall_passed is not None - else "-", + lambda r: ( + "PASS" + if r.overall_passed + else "FAIL" + if r.overall_passed is not None + else "-" + ), ), ("Started", None, lambda r: r.started_at[:19] if r.started_at else "-"), ] diff --git a/tests/unit/test_foundry_backend.py b/tests/unit/test_foundry_backend.py index a6b2c0f8..128a3878 100644 --- a/tests/unit/test_foundry_backend.py +++ b/tests/unit/test_foundry_backend.py @@ -160,15 +160,17 @@ def test_foundry_backend_agent_service_target(tmp_path: Path) -> None: ), ] - with patch( - "agentops.backends.foundry_backend._acquire_token", - return_value="fake-agent-token", - ): - with patch( + with ( + patch( + "agentops.backends.foundry_backend._acquire_token", + return_value="fake-agent-token", + ), + patch( "agentops.backends.foundry_backend.urllib.request.urlopen", side_effect=responses, - ): - result = FoundryBackend().execute(context) + ), + ): + result = FoundryBackend().execute(context) assert result.backend == "foundry" assert result.exit_code == 0 @@ -235,11 +237,12 @@ def __call__(self, **kwargs): assert "ground_truth" in kwargs return {"similarity": 4.0} - with patch( - "agentops.backends.foundry_backend._acquire_token", - return_value="fake-agent-token", - ): - with patch( + with ( + patch( + "agentops.backends.foundry_backend._acquire_token", + return_value="fake-agent-token", + ), + patch( "agentops.backends.foundry_backend._build_foundry_evaluator_runtimes", return_value=[ FoundryEvaluatorRuntime( @@ -253,12 +256,13 @@ def __call__(self, **kwargs): score_keys=["similarity"], ) ], - ): - with patch( - "agentops.backends.foundry_backend.urllib.request.urlopen", - side_effect=responses, - ): - result = FoundryBackend().execute(context) + ), + patch( + "agentops.backends.foundry_backend.urllib.request.urlopen", + side_effect=responses, + ), + ): + result = FoundryBackend().execute(context) assert result.backend == "foundry" assert result.exit_code == 0 @@ -324,13 +328,14 @@ def _fake_invoke_model_direct(self_backend, settings, prompt): return "4" return "8" - with patch( - "agentops.backends.foundry_backend._acquire_token", return_value="fake-token" + with ( + patch( + "agentops.backends.foundry_backend._acquire_token", + return_value="fake-token", + ), + patch.object(FoundryBackend, "_invoke_model_direct", _fake_invoke_model_direct), ): - with patch.object( - FoundryBackend, "_invoke_model_direct", _fake_invoke_model_direct - ): - result = FoundryBackend().execute(context) + result = FoundryBackend().execute(context) assert result.backend == "foundry" assert result.exit_code == 0