diff --git a/.github/actions/azure-oidc-login/action.yml b/.github/actions/azure-oidc-login/action.yml new file mode 100644 index 00000000..dbef7b1b --- /dev/null +++ b/.github/actions/azure-oidc-login/action.yml @@ -0,0 +1,76 @@ +name: Azure OIDC login (composite, no node20 deps) +description: | + Drop-in replacement for azure/login@v2 that performs the OIDC federated + token exchange entirely in bash, so it does not pull any Node.js 20 + JavaScript actions and keeps the Actions run free of "Node.js 20 is + deprecated" annotations. + + After this step runs, the az CLI is authenticated and AZURE_* environment + variables are exported for downstream tools (azure-identity etc). + +inputs: + client-id: + description: "Microsoft Entra application (client) ID" + required: true + tenant-id: + description: "Microsoft Entra tenant ID" + required: true + subscription-id: + description: "Azure subscription ID" + required: true + audience: + description: "Federated identity audience" + required: false + default: "api://AzureADTokenExchange" + +runs: + using: composite + steps: + - name: Federated OIDC login (bash) + shell: bash + env: + AZURE_CLIENT_ID: ${{ inputs.client-id }} + AZURE_TENANT_ID: ${{ inputs.tenant-id }} + AZURE_SUBSCRIPTION_ID: ${{ inputs.subscription-id }} + OIDC_AUDIENCE: ${{ inputs.audience }} + run: | + set -euo pipefail + + : "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?id-token permission missing on the job}" + : "${ACTIONS_ID_TOKEN_REQUEST_URL:?id-token permission missing on the job}" + + echo "::group::Requesting OIDC ID token from GitHub" + ID_TOKEN_JSON=$(curl -sS \ + -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + -H "Accept: application/json" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${OIDC_AUDIENCE}") + ID_TOKEN=$(printf '%s' "$ID_TOKEN_JSON" | python3 -c 'import sys,json;print(json.load(sys.stdin)["value"])') + if [[ -z "${ID_TOKEN}" || "${ID_TOKEN}" == "null" ]]; then + echo "Failed to obtain GitHub OIDC ID token. Response was:" >&2 + echo "$ID_TOKEN_JSON" >&2 + exit 1 + fi + echo "::endgroup::" + + echo "::group::az login --federated-token" + az login \ + --service-principal \ + --username "${AZURE_CLIENT_ID}" \ + --tenant "${AZURE_TENANT_ID}" \ + --federated-token "${ID_TOKEN}" \ + --allow-no-subscriptions \ + --output none + az account set --subscription "${AZURE_SUBSCRIPTION_ID}" + echo "::endgroup::" + + # Export the same env vars azure/login@v2 sets for DefaultAzureCredential + # and other downstream Azure SDKs. + { + echo "AZURE_CLIENT_ID=${AZURE_CLIENT_ID}" + echo "AZURE_TENANT_ID=${AZURE_TENANT_ID}" + echo "AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}" + echo "AZURE_FEDERATED_TOKEN=${ID_TOKEN}" + } >> "${GITHUB_ENV}" + + # The federated token is short-lived and a secret; mask it. + echo "::add-mask::${ID_TOKEN}" diff --git a/.github/skills/release-management/SKILL.md b/.github/skills/release-management/SKILL.md index bfe84f56..a6ec4b73 100644 --- a/.github/skills/release-management/SKILL.md +++ b/.github/skills/release-management/SKILL.md @@ -1,6 +1,6 @@ --- name: release-management -description: Guide maintainers and contributors through branching, versioning, changelog updates, and publishing agentops-toolkit. Trigger when users ask about branching strategy, creating a release, version tagging, publishing to PyPI, updating the changelog, cutting a release, opening a PR, or syncing a fork. Common phrases: "cut a release", "how do I publish", "create release branch", "tag a version", "update changelog", "release process", "bump version", "what branch should I use", "feature branch", "prepare release". +description: 'Guide maintainers and contributors through branching, versioning, changelog updates, and publishing agentops-toolkit. Trigger when users ask about branching strategy, creating a release, version tagging, publishing to PyPI, updating the changelog, cutting a release, opening a PR, or syncing a fork. Common phrases include "cut a release", "how do I publish", "create release branch", "tag a version", "update changelog", "release process", "bump version", "what branch should I use", "feature branch", "prepare release".' --- # Release Management diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..7aefdab6 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,540 @@ +# AgentOps end-to-end demo workflow. +# +# Manual-only: trigger via "Run workflow" on the Actions page. +# Uploads the resulting evidence/ folder as a downloadable artifact and +# writes a Markdown summary to the run page. +# +# This workflow does NOT contact Azure or Foundry. It validates the +# CLI surface (init, eval run, eval run --baseline, report generate) +# against a local in-process HTTP echo agent. + +name: E2E + +on: + workflow_dispatch: + inputs: + scenarios: + description: "Which live scenario(s) to run" + type: choice + default: offline-only + options: + - offline-only + - all + - foundry-prompt + - foundry-hosted + - http-aca + - model-direct + keep_resources: + description: "Skip teardown of per-run resources (debug)" + type: boolean + default: false + +permissions: + contents: read + +jobs: + offline-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: ">=0.9.0" + + - name: Install AgentOps + run: uv sync --group dev + + - name: Install runtime evaluator dependencies + run: uv pip install azure-ai-evaluation pandas + + - name: Run offline CLI smoke test + run: uv run python scripts/e2e_demo.py + + - name: Upload evidence artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: offline-smoke + path: evidence/ + if-no-files-found: error + retention-days: 14 + + unit-tests-with-coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: ">=0.9.0" + + - name: Install dependencies + run: uv sync --group dev + + - name: Install runtime evaluator dependencies + run: uv pip install azure-ai-evaluation pandas + + - name: Run pytest with coverage + run: | + uv run pytest tests/ \ + --ignore=tests/unit/test_browse.py \ + --junitxml=test-results.xml \ + --cov=agentops \ + --cov-report=xml \ + --cov-report=term-missing + + - name: Upload coverage + JUnit + if: always() + uses: actions/upload-artifact@v7 + with: + name: unit-tests-with-coverage + path: | + test-results.xml + coverage.xml + if-no-files-found: warn + retention-days: 14 + + # ===================================================================== + # Live Azure scenarios — opt-in via the `scenarios` workflow input. + # Auth: OIDC federated credential. No secrets stored in the repo. + # See docs/e2e-live-setup.md for the one-time configuration steps. + # ===================================================================== + + bootstrap-live: + if: ${{ github.event.inputs.scenarios != 'offline-only' }} + runs-on: ubuntu-latest + environment: e2e + permissions: + id-token: write + contents: read + outputs: + aca_url: ${{ steps.deploy_perrun.outputs.aca_url }} + aca_app_name: ${{ steps.deploy_perrun.outputs.aca_app_name }} + hosted_agent_id: ${{ steps.create_hosted_agent.outputs.agent_id }} + hosted_agent_name: ${{ steps.create_hosted_agent.outputs.agent_name }} + suffix: ${{ steps.suffix.outputs.value }} + steps: + - uses: actions/checkout@v6 + + - id: suffix + name: Compute per-run suffix + run: echo "value=run${{ github.run_id }}" >> "$GITHUB_OUTPUT" + + - name: Azure login (OIDC) + uses: ./.github/actions/azure-oidc-login + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - id: discover + name: Discover ACR + AI Services + UAMI + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'http-aca' }} + run: | + set -euo pipefail + rg="${{ vars.AZURE_E2E_RESOURCE_GROUP }}" + acr_name=$(az acr list -g "$rg" --query "[0].name" -o tsv) + acr_login=$(az acr show -n "$acr_name" --query loginServer -o tsv) + ai_name=$(az cognitiveservices account list -g "$rg" \ + --query "[?kind=='AIServices'] | [0].name" -o tsv) + uami_name=$(az identity list -g "$rg" \ + --query "[?ends_with(name,'-aca-uami-${{ vars.AZURE_E2E_SUFFIX }}')] | [0].name" -o tsv) + if [ -z "$uami_name" ]; then + # Fallback: any *-aca-uami-* in the RG (bootstrap creates exactly one) + uami_name=$(az identity list -g "$rg" \ + --query "[?contains(name,'-aca-uami-')] | [0].name" -o tsv) + fi + if [ -z "$acr_name" ] || [ -z "$ai_name" ] || [ -z "$uami_name" ]; then + echo "Could not discover ACR / AI Services / aca-uami in $rg" >&2 + exit 1 + fi + uami_id=$(az identity show -g "$rg" -n "$uami_name" --query id -o tsv) + uami_client_id=$(az identity show -g "$rg" -n "$uami_name" --query clientId -o tsv) + echo "acr_name=$acr_name" >> "$GITHUB_OUTPUT" + echo "acr_login=$acr_login" >> "$GITHUB_OUTPUT" + echo "ai_name=$ai_name" >> "$GITHUB_OUTPUT" + echo "uami_id=$uami_id" >> "$GITHUB_OUTPUT" + echo "uami_client_id=$uami_client_id" >> "$GITHUB_OUTPUT" + echo "Discovered ACR=$acr_login, AI Services=$ai_name, UAMI=$uami_name" + + - name: Build & push hello-agent image (server-side ACR build) + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'http-aca' }} + run: | + set -euo pipefail + tag="run${{ github.run_id }}" + az acr build \ + --registry "${{ steps.discover.outputs.acr_name }}" \ + --image "agentops-e2e/hello-agent:$tag" \ + infra/e2e/agent-app + echo "Image: ${{ steps.discover.outputs.acr_login }}/agentops-e2e/hello-agent:$tag" + + - id: deploy_perrun + name: Deploy per-run ACA hello-agent app + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'http-aca' }} + run: | + set -euo pipefail + deployment_name="agentops-e2e-perrun-${{ github.run_id }}" + image="${{ steps.discover.outputs.acr_login }}/agentops-e2e/hello-agent:run${{ github.run_id }}" + az deployment group create \ + --resource-group "${{ vars.AZURE_E2E_RESOURCE_GROUP }}" \ + --name "$deployment_name" \ + --template-file infra/e2e/perrun.bicep \ + --parameters \ + acaEnvironmentId="${{ vars.AZURE_E2E_ACA_ENV_ID }}" \ + suffix="${{ steps.suffix.outputs.value }}" \ + image="$image" \ + acrLoginServer="${{ steps.discover.outputs.acr_login }}" \ + uamiResourceId="${{ steps.discover.outputs.uami_id }}" \ + uamiClientId="${{ steps.discover.outputs.uami_client_id }}" \ + azureOpenAiEndpoint="${{ vars.AZURE_E2E_OPENAI_ENDPOINT }}" \ + azureOpenAiDeployment="${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }}" \ + --output json > deployment.json + aca_url=$(jq -r '.properties.outputs.agentUrl.value' deployment.json) + aca_app=$(jq -r '.properties.outputs.appName.value' deployment.json) + echo "aca_url=$aca_url" >> "$GITHUB_OUTPUT" + echo "aca_app_name=$aca_app" >> "$GITHUB_OUTPUT" + echo "Deployed hello-agent ACA: $aca_url" + + - name: Wait for hello-agent to be ready + if: steps.deploy_perrun.outputs.aca_url != '' + run: | + set -e + # Container needs a moment to pull the image, start uvicorn, and + # for DefaultAzureCredential to acquire its first token. The / + # health endpoint reports ``"ready": true`` once the agent is up. + for i in $(seq 1 60); do + body=$(curl -fsS --max-time 5 "${{ steps.deploy_perrun.outputs.aca_url }}/" || true) + if [ -n "$body" ] && echo "$body" | grep -q '"ready":true'; then + echo "hello-agent ready after ${i} attempts: $body" + exit 0 + fi + sleep 5 + done + echo "hello-agent did not become ready" >&2 + curl -sS --max-time 5 "${{ steps.deploy_perrun.outputs.aca_url }}/" || true + exit 1 + + - name: Set up Python (for hosted agent provisioning) + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'foundry-hosted' }} + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install azure-ai-projects (for hosted agent provisioning) + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'foundry-hosted' }} + run: pip install --quiet "azure-ai-projects>=2.0.1" azure-identity + + - id: create_hosted_agent + name: Create transient hosted agent (with get_weather tool) + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'foundry-hosted' }} + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT }} + run: | + python scripts/e2e_hosted_agent.py create \ + --name "e2e-hosted-${{ steps.suffix.outputs.value }}" \ + --model "${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }}" + + live-foundry-prompt: + needs: bootstrap-live + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'foundry-prompt' }} + runs-on: ubuntu-latest + environment: e2e + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 + with: + version: ">=0.9.0" + - name: Azure login (OIDC) + uses: ./.github/actions/azure-oidc-login + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - run: uv sync --group dev + - run: uv pip install azure-ai-evaluation pandas azure-ai-projects azure-identity + - name: Render scenario config + env: + AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT: ${{ vars.AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT }} + AGENTOPS_E2E_MODEL_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + run: uv run python scripts/e2e_render_config.py + - name: Run AgentOps eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_E2E_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + working-directory: e2e-runs/foundry-prompt + run: uv run agentops eval run --config agentops.yaml + - name: Render transcript + if: always() + run: uv run python scripts/e2e_make_transcript.py e2e-runs/foundry-prompt + - uses: actions/upload-artifact@v7 + if: always() + with: + name: live-foundry-prompt + path: | + e2e-runs/foundry-prompt/.agentops/results/ + e2e-runs/foundry-prompt/transcript.md + e2e-runs/foundry-prompt/HEADER.md + e2e-runs/foundry-prompt/agentops.yaml + if-no-files-found: warn + retention-days: 14 + + live-foundry-hosted: + needs: bootstrap-live + if: ${{ (github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'foundry-hosted') && needs.bootstrap-live.outputs.hosted_agent_id != '' }} + runs-on: ubuntu-latest + environment: e2e + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 + with: + version: ">=0.9.0" + - name: Azure login (OIDC) + uses: ./.github/actions/azure-oidc-login + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - run: uv sync --group dev + - run: uv pip install azure-ai-evaluation pandas azure-ai-projects azure-identity + - name: Render scenario config + env: + AGENTOPS_E2E_FOUNDRY_HOSTED_AGENT: ${{ needs.bootstrap-live.outputs.hosted_agent_id }} + AGENTOPS_E2E_MODEL_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + run: uv run python scripts/e2e_render_config.py + - name: Run AgentOps eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_E2E_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + working-directory: e2e-runs/foundry-hosted + run: uv run agentops eval run --config agentops.yaml + - name: Render transcript + if: always() + run: uv run python scripts/e2e_make_transcript.py e2e-runs/foundry-hosted + - uses: actions/upload-artifact@v7 + if: always() + with: + name: live-foundry-hosted + path: | + e2e-runs/foundry-hosted/.agentops/results/ + e2e-runs/foundry-hosted/transcript.md + e2e-runs/foundry-hosted/HEADER.md + e2e-runs/foundry-hosted/agentops.yaml + e2e-runs/foundry-hosted/agent-info.json + if-no-files-found: warn + retention-days: 14 + + live-http-aca: + needs: bootstrap-live + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'http-aca' }} + runs-on: ubuntu-latest + environment: e2e + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 + with: + version: ">=0.9.0" + - name: Azure login (OIDC) + uses: ./.github/actions/azure-oidc-login + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - run: uv sync --group dev + - run: uv pip install azure-ai-evaluation pandas + - name: Render scenario config + env: + AGENTOPS_E2E_ACA_URL: ${{ needs.bootstrap-live.outputs.aca_url }} + AGENTOPS_E2E_MODEL_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + run: uv run python scripts/e2e_render_config.py + - name: Run AgentOps eval + env: + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_E2E_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + working-directory: e2e-runs/http-aca + run: uv run agentops eval run --config agentops.yaml + - name: Render transcript + if: always() + run: uv run python scripts/e2e_make_transcript.py e2e-runs/http-aca + - uses: actions/upload-artifact@v7 + if: always() + with: + name: live-http-aca + path: | + e2e-runs/http-aca/.agentops/results/ + e2e-runs/http-aca/transcript.md + e2e-runs/http-aca/HEADER.md + e2e-runs/http-aca/agentops.yaml + if-no-files-found: warn + retention-days: 14 + + live-model-direct: + needs: bootstrap-live + if: ${{ github.event.inputs.scenarios == 'all' || github.event.inputs.scenarios == 'model-direct' }} + runs-on: ubuntu-latest + environment: e2e + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v7 + with: + version: ">=0.9.0" + - name: Azure login (OIDC) + uses: ./.github/actions/azure-oidc-login + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - run: uv sync --group dev + - run: uv pip install azure-ai-evaluation pandas azure-ai-projects azure-identity + - name: Render scenario config + env: + AGENTOPS_E2E_MODEL_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + run: uv run python scripts/e2e_render_config.py + - name: Run AgentOps eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_E2E_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_E2E_MODEL_DEPLOYMENT }} + working-directory: e2e-runs/model-direct + run: uv run agentops eval run --config agentops.yaml + - name: Render transcript + if: always() + run: uv run python scripts/e2e_make_transcript.py e2e-runs/model-direct + - uses: actions/upload-artifact@v7 + if: always() + with: + name: live-model-direct + path: | + e2e-runs/model-direct/.agentops/results/ + e2e-runs/model-direct/transcript.md + e2e-runs/model-direct/HEADER.md + e2e-runs/model-direct/agentops.yaml + if-no-files-found: warn + retention-days: 14 + + teardown-live: + needs: + - bootstrap-live + - live-foundry-prompt + - live-foundry-hosted + - live-http-aca + - live-model-direct + if: ${{ always() && github.event.inputs.scenarios != 'offline-only' && github.event.inputs.keep_resources != 'true' }} + runs-on: ubuntu-latest + environment: e2e + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + - name: Azure login (OIDC) + uses: ./.github/actions/azure-oidc-login + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Delete per-run ACA app + if: needs.bootstrap-live.outputs.aca_app_name != '' + run: | + set -e + az containerapp delete \ + --resource-group "${{ vars.AZURE_E2E_RESOURCE_GROUP }}" \ + --name "${{ needs.bootstrap-live.outputs.aca_app_name }}" \ + --yes || true + - name: Set up Python (for hosted agent teardown) + if: needs.bootstrap-live.outputs.hosted_agent_name != '' + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install azure-ai-projects (for hosted agent teardown) + if: needs.bootstrap-live.outputs.hosted_agent_name != '' + run: pip install --quiet "azure-ai-projects>=2.0.1" azure-identity + - name: Delete transient hosted agent + if: needs.bootstrap-live.outputs.hosted_agent_name != '' + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT }} + run: | + python scripts/e2e_hosted_agent.py delete \ + --name "${{ needs.bootstrap-live.outputs.hosted_agent_name }}" || true + - name: Sweep stale e2e ACA apps (>1d old) + run: | + set -e + cutoff=$(date -u -d '1 day ago' +%FT%TZ) + # Sweep stale per-run ACA apps (both legacy 'aca-echo-' and new 'aca-agent-' names). + # NOTE: We deliberately do NOT sweep UAMIs here — the *-aca-uami-* identity is + # long-lived (created by bootstrap.bicep) and reused across runs to avoid the + # multi-minute Entra ID role-assignment propagation delay that would otherwise + # 401 every fresh per-run identity against Azure OpenAI. + mapfile -t apps < <(az containerapp list \ + --resource-group "${{ vars.AZURE_E2E_RESOURCE_GROUP }}" \ + --query "[?(starts_with(name,'aca-echo-run') || starts_with(name,'aca-agent-run')) && properties.latestRevisionFqdn!=null].{name:name,created:systemData.createdAt}" \ + -o json | jq -r --arg cutoff "$cutoff" '.[] | select(.created < $cutoff) | .name') + for app in "${apps[@]}"; do + [ -z "$app" ] && continue + echo "Deleting stale app: $app" + az containerapp delete -g "${{ vars.AZURE_E2E_RESOURCE_GROUP }}" -n "$app" --yes || true + done + + summary: + # Aggregates artifacts from every job into a single consolidated summary + # on the workflow run page. Runs even if upstream jobs failed so partial + # outcomes are still visible. + needs: + - offline-smoke + - unit-tests-with-coverage + - bootstrap-live + - live-foundry-prompt + - live-foundry-hosted + - live-http-aca + - live-model-direct + - teardown-live + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Download all artifacts + uses: actions/download-artifact@v8 + with: + path: artifacts + - name: Render consolidated summary + run: | + python scripts/e2e_aggregate_summary.py --root artifacts >> "$GITHUB_STEP_SUMMARY" + diff --git a/.gitignore b/.gitignore index f2140771..58e2d7e7 100644 --- a/.gitignore +++ b/.gitignore @@ -452,3 +452,15 @@ plugins/agentops/icon.png *.msix *.msm *.msp + +evidence/ + +logs.txt +.last_run + + +e2e-runs/ + +tmp-artifacts/ + +tmp/ diff --git a/AGENTS.md b/AGENTS.md index 02fd9fd3..1832c458 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,8 @@ Public CLI contract: - `agentops report generate --in [--out ]` - `agentops workflow generate [--force] [--dir ]` - `agentops skills install [--platform

] [--prompt] [--force]` +- `agentops agent analyze [--workspace ] [--config ] [--out ] [--lookback-days N] [--severity-fail ]` +- `agentops agent serve [--host ] [--port ] [--config ] [--no-verify] [--workers N]` Planned CLI stubs (not implemented in this release): - `agentops run list|show` @@ -143,7 +145,7 @@ src/ ├── data/ # Starter dataset JSONL rows ├── skills/ # Coding agent skill templates └── workflows/ # CI/CD workflow templates - └── agentops-eval.yml # GitHub Actions evaluation workflow + └── agentops-pr.yml # PR + 3 deploy templates (dev/qa/prod) ``` ### Tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b0bddb9..fa616783 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file. This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres to [Semantic Versioning](https://semver.org/). +## [0.1.8] - 2026-04-22 + +### Added +- **Pre-flight checks for `agentops eval run`** — detects common issues (missing `azure-identity` or `azure-ai-evaluation` packages, missing env vars for AI-assisted/safety evaluators, Azure credential failures, unreachable endpoints) *before* backend execution. All detectable issues are reported at once with actionable error messages and `pip install` hints. +- **`--dry-run` / `-n` flag on `eval run`** — runs pre-flight checks without executing the evaluation. Exits 0 if all checks pass, 1 otherwise. Useful for CI gating and fast feedback. +- **Credential warm-up in pre-flight** — acquires and caches the MSAL token once during pre-flight so subsequent evaluator calls don't each cold-start `az.cmd`. + +### Changed +- **Azure CLI credential timeout raised to 30s** — all `DefaultAzureCredential` instantiation sites (`eval_engine.py`, `foundry_backend.py`) now pass `process_timeout=30`. Default (10s) is insufficient for Windows `az.cmd` cold starts and was causing intermittent `AzureCliCredential: Failed to invoke the Azure CLI` errors. + ## [0.1.7] - 2026-04-21 ### Added diff --git a/README.md b/README.md index 379421bb..1a01bb68 100644 --- a/README.md +++ b/README.md @@ -19,24 +19,15 @@ AgentOps CLI for evaluation, observability, and operational workflows for Micros ## Overview -AgentOps Toolkit is a CLI built on Microsoft Foundry that standardizes evaluation and operational workflows for AI agents and models, helping teams run, monitor, and automate AgentOps processes. +AgentOps Toolkit is a CLI built on Microsoft Foundry that standardizes evaluation workflows for AI agents and models, helping teams run and automate evaluations with consistent inputs and outputs. The project enables: - Consistent local and CI execution of agent evaluations -- Reusable evaluation policies through bundles -- Operational observability through tracing, monitoring, and run inspection +- Automatic evaluator selection based on dataset shape (RAG, agent-with-tools, model quality) - Stable machine-readable outputs for automation - Human-readable reports for PR reviews and quality gates - -Operational capabilities include: - -- Standardized evaluation workflows -- Run history and result inspection -- Tracing and observability -- Monitoring (dashboards and alerts) -- CI/CD automation -- Operational reporting and analysis +- Baseline comparison to detect regressions across runs Core outputs: @@ -51,104 +42,93 @@ Exit code contract: ## Quickstart -

-Quickstart demo: agentops init and eval run -

- ### 1) Install ```bash python -m venv .venv -# activate your venv in the current shell python -m pip install -U pip python -m pip install agentops-toolkit ``` -### 2) Initialize and Configure +### 2) Bootstrap ```bash agentops init ``` -This creates `.agentops/` with starter bundles, datasets, and run configs for common scenarios (model quality, RAG, agent workflow, content safety). +This writes a single `agentops.yaml` at the project root and a tiny seed dataset at `.agentops/data/smoke.jsonl`. Edit `agentops.yaml` to point at your agent. -Set your Foundry project endpoint: +### 3) Configure your agent -```bash -export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +Pick one of these forms for the `agent:` field — AgentOps classifies the target automatically: + +```yaml +agent: "my-rag:3" # Foundry prompt agent (name:version) +agent: "https://...services.ai.azure.com/.../agents/" # Foundry hosted endpoint +agent: "https://api.example.com/chat" # any HTTP/JSON agent (ACA, AKS, custom) +agent: "model:gpt-4o" # raw Foundry model deployment ``` -Then edit `.agentops/run.yaml` to set your `agent_id` and `model` deployment name. +Evaluators are inferred from the dataset shape (rows with `context` → RAG evaluators, rows with `tool_calls`/`tool_definitions` → agent-workflow evaluators). The full minimal config is: -> Authentication uses `DefaultAzureCredential` — run `az login` locally, or use service principal env vars in CI. +```yaml +version: 1 +agent: "my-rag:3" +dataset: .agentops/data/smoke.jsonl +``` -### 3) Run Evaluation +### 4) Run ```bash +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" agentops eval run ``` -Results are written to `.agentops/results/latest/`: -- `results.json` — machine-readable scores -- `report.md` — human-readable summary +Outputs land in `.agentops/results/latest/`: -To run a different scenario: +- `results.json` — machine-readable (versioned, stable schema) +- `report.md` — human-readable, PR-friendly -```bash -agentops eval run --config .agentops/run-rag.yaml -``` - -To regenerate the report from existing results: +To compare against a previous run, pass `--baseline`: ```bash -agentops report generate +agentops eval run --baseline .agentops/results/baseline/results.json ``` -See [Concepts](https://github.com/Azure/agentops/blob/main/docs/concepts.md) for an overview of bundles, datasets, evaluators, backends, and the configuration model. +The report grows a `Comparison vs Baseline` section with per-metric deltas. + +--- ## Commands -| Command | Description | Status | -|---|---|---| -| `agentops --version` | Show installed version | ✅ | -| `agentops init [--path DIR]` | Scaffold project workspace, starter files, and coding agent skills | ✅ | -| `agentops eval run [--config PATH]` | Evaluate a dataset against a bundle | ✅ | -| `agentops eval compare --runs ID1,ID2` | Compare two past runs | ✅ | -| `agentops report generate [--in FILE]` | Regenerate `report.md` from `results.json` | ✅ | -| `agentops workflow generate` | Generate GitHub Actions workflow | ✅ | -| `agentops skills install [--platform

]` | Install coding agent skills (Copilot, Claude) | ✅ | -| `agentops run list\|show` | List or inspect past runs | 🚧 | -| `agentops bundle list\|show` | Browse bundle catalog | 🚧 | -| `agentops dataset validate\|describe` | Dataset utilities | 🚧 | -| `agentops trace init` | Tracing setup | 🚧 | -| `agentops monitor setup\|show\|configure` | Monitoring operations | 🚧 | - -Planned commands return a friendly message indicating they are not yet implemented. +| Command | Description | +|---|---| +| `agentops --version` | Show installed version | +| `agentops init` | Bootstrap `agentops.yaml` and a seed dataset | +| `agentops eval run [--config PATH] [--baseline PATH]` | Run an evaluation | +| `agentops report generate [--in FILE]` | Regenerate `report.md` from `results.json` | +| `agentops workflow generate` | Generate GitHub Actions workflow | +| `agentops skills install [--platform

]` | Install coding agent skills (Copilot, Claude) | +| `agentops mcp serve` | Start the AgentOps MCP server (stdio). Requires `pip install agentops-toolkit[mcp]`. | +| `agentops agent analyze` | Run the watchdog over your run history. Requires `pip install agentops-toolkit[agent]`. | +| `agentops agent serve` | Start the watchdog as a FastAPI Copilot Extension. Requires `pip install agentops-toolkit[agent]`. | ## Documentation -### Concepts and Architecture - -- [Concepts](https://github.com/Azure/agentops/blob/main/docs/concepts.md) — bundles, datasets, evaluators, backends, configuration model -- [How It Works](https://github.com/Azure/agentops/blob/main/docs/how-it-works.md) — architecture, request flow, full schema reference -- [Bundles](https://github.com/Azure/agentops/blob/main/docs/bundles.md) — bundle authoring and evaluator configuration - -### Tutorials - -- [Model-direct evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-model-direct.md) -- [Foundry agent evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-basic-foundry-agent.md) -- [RAG evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-rag.md) -- [HTTP-deployed agent evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-http-agent.md) -- [Conversational agent evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-conversational-agent.md) -- [Agent workflow evaluation](https://github.com/Azure/agentops/blob/main/docs/tutorial-agent-workflow.md) +- [Quickstart tutorial](https://github.com/Azure/agentops/blob/main/docs/tutorial-quickstart.md) — bootstrap a workspace and run one evaluation. +- [End-to-end tutorial](https://github.com/Azure/agentops/blob/main/docs/tutorial-end-to-end.md) — full do-it-yourself tour: Foundry hosted agent, baseline comparison, GitFlow CI/CD, watchdog. +- Per-scenario tutorials: + - [Foundry hosted agent](https://github.com/Azure/agentops/blob/main/docs/tutorial-basic-foundry-agent.md) + - [Model-direct](https://github.com/Azure/agentops/blob/main/docs/tutorial-model-direct.md) + - [RAG](https://github.com/Azure/agentops/blob/main/docs/tutorial-rag.md) + - [Conversational agent](https://github.com/Azure/agentops/blob/main/docs/tutorial-conversational-agent.md) + - [Agent with tool calling](https://github.com/Azure/agentops/blob/main/docs/tutorial-agent-workflow.md) + - [HTTP-deployed agent](https://github.com/Azure/agentops/blob/main/docs/tutorial-http-agent.md) - [Baseline comparison](https://github.com/Azure/agentops/blob/main/docs/tutorial-baseline-comparison.md) - -### Operations - +- [Watchdog agent](https://github.com/Azure/agentops/blob/main/docs/tutorial-agent-watchdog.md) - [CI/CD with GitHub Actions](https://github.com/Azure/agentops/blob/main/docs/ci-github-actions.md) -- [Copilot skills installation](https://github.com/Azure/agentops/blob/main/docs/tutorial-copilot-skills.md) -- [Release process](https://github.com/Azure/agentops/blob/main/docs/release-process.md) - [Built-in evaluator reference](https://github.com/Azure/agentops/blob/main/docs/foundry-evaluation-sdk-built-in-evaluators.md) +- [Release process](https://github.com/Azure/agentops/blob/main/docs/release-process.md) ## Contributing diff --git a/docs/analysis-issue-51-cicd-field-insights.md b/docs/analysis-issue-51-cicd-field-insights.md deleted file mode 100644 index 36e51e2a..00000000 --- a/docs/analysis-issue-51-cicd-field-insights.md +++ /dev/null @@ -1,445 +0,0 @@ -# Issue #51 — Review CI/CD Based on Field Insights - -**Date:** 2026-04-03 -**Issue:** https://github.com/Azure/agentops/issues/51 -**Author:** placerda -**Reference repo:** https://github.com/hrprtkaur88/foundrycicdbasic - ---- - -## 1. Executive Summary - -This analysis evaluates how well AgentOps Toolkit serves as a CI/CD-ready -evaluation tool based on real-world pipeline patterns observed in Harpreet's -Foundry CI/CD reference repository. The goal is to identify what prevents teams -like Harpreet's from replacing their custom Python scripts with -`agentops eval run`, and what AgentOps must improve to be viable in real -CI/CD environments. - -**Key finding:** AgentOps has strong CI/CD foundations (exit codes, artifacts, -declarative config, generated workflow) but is missing critical evaluator -coverage and data-source patterns that real-world pipelines require. A team -using Harpreet's pipeline today cannot switch to AgentOps without losing -evaluator coverage. - ---- - -## 2. Task Analysis - -### Task 1: Review Harpreet repository and pipeline structure - -**What the repo is:** -A reference implementation showing how to create, test, evaluate, and red-team -Foundry agents using raw Python scripts orchestrated by CI/CD pipelines. - -**Repository structure:** - -``` -foundrycicdbasic/ -├── createagent.py # Creates a Foundry agent via Agent Framework SDK -├── exagent.py # Smoke-tests an existing agent with a real query -├── agenteval.py # Runs cloud evaluation via OpenAI Evals API -├── agenteval_classic.py # Local evaluation fallback -├── redteam.py # Red-team safety evaluation -├── redteam_classic.py # Red-team local fallback -├── requirements.txt # Unpinned runtime dependencies -├── sample.env # Example environment variables -├── data_folder/ # Red-team taxonomy + output files -├── .github/workflows/ -│ ├── create-agent-multi-env.yml # GitHub Actions: deploy agent (dev→test→prod) -│ └── agent-consumption-multi-env.yml # GitHub Actions: test→eval→redteam (dev→test→prod) -├── cicd/ -│ ├── createagentpipeline.yml # Azure DevOps: deploy agent -│ └── agentconsumptionpipeline.yml # Azure DevOps: test→eval→redteam -└── cicd_patterns/ - └── foundry-cicd-workflow.pptx # Presentation on patterns -``` - -**Pipeline flow (agent-consumption-multi-env.yml):** - -``` -build (validate syntax) - → test-dev (exagent.py — smoke-test agent) - → evaluate-test (agenteval.py — cloud evaluation) - → red-team-test (redteam.py — safety evaluation) - → verify-prod (exagent.py — production verification) -``` - -**Key observations:** - -1. **All evaluation logic is imperative** — evaluator names, data mappings, - test data, and testing criteria are hardcoded in Python scripts. -2. **No thresholds or gating** — every eval/redteam step uses - `continue-on-error: true`. The pipeline never blocks on quality. -3. **Authentication uses service principal JSON blobs** — stored as - `AZURE_CREDENTIALS_*` secrets, not OIDC. -4. **Dual platform** — same pipelines exist for both GitHub Actions and - Azure DevOps (manually duplicated). -5. **Inline test data** — `agenteval.py` has query/response/tool_definitions - hardcoded in the script, not in external data files. - -### Task 2: Identify evaluation patterns used in real scenarios - -The following evaluation patterns are used in Harpreet's pipeline. Each is -mapped to AgentOps support status. - -#### Pattern A: Agent smoke test (exagent.py) - -**What it does:** Retrieves an existing agent by name, sends a real query, -handles MCP approval requests, and prints the response with citations. - -**Purpose in CI/CD:** Validates the agent is alive and responsive before -running expensive evaluations. - -**AgentOps equivalent:** None. AgentOps has no "health check" or "smoke test" -concept. The `agentops eval run` command goes straight to evaluation. - -**Gap severity:** Low. This is a convenience — users can add a custom step -before `agentops eval run` in their pipeline. - -#### Pattern B: Cloud evaluation with inline data (agenteval.py) - -**What it does:** -1. Creates an OpenAI client from the Foundry project client -2. Defines `data_source_config` with `type: custom` and an item schema -3. Defines `testing_criteria` — a list of `azure_ai_evaluator` entries -4. Calls `client.evals.create()` to create an eval group -5. Calls `client.evals.runs.create()` with inline JSONL data -6. Polls until completion -7. Retrieves output items - -**Evaluators used:** - -| Category | Evaluator | Builtin name | AgentOps support | -|---|---|---|---| -| System | Task Completion | `builtin.task_completion` | **Not supported** | -| System | Task Adherence | `builtin.task_adherence` | **Not supported** | -| System | Intent Resolution | `builtin.intent_resolution` | **Not supported** | -| RAG | Groundedness | `builtin.groundedness` | Supported | -| RAG | Relevance | `builtin.relevance` | **Not supported** | -| Process | Tool Call Accuracy | `builtin.tool_call_accuracy` | Supported | -| Process | Tool Selection | `builtin.tool_selection` | **Not supported** | -| Process | Tool Input Accuracy | `builtin.tool_input_accuracy` | **Not supported** | -| Process | Tool Output Utilization | `builtin.tool_output_utilization` | **Not supported** | - -**Data format used:** -- `query`: array of message objects (system + user messages) -- `response`: array of message objects (assistant + tool_call + tool_result) -- `tool_definitions`: array of tool schemas -- `tool_calls`: null (derived from response) - -**AgentOps data format:** -- `input`: string (simple text field from JSONL) -- `expected`: string (simple text field from JSONL) -- `context`: optional string - -**Gap severity:** **Critical.** 7 of 9 evaluators used in the field are not -supported by AgentOps. The data format is also incompatible — Harpreet uses -conversation-format arrays while AgentOps expects simple string fields. - -#### Pattern C: Red-team / safety evaluation (redteam.py) - -**What it does:** -1. Creates an agent version via `project_client.agents.create_version()` -2. Defines safety testing criteria: - - `builtin.prohibited_actions` - - `builtin.task_adherence` - - `builtin.sensitive_data_leakage` - - `builtin.self_harm` - - `builtin.violence` - - `builtin.sexual` - - `builtin.hate_unfairness` -3. Creates evaluation taxonomy via `project_client.evaluation_taxonomies.create()` -4. Creates eval run with `data_source.type: azure_ai_red_team` -5. Uses `attack_strategies: ["Flip", "Base64"]` with generated adversarial inputs -6. Polls until completion, saves results to JSON - -**AgentOps equivalent:** None. AgentOps has no concept of: -- Red-team data sources (`azure_ai_red_team`) -- Safety evaluators (prohibited_actions, sensitive_data_leakage, violence, etc.) -- Attack strategies -- Evaluation taxonomies - -**Gap severity:** **High.** Red-team testing is a major field requirement. -However, this may be better addressed as a separate `agentops redteam` command -rather than extending `agentops eval run`, since the data source model is -fundamentally different (generated adversarial inputs vs. user-provided JSONL). - -#### Pattern D: Multi-environment sequential deployment - -**What it does:** Runs the same scripts across dev → test → prod environments, -with each stage depending on the previous. Production requires manual approval -via GitHub Environment protection rules. - -**AgentOps equivalent:** Not directly relevant to the AgentOps tool — this is -a pipeline orchestration pattern. AgentOps's `project_endpoint_env` config -already supports being called in different environments by varying the -endpoint secret. No tool change needed. - -**Gap severity:** None for the tool. Documentation gap only. - -#### Pattern E: Scheduled security scans - -**What it does:** Weekly cron trigger (`0 2 * * 1`) runs the full -test → eval → redteam pipeline on Monday mornings. - -**AgentOps equivalent:** Not relevant to the tool — this is a pipeline trigger -pattern. `agentops eval run` works fine when invoked by a cron job. - -**Gap severity:** None for the tool. Documentation gap only. - -### Task 3: Define supported CI/CD integration models - -Based on field analysis, AgentOps should support these integration models: - -| Model | Description | Tool readiness | -|---|---|---| -| **PR gating** | `agentops eval run` in a PR workflow; exit code 2 blocks merge | **Ready** — implemented and documented | -| **Scheduled regression** | Cron-triggered eval run to detect drift | **Ready** — CLI works, needs documentation | -| **Post-deployment validation** | Run eval after deploying to an environment | **Ready** — CLI works, needs documentation | -| **Multi-config matrix** | Run multiple eval configs in parallel | **Ready** — documented with matrix strategy | -| **Advisory mode** | Run eval and report results without blocking | **Partially ready** — exit code 2 blocks; no `--no-fail` flag | - -### Task 4: Define best practices for gating deployments based on evaluations - -**What AgentOps provides today:** - -| Capability | Status | Evidence | -|---|---|---| -| Exit code contract (0/1/2) | Implemented | `cli/app.py` raises `typer.Exit(code=2)` on threshold failure | -| Declarative thresholds in YAML | Implemented | `bundles/*.yaml` with `thresholds[]` | -| Per-metric threshold criteria | Implemented | `>=`, `>`, `<=`, `<`, `==`, `true`/`false` in `thresholds.py` | -| Per-row threshold evaluation | Implemented | `runner.py` `_evaluate_item_thresholds()` | -| PR comment with report | Implemented | Workflow template posts/updates PR comment | -| Job summary | Implemented | Workflow writes to `$GITHUB_STEP_SUMMARY` | -| Artifacts on failure | Implemented | `if: always()` on artifact upload step | - -**What's missing for real-world gating:** - -| Gap | Impact | -|---|---| -| No `--no-fail` / `--advisory` flag | Teams can't run eval in "observe only" mode (like Harpreet's `continue-on-error`) | -| `agentops config validate` not implemented | Teams can't fail-fast on bad config before running expensive evaluations | -| No threshold on safety evaluators | Can't gate on red-team results since safety evaluators aren't supported | - -### Task 5: Identify gaps in current CLI for CI/CD usage - -| Gap | Category | Severity | Detail | -|---|---|---|---| -| Missing cloud evaluators | Evaluator coverage | **Critical** | 7 of 9 evaluators used in field are unsupported: `task_completion`, `task_adherence`, `intent_resolution`, `relevance`, `tool_selection`, `tool_input_accuracy`, `tool_output_utilization` | -| No conversation-format data | Data model | **High** | Field uses array-of-messages for query/response; AgentOps only supports simple string fields | -| No red-team support | Feature | **High** | No safety evaluators, no `azure_ai_red_team` data source, no attack strategies | -| No `--no-fail` flag | CLI | **Medium** | Can't run in advisory mode without `continue-on-error` in the pipeline YAML | -| `config validate` not implemented | CLI | **Medium** | Can't pre-validate configs in CI before running eval | -| `dataset validate` not implemented | CLI | **Medium** | Can't verify dataset integrity in CI | -| No Azure DevOps template | Documentation | **Low** | `agentops config cicd` only generates GitHub Actions; ADO users must write their own | - ---- - -## 3. Acceptance Criteria Assessment - -### AC 1: CI/CD integration patterns are clearly defined - -**Verdict: PARTIALLY MET** - -**What exists:** -- `docs/ci-github-actions.md` — comprehensive guide covering triggers, auth, - exit codes, artifacts, PR comments, job summary, troubleshooting -- Generated workflow template via `agentops config cicd` -- Matrix strategy documentation for multi-config runs -- Internal CI/CD workflows documented for contributors - -**What's missing:** -- No documentation for Azure DevOps integration -- No documentation for "advisory mode" (run without gating) -- No documentation for scheduled evaluation pattern -- The patterns are defined for the *simple case* (model-direct with similarity) - but not for the *real-world case* (agent evaluation with process/system - evaluators) - -**To close:** Document Azure DevOps integration pattern. Document advisory -mode. Ensure patterns cover agent evaluation scenarios, not just model-direct. - -### AC 2: Pipelines support evaluation as a gating mechanism - -**Verdict: MET (for supported evaluators)** - -**Evidence:** -- Exit code 0/1/2 contract is implemented and tested -- Workflow template uses `exit $EXIT_CODE` — non-zero fails the job -- Threshold evaluation supports multiple criteria operators -- Per-row and aggregate threshold evaluation is implemented -- CLI propagates exit code 2 via `raise typer.Exit(code=2)` - -**Caveat:** Gating only works for the evaluators AgentOps supports. Since most -field-used evaluators are unsupported, the gating mechanism exists but can't -be applied to the metrics teams actually care about (task_completion, -intent_resolution, etc.). - -### AC 3: Exit codes are correctly interpreted in CI/CD - -**Verdict: MET** - -**Evidence:** -- Workflow template maps exit codes to step summary messages - (0 → pass, 2 → threshold fail, else → error) -- Exit code saved to `$GITHUB_OUTPUT` for downstream consumption -- `test_cicd.py` asserts `EXIT_CODE` and `exit $EXIT_CODE` are in template -- GitHub Actions natively fails on non-zero — no special handling needed -- Exit code semantics documented in `docs/ci-github-actions.md` - -### AC 4: Artifacts are generated and usable in pipeline context - -**Verdict: MET** - -**Evidence:** -- Workflow uploads 6 artifact files: `results.json`, `report.md`, - `backend_metrics.json`, `cloud_evaluation.json`, `backend.stdout.log`, - `backend.stderr.log` -- Upload uses `if: always()` — artifacts available even on failure -- `results.json` has versioned Pydantic schema — machine-readable -- `report.md` is human-readable and posted as PR comment -- `cloud_evaluation.json` includes `report_url` for Foundry portal deep-link -- `agentops report --in results.json` can regenerate reports from artifacts - -### AC 5: At least one reference pipeline is documented - -**Verdict: MET** - -**Evidence:** -- `docs/ci-github-actions.md` is a complete reference pipeline guide -- `agentops config cicd` generates a tested, ready-to-use workflow -- Template includes inline comments explaining every step -- Quick start, auth setup, customization, and troubleshooting covered - -### AC 6: Integration works with real-world scenarios - -**Verdict: NOT MET** - -**Evidence from field analysis:** - -Harpreet's pipeline represents a real-world scenario. To replace their -`agenteval.py` with `agentops eval run`, a user would need to: - -1. **Define evaluators in a bundle YAML** — but 7 of 9 evaluators they use - are not supported by AgentOps -2. **Provide test data in JSONL** — but the field uses conversation-format - arrays (query as message list, response as message list with tool calls), - while AgentOps expects simple string fields -3. **Get evaluation results** — AgentOps produces `results.json` and - `report.md`, which is better than Harpreet's raw stdout, but the results - won't contain the metrics teams need -4. **Gate on results** — AgentOps has threshold gating, which Harpreet's - pipeline lacks, but it can only gate on supported evaluators - -**What a user would need to do today to use AgentOps in Harpreet's pipeline:** - -```yaml -# What they want to write: -bundle: - evaluators: - - name: TaskCompletionEvaluator # ❌ not supported - - name: TaskAdherenceEvaluator # ❌ not supported - - name: IntentResolutionEvaluator # ❌ not supported - - name: GroundednessEvaluator # ✅ supported - - name: RelevanceEvaluator # ❌ not supported - - name: ToolCallAccuracyEvaluator # ✅ supported - - name: ToolSelectionEvaluator # ❌ not supported - -# What they can actually use today: -bundle: - evaluators: - - name: GroundednessEvaluator # ✅ - - name: ToolCallAccuracyEvaluator # ✅ - # ...that's it -``` - -**Blockers preventing real-world adoption:** - -| Blocker | Why it blocks | -|---|---| -| Missing evaluators | Teams can't measure what matters to them | -| String-only data format | Teams can't provide conversation-format test data | -| No red-team | Teams must maintain a separate `redteam.py` alongside AgentOps | - ---- - -## 4. Gap Prioritization for Closing the Issue - -### Priority 1 — Critical (blocks AC 6) - -| Item | What to do | Effort | -|---|---|---| -| Add system evaluators | Add `task_completion`, `task_adherence`, `intent_resolution` to `_cloud_evaluator_data_mapping` | Low — mapping only, no new API calls | -| Add RAG evaluator: relevance | Add `relevance` alongside existing `groundedness` | Low | -| Add process evaluators | Add `tool_selection`, `tool_input_accuracy`, `tool_output_utilization` to `_EVALUATORS_NEEDING_TOOL_CALLS` or a new set | Low-Medium — need to verify data_mapping for each | - -These evaluators all use the same `azure_ai_evaluator` type and -`builtin.` pattern that AgentOps already supports. The gap is in the -`_cloud_evaluator_data_mapping` function, which doesn't know how to build -`data_mapping` for these evaluators. Each new evaluator needs: -- An entry in the appropriate frozenset (or a new one) -- The correct `data_mapping` fields (query, response, tool_calls, tool_definitions, etc.) - -### Priority 2 — High (improves real-world viability) - -| Item | What to do | Effort | -|---|---|---| -| Conversation-format data support | Allow JSONL rows with array-of-messages for query/response fields | Medium — requires dataset format model changes | -| `--no-fail` / `--advisory` flag | Add CLI flag that makes exit code always 0 (report thresholds but don't gate) | Low | -| `config validate` command | Implement the planned command to pre-validate configs in CI | Medium | - -### Priority 3 — Medium (documentation) - -| Item | What to do | Effort | -|---|---|---| -| Azure DevOps integration pattern | Document how to use `agentops eval run` in an ADO pipeline | Low — docs only | -| Scheduled evaluation pattern | Document cron-triggered eval for drift detection | Low — docs only | -| Advisory mode pattern | Document how to run eval without gating (once `--no-fail` exists) | Low — docs only | -| Multi-environment pattern | Document how to use `project_endpoint_env` across environments | Low — docs only | - -### Priority 4 — Future (separate feature) - -| Item | What to do | Effort | -|---|---|---| -| Red-team support | New command or new data source type — fundamentally different flow | High — new feature | -| Safety evaluators | `prohibited_actions`, `sensitive_data_leakage`, `violence`, etc. | Medium — requires red-team data source | - ---- - -## 5. Recommendation - -**To close issue #51, focus on Priority 1 (missing evaluators).** This is the -single biggest blocker for real-world CI/CD adoption. The evaluators all follow -the same `azure_ai_evaluator` / `builtin.` pattern that AgentOps already -implements — the gap is mechanical, not architectural. - -Adding 7 evaluators to `foundry_backend.py` would change the AC 6 verdict from -"NOT MET" to "PARTIALLY MET" (still missing conversation-format data and -red-team, but the core evaluation flow would work for the majority of -field-used evaluators). - -Red-team support (Priority 4) should be tracked as a separate issue — it -requires a different data source model (`azure_ai_red_team` with attack -strategies and taxonomy generation) that doesn't fit the current -`agentops eval run` flow. - ---- - -## 6. Summary Scorecard - -| Acceptance Criterion | Verdict | -|---|---| -| AC 1: CI/CD integration patterns clearly defined | ⚠️ Partially met | -| AC 2: Pipelines support evaluation as gating mechanism | ✅ Met | -| AC 3: Exit codes correctly interpreted in CI/CD | ✅ Met | -| AC 4: Artifacts generated and usable in pipeline context | ✅ Met | -| AC 5: At least one reference pipeline documented | ✅ Met | -| AC 6: Integration works with real-world scenarios | ❌ Not met | - -**Overall: 4/6 met, 1/6 partially met, 1/6 not met.** - -The blocking gap is evaluator coverage. AgentOps has the right architecture -for CI/CD integration — declarative config, exit-code gating, artifact -production, generated workflows — but it cannot evaluate the metrics that -real-world Foundry agent pipelines need. diff --git a/docs/analysis-issue-51-two-track.md b/docs/analysis-issue-51-two-track.md deleted file mode 100644 index b320c71e..00000000 --- a/docs/analysis-issue-51-two-track.md +++ /dev/null @@ -1,447 +0,0 @@ -# Issue #51 — Two-Track Analysis - -**Date:** 2026-04-03 - ---- - -## Track 1: How to Fully Support Foundry Default Evaluators - -### Current Architecture - -The cloud evaluation path in `foundry_backend.py` builds evaluators like this: - -```python -builtin_name = _to_builtin_evaluator_name(evaluator.name) # "SimilarityEvaluator" → "similarity" -criterion = { - "type": "azure_ai_evaluator", - "name": evaluator.name, - "evaluator_name": f"builtin.{builtin_name}", - "data_mapping": _cloud_evaluator_data_mapping(builtin_name, input_field, expected_field, context_field), -} -if _cloud_evaluator_needs_model(builtin_name): - criterion["initialization_parameters"] = {"deployment_name": settings.model} -``` - -The `_cloud_evaluator_data_mapping` function routes evaluators to the correct -`data_mapping` based on frozenset membership: - -``` -default path → {"query": "{{item.X}}", "response": "{{sample.output_text}}"} -_NLP_ONLY_EVALUATORS → no "query", just "response" -_GROUND_TRUTH → adds "ground_truth": "{{item.Y}}" -_CONTEXT → adds "context": "{{item.Z}}" -_TOOL_CALLS → adds "tool_calls": "{{sample.tool_calls}}", "tool_definitions": "{{item.tool_definitions}}" -``` - -### Problem: Only 8 of ~35 evaluators are routed correctly - -Any evaluator NOT in any frozenset falls to the default path (`query` + `response`). -This accidentally works for some evaluators (like `coherence`) but silently sends -wrong data_mappings for many others. - -### What Each Evaluator Actually Needs - -Based on Foundry cloud evaluation docs (2026-04-02), here are the correct -`data_mapping` patterns for every built-in evaluator: - -#### Pattern 1: query + response (simplest — default path) - -Works with current default path. No code change needed. - -| Evaluator | builtin name | Needs model | Status | -|---|---|---|---| -| CoherenceEvaluator | `coherence` | Yes | ✅ Works today (falls to default) | -| FluencyEvaluator | `fluency` | Yes | ✅ Works today | -| RelevanceEvaluator | `relevance` | Yes | ✅ Works today | -| IntentResolutionEvaluator | `intent_resolution` | Yes | ✅ Works today | -| TaskCompletionEvaluator | `task_completion` | Yes | ✅ Works today | -| ViolenceEvaluator | `violence` | Yes | ✅ Works today | -| SexualEvaluator | `sexual` | Yes | ✅ Works today | -| SelfHarmEvaluator | `self_harm` | Yes | ✅ Works today | -| HateUnfairnessEvaluator | `hate_unfairness` | Yes | ✅ Works today | -| ContentSafetyEvaluator | `content_safety` | Yes | ✅ Works today | -| ProtectedMaterialEvaluator | `protected_material` | Yes | ✅ Works today | -| CodeVulnerabilityEvaluator | `code_vulnerability` | Yes | ✅ Works today | -| UngroundedAttributesEvaluator | `ungrounded_attributes` | Yes | ✅ Works today | -| IndirectAttackEvaluator | `indirect_attack` | Yes | ✅ Works today | - -**Verdict:** These 14 evaluators already work with the current code — users -just don't know they can use them because they're not documented/tested. - -#### Pattern 2: query + response (output_items) — agent structured output - -`task_adherence` needs `{{sample.output_items}}` instead of -`{{sample.output_text}}` for the response field, because it needs to see the -full structured agent output (tool calls, intermediate steps). - -| Evaluator | builtin name | response field | Status | -|---|---|---|---| -| TaskAdherenceEvaluator | `task_adherence` | `{{sample.output_items}}` | ❌ **Broken** — sends `output_text` | - -**Fix required:** Add `task_adherence` to a new set -`_EVALUATORS_NEEDING_OUTPUT_ITEMS` and map `response` to -`{{sample.output_items}}` instead of `{{sample.output_text}}`. - -#### Pattern 3: response + ground_truth (existing) - -Already implemented via `_EVALUATORS_NEEDING_GROUND_TRUTH`. - -| Evaluator | builtin name | Status | -|---|---|---| -| SimilarityEvaluator | `similarity` | ✅ Supported | -| ResponseCompletenessEvaluator | `response_completeness` | ❌ Missing from frozenset | - -**Fix required:** Add `response_completeness` to `_EVALUATORS_NEEDING_GROUND_TRUTH`. - -#### Pattern 4: NLP only — no query, no model (existing) - -Already implemented via `_NLP_ONLY_EVALUATORS`. - -| Evaluator | builtin name | Status | -|---|---|---| -| F1ScoreEvaluator | `f1_score` | ✅ Supported | -| BleuScoreEvaluator | `bleu` | ✅ Supported | -| GleuScoreEvaluator | `gleu` | ✅ Supported | -| RougeScoreEvaluator | `rouge` | ✅ Supported | -| MeteorScoreEvaluator | `meteor` | ✅ Supported | - -#### Pattern 5: response + context (existing) - -Already implemented via `_EVALUATORS_NEEDING_CONTEXT`. - -| Evaluator | builtin name | Status | -|---|---|---| -| GroundednessEvaluator | `groundedness` | ✅ Supported | -| GroundednessProEvaluator | `groundedness_pro` | ❌ Missing from frozenset | -| RetrievalEvaluator | `retrieval` | ❌ Missing from frozenset | - -**Fix required:** Add `groundedness_pro` and `retrieval` to -`_EVALUATORS_NEEDING_CONTEXT`. - -#### Pattern 6: tool evaluators (existing) - -Already implemented via `_EVALUATORS_NEEDING_TOOL_CALLS`. - -| Evaluator | builtin name | data_mapping | Status | -|---|---|---|---| -| ToolCallAccuracyEvaluator | `tool_call_accuracy` | query, response, tool_calls, tool_definitions | ✅ Supported | -| ToolSelectionEvaluator | `tool_selection` | query, response, tool_calls, tool_definitions | ❌ Missing from frozenset | -| ToolInputAccuracyEvaluator | `tool_input_accuracy` | query, response, tool_definitions | ❌ Missing (needs tool_definitions but not tool_calls) | -| ToolOutputUtilizationEvaluator | `tool_output_utilization` | query, response, tool_definitions | ❌ Missing | -| ToolCallSuccessEvaluator | `tool_call_success` | response, tool_definitions | ❌ Missing | - -**Fix required:** -- Add `tool_selection` to `_EVALUATORS_NEEDING_TOOL_CALLS` -- For `tool_input_accuracy` and `tool_output_utilization`: need - `tool_definitions` but NOT `tool_calls` — need a new set - `_EVALUATORS_NEEDING_TOOL_DEFINITIONS_ONLY` -- For `tool_call_success`: needs `response` + `tool_definitions` only - -#### Pattern 7: Special — Graders - -Azure OpenAI graders use `type: "azure_openai_grader"` instead of -`type: "azure_ai_evaluator"`. These are a different testing criteria type. - -| Evaluator | Status | -|---|---| -| AzureOpenAILabelGrader | ❌ Not supported — different type | -| AzureOpenAIStringCheckGrader | ❌ Not supported — different type | -| AzureOpenAITextSimilarityGrader | ❌ Not supported — different type | -| AzureOpenAIGrader | ❌ Not supported — different type | - -**Out of scope for now.** Graders require a fundamentally different config -model (rubric templates, scoring criteria). Can be tracked separately. - -#### Pattern 8: Special — Red team - -Red team evaluators use a different data source type -(`azure_ai_red_team`) with attack strategies and taxonomy generation. - -| Evaluator | Status | -|---|---| -| ProhibitedActionsEvaluator | ❌ Different flow | -| SensitiveDataLeakageEvaluator | ❌ Different flow | - -**Out of scope for now.** Red team requires a separate execution flow. - -### Summary: What Needs to Change in `foundry_backend.py` - -| Change | Affected evaluators | Effort | -|---|---|---| -| Add to `_EVALUATORS_NEEDING_GROUND_TRUTH` | `response_completeness` | 1 line | -| Add to `_EVALUATORS_NEEDING_CONTEXT` | `groundedness_pro`, `retrieval` | 1 line | -| Add to `_EVALUATORS_NEEDING_TOOL_CALLS` | `tool_selection` | 1 line | -| New set: `_EVALUATORS_NEEDING_TOOL_DEFS_ONLY` | `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` | ~10 lines | -| New set: `_EVALUATORS_NEEDING_OUTPUT_ITEMS` | `task_adherence` | ~5 lines | -| Document that default path works | `coherence`, `fluency`, `relevance`, `intent_resolution`, `task_completion`, all safety evaluators | 0 lines (docs only) | - -### Data Model Gap: item_schema - -The current code builds `item_schema` with only two string fields: - -```python -item_schema = { - "type": "object", - "properties": { - input_field: {"type": "string"}, - expected_field: {"type": "string"}, - }, - "required": [input_field, expected_field], -} -``` - -For tool evaluators to work, the schema must also declare `tool_definitions` -(and `tool_calls` if present in the dataset). The schema needs to be -dynamically built based on which evaluators are enabled. - -**Fix required:** When any evaluator in `_EVALUATORS_NEEDING_TOOL_CALLS` or -`_EVALUATORS_NEEDING_TOOL_DEFS_ONLY` is enabled, add `tool_definitions` to -`item_schema.properties`. Similarly, add `context_field` when context -evaluators are used. - -### Data Model Gap: DatasetFormat - -`DatasetFormat` currently has `input_field`, `expected_field`, and -`context_field`. It does NOT have: -- `tool_definitions_field` — needed for tool evaluators -- `tool_calls_field` — needed for `tool_call_accuracy`, `tool_selection` - -**Fix required:** Add optional fields to `DatasetFormat` model: - -```python -class DatasetFormat(BaseModel): - type: str - input_field: str - expected_field: str - context_field: Optional[str] = None - tool_definitions_field: Optional[str] = None # NEW - tool_calls_field: Optional[str] = None # NEW -``` - -### Revised Evaluator Support Count - -After the fixes above: - -| Category | Before | After | -|---|---|---| -| Works correctly today | 8 (NLP + similarity + groundedness + tool_call_accuracy) | 8 | -| Accidentally works (default path) | 0 recognized | 14 newly recognized | -| Fixed by adding to frozensets | 0 | 5 (response_completeness, groundedness_pro, retrieval, tool_selection, task_adherence) | -| Fixed by new sets | 0 | 3 (tool_input_accuracy, tool_output_utilization, tool_call_success) | -| **Total supported** | **8** | **30** | -| Remaining unsupported | | 5 (4 graders + documentation_retrieval) | - ---- - -## Track 2: Evaluation Patterns from Real Scenarios (Harpreet) - -### Pattern A: Cloud Agent Evaluation with Inline Data - -**Source:** `agenteval.py` - -**Flow:** -1. Connect to Foundry project via `AIProjectClient` -2. Get OpenAI client via `project_client.get_openai_client()` -3. Define `data_source_config` with `type: custom` and item_schema -4. Define `testing_criteria` — array of `azure_ai_evaluator` entries -5. Call `client.evals.create()` with testing_criteria -6. Call `client.evals.runs.create()` with inline JSONL data -7. Poll `client.evals.runs.retrieve()` until completed/failed -8. Retrieve output items via `client.evals.runs.output_items.list()` - -**Data format used:** - -```python -data_source_config = { - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"anyOf": [{"type": "string"}, {"type": "array"}]}, - "tool_definitions": {"anyOf": [{"type": "object"}, {"type": "array"}]}, - "tool_calls": {"anyOf": [{"type": "object"}, {"type": "array"}]}, - "response": {"anyOf": [{"type": "string"}, {"type": "array"}]}, - }, - "required": ["query", "response", "tool_definitions"], - }, - "include_sample_schema": True, -} -``` - -**Key observation:** The field types use `anyOf` with string OR array. This -allows both simple string queries AND structured conversation-format arrays. -AgentOps hardcodes `{"type": "string"}` — this works for simple eval but -blocks conversation-format data. - -**Evaluators used (9 total):** - -| # | Name | Category | data_mapping | -|---|---|---|---| -| 1 | task_completion | System | query, response, tool_definitions | -| 2 | task_adherence | System | query, response, tool_definitions | -| 3 | intent_resolution | System | query, response, tool_definitions | -| 4 | groundedness | RAG | query, tool_definitions, response | -| 5 | relevance | RAG | query, response | -| 6 | tool_call_accuracy | Process | query, tool_definitions, tool_calls, response | -| 7 | tool_selection | Process | query, response, tool_calls, tool_definitions | -| 8 | tool_input_accuracy | Process | query, response, tool_definitions | -| 9 | tool_output_utilization | Process | query, response, tool_definitions | - -**AgentOps compatibility after Track 1 fixes:** 9/9 evaluators would be -supported. The remaining gap is the `item_schema` format — Harpreet uses -`anyOf` types while AgentOps hardcodes `string`. - -### Pattern B: Red Team Safety Evaluation - -**Source:** `redteam.py` - -**Flow:** -1. Connect to Foundry project client -2. Create an agent version via `project_client.agents.create_version()` -3. Define safety testing criteria (7 evaluators) -4. Create evaluation taxonomy via `project_client.evaluation_taxonomies.create()` -5. Create eval run with `data_source.type: azure_ai_red_team` -6. Uses generated adversarial inputs with attack strategies `["Flip", "Base64"]` -7. Poll until completion, save results to JSON - -**Data source:** `azure_ai_red_team` — fundamentally different from the -`custom`/`completions`/`azure_ai_target_completions` data sources that -AgentOps supports. - -**Safety evaluators used (7 total):** - -| # | Name | builtin name | -|---|---|---| -| 1 | Prohibited Actions | `builtin.prohibited_actions` | -| 2 | Task Adherence | `builtin.task_adherence` | -| 3 | Sensitive Data Leakage | `builtin.sensitive_data_leakage` | -| 4 | Self Harm | `builtin.self_harm` | -| 5 | Violence | `builtin.violence` | -| 6 | Sexual | `builtin.sexual` | -| 7 | Hate Unfairness | `builtin.hate_unfairness` | - -**Key observations:** -- Safety evaluators like `violence`, `self_harm`, `sexual`, `hate_unfairness` - CAN be used in normal cloud evaluation (Pattern A) with `query + response` - data mapping — they don't REQUIRE the red team data source. -- `prohibited_actions` and `sensitive_data_leakage` are red-team-specific. -- `task_adherence` is reused across both patterns. - -**AgentOps compatibility:** The safety evaluators (items 4-7) would work in -normal eval after Track 1 (they use the default `query + response` pattern). -The red-team flow itself (attack strategies, taxonomy generation) is a -separate feature. - -### Pattern C: Agent Smoke Test - -**Source:** `exagent.py` - -**Flow:** -1. Connect to Foundry project client -2. Get existing agent by name via `project_client.agents.get()` -3. Get OpenAI client via `project_client.get_openai_client()` -4. Send a query via `openai_client.responses.create()` with agent reference -5. Handle MCP approval requests (auto-approve) -6. Poll for response completion -7. Display response text and citations - -**AgentOps compatibility:** Not relevant to evaluation. This is a -pre-evaluation health check. Users can add this as a custom pipeline step -before `agentops eval run`. No tool change needed. - -### Pattern D: Data Format — Conversation vs. String - -**The critical data model difference:** - -Harpreet's `agenteval.py` provides data in **conversation format**: - -```python -query = [ - {"role": "system", "content": "You are a weather report agent."}, - {"role": "user", "content": [{"type": "text", "text": "Can you send me..."}]}, -] - -response = [ - {"role": "assistant", "content": [{"type": "tool_call", "name": "fetch_weather", ...}]}, - {"role": "tool", "content": [{"type": "tool_result", ...}]}, - {"role": "assistant", "content": [{"type": "text", "text": "I have successfully..."}]}, -] - -tool_definitions = [ - {"name": "fetch_weather", "description": "...", "parameters": {...}}, - {"name": "send_email", "description": "...", "parameters": {...}}, -] -``` - -AgentOps datasets use **simple string format**: - -```jsonl -{"input": "What is the weather?", "expected": "Sunny, 25°C"} -``` - -**When does this matter?** - -- **For model-direct evaluation:** Simple strings work fine. The model receives - the query and generates a response — evaluators compare output_text. -- **For agent evaluation with tool calls:** The conversation format is needed - when evaluating tool-using agents on pre-computed responses. But when using - `azure_ai_target_completions` with a live agent target, the agent generates - structured responses at runtime — so simple string queries work. -- **For dataset (offline) evaluation:** If users want to evaluate - pre-computed agent conversations (not calling the agent at runtime), - they need conversation-format JSONL rows. - -**Impact on AgentOps:** - -The current `item_schema` hardcodes `{"type": "string"}`. This blocks: -1. Dataset evaluation with pre-computed structured responses -2. Tool evaluators that need `tool_definitions` in the dataset rows - -It does NOT block: -1. Live agent evaluation (agent generates structured output at runtime) -2. Live model evaluation (model generates text at runtime) - -**Fix:** Make `item_schema.properties` type flexible — use `anyOf` when the -evaluator requires structured data, or infer from JSONL row content. - ---- - -## Synthesis: Combined Gap Map - -| # | Gap | Track | Severity | Fix | -|---|---|---|---|---| -| 1 | 14 evaluators work but aren't documented | Track 1 | Low | Document and add tests | -| 2 | `response_completeness` missing from ground_truth set | Track 1 | Low | 1 line | -| 3 | `groundedness_pro`, `retrieval` missing from context set | Track 1 | Low | 1 line | -| 4 | `tool_selection` missing from tool_calls set | Track 1 | Low | 1 line | -| 5 | `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` need new set | Track 1 | Medium | ~10 lines | -| 6 | `task_adherence` needs `{{sample.output_items}}` response mapping | Track 1 | Medium | ~5 lines | -| 7 | `item_schema` hardcodes `{"type": "string"}` | Track 1+2 | High | Dynamic schema building | -| 8 | `DatasetFormat` lacks `tool_definitions_field` | Track 1+2 | High | Model change + wire through | -| 9 | `item_schema` doesn't include context_field | Track 1 | Medium | Dynamic schema building | -| 10 | Red team flow not supported | Track 2 | Future | Separate feature | -| 11 | Graders not supported | Track 1 | Future | Different testing_criteria type | - -### Recommended Implementation Order - -**Phase 1 — Quick wins (unblock 14 more evaluators):** -- Add evaluators to existing frozensets (#2, #3, #4) -- Create new frozensets (#5, #6) -- Update `_cloud_evaluator_data_mapping` for new patterns -- Add unit tests -- Update evaluator reference doc - -**Phase 2 — Schema flexibility (unblock tool evaluators with dataset data):** -- Add `tool_definitions_field` and `tool_calls_field` to `DatasetFormat` -- Build `item_schema` dynamically based on enabled evaluators -- Add `context_field` to `item_schema` when context evaluators are used -- Use `anyOf` types when field content may be structured - -**Phase 3 — Documentation (confirm patterns work end-to-end):** -- Document which evaluators work for each scenario -- Add bundle examples for agent evaluation with tool evaluators -- Document conversation-format dataset rows - -**Phase 4 — Future:** -- Red team data source support -- Azure OpenAI grader support diff --git a/docs/ci-github-actions.md b/docs/ci-github-actions.md index 37ee1a98..f454ab0b 100644 --- a/docs/ci-github-actions.md +++ b/docs/ci-github-actions.md @@ -1,379 +1,245 @@ -# Running AgentOps Evaluations in GitHub Actions - -This guide explains how to add AgentOps evaluation to your CI/CD pipeline using GitHub Actions. Inspired by [GenAIOps Git Workflow](https://github.com/Azure/GenAIOps/blob/main/documentation/git_workflow.md) and [Foundry CI/CD patterns](https://github.com/balakreshnan/foundrycicdbasic), AgentOps generates up to three pipeline types tailored to your project. - -## Pipeline Types - -`agentops workflow generate` auto-detects which pipelines to create based on your `.agentops/` workspace: - -| Pipeline | File | Trigger | Purpose | -| -------- | ---- | ------- | ------- | -| **PR Evaluation** | `agentops-eval.yml` | Pull requests to main/develop | Gate PRs on evaluation thresholds | -| **CI Evaluation** | `agentops-eval-ci.yml` | Push to develop/main | Post-merge comprehensive evaluation with optional matrix strategy | -| **CD Pipeline** | `agentops-eval-cd.yml` | Push to main | Safety QA evaluation gate + deploy placeholder | - -### Auto-Detection Rules - -- **PR pipeline** — always generated. -- **CI pipeline** — generated when multiple bundles or run configs exist in `.agentops/`. -- **CD pipeline** — generated alongside the CI pipeline (same detection rule). - -To override auto-detection, simply delete any unwanted workflow file after generation. - -### Branching Strategy - -The pipeline suite maps to the Git Flow branching model: - -``` -feature/* → PR to develop → agentops-eval.yml (PR gate) - merge to develop → agentops-eval-ci.yml (CI evaluation) - release/* → PR to main → agentops-eval.yml (PR gate) - merge to main → agentops-eval-cd.yml (safety QA → deploy) +# AgentOps GenAIOps GitFlow on GitHub Actions + +This guide shows how to wire AgentOps into a complete GenAIOps CI/CD +pipeline on GitHub Actions, mapped to a classic GitFlow branching model +with three deployment environments (`dev`, `qa`, `production`). + +`agentops workflow generate` ships **four** ready-to-use templates that +form the full scaffold: + +| File | Trigger | GitHub Environment | Purpose | +|---|---|---|---| +| `agentops-pr.yml` | PRs to `develop`, `release/**`, `main` | (none) | Eval gate. Fails the PR if thresholds drop. Comments report on PR. | +| `agentops-deploy-dev.yml` | push to `develop` | `dev` | Eval → build → deploy DEV | +| `agentops-deploy-qa.yml` | push to `release/**` | `qa` | Eval → build → deploy QA | +| `agentops-deploy-prod.yml` | push to `main` | `production` | Safety eval → build → deploy PROD (gated by required reviewers) | + +## GitFlow assumed + +```mermaid +flowchart LR + feat["feature/*"] -->|PR| prGate1{{"agentops-pr.yml
(gate)"}} + prGate1 -->|merge| dev["develop"] + dev --> deployDev["agentops-deploy-dev.yml"] + deployDev --> DEV(["DEV"]) + + rel["release/*"] -->|push| deployQa["agentops-deploy-qa.yml"] + deployQa --> QA(["QA"]) + + rel -->|PR| prGate2{{"agentops-pr.yml
(gate)"}} + prGate2 -->|merge| main["main"] + main --> deployProd["agentops-deploy-prod.yml"] + deployProd --> PROD(["PROD
(required reviewers)"]) + + classDef gate fill:#fff3cd,stroke:#856404,color:#000; + classDef env fill:#d1ecf1,stroke:#0c5460,color:#000; + class prGate1,prGate2 gate; + class DEV,QA,PROD env; ``` -## Quick Start - -1. **Initialise your workspace** (if you haven't already): +If you are on trunk-based development, generate only the templates you +need: `agentops workflow generate --kinds pr,dev,prod`. - ```bash - agentops init - ``` +## Quick start - This creates the `.agentops/` directory with starter configs, bundles, and datasets. - -2. **Generate the workflow files**: - - ```bash - agentops workflow generate - ``` - - This creates one or more files in `.github/workflows/` based on your workspace content. - -3. **Configure GitHub Secrets** (see [Authentication](#authentication) below). - -4. **Push a PR** — the PR evaluation runs automatically. Merge to trigger the CI evaluation. - -## Required Files - -Your repository must contain these files for the workflow to succeed: - -| File | Purpose | -| --------------------------------- | --------------------------------------------------------------- | -| `.agentops/run.yaml` | Run specification — references the bundle, dataset, and backend | -| `.agentops/bundles/.yaml` | Evaluation bundle — evaluators + thresholds | -| `.agentops/datasets/.yaml` | Dataset metadata | -| `.agentops/datasets/.jsonl` | Dataset rows (JSONL format) | +```bash +# 1. Make sure your eval works locally first. +agentops eval run -All paths in `run.yaml` are relative to the `.agentops/` directory. +# 2. Generate the four workflows. +agentops workflow generate -### Example `run.yaml` +# 3. Configure GitHub (see sections below): +# - OIDC repo variables +# - dev / qa / production environments +# - branch protection on develop and main +# - fill in Build / Deploy placeholders -```yaml -version: 1 -target: - type: model - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - model: gpt-4o - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -execution: - timeout_seconds: 1800 -output: - write_report: true +# 4. Commit and push. ``` -## Authentication - -The workflow uses **Workload Identity Federation (OIDC)** — no client secrets to manage or rotate. The GitHub Actions runner exchanges a short-lived OIDC token for an Azure access token at runtime. - -#### Azure setup (one-time) - -1. **Create or reuse an App Registration** in Azure AD (Microsoft Entra ID). -2. **Add a Federated Credential**: - - Go to the App Registration → **Certificates & secrets** → **Federated credentials** → **Add credential** - - Organization: your GitHub org/user - - Repository: your repo name - - Entity type: `Pull Request` (for PR triggers) **and** `Branch` (for CI, CD, and workflow_dispatch triggers) - - Name: e.g. `github-agentops-eval` -3. **Grant the app** the required roles on your Foundry project: - - `Cognitive Services User` — invoke agents and evaluator models - - `Azure AI Developer` — access evaluation APIs and Foundry features -4. Note the **Application (client) ID**, **Directory (tenant) ID**, and **Subscription ID**. - -#### GitHub setup - -Set these as **repository variables** (not secrets — they are not confidential): - -| Variable | Value | -| ----------------------- | ----------------------- | -| `AZURE_CLIENT_ID` | Application (client) ID | -| `AZURE_TENANT_ID` | Directory (tenant) ID | -| `AZURE_SUBSCRIPTION_ID` | Azure subscription ID | - -Set this as a **repository secret**: - -| Secret | Value | -| ----------------------------------- | ---------------------------- | -| `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Foundry project endpoint URL | - -Go to **Settings** → **Secrets and variables** → **Actions** → **Variables** tab (for variables) or **Secrets** tab (for the endpoint). - -## Workflow Triggers - -Each pipeline type has different triggers: - -### PR Evaluation (`agentops-eval.yml`) - -| Trigger | When | -| ------------------- | ---------------------------------------------------------------------------------- | -| `pull_request` | Any PR targeting `main` or `develop` | -| `workflow_dispatch` | Manual run from the Actions tab (supports custom config path and output directory) | - -### CI Evaluation (`agentops-eval-ci.yml`) - -| Trigger | When | -| ------------------- | ---------------------------------------------------------------------------------- | -| `push` | Push to `develop` or `main` (path filter: `.agentops/**`, `src/**`, `pyproject.toml`) | -| `workflow_dispatch` | Manual run from the Actions tab | - -### CD Pipeline (`agentops-eval-cd.yml`) - -| Trigger | When | -| ------------------- | ---------------------------------------------------------------------------------- | -| `push` | Push to `main` | -| `workflow_dispatch` | Manual run from the Actions tab (supports `skip_safety` input) | - -The CD pipeline has two jobs: **safety-qa** (runs evaluation as a quality gate) and **deploy** (placeholder for deployment commands). The deploy job only runs if the safety-qa job passes. - -To change which branches trigger evaluations, edit the branch arrays in the workflow files. - -## Exit Codes and CI Behaviour - -AgentOps returns CI-friendly exit codes that GitHub Actions interprets directly: - -| Exit Code | Meaning | CI Result | -| --------- | --------------------------------------------------- | ------------ | -| `0` | Evaluation succeeded, all thresholds passed | ✅ Job passes | -| `2` | Evaluation succeeded, one or more thresholds failed | ❌ Job fails | -| `1` | Runtime or configuration error | ❌ Job fails | - -No special handling is needed — GitHub Actions fails the job on any non-zero exit code. +## Configuration walkthrough -## Artifacts - -Each pipeline uploads files as GitHub Actions artifacts: - -| Pipeline | Artifact name | Contents | -| -------- | ------------- | -------- | -| PR Evaluation | `agentops-eval-results` | results.json, report.md, backend_metrics.json, cloud_evaluation.json, logs | -| CI Evaluation | `agentops-ci-eval-results` | Same as above | -| CD Pipeline | `agentops-cd-safety-results` | Same as above (from safety-qa job) | - -Individual files in the artifact: - -| File | Description | -| ----------------------- | -------------------------------------------------------------- | -| `results.json` | Machine-readable evaluation results (versioned schema) | -| `report.md` | Human-readable Markdown summary | -| `backend_metrics.json` | Raw backend scores per row | -| `cloud_evaluation.json` | Cloud eval metadata with Foundry portal link (cloud mode only) | -| `backend.stdout.log` | Backend stdout capture | -| `backend.stderr.log` | Backend stderr capture | - -Artifacts are uploaded even when the evaluation fails (`if: always()`), so you can always inspect results. +### 1. Repository variables (OIDC) -### Downloading artifacts +In Settings → Secrets and variables → Actions → **Variables**, add: -From the **Actions** tab → select the workflow run → scroll to **Artifacts** → click to download. +| Variable | Purpose | +|---|---| +| `AZURE_CLIENT_ID` | App registration / managed identity used for federated login | +| `AZURE_TENANT_ID` | Azure AD tenant | +| `AZURE_SUBSCRIPTION_ID` | Target subscription | +| `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Foundry project URL (used by the eval step) | -## PR Comments +Then on the Azure side, configure Workload Identity Federation +(federated credentials) on the app registration so it can be assumed +from GitHub Actions runs. See +[Microsoft's WIF docs](https://learn.microsoft.com/azure/active-directory/workload-identities/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp). -When triggered by a pull request, the workflow automatically posts (or updates) a PR comment containing the full `report.md` content. This gives reviewers immediate visibility into evaluation results without downloading artifacts. +### 2. GitHub Environments -The comment is identified by a hidden HTML marker (``) so subsequent pushes to the same PR update the existing comment rather than creating duplicates. +In Settings → Environments, create three: -## Job Summary +#### `dev` +- Usually no protection rules. +- Override env-specific variables here (e.g. dev resource group, dev + ACA app name). -The workflow writes a [GitHub Actions Job Summary](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary) that includes: +#### `qa` +- Optional: restrict deployment branches to `release/**`. +- Override env-specific variables for QA infra. -- Pass/fail status banner -- Full `report.md` content (when available) +#### `production` +- **Required reviewers**: at least one. Deploys to PROD pause until + approved. +- Optional: **Wait timer** for an extra cool-down. +- Optional: **Deployment branches**: restrict to `main`. +- Override env-specific variables for production infra. -This is visible on the workflow run page without downloading artifacts. - -## CLI Command Reference - -### Generate the workflows - -```bash -agentops workflow generate -``` +Environment-level variables override repo-level ones automatically +when the workflow's `environment:` matches. -This auto-detects which pipelines to generate based on your `.agentops/` workspace content. +### 3. Fill in Build and Deploy -Options: +Each `agentops-deploy-*.yml` ships with `Build (placeholder)` and +`Deploy (placeholder)` steps. The DEV template lists commented example +snippets for the most common patterns. Copy the relevant one into all +three deploy templates. -| Flag | Description | Default | -| ------------ | --------------------------------- | ----------------------- | -| `--dir PATH` | Target repository root directory | `.` (current directory) | -| `--force` | Overwrite existing workflow files | `false` | +#### Container Apps -### Regenerate (overwrite) - -```bash -agentops workflow generate --force -``` - -## Customisation - -### Using a different config path - -With `workflow_dispatch`, you can specify a custom config path: +```yaml +# Build +- name: Build image + run: | + az acr build \ + --registry "${{ vars.ACR_NAME }}" \ + --image "myapp:${{ github.sha }}" \ + . -```bash -agentops eval run --config path/to/custom-run.yaml +# Deploy +- name: Deploy to ACA + run: | + az containerapp update \ + --name "${{ vars.ACA_APP_NAME }}" \ + --resource-group "${{ vars.AZURE_RESOURCE_GROUP }}" \ + --image "${{ vars.ACR_NAME }}.azurecr.io/myapp:${{ github.sha }}" ``` -Or modify the workflow's default: +#### App Service ```yaml -steps: - - name: Run evaluation - run: agentops eval run --config .agentops/my-custom-run.yaml +# Build +- uses: actions/setup-python@v5 + with: { python-version: "3.11" } +- run: pip install -r requirements.txt -t ./dist +- run: cp -r src ./dist/ + +# Deploy +- uses: azure/webapps-deploy@v3 + with: + app-name: ${{ vars.WEBAPP_NAME }} + package: ./dist ``` -### Using a custom output directory +#### Foundry hosted agent ```yaml -steps: - - name: Run evaluation - run: agentops eval run --config .agentops/run.yaml --output ./eval-output -``` +# Build is typically empty: hosted agents are configured, not packaged. -Update the artifact upload paths accordingly. - -### Running multiple evaluations - -To run several evaluation configs in a single workflow, use a matrix strategy: - -```yaml -jobs: - evaluate: - strategy: - fail-fast: false - matrix: - config: - - .agentops/runs/model-direct.yaml - - .agentops/runs/rag-retrieval.yaml - - .agentops/runs/agent-tools.yaml - steps: - # ... - - name: Run evaluation - run: agentops eval run --config ${{ matrix.config }} +# Deploy: publish a new agent version with whatever your project uses +# to manage Foundry agents (project-specific tooling). ``` -### Skipping the PR comment - -Remove or comment out the "Post report as PR comment" step in the workflow. - -## CD Pipeline - -The CD pipeline (`agentops-eval-cd.yml`) is generated alongside the CI pipeline when multiple bundles or run configs exist in the workspace. It runs on pushes to `main` and acts as a deployment gate. - -### How it works - -1. The **safety-qa** job runs `agentops eval run` to evaluate the model/agent. -2. If evaluation passes (exit code 0), the **deploy** job runs. -3. If thresholds fail (exit code 2) or an error occurs (exit code 1), the deploy job is skipped. -4. The deploy job is a **placeholder** — fill it in with your deployment commands. - -### Skipping safety checks - -For emergency deployments, use `workflow_dispatch` with the `skip_safety` input set to `true`. This skips the safety-qa job and runs the deploy job directly. - -### Adding deployment steps - -Edit the `deploy` job in `agentops-eval-cd.yml` and replace the placeholder with your deployment commands: +#### azd-managed app ```yaml -deploy: - name: Deploy - needs: safety-qa - runs-on: ubuntu-latest - # environment: production # Uncomment for manual approval gate - steps: - - uses: actions/checkout@v4 - - name: Deploy to production - run: | - # Your deployment commands here, e.g.: - # az webapp deploy ... - # kubectl apply ... - # azd deploy ... +# Build +- uses: Azure/setup-azd@v2 +- run: azd package --no-prompt + +# Deploy +- run: azd deploy --no-prompt + env: + AZURE_ENV_NAME: dev # or qa / prod ``` -### Adding environment approval - -Uncomment `environment: production` in the deploy job to require manual approval before deployment. Configure the environment in GitHub Settings → Environments. - -## CI Evaluation Pipeline +### 4. Branch protection -The CI pipeline (`agentops-eval-ci.yml`) is generated when multiple bundles or run configs exist. It runs after merges for comprehensive evaluation. +In Settings → Branches, add a rule for **both `develop` and `main`**: -### Enabling matrix strategy +- ✅ Require a pull request before merging. +- ✅ Require status checks to pass: select + **`AgentOps PR / Eval (PR gate)`**. +- (Optional) Require linear history. -Uncomment the matrix block in the CI workflow and list your run configs: +This makes the AgentOps eval a hard merge requirement. -```yaml -strategy: - fail-fast: false - matrix: - config: - - .agentops/run.yaml - - .agentops/runs/rag-retrieval.yaml - - .agentops/runs/agent-tools.yaml -``` +## Exit codes -### Enabling baseline comparison +The eval step uses the AgentOps exit code contract to gate deploys: -Uncomment the comparison step in the CI workflow. Store a baseline run ID and compare automatically: +| Exit code | Meaning | Job result | +|---|---|---| +| `0` | Eval ran, all thresholds passed | ✅ pass | +| `2` | Eval ran, one or more thresholds failed | ❌ fail (deploy never runs) | +| `1` | Runtime / config error | ❌ fail | -```yaml -- name: Compare against baseline - run: | - BASELINE=$(cat .agentops/results/baseline_id.txt) - CURRENT=$(jq -r '.run_id' .agentops/results/latest/results.json) - agentops eval compare --runs "$BASELINE,$CURRENT" -f md -``` +## Artifacts -## Troubleshooting +Each workflow uploads (always — even on failure): -| Problem | Solution | -| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Error: evaluation failed: ...` (exit 1) | Check that `.agentops/run.yaml` exists, config is valid YAML, and secrets are set | -| `Threshold status: FAILED` (exit 2) | Review `report.md` — thresholds are too strict or model quality regressed | -| Missing artifacts | Ensure `.agentops/results/latest/` is not in `.gitignore` — the workflow reads this path | -| Authentication errors | Verify the federated credential entity matches your repo/branch; check that `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` are set as repository variables; confirm the app registration has access to the Foundry project | -| `agentops: command not found` | Ensure `pip install agentops-toolkit` runs before the eval step | -| Only PR workflow generated | Auto-detection found a single bundle — this is expected; add bundles or run configs to trigger CI/CD pipelines | +- `results.json` — machine-readable, versioned +- `report.md` — human-readable +- `cloud_evaluation.json` — present when using Foundry cloud evaluation; + contains a deep link to the New Foundry Experience Evaluations page -## Internal CI/CD Workflows (Contributors) +Artifact names per workflow: -If you are contributing to the agentops-toolkit repository itself, the project has separate CI/CD workflows for building and releasing the package: +| Workflow | Artifact name | +|---|---| +| `agentops-pr.yml` | `agentops-pr-results` | +| `agentops-deploy-dev.yml` | `agentops-dev-results` | +| `agentops-deploy-qa.yml` | `agentops-qa-results` | +| `agentops-deploy-prod.yml` | `agentops-prod-results` | -| Workflow | Trigger | Purpose | -| ----------------- | ------------------------------------------ | ------------------------------------------------------------------------- | -| `ci.yml` | Push to `develop`, PRs to `main`/`develop` | Lint (ruff) + test (matrix) + coverage | -| `_build.yml` | Called by staging/release | Reusable lint + test + build package | -| `staging.yml` | Push to `release/**` | Build → TestPyPI → verify install | -| `release.yml` | Push `v*` tag | TestPyPI → PyPI (with approval) → GitHub Release | -| `cut-release.yml` | Manual dispatch (Actions tab button) | Create release branch from `develop`, update CHANGELOG, open PR to `main` | +## CLI reference -The **Cut Release** workflow provides a one-click way to start a release: enter a version number in the Actions UI, and it creates the release branch, updates the changelog, and opens the PR automatically. +```bash +agentops workflow generate # all four templates (default) +agentops workflow generate --kinds pr,dev,prod # subset (trunk-based) +agentops workflow generate --force # overwrite existing files +agentops workflow generate --dir # different repo root +``` -For full details, see [release-process.md](release-process.md). +| Flag | Description | Default | +|---|---|---| +| `--kinds` | Comma-separated subset of `pr,dev,qa,prod` | all four | +| `--force` | Overwrite existing workflow files | `false` | +| `--dir` | Repository root | `.` | + +## Customisation tips + +- **Tighten thresholds for QA / PROD** — copy `.agentops/run.yaml` to + `.agentops/run-qa.yaml` / `.agentops/run-prod.yaml` and tighten + thresholds in the bundle. Update the `inputs.config` default in the + matching workflow file. +- **Scheduled runs** — add a `schedule:` entry in `agentops-pr.yml` (or + a new file) to evaluate against `main` nightly. +- **Matrix per scenario** — if you have multiple `runs/*.yaml`, extend + the eval job with `strategy.matrix.config:` and reference + `${{ matrix.config }}` in the eval step. +- **Regression baseline** — wire deploy templates to download the + previous run's `results.json` artifact and call + `agentops eval compare` between the two. + +## Migration from the older 3-template layout + +If your repository still has `agentops-eval.yml`, `agentops-eval-ci.yml`, +or `agentops-eval-cd.yml` from a prior version of AgentOps: + +1. Delete the three old files. +2. Run `agentops workflow generate`. +3. Re-add Build / Deploy commands you had customised. +4. Update branch-protection status checks to point at the new + `AgentOps PR` job. diff --git a/docs/concepts.md b/docs/concepts.md index 8f7d093c..ffe692f4 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -4,53 +4,34 @@ This page explains the core building blocks of AgentOps and how they fit togethe ## How an Evaluation Works +```mermaid +flowchart TD + run["run.yaml
what, where, how to eval"] + bundle["Bundle
evaluators + thresholds"] + dataset["Dataset
JSONL rows: input, expected"] + runner(["Runner
resolves backend"]) + foundry["Foundry
Backend"] + http["HTTP
Backend"] + local["Local
Adapter"] + evals(["Evaluators
score each response"]) + results[/"results.json
(machine)"/] + report[/"report.md
(human)"/] + + run --> bundle + run --> dataset + bundle --> runner + dataset --> runner + runner --> foundry + runner --> http + runner --> local + foundry --> evals + http --> evals + local --> evals + evals --> results + evals --> report ``` - ┌─────────────────────────────┐ - │ run.yaml │ - │ (what, where, how to eval) │ - └──────┬──────────┬───────────┘ - │ │ - ┌────────────┘ └────────────┐ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ Bundle │ │ Dataset │ - │ (evaluators + │ │ (JSONL rows: │ - │ thresholds) │ │ input, │ - └────────┬────────┘ │ expected) │ - │ └────────┬────────┘ - │ │ - └──────────┐ ┌────────────────────┘ - ▼ ▼ - ┌──────────────┐ - │ Runner │ - │ (resolves │ - │ backend) │ - └──────┬───────┘ - │ - ┌──────────────┼──────────────┐ - ▼ ▼ ▼ - ┌────────────┐ ┌────────────┐ ┌────────────┐ - │ Foundry │ │ HTTP │ │ Local │ - │ Backend │ │ Backend │ │ Adapter │ - └──────┬─────┘ └──────┬─────┘ └──────┬─────┘ - │ │ │ - └──────────────┼──────────────┘ - ▼ - ┌──────────────────┐ - │ Evaluators │ - │ (score each │ - │ response) │ - └────────┬─────────┘ - │ - ┌────────────┴────────────┐ - ▼ ▼ - ┌──────────────┐ ┌──────────────┐ - │ results.json │ │ report.md │ - │ (machine) │ │ (human) │ - └──────────────┘ └──────────────┘ - - Exit code: 0 = pass, 2 = threshold fail, 1 = error -``` + +> Exit code: `0` = pass, `2` = threshold fail, `1` = error ## Core Concepts diff --git a/docs/e2e-live-architecture.md b/docs/e2e-live-architecture.md new file mode 100644 index 00000000..f4384ef6 --- /dev/null +++ b/docs/e2e-live-architecture.md @@ -0,0 +1,82 @@ +# Live E2E architecture + +Short architectural note for `infra/e2e/` and the live jobs in +`.github/workflows/e2e.yml`. The end-user setup guide is in +[`e2e-live-setup.md`](e2e-live-setup.md). + +## Goal + +Exercise all four AgentOps backends against real Azure resources on every +manual workflow dispatch, while keeping per-run cost and time bounded. + +## Layered provisioning (hybrid efemeridade) + +| Layer | When | What | Why | +|---|---|---|---| +| **Bootstrap** | One-time, by the user (`infra/e2e/bootstrap.bicep`) | AI Services account, Foundry project, `gpt-4o-mini` deployment, ACA managed environment, Log Analytics, ACR | Heavy/slow resources whose creation dwarfs the actual eval. Idle cost ≈ $0. | +| **Per-run** | Every workflow dispatch (`infra/e2e/perrun.bicep`) | One ACA echo app named `aca-echo-run` | Cheap, fast (<30s) artifact tied to the specific run. Lets `http-aca` test a fresh URL each time. | +| **Manual** | One-time, in the Foundry portal | Prompt agent (`e2e-prompt:1`) and (optional) hosted agent endpoint | Foundry agent CRUD has no Bicep coverage and the SDK surface is moving fast. Stable Variables in the repo are simpler than dynamic creation today. | + +## Auth + +GitHub Actions ↔ Entra federated credential. The workflow declares +`permissions: id-token: write` and uses `azure/login@v2` with +`client-id` / `tenant-id` / `subscription-id` from repo Actions +**Variables**. No client secrets exist anywhere. The login propagates to +`az`, `DefaultAzureCredential` (Python), and `azure-ai-projects` via the +env vars `azure/login` exports. + +The repo never holds Azure credentials. Compromising the repo cannot +exfiltrate any usable Azure credential — only the trust policy on the +Entra app needs to be revoked to cut access. + +## Job graph + +```mermaid +flowchart TD + bootstrap(["bootstrap-live
deploys per-run ACA"]) + foundryPrompt["live-foundry-prompt"] + foundryHosted["live-foundry-hosted"] + httpAca["live-http-aca"] + modelDirect["live-model-direct"] + teardown(["teardown
always() — sweeps stale"]) + + bootstrap --> foundryPrompt + bootstrap --> foundryHosted + bootstrap --> httpAca + bootstrap --> modelDirect + foundryPrompt --> teardown + foundryHosted --> teardown + httpAca --> teardown + modelDirect --> teardown +``` + +`bootstrap-live` only runs `perrun.bicep` if the requested scenarios +include `http-aca`. The four scenario jobs each render their own +`agentops.yaml` via `scripts/e2e_render_config.py`, run `agentops eval`, +and upload `.agentops/results/` as an artifact. `teardown-live` deletes +the per-run app and runs a defensive sweep for any `aca-echo-run*` older +than one day. + +## Why no agent provisioning script + +Earlier drafts included `scripts/e2e_create_agents.py` to create Foundry +agents on every run. We pulled it because: + +1. Agent CRUD APIs in `azure-ai-projects` are still preview-shaped. +2. Agent creation is fast in the portal and only needed once. +3. A failure in agent creation would block all live scenarios, even the + non-Foundry ones. + +If a future SDK release stabilises the agent management surface, an +opt-in `--ephemeral-agent` mode is straightforward to add: it would slot +into `bootstrap-live` and write its own GitHub Output that the prompt +scenario consumes. + +## Out of scope for v1 + +- Hosted Foundry agent image build + ACR push + endpoint provisioning. +- Async hosted agent lifecycle (`background: true`). +- Per-event SSE evaluation for Invocations agents. + +These are tracked in the broader 1.0 plan and will move to v1.1. diff --git a/docs/e2e-live-setup.md b/docs/e2e-live-setup.md new file mode 100644 index 00000000..a8369be4 --- /dev/null +++ b/docs/e2e-live-setup.md @@ -0,0 +1,226 @@ +# Live Azure E2E — one-time setup + +This guide walks through the human-only steps needed to enable the **live** +jobs in [`.github/workflows/e2e.yml`](../.github/workflows/e2e.yml). Once +completed, anyone with `Run workflow` permission can dispatch the workflow +and pick which scenario(s) to execute against real Azure resources. + +> **Auth model:** GitHub OIDC + Entra federated credential. **No client +> secrets** are stored in the repository. + +--- + +## Prerequisites + +- An Azure subscription you control (the test subscription is fine). +- A pre-created resource group in your subscription. Any RG works — the + workflow reads its name from the `AZURE_E2E_RESOURCE_GROUP` Variable. + Examples in this guide use `` as a placeholder. +- Sufficient role on that RG to assign roles (Owner or `User Access + Administrator`). +- `az` CLI ≥ 2.60 and `bicep` ≥ 0.27 installed locally. +- Admin access to **GitHub repository settings** for `Azure/agentops` + (to add Actions Variables and review the federated credential). + +--- + +## Step 1 — One-time shared infra (`bootstrap.bicep`) + +Deploys the long-lived resources that every workflow run reuses: +AI Services + Foundry project + `gpt-4o-mini` deployment + Container Apps +managed environment + Log Analytics + ACR. + +```bash +az login +az account set --subscription + +az deployment group create \ + --resource-group \ + --name agentops-e2e-bootstrap \ + --template-file infra/e2e/bootstrap.bicep \ + --parameters @infra/e2e/bootstrap.parameters.example.json +``` + +Capture the outputs (printed at the end of the deployment): + +```bash +az deployment group show \ + -g \ + -n agentops-e2e-bootstrap \ + --query properties.outputs +``` + +You will use these values in **Step 4**. + +--- + +## Step 2 — Foundry agents (manual, in the portal) + +Bicep does not yet declaratively manage Foundry agents, so create them +once via the [Azure AI Foundry portal](https://ai.azure.com): + +1. Open the project created by `bootstrap.bicep`. +2. **Prompt agent** (covers the `foundry-prompt` scenario) + - Create a new prompt-based agent. + - Name it `e2e-prompt`. + - Use `gpt-4o-mini` as the model. + - Save and **publish**. Note the version (usually `1`). + - The agent ID is `e2e-prompt:` (e.g. `e2e-prompt:1`). +3. **Hosted agent** (covers the `foundry-hosted` scenario, optional) + - Deploy any agent that exposes the **Responses** protocol. + - Copy its endpoint URL (`https://...`). + - This scenario is opt-in: leave the related Variable empty to skip it. + +--- + +## Step 3 — Entra app + federated credential + +The workflow authenticates to Azure with OIDC. No secrets, just a trust +relationship between the repo and an Entra app. + +```bash +APP_NAME=agentops-e2e +SUBSCRIPTION_ID= +RG= + +# 1. Create the app registration (no client secret). +APP_ID=$(az ad app create --display-name "$APP_NAME" --query appId -o tsv) +SP_ID=$(az ad sp create --id "$APP_ID" --query id -o tsv) + +# 2. Assign roles on the RG only (least privilege you can get away with). +# `User Access Administrator` is needed because Foundry agent operations +# can trigger role assignments to managed identities. If you want a +# tighter role, swap UAA for a custom role with just +# `Microsoft.Authorization/roleAssignments/*` over the AI Services scope. +az role assignment create \ + --assignee-object-id "$SP_ID" \ + --assignee-principal-type ServicePrincipal \ + --role Contributor \ + --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RG" + +az role assignment create \ + --assignee-object-id "$SP_ID" \ + --assignee-principal-type ServicePrincipal \ + --role "User Access Administrator" \ + --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RG" + +# 3. Federated credential — bind the app to runs of e2e.yml on the +# feature/revamp-1.0 branch. Add another credential for `main` later. +cat > /tmp/fic.json < If you ever rename the branch or merge to `main`, add another federated +> credential entry with the new `subject:` value. + +--- + +## Step 4 — Add Actions Variables + +GitHub → **Settings → Secrets and variables → Actions → Variables** +(repo-level). Add the following keys. + +**Identity:** + +| Variable | Value | +|---|---| +| `AZURE_CLIENT_ID` | App ID from Step 3 | +| `AZURE_TENANT_ID` | Tenant ID from Step 3 | +| `AZURE_SUBSCRIPTION_ID` | Subscription ID | +| `AZURE_E2E_RESOURCE_GROUP` | name of the RG you deployed `bootstrap.bicep` into | + +**Bootstrap outputs (Step 1):** + +| Variable | Source | +|---|---| +| `AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT` | `outputs.foundryProjectEndpoint.value` | +| `AZURE_E2E_MODEL_DEPLOYMENT` | `outputs.modelDeployment.value` | +| `AZURE_E2E_ACA_ENV_ID` | `outputs.acaEnvironmentId.value` | +| `AZURE_E2E_ACR_LOGIN_SERVER` | `outputs.acrLoginServer.value` | + +**Foundry agents (Step 2):** + +| Variable | Value | +|---|---| +| `AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT` | `e2e-prompt:1` | +| `AGENTOPS_E2E_FOUNDRY_HOSTED_URL` | hosted agent URL, or leave unset to skip | + +No GitHub Secrets are required. + +--- + +## Step 5 — Trigger the workflow + +The workflow is manual-only: + +```bash +# Default: run only the offline demo + unit tests. +gh workflow run e2e.yml --ref feature/revamp-1.0 + +# All four live scenarios. +gh workflow run e2e.yml --ref feature/revamp-1.0 -f scenarios=all + +# Just one scenario (foundry-prompt | foundry-hosted | http-aca | model-direct). +gh workflow run e2e.yml --ref feature/revamp-1.0 -f scenarios=http-aca + +# Keep the per-run ACA app around for debugging. +gh workflow run e2e.yml --ref feature/revamp-1.0 \ + -f scenarios=http-aca -f keep_resources=true +``` + +> The `Run workflow` button only renders on the repository's **default +> branch**. While `e2e.yml` lives on `feature/revamp-1.0`, use the `gh` +> CLI as shown above. Once PR #108 merges to `main`, the button will +> appear in the Actions tab. + +--- + +## Cost & lifecycle notes + +- **Bootstrap (one-time):** ~5 minutes to deploy. Idle costs are minimal — + AI Services and ACA are pay-per-request, ACR Basic is ~$5/mo, Log + Analytics has a generous free tier for low ingestion. +- **Per run:** ~3–5 minutes total (ACA app comes up in ~30s, scenarios run + in parallel, teardown is fast). Token cost is a few cents per run with + the small datasets shipped in `scripts/e2e_data/`. +- **Teardown:** the workflow always deletes the per-run ACA app on exit + unless `keep_resources=true`. A second pass sweeps any `aca-echo-run*` + app older than one day to catch leftovers from runs that aborted before + teardown could register. + +--- + +## Troubleshooting + +| Symptom | Likely cause | +|---|---| +| `AADSTS70021: No matching federated identity record found` | Branch name in workflow run does not match the `subject:` of any federated credential. Add a credential for the new ref. | +| `AuthorizationFailed` on Bicep deployment | App registration is missing `Contributor` on the RG. | +| `RoleAssignmentRequiresElevation` during bootstrap | App registration is missing `User Access Administrator`. | +| `live-foundry-prompt` fails with 404 on agent | `AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT` does not match a real agent in the project. Re-publish in the portal and update the Variable. | +| ACA echo URL returns HTML instead of JSON | The container failed to start. Check logs in `Microsoft.App/containerApps//logs`. | + +--- + +## What's next + +- A future iteration may declaratively create Foundry agents via the + `azure-ai-projects` SDK from a one-time bootstrap script, removing + Step 2. +- Hosted Foundry agent provisioning via container build + push to ACR is + tracked under the v1.1 follow-ups (see `docs/concepts.md` deferred + list). diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 0910d359..4f2a5624 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -31,79 +31,75 @@ src/ ├── __main__.py # Enables `python -m agentops` │ ├── cli/ - │ └── app.py # Typer CLI definition (init, eval run, report) + │ └── app.py # Typer CLI definition (init, eval run, report, + │ # workflow, skills, mcp, agent) │ - ├── core/ # Pure data logic — ADD models, loaders, threshold rules here - │ ├── models.py # All Pydantic schemas - │ ├── config_loader.py # YAML → Pydantic model + ├── core/ # Pure data layer — no Azure imports, no I/O + │ ├── agentops_config.py # Flat 1.0 `agentops.yaml` Pydantic schema + │ ├── config_loader.py # YAML → AgentOpsConfig + │ ├── evaluators.py # Evaluator catalog (presets + auto-selection) + │ └── results.py # RunResult / RowResult / TargetInfo / RunSummary + │ + ├── pipeline/ # Run orchestration — ADD execution flows here + │ ├── orchestrator.py # End-to-end `eval run` driver + │ ├── runtime.py # Pre-flight checks (deps, creds, endpoints) + │ ├── invocations.py # Per-row agent / model invocation strategies │ ├── thresholds.py # Threshold pass/fail evaluation - │ └── reporter.py # Markdown report generation + │ ├── reporter.py # Markdown report generation + │ ├── comparison.py # `eval compare` two runs + │ ├── publisher.py # Classic Foundry publish (OneDP upload of metrics) + │ └── cloud_publisher.py # New Foundry publish (server-side via OpenAI Evals API) │ - ├── services/ # Orchestration — ADD workflows here - │ ├── runner.py # Main evaluation orchestrator - │ ├── reporting.py # Report regeneration service - │ ├── initializer.py # Workspace scaffolding (agentops init) - │ ├── skills.py # Coding agent skills installation - │ └── foundry_evals.py # Foundry Evaluations panel publishing + ├── services/ # Workspace / project tooling + │ ├── initializer.py # `agentops init` workspace scaffolding + │ ├── skills.py # Coding agent skill installation + │ └── cicd.py # CI/CD workflow generation │ - ├── backends/ # Execution engines — ADD new backends here - │ ├── base.py # Backend Protocol + shared dataclasses - │ ├── eval_engine.py # Shared evaluation engine (evaluators, scoring, dataset utils) - │ ├── foundry_backend.py # Foundry Agent Service (cloud + local) - │ ├── http_backend.py # HTTP endpoint execution - │ └── local_adapter_backend.py # Local adapter (subprocess + callable modes) + ├── agent/ # `agentops agent analyze|serve` watchdog + ├── mcp/ # `agentops mcp serve` Model Context Protocol server │ - ├── utils/ # Shared helpers - │ ├── yaml.py # YAML load + env-var interpolation - │ └── logging.py # Logger factory and setup + ├── utils/ # Shared helpers (yaml load, logging, colors) │ └── templates/ # Starter files for `agentops init` - ├── config.yaml - ├── run.yaml - ├── run-rag.yaml - ├── run-agent.yaml - ├── run-http-model.yaml - ├── run-http-rag.yaml - ├── run-http-agent-tools.yaml - ├── run-callable.yaml + ├── agentops.yaml # Minimal flat config (the single config file) ├── callable_adapter.py - ├── bundles/ # Pre-built evaluation bundles - ├── datasets/ # Dataset definitions (.yaml) - ├── data/ # Sample dataset rows (.jsonl) - └── skills/ # Coding agent skill templates + ├── data/ # Sample dataset rows (.jsonl) + ├── skills/ # Coding agent skill templates + └── workflows/ # CI/CD workflow templates ``` ### Where to Add New Code | I want to… | Directory / File | |---|---| -| Add a new Pydantic model or schema field | `core/models.py` | -| Add a new config file type | `core/config_loader.py` (new loader) + `core/models.py` (new model) | -| Add a new local evaluator | `backends/eval_engine.py` (shared eval engine) + update bundle docs | -| Add a new execution backend | `backends/` (new file implementing `Backend` protocol from `base.py`) + register in `services/runner.py` | -| Support a new endpoint kind | `core/models.py` (`EndpointKind` literal) + `services/runner.py` (resolution) + `backends/` | -| Add a new CLI command | `cli/app.py` (keep it thin — delegate to `services/`) | -| Add a new workflow/service | `services/` (new file) | +| Add a field to `agentops.yaml` | `core/agentops_config.py` | +| Add a new evaluator preset | `core/evaluators.py` (catalog) | +| Change pre-flight checks | `pipeline/runtime.py` | +| Add a new invocation strategy (new target kind) | `pipeline/invocations.py` + `core/agentops_config.py::classify_agent` | +| Tweak the report layout | `pipeline/reporter.py` | +| Add or change a publish destination | `pipeline/publisher.py` (Classic) or `pipeline/cloud_publisher.py` (New Foundry); register in `pipeline/orchestrator.py` | +| Add a new CLI command | `cli/app.py` (keep it thin — delegate to `pipeline/` or `services/`) | | Add a starter template | `templates/` + update `pyproject.toml` package-data | -| Add a new coding agent skill | `templates/skills//SKILL.md` + update `_SKILLS` in `services/skills.py` | +| Add a coding agent skill | `templates/skills//SKILL.md` + sync to `plugins/agentops/skills/` (`scripts/sync-skills.{sh,ps1}`) | ## Request Flow (eval run) When you run `agentops eval run`, the following happens step by step: ``` -1. CLI parses args (cli/app.py → cmd_eval_run) -2. Runner loads config (services/runner.py → load_run_config, load_bundle_config, load_dataset_config) -3. Runner selects backend (FoundryBackend, HttpBackend, or LocalAdapterBackend based on execution_mode + endpoint.kind) -4. Backend executes evaluation (backends/ → invokes agent/model, collects responses) -5. Backend writes backend_metrics.json (raw scores per row) -6. Runner loads backend metrics (services/runner.py → _load_backend_metrics) -7. Runner evaluates thresholds (core/thresholds.py → pass/fail per metric per row) -8. Runner consolidates results (services/runner.py → builds RunResult) -9. Runner writes results.json (normalized, versioned output) -10. Runner generates report.md (core/reporter.py → Markdown from RunResult) -11. Runner syncs latest/ dir (copies to .agentops/results/latest/) -12. CLI returns exit code (0 = pass, 2 = threshold fail, 1 = error) + 1. CLI parses args (cli/app.py → cmd_eval_run) + 2. Loader parses agentops.yaml (core/config_loader.py → AgentOpsConfig) + 3. classify_agent resolves kind (foundry_prompt | foundry_hosted | http_json | model_direct) + 4. Pre-flight checks run (pipeline/runtime.py — deps, creds, endpoint reachability) + 5. Orchestrator iterates dataset (pipeline/orchestrator.py) + 6. Per row: invoke target (pipeline/invocations.py — picks Foundry / HTTP / model API) + 7. Per row: run evaluators (core/evaluators.py — auto-selected from row shape) + 8. Aggregate metrics (orchestrator builds RunResult) + 9. Evaluate thresholds (pipeline/thresholds.py — pass/fail per metric) +10. Write results.json + report.md (pipeline/reporter.py) +11. Sync .agentops/results/latest/ +12. (Optional) Publish to Foundry (pipeline/publisher.py or cloud_publisher.py) +13. CLI returns exit code (0 = pass, 2 = threshold fail, 1 = error) ``` ## CLI Commands @@ -137,579 +133,279 @@ Exit codes are part of the public API. **Do not change their meaning.** | `2` | Execution succeeded **but** one or more thresholds failed | | `1` | Runtime or configuration error | -## User Workspace Structure (`.agentops/`) - -The `.agentops/` directory lives in your project root and stores all evaluation configuration and outputs. - -``` -.agentops/ -├── config.yaml # Workspace-level defaults -├── run.yaml # Default model-direct run specification -├── run-rag.yaml # Example run for RAG scenario -├── run-agent.yaml # Example run for Agent-with-tools scenario -├── bundles/ - ├── model_quality_baseline.yaml - ├── rag_quality_baseline.yaml - ├── conversational_agent_baseline.yaml - ├── agent_workflow_baseline.yaml - └── safe_agent_baseline.yaml -├── datasets/ -│ ├── smoke-rag.yaml # Dataset metadata and source mapping -│ └── ... -├── data/ -│ ├── smoke-rag.jsonl # Actual data rows -│ └── ... -└── results/ - ├── 2026-03-03_143022/ # Timestamped run (immutable) - │ ├── results.json - │ ├── report.md - │ └── backend_metrics.json - └── latest/ # Always points to the most recent run - ├── results.json - └── report.md -``` +## User Workspace Structure (`agentops.yaml` + `.agentops/`) -## Bundle (`.agentops/bundles/*.yaml`) - -- Defines *what quality means* for a scenario. -- Contains evaluators and threshold rules. -- Evaluators are explicit score producers: - - `source: local` for AgentOps-native evaluators (for example `exact_match`, `avg_latency_seconds`) - - `source: foundry` for Foundry SDK evaluators (name must match evaluator class name, for example `GroundednessEvaluator`) -- Supported local evaluators are explicit: `exact_match`, `latency_seconds`, `avg_latency_seconds`. -- AgentOps does not emulate Foundry evaluators locally; if you configure `SimilarityEvaluator`/`GroundednessEvaluator`, use `source: foundry`. -- Foundry evaluators support generic configuration via `evaluators[].config`: - - `kind`: `builtin` (default) or `custom` - - `class_name`: built-in class name from `azure.ai.evaluation` (optional; defaults to evaluator `name`) - - `callable_path`: required when `kind: custom`, format `:` - - `init`: constructor kwargs (supports `${env:VAR}` placeholders) - - `input_mapping`: maps evaluator args to runtime values (for example `$prompt`, `$prediction`, `$expected`, `$row.`, `${env:VAR}`) - - `score_keys`: ordered list of candidate keys used to extract numeric score from evaluator output -- Create a new bundle when you need a different quality policy (for example: stricter production gate vs. smoke gate). -- Minimal shape: +The flat 1.0 schema places **one config file** at the project root and a +small directory for datasets, run history, and (optionally) skills. -```yaml -version: 1 -name: rag_strict -evaluators: - - name: GroundednessEvaluator - source: foundry - enabled: true - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: GroundednessEvaluator - criteria: ">=" - value: 3 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 10.0 ``` - -Example with explicit Foundry evaluator config: - -```yaml -version: 1 -name: qa_similarity -evaluators: - - name: SimilarityEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: SimilarityEvaluator - init: - model_config: - azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT} - azure_deployment: ${env:AZURE_OPENAI_DEPLOYMENT} - input_mapping: - query: $prompt - response: $prediction - ground_truth: $expected - score_keys: - - similarity - - score - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: SimilarityEvaluator - criteria: ">=" - value: 3 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 10.0 +/ +├── agentops.yaml # Single source of truth (flat 1.0 schema) +├── .agentops/ +│ ├── data/ +│ │ └── smoke.jsonl # Sample dataset (created by `agentops init`) +│ └── results/ +│ ├── 2026-05-06T14-30-22Z/ # Timestamped run (immutable history) +│ │ ├── results.json +│ │ ├── report.md +│ │ └── cloud_evaluation.json # only when `publish:` was set +│ └── latest/ # Mirror of the most recent run +└── .github/skills/ # Coding agent skills (Copilot) + ├── agentops-config/SKILL.md + ├── agentops-eval/SKILL.md + └── ... ``` -For built-in Foundry evaluators, AgentOps uses `DefaultAzureCredential` by default (passwordless). Prefer managed identity in Azure environments and avoid API keys. - -- Recommended evaluation scenario bundles: - - `model_quality_baseline`: Model quality — SimilarityEvaluator, CoherenceEvaluator, FluencyEvaluator, F1ScoreEvaluator - - `rag_quality_baseline`: RAG — GroundednessEvaluator, RelevanceEvaluator, RetrievalEvaluator, ResponseCompletenessEvaluator - - `conversational_agent_baseline`: Conversational — CoherenceEvaluator, FluencyEvaluator, RelevanceEvaluator, SimilarityEvaluator - - `agent_workflow_baseline`: Agent with Tools — TaskCompletionEvaluator, ToolCallAccuracyEvaluator, IntentResolutionEvaluator - - `safe_agent_baseline`: Content Safety — ViolenceEvaluator, SexualEvaluator, SelfHarmEvaluator, HateUnfairnessEvaluator, ProtectedMaterialEvaluator +The legacy layered layout (`.agentops/config.yaml` + `bundles/` + +`datasets/*.yaml` + `run.yaml`) **no longer exists**. The new schema is +declared by [src/agentops/core/agentops_config.py](../src/agentops/core/agentops_config.py) +and rejects any of the legacy top-level keys (`target`, `bundle`, +`execution`, `output`, `scenario`, `backend`, `run`) at parse time with +an actionable error. -- Threshold criteria: - - Numeric: `>=`, `>`, `<=`, `<`, `==` (requires `value`) - - Boolean: `true`, `false` (do not set `value`) +## `agentops.yaml` (flat 1.0 schema) -## Dataset (`.agentops/datasets/*.yaml`) +### Minimal config -- Describes the dataset source and format metadata used in evaluation. -- Create a new dataset config when you want to evaluate another file/source (for example: regression set, domain-specific set). -- Minimal shape: +The minimum is three lines: ```yaml version: 1 -name: regression_set -source: - type: file - path: ../data/regression.jsonl -format: - type: jsonl - input_field: input - expected_field: expected +agent: my-rag:3 +dataset: ./qa.jsonl ``` -- `path` is resolved relative to the dataset config file location. -- Keep dataset YAML definitions in `.agentops/datasets/` and `.jsonl` rows in `.agentops/data/` so definitions and data stay separate. - -## Run config (`.agentops/run.yaml`) - -- Connects one bundle + one dataset + backend execution details. -- This is the default run file loaded by `agentops eval run`. -- This is the file you change most often to point to your target (Foundry agent, HTTP endpoint, or local adapter). -- Create additional run files when you need different execution modes (for example: Foundry vs HTTP vs local adapter). - -`agentops init` seeds three scenario-oriented run files: -- `.agentops/run.yaml` (model-direct, default) -- `.agentops/run-rag.yaml` (agent + rag baseline) -- `.agentops/run-agent.yaml` (agent + tools baseline) -- `.agentops/run-http-model.yaml` (model via HTTP endpoint) -- `.agentops/run-http-rag.yaml` (RAG via HTTP endpoint) -- `.agentops/run-http-agent-tools.yaml` (agent-with-tools via HTTP endpoint) - -### run.yaml schema - -Run configs use `version: 1`. - -#### Top-level structure - -- `version: 1` — Required -- `run` — Optional metadata (`name`, `description`) -- `target` — What is being evaluated and how (required) -- `bundle` — Evaluator bundle reference (required) -- `dataset` — Dataset reference (required) -- `execution` — Execution settings (optional, defaults provided) -- `output` — Output settings (optional, defaults provided) - -#### `target` section - -- `type` — `agent` or `model` -- `hosting` — `local`, `foundry`, `aks`, or `containerapps` -- `execution_mode` — `local` or `remote` -- `agent_mode` — `prompt` or `hosted` (Foundry-only, optional) -- `framework` — `agent_framework`, `langgraph`, or `custom` (agent-only, optional) -- `endpoint` — Remote endpoint config (required when `execution_mode: remote`) -- `local` — Local adapter config (required when `execution_mode: local`) - -#### `target.endpoint` fields (remote execution) - -- `kind` — `foundry_agent` or `http` - -Foundry agent endpoint fields: -- `agent_id` — Agent identifier, e.g. `my-agent:3` (name:version) -- `project_endpoint` — Foundry project URL (inline value) -- `project_endpoint_env` — Env var name holding the project URL (default: `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`) -- `api_version` — Agent Service API version -- `poll_interval_seconds` — Polling interval for cloud eval -- `max_poll_attempts` — Max polling attempts -- `model` — Deployment name for evaluators - -HTTP endpoint fields: -- `url` — Direct URL to the agent endpoint -- `url_env` — Environment variable name holding the URL (default: `AGENT_HTTP_URL`) -- `request_field` — JSON key for the user prompt (default: `message`) -- `response_field` — Dot-path to extract response text (default: `text`) -- `headers` — Static extra HTTP headers -- `auth_header_env` — Environment variable for Bearer token -- `tool_calls_field` — Dot-path to extract tool calls from response -- `extra_fields` — JSONL row field names to forward in the request body - -#### `target.local` fields (local execution) - -Exactly one of `adapter` or `callable` must be provided: - -- `adapter` — Command string to spawn the local adapter process (subprocess mode). Receives JSON on stdin per row, emits JSON on stdout per row. -- `callable` — Python function path as `module:function` (callable mode). The function receives `(input_text: str, context: dict) -> dict` and must return `{"response": "..."}`. - -#### `bundle` and `dataset` references - -Both support two resolution modes (at least one required): -- `name` — Convention-based: resolves to `/bundles/.yaml` or `/datasets/.yaml` -- `path` — Explicit path (relative to config file directory) - -#### `execution` section - -- `concurrency` — Max parallel evaluations (default: `1`; schema-only, executes sequentially for now) -- `timeout_seconds` — Overall timeout (default: `300`) - -#### `output` section - -- `path` — Output directory -- `write_report` — Generate `report.md` (default: `true`) -- `publish_foundry_evaluation` — Publish results to Foundry (default: `true`) -- `fail_on_foundry_publish_error` — Fail if Foundry publish fails (default: `false`) - -#### Validation rules - -- `agent_mode` is only valid when `hosting == "foundry"` -- `framework` is only valid when `type == "agent"` -- `endpoint` is required when `execution_mode == "remote"` -- `local.adapter` is required when `execution_mode == "local"` -- Thresholds are **exclusively in bundles** — no run-level threshold overrides - -#### Valid combinations - -Not every combination of dimensions is valid. The table below lists all supported configurations: - -| `type` | `hosting` | `execution_mode` | `endpoint.kind` | `framework` | `agent_mode` | Starter config | -|---|---|---|---|---|---|---| -| `model` | `foundry` | `remote` | `foundry_agent` | — | — | `run.yaml` | -| `agent` | `foundry` | `remote` | `foundry_agent` | — | `prompt` or `hosted` | `run-rag.yaml`, `run-agent.yaml` | -| `model` | `aks` | `remote` | `http` | — | — | `run-http-model.yaml` | -| `model` | `containerapps` | `remote` | `http` | — | — | `run-http-model.yaml` | -| `agent` | `aks` | `remote` | `http` | `langgraph`, `custom`, … | — | `run-http-rag.yaml`, `run-http-agent-tools.yaml` | -| `agent` | `containerapps` | `remote` | `http` | `agent_framework`, `custom`, … | — | `run-http-rag.yaml`, `run-http-agent-tools.yaml` | -| `model` | `local` | `local` | — | — | — | — (custom) | -| `agent` | `local` | `local` | — | `custom` | — | — (custom) | - -### Backend resolution - -The runner resolves the execution backend from the run config: -- `execution_mode: local` → `LocalAdapterBackend` -- `execution_mode: remote` + `endpoint.kind: foundry_agent` → `FoundryBackend` -- `execution_mode: remote` + `endpoint.kind: http` → `HttpBackend` - -### Config validation - -Configs missing a `version` field or containing a legacy `backend` key are **rejected** with an actionable error message. The error includes a migration hint suggesting `target.hosting` as the replacement. - -> **Note:** Do NOT include a `backend:` key at the top level of `run.yaml`. The backend is determined by `target.hosting` and `target.execution_mode`. See [docs/run-yaml-schema.md](run-yaml-schema.md) for the complete schema reference. - -### Evaluator model configuration +That's a complete config. AgentOps: -AI-assisted evaluators (GroundednessEvaluator, RelevanceEvaluator, CoherenceEvaluator, FluencyEvaluator, SimilarityEvaluator, RetrievalEvaluator, ResponseCompletenessEvaluator, etc.) use an LLM as a judge. They require an Azure OpenAI model deployment to run. +* Resolves `agent` into one of four target kinds (see below). +* Auto-selects evaluators from the dataset row shape (presence of + `context`, `tool_calls`, `tool_definitions`). +* Applies sensible default thresholds from the evaluator catalog. -**For Foundry remote execution:** Set `target.endpoint.model` in `run.yaml` to a deployment name that exists in your Foundry project. +### Top-level fields -**For local/callable execution:** Set these environment variables before running: -```bash -export AZURE_OPENAI_ENDPOINT="https://.openai.azure.com/" -export AZURE_OPENAI_DEPLOYMENT="gpt-4o-mini" -``` - -The toolkit auto-injects `model_config` for all AI-assisted evaluators. You do not need to configure `model_config` manually in bundle YAML unless you want to override the defaults. - -**Recommended models for evaluation judges:** Use instruction-following models like `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`. Avoid reasoning models (`o1`, `o3`, `o4`, `gpt-5`, `gpt-5-nano`) — they are slower, more expensive, and may not follow the evaluator prompt format reliably. - -### Callable adapter import requirements - -The callable adapter module must be importable from your project root directory or from the `.agentops/` directory. Both locations are automatically added to the Python path when the CLI runs. - -- Place the file at the project root (`callable_adapter.py`) or inside `.agentops/callable_adapter.py`. -- Use `callable_adapter:run_evaluation` as the callable path in `run.yaml` — no directory prefix needed. -- Do **not** use dotted paths like `.agentops.callable_adapter` — relative imports do not work. - -After generating an adapter, verify importability: -```bash -python -c "from callable_adapter import run_evaluation; print('OK')" -``` +| Field | Required | Description | +|---|---|---| +| `version` | yes | Schema version. Must be `1`. | +| `agent` | yes | Target identifier. See "Target kinds" below. | +| `dataset` | yes | Relative path to a JSONL file with one evaluation row per line. | +| `thresholds` | no | Dict of `metric_name: criteria_expression`. Examples: `">=3"`, `"<=10"`, `"true"`, raw number `3` (treated as `>=3`). Defaults from catalog if omitted. | +| `protocol` | no | Wire protocol for URL-based agents: `responses` (Foundry hosted), `invocations` (Knative), `http-json` (default for arbitrary HTTPS). | +| `request_field` / `response_field` / `tool_calls_field` | no | JSON keys / dot-paths used to marshal each row into the request and extract the response. Defaults are sensible for OpenAI-compatible / ACA endpoints. | +| `headers` | no | Static HTTP headers (dict). | +| `auth_header_env` | no | Env var name holding a Bearer token. | +| `evaluators` | no | Escape-hatch list of evaluator names that overrides auto-selection. | +| `publish` | no | `foundry` (Classic) or `foundry_cloud` (preview, server-side). See [Publishing](#publishing-to-foundry-evaluations). | +| `project_endpoint` | no | Foundry project URL used by `publish:`. Falls back to `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`. | + +### Target kinds + +`classify_agent()` resolves `agent` into one of four kinds based on shape: + +| Kind | Trigger | Example `agent` value | +|---|---|---| +| `foundry_prompt` | `name:version` | `my-rag:3` | +| `foundry_hosted` | URL on a Foundry domain | `https://contoso.services.ai.azure.com/.../agents/` | +| `http_json` | Any other HTTPS URL | `https://my-aca-app.eastus2.azurecontainerapps.io/chat` | +| `model_direct` | `model:` | `model:gpt-4o-mini` | -### Callable adapter authentication patterns +The kind drives both invocation strategy (`pipeline/invocations.py`) and +which fields make sense (e.g. `protocol` is rejected for +`foundry_prompt` and `model_direct`). -If your agent endpoint requires authentication, include the appropriate headers in the callable adapter. Use environment variables for token values — never hardcode credentials. +### Examples -**Dapr token (Azure Container Apps):** -```python -API_TOKEN = os.environ.get("APP_API_TOKEN", "") -if API_TOKEN: - headers["dapr-api-token"] = API_TOKEN -``` +**Foundry prompt agent (RAG bundle auto-selected from dataset rows):** -**API Key:** -```python -API_KEY = os.environ.get("API_KEY", "") -if API_KEY: - headers["X-API-KEY"] = API_KEY +```yaml +version: 1 +agent: my-rag:3 +dataset: .agentops/data/qa.jsonl +thresholds: + groundedness: ">=3" + coherence: ">=3" + avg_latency_seconds: "<=10" +publish: foundry # Classic Foundry Evaluations panel (best-effort) ``` -**Bearer token (Entra ID / OAuth):** For Bearer token authentication, consider using the HTTP backend with `auth_header_env` instead of a callable adapter, as the HTTP backend handles this natively. - -### azd integration - -If you deployed your Azure resources with `azd` (Azure Developer CLI), your `.azure//.env` file contains resource metadata (subscription ID, resource group, resource names) that can be used to auto-configure endpoints. The evaluation skills (`/agentops-config`, `/agentops-eval`) can auto-discover these values via Azure CLI queries. - -### Minimal run.yaml example (Foundry agent) +**HTTP-deployed agent (LangGraph / ACA / custom REST):** ```yaml version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: my-agent:1 - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: rag_quality_baseline -dataset: - name: smoke-rag -execution: - timeout_seconds: 300 -output: - write_report: true +agent: https://my-aca-app.eastus2.azurecontainerapps.io/chat +dataset: .agentops/data/qa.jsonl +request_field: message # default is "message" +response_field: text # dot-path; default is "text" +auth_header_env: APP_API_TOKEN # value used as Bearer token ``` -### Minimal run.yaml example (HTTP endpoint) +**Raw model deployment:** ```yaml version: 1 -target: - type: model - hosting: aks - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - request_field: message - response_field: text -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -output: - write_report: true +agent: model:gpt-4o-mini +dataset: .agentops/data/qa.jsonl +thresholds: + similarity: ">=4" + avg_latency_seconds: "<=8" ``` -### Minimal run.yaml example (local adapter) +**New Foundry server-side run (preview):** ```yaml version: 1 -target: - type: model - hosting: local - execution_mode: local - local: - adapter: python my_adapter.py -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -output: - write_report: true +agent: my-rag:3 # name:version is required for cloud mode +dataset: .agentops/data/qa.jsonl +publish: foundry_cloud +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" ``` -## Evaluation scenarios +## Datasets + +A dataset is a plain JSONL file. One row per line. No companion YAML. -AgentOps supports five evaluation scenarios: +Required fields: -### Model Quality +| Field | Type | Notes | +|---|---|---| +| `input` | string | The prompt sent to the target. | +| `expected` | string | Ground-truth response used by reference-based evaluators. | -- Evaluates raw model output quality for any model deployment -- Uses `SimilarityEvaluator`, `CoherenceEvaluator`, `FluencyEvaluator`, `F1ScoreEvaluator` -- Bundle: `model_quality_baseline.yaml` -- Dataset: rows with `input` and `expected` fields -- Target config: `type: model` +Optional fields drive evaluator auto-selection: -### RAG Quality +| Field | Triggers | +|---|---| +| `context` | RAG evaluators (`GroundednessEvaluator`, `RelevanceEvaluator`, `RetrievalEvaluator`, `ResponseCompletenessEvaluator`) | +| `tool_calls` + `tool_definitions` | Tool-use evaluators (`ToolCallAccuracyEvaluator`, `IntentResolutionEvaluator`, `TaskAdherenceEvaluator`, …) | -- Evaluates grounding of responses against context/retrieved documents -- Uses `GroundednessEvaluator`, `RelevanceEvaluator`, `RetrievalEvaluator`, `ResponseCompletenessEvaluator`, `CoherenceEvaluator` -- Bundle: `rag_quality_baseline.yaml` -- Dataset: rows with `input`, `expected`, and `context` fields -- Target config: `type: agent` (agent with knowledge base / retrieval) +Example RAG row: -### Conversational Agent +```json +{"input": "What is the refund policy?", "expected": "Refunds within 30 days.", "context": "Our policy: refunds available within 30 days of purchase."} +``` -- Evaluates chatbots, assistants, and Q&A agents -- Uses `CoherenceEvaluator`, `FluencyEvaluator`, `RelevanceEvaluator`, `SimilarityEvaluator` -- Bundle: `conversational_agent_baseline.yaml` -- Dataset: rows with `input` and `expected` fields -- Target config: `type: agent` +## Evaluator auto-selection -### Agent Workflow (Tools) +The catalog is defined in [src/agentops/core/evaluators.py](../src/agentops/core/evaluators.py). +Selection rules (in order): -- Evaluates agents that use tool calls (function calling) -- Uses `TaskCompletionEvaluator`, `ToolCallAccuracyEvaluator`, `IntentResolutionEvaluator`, `TaskAdherenceEvaluator`, `ToolSelectionEvaluator`, `ToolInputAccuracyEvaluator` -- Bundle: `agent_workflow_baseline.yaml` -- Dataset: rows with `input`, `expected`, `tool_definitions`, and `tool_calls` fields -- Target config: `type: agent` +1. If `evaluators:` is set in `agentops.yaml`, use it verbatim (escape hatch). +2. Otherwise, start from the **quality baseline** for the resolved target + kind (e.g. `Coherence + Fluency + Similarity + F1Score` for chat-style agents). +3. If dataset rows include `context`, add the **RAG bundle** + (`Groundedness`, `Relevance`, `Retrieval`, `ResponseCompleteness`). +4. If rows include `tool_calls` + `tool_definitions`, add the **tool-use + bundle** (`ToolCallAccuracy`, `IntentResolution`, `TaskAdherence`, …). +5. `avg_latency_seconds` is always included as a runtime metric. -See [bundles.md](bundles.md) for detailed evaluator descriptions and configuration. +### Recommended judge models -## Backend behavior +AI-assisted evaluators use an LLM as a judge. Use instruction-following +models like `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`. **Avoid +reasoning models** (`o1`, `o3`, `o4`, `gpt-5`, `gpt-5-nano`) — they are +slower, more expensive, and may not follow the evaluator prompt format. -- AgentOps Toolkit provides backend orchestration with multiple execution backends. -- The backend is selected automatically based on `execution_mode` and `endpoint.kind` in the run config. -- In `foundry` mode, AgentOps uses **Foundry Cloud Evaluation** (project-native eval/run lifecycle). -- Cloud runs are persisted in the Foundry project and visible in **Build > Evaluations** (New Foundry Experience). -- The `http` backend supports any HTTP-deployed agent (LangGraph, LangChain, OpenAI, ACA, custom REST). -- The `local` adapter backend supports custom evaluation pipelines via a stdin/stdout JSON protocol. -- All backends write `backend_metrics.json` automatically. -- AgentOps then writes normalized `results.json` (stable contract for CI/reporting). +Set the deployment via env vars before running: -## Foundry target mode +```bash +export AZURE_OPENAI_ENDPOINT="https://.openai.azure.com/" +export AZURE_OPENAI_DEPLOYMENT="gpt-4o-mini" +``` -- `target.type: agent` with `endpoint.kind: foundry_agent` - - Required in endpoint config: `agent_id` - - Required env: `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` - - Authentication: automatic via `DefaultAzureCredential` (supports `az login`, managed identity, service principal) - - Optional tuning: `poll_interval_seconds`, `max_poll_attempts` +## Thresholds -- `target.type: model` with `endpoint.kind: foundry_agent` - - Sends prompts directly to a model deployment (no agent involved) - - Required in endpoint config: `model` (deployment name that already exists in the Foundry project) - - Required env: `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` - - Does **not** require `agent_id` - - Cloud evaluation uses `completions` data source type - - Local evaluation uses OpenAI chat completions API via the Foundry project client +Threshold expressions accept: -## Main Foundry testing flow +| Form | Meaning | +|---|---| +| `">=3"`, `">3"`, `"<=10"`, `"<10"`, `"==1"` | Numeric comparison | +| `"true"` / `"false"` | Boolean expectation (used by safety evaluators) | +| Raw number `3` | Shorthand for `>=3` | -- Authenticate (pick one): - - Local dev: `az login` - - CI/CD: set `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_SECRET` - - Azure hosted: managed identity (no config needed) -- Set project endpoint: - - `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/` -- Configure the run file for your scenario (`.agentops/run.yaml`, `.agentops/run-rag.yaml`, or `.agentops/run-agent.yaml`): +Each row is judged against every applicable threshold. A row passes only +if every threshold passes. The run passes only if every row passes +(this is the only condition that maps to exit code `0`; otherwise `2`). -Example for agent target: +## Publishing to Foundry Evaluations -```yaml -version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: my-agent:1 - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: rag_quality_baseline -dataset: - name: smoke-rag -output: - write_report: true -``` +`publish:` is opt-in. Both modes are best-effort: if publish fails, the +local `results.json` and `report.md` remain the canonical record and the +exit code reflects only thresholds, not publish failures. -Example for model-direct target: +| Mode | What it does | Where results land | Target restriction | +|---|---|---|---| +| `publish: foundry` | Uploads metrics computed locally via OneDP. | **Classic** Foundry Evaluations panel. | Any target kind. | +| `publish: foundry_cloud` (preview) | Re-runs the agent + builtin evaluators **server-side** via the OpenAI Evals API. | **New** Foundry Evaluations panel. | `foundry_prompt` only (`name:version` Foundry agents). | -```yaml -version: 1 -target: - type: model - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -output: - write_report: true -``` +Both modes: -- Run `agentops eval run` for the default model-direct config, or `agentops eval run --config .agentops/run-rag.yaml` / `agentops eval run --config .agentops/run-agent.yaml` for scenario-specific files. -- AgentOps creates one thread/run per dataset row, fetches the assistant response, computes metrics, and writes artifacts. +* Require either `project_endpoint` in `agentops.yaml` or + `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in the environment. +* Authenticate with `DefaultAzureCredential` (passwordless: `az login`, + managed identity, or service principal). +* Write `cloud_evaluation.json` next to `results.json` containing + `mode` (`classic` or `cloud`), `evaluation_name`, `report_url`, and + (for `foundry_cloud`) the `eval_id` / `run_id` / terminal `status`. -## Foundry backend inputs +The cloud-mode trade-offs (so you can decide consciously): -- Dataset config must point to a JSONL file. -- Each row must include the fields configured in dataset format (`input_field`, `expected_field`). -- For `target: agent`, each row input is sent as a user message to the configured Foundry agent. -- The backend computes only the metrics configured in the bundle: - - **Foundry evaluators** (`source: foundry`) are executed by the cloud evaluation API. - - **Local evaluators** (`source: local`) such as `exact_match`, `latency_seconds`, and `avg_latency_seconds` are computed by AgentOps only when explicitly enabled in the bundle. - - `samples_evaluated` is always emitted. +* Foundry-side latency replaces the locally-measured wall-clock latency. +* Judges are opaque (Foundry-managed); custom evaluators are skipped. +* The dataset is uploaded as an OpenAI file (egress + transient storage + in your project). +* Evaluator runs cost against your Azure OpenAI deployment. +* Polling adds ~5 s × N to the total wall-clock time. -## Backend metrics contract (`backend_metrics.json`) +Implementation lives in [src/agentops/pipeline/publisher.py](../src/agentops/pipeline/publisher.py) +(Classic) and [src/agentops/pipeline/cloud_publisher.py](../src/agentops/pipeline/cloud_publisher.py) +(New Foundry). Dispatch happens in +[src/agentops/pipeline/orchestrator.py](../src/agentops/pipeline/orchestrator.py). -- This is the file consumed by AgentOps to build `results.json`. -- In `foundry` mode AgentOps generates it automatically. -- The `local` adapter backend also generates it automatically. -- If writing a custom adapter, the output should match this shape: +## Pre-flight checks -```json -{ - "metrics": [ - { "name": "exact_match", "value": 0.84 }, - { "name": "avg_latency_seconds", "value": 1.21 } - ], - "row_metrics": [ - { - "row_index": 1, - "input": "What is the refund policy?", - "response": "Refunds are available within 30 days.", - "metrics": [ - { "name": "exact_match", "value": 1.0 }, - { "name": "avg_latency_seconds", "value": 1.21 } - ] - }, - { - "row_index": 2, - "input": "How do I reset my password?", - "response": "Go to Settings > Security > Reset.", - "metrics": [ - { "name": "exact_match", "value": 0.0 }, - { "name": "avg_latency_seconds", "value": 0.98 } - ] - } - ] -} -``` +Before any agent invocation, [pipeline/runtime.py](../src/agentops/pipeline/runtime.py) +runs a short series of checks and reports **all** failures at once: + +* Required Python packages installed (`azure-identity`, + `azure-ai-evaluation` for AI-assisted evaluators, `azure-ai-projects` + if `publish: foundry_cloud`). +* Required env vars set (`AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`, + `AZURE_OPENAI_*` deployment fields). +* Azure CLI credential acquires a token within 30 s + (`process_timeout=30` is set everywhere `DefaultAzureCredential` is + instantiated to absorb Windows `az.cmd` cold starts). +* For URL agents, the endpoint resolves and accepts a TCP connection. + +`agentops eval run --dry-run` runs only the pre-flight phase and exits +`0` (all clear) or `1` (something to fix). Useful for CI gating. + +## Invocation strategies (target kind → wire call) + +There is no longer a free-form `backend:` field. The invocation +strategy is derived from the target kind resolved by `classify_agent()`: + +| Target kind | Invocation strategy | +|---|---| +| `foundry_prompt` | Foundry Agent Service threads/runs API via `AIProjectClient` | +| `foundry_hosted` | Direct call to the hosted endpoint with the configured `protocol` | +| `http_json` | POST `{request_field: input, ...}` and extract `response_field` (dot-path) | +| `model_direct` | Azure OpenAI chat completions via `AIProjectClient.get_openai_client()` | -- Required rules: - - root JSON object - - `metrics` must be a list - - each metric entry must include `name` (string) and `value` (number) -- `row_metrics` is optional, but recommended for dataset-native consolidation. -- when present, each row entry must include: - - `row_index` (1-based) - - `metrics` list with `{name, value}` entries - - `input` (string, optional) — the user prompt sent to the agent/model - - `response` (string, optional) — the agent/model output text -- Each metric `name` must match the evaluator `name` referenced in bundle thresholds. -- AgentOps applies thresholds per item and then consolidates item verdicts into run-level outputs. -- AgentOps validates that every enabled evaluator in the bundle has produced scores in `row_metrics`. +`AIProjectClient.get_openai_client()` is **always called without +`api_version`** — passing one explicitly has historically caused 404s +in this codebase. ## How evaluators and metrics work - Evaluator execution is row-first: - each dataset row is evaluated and can produce one or more row scores. -- Threshold evaluation is bundle-driven: - - each threshold references one evaluator score (`thresholds[].evaluator`) - - each row receives a threshold verdict per evaluator - - if a row passes all threshold rules, the row verdict is PASS +- Threshold evaluation is config-driven: + - each entry in `thresholds:` maps an evaluator's score key to a comparison expression + - each row receives a verdict per threshold + - a row passes only if every applicable threshold passes - run-level threshold status is consolidated from item verdicts. - Metrics have three levels in `results.json`: - - `metrics`: backend/global metrics (already aggregated by backend) + - `metrics`: backend/global metrics (already aggregated) - `row_metrics`: per-row evaluator outputs (`row_index` + metric list + optional `input`/`response` text) - `item_evaluations`: per-row threshold verdicts (per evaluator + final row PASS/FAIL) - `run_metrics`: consolidated execution metrics derived by AgentOps @@ -738,8 +434,9 @@ In short: ## Outputs and history -- Every run stores artifacts in `.agentops/results//`. -- AgentOps also refreshes `.agentops/results/latest/` with a copy of the most recent run. +- Every run writes its artifacts to `.agentops/results//` (immutable history). +- AgentOps then refreshes `.agentops/results/latest/` with a copy of that run, so `latest/` always points at the most recent results. +- Pass `--output

` to skip the default layout and write only to that path (useful for named baselines or CI artifacts). - `results.json`: normalized, machine-readable result for CI/automation. - `report.md`: human-readable summary for review. diff --git a/docs/media/agentops-diagrams.vsdx b/docs/media/agentops-diagrams.vsdx new file mode 100644 index 00000000..0d4a9d0d Binary files /dev/null and b/docs/media/agentops-diagrams.vsdx differ diff --git a/docs/release-process.md b/docs/release-process.md index b1c2e5e0..24b3b144 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -274,26 +274,14 @@ The staging pipeline validates a release candidate by publishing to TestPyPI and ### Pipeline Flow -``` -push to release/v0.2.0 - │ - ┌────▼────────┐ - │ _build │ ← Reusable workflow: test + build package - │ (tests + │ Version: 0.2.1.dev3 (from setuptools-scm) - │ package) │ - └────┬────────┘ - │ - ┌────▼───────────┐ - │ publish-testpypi │ ← Upload to TestPyPI (staging environment) - │ │ Uses TEST_PYPI_TOKEN secret - └────┬───────────┘ - │ - ┌────▼───────────┐ - │ verify-testpypi │ ← Install from TestPyPI in a fresh environment - │ │ Run: agentops --version - │ │ Run: agentops --help - │ │ Run: agentops init (in temp directory) - └─────────────────┘ +```mermaid +flowchart TD + push(["push to release/v0.2.0"]) + build["_build
tests + package
Version: 0.2.1.dev3 (setuptools-scm)"] + publish["publish-testpypi
Upload to TestPyPI (staging environment)
Uses TEST_PYPI_TOKEN secret"] + verify["verify-testpypi
Install from TestPyPI in fresh environment
agentops --version / --help / init"] + + push --> build --> publish --> verify ``` ### What Gets Validated @@ -522,34 +510,19 @@ The production pipeline publishes a final release to PyPI and creates a GitHub R ### Pipeline Flow -``` -push tag v0.2.0 - │ - ┌────▼────────┐ - │ _build │ ← Same reusable build as staging - │ (tests + │ Version: 0.2.0 (clean, from tag) - │ package) │ - └────┬────────┘ - │ - ┌────▼───────────┐ - │ publish-testpypi │ ← Final TestPyPI upload (clean version) - └────┬───────────┘ - │ - ┌────▼───────────┐ - │ verify-testpypi │ ← Smoke test from TestPyPI - └────┬───────────┘ - │ - ┌────▼───────────┐ - │ publish-pypi │ ← ⏸️ PAUSES HERE — requires approval - │ │ Uses PYPI_TOKEN secret - │ (environment: │ Designated reviewers must approve - │ release) │ - └────┬───────────┘ - │ - ┌────▼───────────┐ - │ github-release │ ← Creates GitHub Release with artifacts - │ │ Generates release notes automatically - └─────────────────┘ +```mermaid +flowchart TD + tag(["push tag v0.2.0"]) + build["_build
tests + package
Version: 0.2.0 (clean, from tag)"] + publishTest["publish-testpypi
Final TestPyPI upload (clean version)"] + verifyTest["verify-testpypi
Smoke test from TestPyPI"] + publishPypi{{"publish-pypi ⏸
PAUSES — requires approval
Uses PYPI_TOKEN
environment: release"}} + ghRelease["github-release
Creates GitHub Release with artifacts
Auto-generated release notes"] + + tag --> build --> publishTest --> verifyTest --> publishPypi --> ghRelease + + classDef gate fill:#fff3cd,stroke:#856404,color:#000; + class publishPypi gate; ``` ### Step-by-Step: Cutting a Release @@ -847,63 +820,42 @@ Use this checklist when cutting a release: ## Architecture Diagram -``` - Feature Development Staging Production Release - ───────────────── ─────── ────────────────── - - feature/* ──PR──→ develop - │ - ├──→ CI (ci.yml) - │ lint + test + coverage - │ + publish-dev → TestPyPI (dev version) - │ - └──→ Cut Release (cut-release.yml) - manual dispatch → enter version - │ - └──→ release/v0.2.0 - │ - ├──→ Staging (staging.yml) - │ - │ ┌──────────┐ - │ │ _build │ - │ │ test+build│ - │ └────┬─────┘ - │ │ - │ ┌────▼────────┐ - │ │ TestPyPI │ - │ │ publish │ - │ └────┬────────┘ - │ │ - │ ┌────▼────────┐ - │ │ Verify │ - │ │ install │ - │ └─────────────┘ - │ - └──PR──→ main ──tag──→ v0.2.0 - │ - ├──→ Release (release.yml) - │ - │ ┌──────────┐ - │ │ _build │ - │ └────┬─────┘ - │ │ - │ ┌────▼────────┐ - │ │ TestPyPI │ - │ └────┬────────┘ - │ │ - │ ┌────▼────────┐ - │ │ Verify │ - │ └────┬────────┘ - │ │ - │ ┌────▼────────┐ - │ │ PyPI │ - │ │ (approval) │ - │ └────┬────────┘ - │ │ - │ ┌────▼────────┐ - │ │ GitHub │ - │ │ Release │ - │ └────────────┘ - │ - main ──merge──→ develop +```mermaid +flowchart TD + feat["feature/*"] -->|PR| develop(["develop"]) + develop --> ci["CI (ci.yml)
lint + test + coverage
publish-dev → TestPyPI (dev version)"] + develop --> cut{{"Cut Release (cut-release.yml)
manual dispatch — enter version"}} + cut --> rel(["release/v0.2.0"]) + + rel --> stagingBuild["_build
test + build"] + stagingBuild --> stagingTest["TestPyPI publish"] + stagingTest --> stagingVerify["Verify install"] + + rel -->|PR| main(["main"]) + main -->|tag| tag(["v0.2.0"]) + + tag --> relBuild["_build"] + relBuild --> relTest["TestPyPI"] + relTest --> relVerify["Verify"] + relVerify --> relPypi{{"PyPI
(approval)"}} + relPypi --> relGh["GitHub Release"] + + main -->|merge back| develop + + subgraph Staging["Staging (staging.yml)"] + stagingBuild + stagingTest + stagingVerify + end + + subgraph Release["Release (release.yml)"] + relBuild + relTest + relVerify + relPypi + relGh + end + + classDef gate fill:#fff3cd,stroke:#856404,color:#000; + class cut,relPypi gate; ``` diff --git a/docs/run-yaml-schema.md b/docs/run-yaml-schema.md deleted file mode 100644 index 03ad42df..00000000 --- a/docs/run-yaml-schema.md +++ /dev/null @@ -1,274 +0,0 @@ -# run.yaml Schema Reference - -Complete reference for the `run.yaml` configuration file used by `agentops eval run`. - -## Top-Level Structure - -```yaml -version: 1 # Required — schema version -run: # Optional — run metadata - name: "my evaluation" - description: "..." -target: # Required — what is being evaluated - ... -bundle: # Required — evaluator bundle reference - ... -dataset: # Required — dataset reference - ... -execution: # Optional — execution settings - ... -output: # Optional — output settings - ... -``` - -> **IMPORTANT:** Do NOT include a `backend:` key at the top level. The backend is determined by `target.hosting` and `target.execution_mode`. A `backend:` key will cause a runtime error. - ---- - -## `target` Section (required) - -Defines what is being evaluated and how the toolkit connects to it. - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `type` | `"agent"` \| `"model"` | Yes | — | What is being evaluated | -| `hosting` | `"local"` \| `"foundry"` \| `"aks"` \| `"containerapps"` | Yes | — | Where the target is hosted | -| `execution_mode` | `"local"` \| `"remote"` | Yes | — | How the toolkit connects to the target | -| `agent_mode` | `"prompt"` \| `"hosted"` | No | — | Foundry-only: agent interaction mode | -| `framework` | `"agent_framework"` \| `"langgraph"` \| `"custom"` | No | — | Agent-only: agent framework | -| `endpoint` | object | When `execution_mode: remote` | — | Remote endpoint configuration | -| `local` | object | When `execution_mode: local` | — | Local adapter configuration | - -### Validation Rules - -- `agent_mode` is only valid when `hosting == "foundry"` -- `framework` is only valid when `type == "agent"` -- `endpoint` is required when `execution_mode == "remote"` -- `local` is required when `execution_mode == "local"` - -### Backend Resolution - -The execution backend is determined automatically: - -| `execution_mode` | `endpoint.kind` | Backend | -|---|---|---| -| `local` | — | `LocalAdapterBackend` | -| `remote` | `foundry_agent` | `FoundryBackend` | -| `remote` | `http` | `HttpBackend` | - ---- - -## `target.endpoint` Section (remote execution) - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `kind` | `"foundry_agent"` \| `"http"` | Yes | — | Endpoint type | - -### Foundry Agent Endpoint Fields (`kind: foundry_agent`) - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `agent_id` | string | No | — | Agent identifier (e.g., `my-agent:3`) | -| `project_endpoint` | string | No | — | Foundry project URL (inline value) | -| `project_endpoint_env` | string | No | `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Env var name holding the project URL | -| `api_version` | string | No | `"2025-05-01"` | Agent Service API version | -| `poll_interval_seconds` | float | No | — | Polling interval for cloud eval | -| `max_poll_attempts` | int | No | — | Max polling attempts | -| `model` | string | No | — | Model deployment name for evaluators | - -> **Evaluator Model:** When using AI-assisted evaluators (Groundedness, Relevance, Coherence, etc.), set `model` to an instruction-following deployment like `gpt-4o-mini` or `gpt-4.1-mini`. Avoid reasoning models (`o1`, `o3`, `o4`, `gpt-5`) — they are slower, more expensive, and may not follow evaluator prompts reliably. - -### HTTP Endpoint Fields (`kind: http`) - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `url` | string | No* | — | Direct URL to the agent endpoint | -| `url_env` | string | No* | `AGENT_HTTP_URL` | Env var name holding the URL | -| `request_field` | string | No | `"message"` | JSON key for the user prompt | -| `response_field` | string | No | `"text"` | Dot-path to extract response text | -| `headers` | object | No | `{}` | Static extra HTTP headers | -| `auth_header_env` | string | No | — | Env var for Bearer token | -| `tool_calls_field` | string | No | — | Dot-path to extract tool calls | -| `extra_fields` | list[string] | No | — | JSONL row fields to forward in request | - -*At least one of `url` or `url_env` is required. - ---- - -## `target.local` Section (local execution) - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `adapter` | string | No* | — | Command string for subprocess adapter | -| `callable` | string | No* | — | Python function as `module:function` | - -*Exactly one of `adapter` or `callable` must be provided. - -### Callable Adapter - -The `callable` field references a Python function using `module:function` syntax. The module must be importable from the project root or from `.agentops/`. - -```yaml -local: - callable: callable_adapter:run_evaluation -``` - -The function signature must be: -```python -def run_evaluation(input_text: str, context: dict) -> dict: - return {"response": "the model/agent output text"} -``` - -### Subprocess Adapter - -The `adapter` field specifies a shell command. The subprocess receives JSON on stdin per row and emits JSON on stdout. - -```yaml -local: - adapter: "python my_adapter.py" -``` - ---- - -## `bundle` Section (required) - -References the evaluator bundle. At least one of `name` or `path` is required. - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `name` | string | No* | — | Resolves to `/bundles/.yaml` | -| `path` | path | No* | — | Explicit path (relative to config file directory) | - ---- - -## `dataset` Section (required) - -References the evaluation dataset. At least one of `name` or `path` is required. - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `name` | string | No* | — | Resolves to `/datasets/.yaml` | -| `path` | path | No* | — | Explicit path (relative to config file directory) | - ---- - -## `execution` Section (optional) - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `concurrency` | int | No | `1` | Max parallel evaluations (schema-only for now) | -| `timeout_seconds` | int | No | `300` | Overall timeout in seconds | - ---- - -## `output` Section (optional) - -| Field | Type | Required | Default | Description | -|---|---|---|---|---| -| `path` | path | No | — | Output directory override | -| `write_report` | bool | No | `true` | Generate `report.md` | -| `publish_foundry_evaluation` | bool | No | `true` | Publish results to Foundry | -| `fail_on_foundry_publish_error` | bool | No | `false` | Fail if Foundry publish fails | - ---- - -## Environment Variables - -### Required for Foundry Backend - -| Variable | Purpose | Default | -|---|---|---| -| `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Foundry project endpoint URL | Required | - -### Evaluator Model (for AI-assisted evaluators) - -| Variable | Purpose | Default | -|---|---|---| -| `AZURE_OPENAI_ENDPOINT` | Azure OpenAI endpoint | Auto-derived from project endpoint | -| `AZURE_OPENAI_DEPLOYMENT` | Model deployment name | — | -| `AZURE_AI_MODEL_DEPLOYMENT_NAME` | Explicit deployment name override | — | -| `AZURE_OPENAI_API_VERSION` | OpenAI API version | SDK default | - -### Execution Mode - -| Variable | Purpose | Default | -|---|---|---| -| `AGENTOPS_FOUNDRY_MODE` | `cloud` or `local` execution | `cloud` | - -### Authentication - -| Variable | Purpose | -|---|---| -| `AZURE_CLIENT_ID` | Service principal client ID | -| `AZURE_TENANT_ID` | Service principal tenant ID | -| `AZURE_CLIENT_SECRET` | Service principal secret | -| `AZURE_OPENAI_API_KEY` | API key (alternative to credential) | - ---- - -## Examples - -### Model Quality (Foundry remote) - -```yaml -version: 1 -target: - type: model - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - model: gpt-4o-mini - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -output: - write_report: true -``` - -### RAG Quality (callable adapter) - -```yaml -version: 1 -target: - type: agent - hosting: containerapps - execution_mode: local - local: - callable: callable_adapter:run_evaluation -bundle: - name: rag_quality_baseline -dataset: - path: .agentops/datasets/dataset.yaml -output: - write_report: true -``` - -### HTTP Agent with Tools - -```yaml -version: 1 -target: - type: agent - hosting: aks - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - request_field: message - response_field: response.text - tool_calls_field: response.tool_calls - auth_header_env: AGENT_API_KEY -bundle: - name: agent_workflow_baseline -dataset: - path: .agentops/datasets/dataset.yaml -output: - write_report: true -``` - -### azd Integration - -If you deployed your resources with `azd` (Azure Developer CLI), your `.azure//.env` file contains resource metadata (subscription, resource group, resource names) that can be used to auto-configure endpoints via Azure CLI queries. The skills (`/agentops-config`, `/agentops-eval`) can auto-discover these values. diff --git a/docs/telemetry.md b/docs/telemetry.md deleted file mode 100644 index 04802ee9..00000000 --- a/docs/telemetry.md +++ /dev/null @@ -1,446 +0,0 @@ -# Telemetry — Observability for Evaluation Runs - -This document explains how AgentOps uses **OpenTelemetry (OTel)** to give you visibility into every evaluation run. It is written for developers who have never used OTel before, so we start with the basics. - ---- - -## Table of Contents - -1. [Why Telemetry?](#why-telemetry) -2. [Concepts You Need to Know](#concepts-you-need-to-know) -3. [How AgentOps Uses OTel](#how-agentops-uses-otel) -4. [Quick Start — Local Setup with Jaeger](#quick-start--local-setup-with-jaeger) -5. [Environment Variables](#environment-variables) -6. [The Trace Tree](#the-trace-tree) -7. [Semantic Conventions (Attributes)](#semantic-conventions-attributes) -8. [Viewing Traces in Jaeger](#viewing-traces-in-jaeger) -9. [Sending Traces to Azure Monitor](#sending-traces-to-azure-monitor) -10. [FAQ](#faq) - ---- - -## Why Telemetry? - -When you run `agentops eval run`, a lot happens under the hood — dataset rows are loaded, agents are invoked, evaluators score responses, thresholds are checked. If something is slow, fails, or produces surprising scores, you want to **see exactly what happened**. - -Telemetry records the full timeline of every evaluation run so you can: - -- **Debug slow evaluations** — see which rows or evaluators took the longest. -- **Trace failures** — pinpoint exactly where an error occurred. -- **Monitor quality over time** — forward data to dashboards in Azure Monitor, Grafana, or Datadog. -- **Audit runs** — keep a detailed, machine-readable record of what happened. - ---- - -## Concepts You Need to Know - -If you are already familiar with OpenTelemetry, skip to [How AgentOps Uses OTel](#how-agentops-uses-otel). - -### What Is OpenTelemetry? - -OpenTelemetry (OTel) is an **open standard** for collecting diagnostic data from software. Think of it as a universal language that lets your application say: "I started doing X, it took 200ms, here are the details, and it succeeded." Any tool that speaks OTel (Jaeger, Azure Monitor, Datadog, Grafana Tempo, etc.) can receive and display that data. - -### What Is a Trace? - -A **trace** represents a single end-to-end operation. In AgentOps, one evaluation run = one trace. A trace is made up of **spans**. - -### What Is a Span? - -A **span** is a unit of work with a start time, end time, a name, and key-value attributes. Spans nest inside each other to form a tree. Example: - -``` -RUN conversational_agent_baseline ← root span (the whole run) -├── eval_item 0 ← child span (one dataset row) -│ ├── invoke_agent my-agent ← grandchild (the agent call) -│ ├── evaluator builtin.similarity ← grandchild (scoring) -│ └── evaluator builtin.coherence ← grandchild (scoring) -├── eval_item 1 -│ ├── invoke_agent my-agent -│ ├── evaluator builtin.similarity -│ └── evaluator builtin.coherence -└── ... -``` - -Each span records **attributes** — structured key-value pairs like `agentops.eval.evaluator.score = 0.87`. - -### What Is OTLP? - -**OTLP** (OpenTelemetry Protocol) is the wire format used to send traces from your application to a backend. AgentOps uses **OTLP/HTTP** with **Protobuf** encoding — which simply means it sends a compact binary HTTP POST to a collector URL. - -### What Is an Exporter? - -An **exporter** is the component that ships span data out of your process. AgentOps uses the `OTLPSpanExporter` from the `opentelemetry-exporter-otlp-proto-http` package, which sends spans over HTTP. - -### What Is a Collector / Backend? - -A **collector** (or backend) is the server that receives spans. Popular options: - -| Collector | Runs Locally? | Cloud? | Best For | -|---|---|---|---| -| [Jaeger](https://www.jaegertracing.io/) | Yes (Docker) | No | Local development, free | -| [Azure Monitor / App Insights](https://learn.microsoft.com/azure/azure-monitor/) | No | Yes | Production on Azure | -| [Grafana Tempo](https://grafana.com/oss/tempo/) | Yes | Yes | Teams already using Grafana | -| [Datadog](https://www.datadoghq.com/) | No | Yes | Multi-cloud SaaS | - -You pick one, point `AGENTOPS_OTLP_ENDPOINT` at it, and spans start flowing. - ---- - -## How AgentOps Uses OTel - -All telemetry logic lives in **one file**: `src/agentops/utils/telemetry.py`. - -### Design Principles - -1. **Opt-in** — Tracing is disabled by default. Set `AGENTOPS_OTLP_ENDPOINT` to turn it on. -2. **Zero-cost when off** — Every function checks `_tracing_enabled` first and returns immediately if `False`. No OTel packages are imported. -3. **Lazy imports** — `opentelemetry` is imported inside `init_tracing()`, not at the top of the file. If you don't have OTel installed and tracing is off, everything still works. -4. **Graceful degradation** — If OTel packages are missing and you set the env var, the `ImportError` is caught silently. No crash. - -### Lifecycle - -``` -1. runner.py calls init_tracing() -2. → reads AGENTOPS_OTLP_ENDPOINT -3. → if empty: return (no-op mode) -4. → if set: import opentelemetry, create TracerProvider, attach OTLP exporter -5. runner.py opens eval_run_span() ← root span starts -6. for each row: -7. open eval_item_span() ← child span -8. open agent_invoke_span() ← grandchild span -9. set_agent_invoke_result() ← record tokens, model -10. record_evaluator_span() × N ← one per evaluator -11. set_eval_item_result() ← mark row pass/fail -12. set_eval_run_result() ← mark run pass/fail -13. runner.py calls shutdown() ← flush & close -``` - -Each of these functions is a **context manager** (using Python's `with` statement), so spans are automatically closed even if an exception occurs. - ---- - -## Quick Start — Local Setup with Jaeger - -The fastest way to see traces is to run [Jaeger](https://www.jaegertracing.io/) locally with Docker. - -### 1. Start Jaeger - -```bash -docker run -d --name jaeger \ - -p 16686:16686 \ - -p 4318:4318 \ - jaegertracing/jaeger:latest -``` - -| Port | Purpose | -|---|---| -| `16686` | Jaeger Web UI | -| `4318` | OTLP/HTTP receiver (this is what AgentOps talks to) | - -### 2. Install the OTel packages - -```bash -pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http -``` - -### 3. Set the environment variable - -```bash -# Linux / macOS -export AGENTOPS_OTLP_ENDPOINT=http://localhost:4318 - -# Windows PowerShell -$env:AGENTOPS_OTLP_ENDPOINT = "http://localhost:4318" -``` - -### 4. Run an evaluation - -```bash -agentops eval run --config .agentops/run.yaml -``` - -### 5. Open Jaeger - -Go to [http://localhost:16686](http://localhost:16686), select the **agentops** service, and click **Find Traces**. You will see the full trace tree for your evaluation run. - ---- - -## Environment Variables - -| Variable | Required? | Default | Description | -|---|---|---|---| -| `AGENTOPS_OTLP_ENDPOINT` | No | *(unset — tracing disabled)* | Base URL of the OTLP/HTTP collector. AgentOps appends `/v1/traces` automatically. | - -That's it — one variable controls everything. - -When unset: -- No OTel packages are imported. -- All telemetry functions are no-ops. -- Zero performance overhead. - ---- - -## The Trace Tree - -Every `agentops eval run` produces one trace with the following span hierarchy: - -``` -RUN kind=SERVER -│ -│ Attributes: -│ cicd.pipeline.name = -│ cicd.pipeline.action.name = "RUN" -│ agentops.eval.dataset = -│ agentops.eval.backend = -│ agentops.eval.target = -│ agentops.eval.model = (if applicable) -│ agentops.eval.agent_id = (if applicable) -│ -├── eval_item 0 kind=INTERNAL -│ │ cicd.pipeline.task.name = "eval_item" -│ │ agentops.eval.item.index = 0 -│ │ agentops.eval.item.input = "..." -│ │ agentops.eval.item.expected = "..." -│ │ agentops.eval.item.passed = true -│ │ -│ ├── invoke_agent my-agent kind=CLIENT -│ │ gen_ai.operation.name = "invoke_agent" -│ │ gen_ai.provider.name = "azure.ai.inference" -│ │ gen_ai.request.model = "gpt-4o" -│ │ gen_ai.agent.id = "my-agent:3" -│ │ gen_ai.usage.input_tokens = 142 -│ │ gen_ai.usage.output_tokens = 87 -│ │ -│ ├── evaluator builtin.similarity kind=INTERNAL -│ │ agentops.eval.evaluator.name = "SimilarityEvaluator" -│ │ agentops.eval.evaluator.builtin = "builtin.similarity" -│ │ agentops.eval.evaluator.score = 0.91 -│ │ agentops.eval.evaluator.threshold = 0.7 -│ │ agentops.eval.evaluator.passed = true -│ │ -│ └── evaluator builtin.coherence kind=INTERNAL -│ agentops.eval.evaluator.score = 0.85 -│ ... -│ -├── eval_item 1 -│ └── ... -│ -└── (final attributes on root span) - cicd.pipeline.result = "success" - agentops.eval.items_total = 10 - agentops.eval.items_passed = 9 - agentops.eval.pass_rate = 0.9 -``` - -### Span Kinds Explained - -| Kind | Meaning | Used For | -|---|---|---| -| `SERVER` | Receives and processes a request | The root eval run span | -| `CLIENT` | Makes an outbound call | Agent/model invocation | -| `INTERNAL` | Internal operation within the service | Eval items, evaluators | - ---- - -## Semantic Conventions (Attributes) - -AgentOps uses three layers of semantic conventions to make traces interoperable with standard OTel tooling. - -### 1. CICD Layer (`cicd.pipeline.*`) - -Maps evaluation runs to the standard CI/CD semantic convention, so tools like Azure Monitor pipelines can understand the structure. - -| Attribute | Example | Description | -|---|---|---| -| `cicd.pipeline.name` | `conversational_agent_baseline` | Bundle name | -| `cicd.pipeline.action.name` | `RUN` | Fixed action type | -| `cicd.pipeline.result` | `success` / `failure` | Overall run outcome | -| `cicd.pipeline.task.name` | `eval_item` | Task type for item spans | -| `cicd.pipeline.task.run.id` | `0` | Row index | -| `cicd.pipeline.task.run.result` | `success` / `failure` | Item outcome | - -### 2. GenAI Layer (`gen_ai.*`) - -Follows the [OTel GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) for agent and model invocation spans. - -| Attribute | Example | Description | -|---|---|---| -| `gen_ai.operation.name` | `invoke_agent` / `chat` | Operation type | -| `gen_ai.provider.name` | `azure.ai.inference` | Provider | -| `gen_ai.request.model` | `gpt-4o` | Requested model deployment | -| `gen_ai.response.model` | `gpt-4o-2024-08-06` | Actual model version | -| `gen_ai.agent.id` | `my-agent:3` | Foundry agent identifier | -| `gen_ai.agent.name` | `my-agent` | Agent display name | -| `gen_ai.agent.version` | `3` | Agent version | -| `gen_ai.usage.input_tokens` | `142` | Input token count | -| `gen_ai.usage.output_tokens` | `87` | Output token count | - -### 3. AgentOps Layer (`agentops.eval.*`) - -Custom attributes for evaluation-specific data that has no standard equivalent. - -| Attribute | Example | Description | -|---|---|---| -| `agentops.eval.dataset` | `smoke-model-direct` | Dataset name | -| `agentops.eval.backend` | `foundry` | Execution backend | -| `agentops.eval.target` | `agent` | Target type | -| `agentops.eval.model` | `gpt-4o` | Model deployment | -| `agentops.eval.agent_id` | `my-agent:3` | Agent ID | -| `agentops.eval.items_total` | `10` | Total rows evaluated | -| `agentops.eval.items_passed` | `9` | Rows passing thresholds | -| `agentops.eval.pass_rate` | `0.9` | Pass rate | -| `agentops.eval.item.index` | `0` | Row index | -| `agentops.eval.item.input` | `"What is 2+2?"` | Input text | -| `agentops.eval.item.expected` | `"4"` | Expected answer | -| `agentops.eval.item.passed` | `true` | Row pass/fail | -| `agentops.eval.evaluator.name` | `SimilarityEvaluator` | Class name | -| `agentops.eval.evaluator.builtin` | `builtin.similarity` | Builtin name | -| `agentops.eval.evaluator.source` | `local` / `foundry` | Where evaluator runs | -| `agentops.eval.evaluator.score` | `0.91` | Numeric score | -| `agentops.eval.evaluator.threshold` | `0.7` | Configured threshold | -| `agentops.eval.evaluator.passed` | `true` | Score vs threshold | - ---- - -## Viewing Traces in Jaeger - -Once you run an evaluation with `AGENTOPS_OTLP_ENDPOINT` set, open Jaeger at [http://localhost:16686](http://localhost:16686). - -### Finding Your Trace - -1. In the **Service** dropdown, select `agentops`. -2. Click **Find Traces**. -3. You will see one trace per evaluation run, named `RUN `. - -### Reading the Timeline - -Jaeger shows spans as horizontal bars on a timeline: - -``` -|============ RUN conversational_agent_baseline (1.2s) ============| - |=== eval_item 0 (400ms) ===| - |= invoke_agent (350ms) =| - |= similarity (20ms) =| - |= coherence (15ms) =| - |=== eval_item 1 (380ms) ===| - |= invoke_agent (330ms) =| - ... -``` - -- **Longer bars** = more time. This immediately shows you where time is spent. -- Click any span to see its **attributes** (the key-value pairs listed above). -- Look for spans with **red** or **error** status to find failures. - -### Common Questions You Can Answer - -| Question | Where to Look | -|---|---| -| Which row was slowest? | Sort `eval_item` spans by duration | -| Why did a row fail? | Check `agentops.eval.item.passed` and evaluator scores | -| How many tokens did the agent use? | Check `gen_ai.usage.input_tokens` + `output_tokens` | -| What was the overall pass rate? | Root span → `agentops.eval.pass_rate` | -| Which evaluator scored lowest? | Compare `agentops.eval.evaluator.score` across evaluator spans | - ---- - -## Sending Traces to Azure Monitor - -For production, you may want traces in Azure Monitor / Application Insights instead of local Jaeger. - -### Option A: Use the OTel Collector as a Proxy - -Run the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) with an Azure Monitor exporter: - -```yaml -# otel-collector-config.yaml -receivers: - otlp: - protocols: - http: - endpoint: 0.0.0.0:4318 - -exporters: - azuremonitor: - connection_string: "InstrumentationKey=;..." - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [azuremonitor] -``` - -Then set `AGENTOPS_OTLP_ENDPOINT=http://localhost:4318`. - -### Option B: Use Azure Monitor's OTLP Endpoint Directly - -Azure Monitor now supports OTLP ingestion natively. Set the endpoint to your Application Insights OTLP ingestion URL: - -```bash -export AGENTOPS_OTLP_ENDPOINT=https://.applicationinsights.azure.com -``` - -Refer to the [Azure Monitor OTLP documentation](https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-configuration) for details. - ---- - -## Evaluation Tracing vs. Agent Execution Tracing - -It is important to understand that AgentOps telemetry covers **evaluation observability** — not agent execution tracing. These are two different things: - -| | Evaluation Tracing (AgentOps) | Agent Execution Tracing (Foundry / Agent Framework) | -|---|---|---| -| **What it traces** | The eval run: which rows were evaluated, what scores each evaluator gave, pass/fail, timing | What the agent did step-by-step: tool calls, LLM calls, retrieval, reasoning | -| **Who provides it** | AgentOps (`telemetry.py` → `runner.py`) | Foundry portal, Agent Framework SDK, Azure Monitor | -| **Where to see it** | Jaeger, Azure Monitor, any OTLP backend | Foundry portal → Agent → Traces tab, Azure Monitor | -| **Activation** | `AGENTOPS_OTLP_ENDPOINT` env var | Automatic for Foundry agents; `configure_azure_monitor()` for custom agents | - -**AgentOps does not reimplement agent execution tracing** — Foundry and the Agent Framework already do that natively. If your agent runs on Foundry or uses the Agent Framework SDK, execution traces are generated automatically and visible in the Foundry portal. - -For custom agents (HTTP or local), make sure your agent code has OTel instrumentation configured (e.g., `azure-monitor-opentelemetry` with `configure_azure_monitor()`). The `agentops-trace` skill can help verify this. - ---- - -## FAQ - -### Do I need OpenTelemetry installed to use AgentOps? - -**No.** OTel is completely optional. If the packages are not installed, or `AGENTOPS_OTLP_ENDPOINT` is not set, everything works normally with zero overhead. - -### What packages do I need for tracing? - -```bash -pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http -``` - -### Is there any performance overhead? - -When tracing is **disabled** (the default), overhead is effectively zero — just a boolean check per function call. - -When tracing is **enabled**, spans are batched and sent asynchronously by the `BatchSpanProcessor`, so the impact on evaluation runtime is minimal. - -### Can I use a different backend (not Jaeger)? - -Yes. Any OTLP-compatible backend works. Just point `AGENTOPS_OTLP_ENDPOINT` at it. Popular options: Grafana Tempo, Datadog, Honeycomb, Zipkin (with an OTLP adapter). - -### Where is the telemetry code? - -One file: [`src/agentops/utils/telemetry.py`](../src/agentops/utils/telemetry.py). - -### Can I extend the spans with custom attributes? - -Not currently via configuration. If you need custom attributes, you can modify `telemetry.py` directly — the API is straightforward. Each span is a standard OTel span, so you can call `span.set_attribute("my.custom.key", value)` anywhere inside a span context. - ---- - -## Summary - -| Topic | Key Point | -|---|---| -| **Activation** | Set `AGENTOPS_OTLP_ENDPOINT` — that's it | -| **Dependencies** | `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http` | -| **Local viewer** | Jaeger via Docker on port `16686` | -| **Production** | Azure Monitor, Grafana Tempo, or any OTLP backend | -| **Overhead** | Zero when disabled, minimal when enabled | -| **Code** | `src/agentops/utils/telemetry.py` (one file) | -| **Standards** | CICD semconv + GenAI semconv + AgentOps custom attributes | diff --git a/docs/tutorial-agent-watchdog.md b/docs/tutorial-agent-watchdog.md new file mode 100644 index 00000000..4e5f718a --- /dev/null +++ b/docs/tutorial-agent-watchdog.md @@ -0,0 +1,237 @@ +# Tutorial — AgentOps Watchdog Agent + +The watchdog agent gives the GenAIOps / DevOps engineer a single +command (and a Copilot Chat extension) that answers the question +*"are my agents healthy in production?"* by combining three signal +sources: + +1. **AgentOps eval history** — every `.agentops/results/*/results.json` + the pipeline has produced. +2. **Azure Monitor / Application Insights** — Foundry agent telemetry + queried via KQL. +3. **Foundry control plane** — agent metadata and recent runs read + through `azure-ai-projects`. + +The agent runs the same checks (regression, latency, errors, safety) +in three form factors: + +| Form factor | Use it when… | +|---|---| +| `agentops agent analyze` (CLI) | You want a Markdown report locally or in CI. | +| `agentops agent serve` (FastAPI Copilot Extension) | You want a chat-driven watchdog inside GitHub Copilot Chat. | +| Container Apps deploy (`templates/agent-server/`) | You want the same Copilot Extension hosted publicly. | + +## 1. Local dry-run + +```bash +pip install agentops-toolkit +agentops init # if you don't already have .agentops/ + +# Optional: drop a starter agent.yaml into the workspace. +cp $(python -c "import agentops, pathlib; print(pathlib.Path(agentops.__file__).parent / 'templates' / 'agent.yaml')") .agentops/agent.yaml + +agentops agent analyze +``` + +The first run produces `.agentops/agent/report.md`. With no +`agent.yaml` the analyzer uses defaults: results-history is the only +active source, Azure Monitor and Foundry control are reported as +`skipped` in the diagnostics block. + +## 2. Wire production telemetry + +Edit `.agentops/agent.yaml`: + +```yaml +sources: + azure_monitor: + enabled: true + app_insights_resource_id: /subscriptions/.../components/myappi + foundry_control: + enabled: true + project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT +``` + +Install the agent extras (lazy SDKs only loaded when sources are +enabled): + +```bash +pip install "agentops-toolkit[agent]" +az login +agentops agent analyze --severity-fail critical +``` + +Exit codes are CI-friendly: + +- `0` — analyzer ran clean +- `2` — a finding meets the configured `--severity-fail` floor +- `1` — runtime / configuration error + +## 2b. Security posture audit (WAF-AI) + +The watchdog can also run a **read-only audit of the Azure footprint** +hosting your agent against the [Microsoft Well-Architected Framework +for AI workloads — Security pillar][waf-ai]. This is opt-in: the +findings live in their own `security` category and are skipped unless +both the `azure_resources` source and the `posture` check are enabled. + +The audit runs five high-impact rules against the Cognitive Services / +Azure OpenAI account: + +| Rule id | Severity | What it checks | +|---|---|---| +| `waf.security.local_auth_disabled` | critical | `disableLocalAuth: true` (Entra ID only, no API keys) | +| `waf.security.public_network_access` | warning | Public access disabled, private endpoint, **or** ACL `defaultAction: Deny` | +| `waf.security.managed_identity` | warning | System- or user-assigned MI present on the account | +| `waf.security.diagnostic_settings` | warning | Diagnostic logs flowing to Log Analytics / storage / event hub | +| `waf.security.content_filter` | critical | Every model deployment has a RAI policy applied | + +Required RBAC: **Reader** on the resource group (or on each +individual resource), granted to whoever runs `agentops agent analyze` +(your local identity locally, or the OIDC-federated identity in CI). + +Enable in `.agentops/agent.yaml`: + +```yaml +sources: + azure_resources: + enabled: true + subscription_id_env: AZURE_SUBSCRIPTION_ID # or set subscription_id directly + resource_group: rg-myproject + cognitive_services_account: ai-services-myproject + +checks: + posture: + enabled: true + pillar: security + # Skip individual rules without disabling the whole check, e.g. + # exclude_rules: + # - waf.security.diagnostic_settings + exclude_rules: [] +``` + +Run only the security category, or skip a specific rule from the CLI: + +```bash +# Run every check, including the WAF audit (the default once enabled). +agentops agent analyze + +# Only run the security audit. +agentops agent analyze --categories security + +# Skip a specific rule on top of any YAML excludes. +agentops agent analyze --exclude-rules waf.security.diagnostic_settings + +# Skip multiple rules. +agentops agent analyze --exclude-rules waf.security.diagnostic_settings,waf.security.managed_identity +``` + +The Markdown report groups findings by category, so security findings +appear under their own `### 🔐 Security` heading with a footer link +back to the WAF-AI guidance. + +[waf-ai]: https://learn.microsoft.com/azure/well-architected/ai/security + +## 3. CI scheduled run + +Pair the analyzer with a GitHub Actions schedule: + +```yaml +on: + schedule: [{ cron: "0 7 * * *" }] + workflow_dispatch: +jobs: + watchdog: + runs-on: ubuntu-latest + permissions: { id-token: write, contents: read } + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: "3.11" } + - run: pip install "agentops-toolkit[agent]" + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - run: agentops agent analyze --severity-fail critical + - uses: actions/upload-artifact@v4 + with: + name: agentops-watchdog-report + path: .agentops/agent/report.md +``` + +## 4. Copilot Chat extension (local) + +```bash +pip install "agentops-toolkit[agent]" +agentops agent serve --no-verify --port 8080 +``` + +Then point a GitHub App's Copilot Extension webhook at +`http://localhost:8080/agents/messages`. **`--no-verify` is +local-only** — never expose that endpoint publicly without signature +validation. + +## 5. Hosted Copilot Extension on Azure Container Apps + +The repo ships a minimal scaffold: + +``` +src/agentops/templates/agent-server/ +├── Dockerfile +├── main.bicep +└── README.md +``` + +Workflow: + +```bash +az acr build --registry --image agentops-watchdog:1.0.0 \ + --file Dockerfile . + +az deployment group create \ + --resource-group \ + --template-file main.bicep \ + --parameters \ + environmentName= \ + image=.azurecr.io/agentops-watchdog:1.0.0 \ + userAssignedIdentityId= \ + appInsightsResourceId= \ + foundryProjectEndpoint= +``` + +The user-assigned identity needs `Monitoring Reader` on the App +Insights resource and `Azure AI Developer` on the Foundry project. + +## What the report looks like + +``` +# AgentOps Watchdog Report + +## Verdict: 🚨 CRITICAL issues found + +## Summary +| Severity | Count | +|---|---| +| 🚨 Critical | 1 | +| ⚠️ Warning | 1 | +| ℹ️ Info | 0 | + +## Sources +| Source | Status | Detail | +|---|---|---| +| `results_history` | `ok` | 7 | +| `azure_monitor` | `ok` | | +| `foundry_control` | `skipped` | no project_endpoint configured | + +## Findings +| Severity | ID | Title | Source | +|---|---|---|---| +| 🚨 `critical` | `regression.coherence` | Regression detected on `coherence` | results_history | +| ⚠️ `warning` | `latency.p95_production` | Production p95 latency exceeds threshold | azure_monitor | +``` + +Each finding has its own *Details* section with a recommendation and +an *Evidence* JSON block — that is the bit you copy/paste into a PR +or an incident. diff --git a/docs/tutorial-agent-workflow.md b/docs/tutorial-agent-workflow.md index 1cc00cb3..b387cc50 100644 --- a/docs/tutorial-agent-workflow.md +++ b/docs/tutorial-agent-workflow.md @@ -1,313 +1,110 @@ -# Tutorial: Evaluating an Agent Workflow with Tools (Agent Framework) +# Tutorial — agent workflow with tool calling -This tutorial shows how to evaluate an **agent with tool calling** built with Microsoft Agent Framework using AgentOps. +Evaluate an agent that calls **tools** (function calls / actions). +AgentOps grades both the **final natural-language answer** *and* the +**tool selection / arguments** the agent chose along the way. -Workflow agents orchestrate multi-step tasks: they interpret user intent, select the right tools, call them with correct arguments, and synthesize a final response. The evaluation measures **task completion, tool selection accuracy, and intent resolution**. +## Required dataset shape -## When to Use This Scenario +What turns a regular dataset into a tool-calling dataset is one or +both of these row fields: -Use the **agent workflow** evaluation when: +| Field | What it is | +|---|---| +| `tool_definitions` | The tools the agent has access to (OpenAI tool-call schema). | +| `tool_calls` | The expected tool calls (name + arguments). | -- Your agent calls external tools or functions (APIs, databases, search, calculations) -- You want to verify the agent selects the correct tool for each task -- You want to check that tool call arguments are accurate -- Your agent is built with Microsoft Agent Framework and runs as local Python code -- You need CI-friendly quality gates for tool-calling agents +When AgentOps sees `tool_calls` (or `tool_definitions`) in the +dataset rows, it auto-selects the **agent workflow** evaluators: +TaskCompletion, ToolCallAccuracy, IntentResolution, TaskAdherence, +plus the conversational baseline (Coherence, Fluency, Similarity, +F1Score, latency). -This tutorial uses the **callable adapter** to invoke the agent directly as a Python function. - -## Prerequisites - -- Python 3.11+ -- AgentOps installed: `pip install agentops-toolkit` -- Microsoft Agent Framework SDK installed (for your agent code) -- An Azure OpenAI deployment for AI-assisted evaluators -- `az login` completed - -## Part 1: Initialize the Workspace +## 1. Bootstrap ```bash -cd your-project-root +pip install agentops-toolkit agentops init +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" ``` -Confirm the agent workflow bundle and dataset exist: - -``` -.agentops/ -├── bundles/ -│ └── agent_workflow_baseline.yaml -├── datasets/ -│ └── smoke-agent-tools.yaml -├── data/ -│ └── smoke-agent-tools.jsonl -└── callable_adapter.py -``` - -## Part 2: Understand the Dataset Format - -Agent workflow evaluation requires richer dataset rows. Review `.agentops/data/smoke-agent-tools.jsonl`: - -```json -{ - "id": "1", - "input": "What is the weather in Seattle today?", - "expected": "I'll check the weather for Seattle. The current temperature is 55°F with partly cloudy skies.", - "tool_definitions": [ - { - "name": "get_weather", - "description": "Get current weather for a city", - "parameters": { - "type": "object", - "properties": { "city": { "type": "string" } }, - "required": ["city"] - } - } - ], - "tool_calls": [ - { "name": "get_weather", "arguments": { "city": "Seattle" } } - ] -} -``` - -Each row contains: -- `input` — The user request -- `expected` — The expected final response -- `tool_definitions` — Available tools the agent can choose from -- `tool_calls` — The expected tool calls (name + arguments) - -The evaluators compare what tools the agent **should have called** vs. what it **actually called**. +## 2. Edit `agentops.yaml` -## Part 3: Point to Your Agent Function - -The callable adapter lets you point AgentOps directly to a Python function in your project — no wrapper code needed. Your function just needs to follow this contract: - -``` -(input_text: str, context: dict) -> dict -``` - -Where the returned dict has at least `{"response": "..."}`, and optionally `{"tool_calls": [...]}`. - -### Option A: Point directly to your existing function - -If your project already has a function with the right signature (or close to it), just reference it in `run.yaml`: +For a Foundry prompt agent that already has tools registered: ```yaml -local: - callable: my_agent.workflow:run_evaluation -``` - -For example, if your Agent Framework code lives in `my_agent/workflow.py`: - -```python -# my_agent/workflow.py - -def run_evaluation(input_text: str, context: dict) -> dict: - """Entry point called by AgentOps for each dataset row.""" - result = my_workflow.invoke( - user_message=input_text, - available_tools=context.get("tool_definitions", []), - ) - return { - "response": result.final_answer, - "tool_calls": [ - {"name": tc.name, "arguments": tc.arguments} - for tc in result.tool_calls - ], - } -``` - -### Option B: Use the starter template - -`agentops init` already creates `.agentops/callable_adapter.py` with the correct signature and placeholder code. Open it and replace the body with your agent invocation — typically 2-3 lines: - -```python -# .agentops/callable_adapter.py (created by agentops init) - -def run_evaluation(input_text: str, context: dict) -> dict: - from my_agent.workflow import run_workflow - - result = run_workflow( - user_message=input_text, - available_tools=context.get("tool_definitions", []), - ) - return { - "response": result.final_answer, - "tool_calls": [ - {"name": tc.name, "arguments": tc.arguments} - for tc in result.tool_calls - ], - } +version: 1 +agent: "weather-bot:2" +dataset: .agentops/data/tools.jsonl ``` -### Return contract - -For the `agent_workflow_baseline` evaluators to work, the return dict should include: -- `"response"` — The agent's final text response (required) -- `"tool_calls"` — A list of tool calls the agent made (optional but recommended for tool accuracy evaluators) - -## Part 4: Configure the Run - -Edit `.agentops/run.yaml` to point to your function and select the workflow bundle: +For an HTTP-deployed agent that returns tool calls in its response +body: ```yaml version: 1 - -target: - type: agent - hosting: local - execution_mode: local - local: - # Point to your function: module.path:function_name - callable: my_agent.workflow:run_evaluation - -bundle: - name: agent_workflow_baseline - -dataset: - name: smoke-agent-tools - -execution: - timeout_seconds: 300 - -output: - write_report: true +agent: "https://aca-weather-bot.example.com/" +http: + request_field: message + response_field: text + tool_calls_field: tool_calls +dataset: .agentops/data/tools.jsonl ``` -Key fields: -- `local.callable` — The `module:function` path to your agent function. Use your project's module path (e.g. `my_agent.workflow:run_evaluation`) or point to the starter template (`callable_adapter:run_evaluation`). -- `target.type: agent` — Identifies this as an agent (not a model) -- `bundle.name: agent_workflow_baseline` — Uses tool-calling evaluators -- `dataset.name: smoke-agent-tools` — Dataset with `tool_definitions` and `tool_calls` fields - -## Part 5: Set Up AI-Assisted Evaluator Credentials +`tool_calls_field` tells AgentOps where in the response JSON to find +the structured tool calls (dot-path notation supported). -The workflow evaluators (TaskCompletionEvaluator, IntentResolutionEvaluator, etc.) are **AI-assisted**. +## 3. Dataset shape (`tools.jsonl`) -```bash -export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://your-project.services.ai.azure.com" -export AZURE_OPENAI_ENDPOINT="https://your-openai.openai.azure.com/" -export AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o" +```jsonl +{"id":"1","input":"What's the weather in Paris, France?","expected":"Calls get_weather with location='Paris, France'.","tool_calls":[{"type":"function_call","name":"get_weather","arguments":{"location":"Paris, France"}}]} +{"id":"2","input":"How is the weather in Tokyo, Japan?","expected":"Calls get_weather with location='Tokyo, Japan'.","tool_calls":[{"type":"function_call","name":"get_weather","arguments":{"location":"Tokyo, Japan"}}]} ``` -Or on Windows (PowerShell): - -```powershell -$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT = "https://your-project.services.ai.azure.com" -$env:AZURE_OPENAI_ENDPOINT = "https://your-openai.openai.azure.com/" -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME = "gpt-4o" -``` +You can additionally include `tool_definitions` to give the evaluator +the schema of every tool the agent should know about. This sharpens +the **ToolSelectionEvaluator** judgement. -## Part 6: Run the Evaluation +## 4. Run ```bash -agentops eval run --config .agentops/run.yaml -``` - -### Output - +agentops eval run ``` -AgentOps evaluation run - Config: .agentops/run.yaml - Bundle: agent_workflow_baseline - Dataset: smoke-agent-tools (5 rows) - Backend: local_adapter (callable) -Processing row 1/5 -Processing row 2/5 -... +The report's per-row block shows: -Results: .agentops/results/latest/results.json -Report: .agentops/results/latest/report.md - -Summary: - Overall: PASSED - Thresholds: 6/6 passed - TaskCompletionEvaluator avg: 4.0 - ToolCallAccuracyEvaluator avg: 4.5 - IntentResolutionEvaluator avg: 4.2 - TaskAdherenceEvaluator avg: 3.8 - ToolSelectionEvaluator avg: 4.1 - ToolInputAccuracyEvaluator avg: 4.3 -``` +- The agent's final text response +- The structured tool calls the agent emitted +- ToolCallAccuracy / IntentResolution / TaskAdherence scores -### Exit Codes +## 5. CI gate -- `0` — All thresholds passed -- `2` — One or more thresholds failed -- `1` — Runtime or configuration error +In a PR check, fail when tool quality regresses. After your first +run, diff every subsequent run against it: -## Thresholds - -The `agent_workflow_baseline` bundle enforces: - -| Evaluator | Criteria | Threshold | -|---|---|---| -| TaskCompletionEvaluator | ≥ | 3.0 | -| ToolCallAccuracyEvaluator | ≥ | 3.0 | -| IntentResolutionEvaluator | ≥ | 3.0 | -| TaskAdherenceEvaluator | ≥ | 3.0 | -| ToolSelectionEvaluator | ≥ | 3.0 | -| ToolInputAccuracyEvaluator | ≥ | 3.0 | -| avg_latency_seconds | ≤ | 15.0 | - -Scores range from 1 to 5. Adjust thresholds in `.agentops/bundles/agent_workflow_baseline.yaml`. - -## Building Your Dataset - -When creating your own dataset for agent workflow evaluation: - -1. **Identify representative tasks** — Cover the main use cases your agent handles -2. **Define tool definitions** — List all tools the agent has access to for each row -3. **Specify expected tool calls** — What tools should be called and with what arguments -4. **Write expected responses** — The ideal final response after tool execution -5. **Include edge cases** — Tasks where no tool should be called, or multiple tools are needed - -Example with multiple tools: - -```json -{ - "id": "multi-tool-1", - "input": "Book a flight from NYC to London and check the weather there", - "expected": "I've found flights from NYC to London and the weather in London is 12°C with rain.", - "tool_definitions": [ - {"name": "search_flights", "description": "Search flights", "parameters": {"type": "object", "properties": {"origin": {"type": "string"}, "destination": {"type": "string"}}, "required": ["origin", "destination"]}}, - {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}} - ], - "tool_calls": [ - {"name": "search_flights", "arguments": {"origin": "NYC", "destination": "London"}}, - {"name": "get_weather", "arguments": {"city": "London"}} - ] -} +```bash +agentops eval run --baseline .agentops/results/latest/results.json ``` -## Comparing with Foundry Agent Evaluation - -If your agent is also deployed to Foundry, you can run the **same bundle** against different targets: +AgentOps loads the baseline into memory before refreshing `latest/`, +so `latest/results.json` is shorthand for "the run before this one". +For CI, commit a stable baseline file (see +[tutorial-baseline-comparison.md](tutorial-baseline-comparison.md)). -| Target | Run Config | Execution | -|---|---|---| -| Local Agent Framework | `local.callable: my_adapter:run_eval` | In-process, fast | -| Foundry Agent | `endpoint.kind: foundry_agent` | Cloud, production-like | +## Build a real tool-calling agent -Use `agentops eval compare` to compare results across targets: +The repo's E2E test deploys a real Microsoft Agent Framework agent +(FastAPI on Container Apps) with a `get_weather` tool. See: -```bash -agentops eval compare --runs .agentops/results/local-run,.agentops/results/foundry-run -``` +- `infra/e2e/agent-app/app.py` — minimal Agent Framework + FastAPI app +- `infra/e2e/perrun.bicep` — per-run ACA deployment +- `scripts/e2e_data/tools.jsonl` — the dataset used to grade it -## CI/CD Integration - -```yaml -- name: Run agent workflow evaluation - run: | - pip install agentops-toolkit - agentops eval run --config .agentops/run.yaml -``` +That same setup is what `tutorial-http-agent.md` walks through. -## Notes +## See also -- **Callable vs HTTP**: Use callable for Agent Framework code that runs in-process. Use HTTP backend (`endpoint.kind: http`) if your agent is deployed as a REST service (LangGraph, ACA, etc.). -- **Tool calls in response**: If your agent framework provides tool call metadata, include it in the callable return dict. The `ToolCallAccuracyEvaluator` and `ToolSelectionEvaluator` use this data. -- **Timeout**: The default timeout is 15 seconds per row for agent workflows. Increase `execution.timeout_seconds` if your agent makes slow external calls. -- **Safety evaluation**: Add the `safe_agent_baseline` bundle as a second evaluation pass to check for content safety issues. +- [tutorial-conversational-agent.md](tutorial-conversational-agent.md) — same shape, no tools +- [tutorial-http-agent.md](tutorial-http-agent.md) — deploying an HTTP agent +- [tutorial-rag.md](tutorial-rag.md) — RAG instead of tools +- [foundry-evaluation-sdk-built-in-evaluators.md](foundry-evaluation-sdk-built-in-evaluators.md) — full evaluator reference diff --git a/docs/tutorial-baseline-comparison.md b/docs/tutorial-baseline-comparison.md index 4b3978c8..17714d48 100644 --- a/docs/tutorial-baseline-comparison.md +++ b/docs/tutorial-baseline-comparison.md @@ -1,258 +1,114 @@ -# Tutorial: Baseline Comparison +# Tutorial — baseline comparison -This tutorial walks through comparing evaluation runs to catch regressions before they reach production. It covers the mechanics of the compare command, but also explores how comparisons behave differently depending on whether you are evaluating a model deployment directly or an agent — and when each approach makes sense. +Detect regressions between two AgentOps runs. This is the workflow +that turns AgentOps from a one-shot evaluation into a quality gate. -## Why compare runs? +## The contract -Every time you change something — a model deployment, an agent's instructions, a retrieval pipeline, or even the evaluation dataset itself — you risk degrading quality without realizing it. A single evaluation run tells you where you stand *now*. Comparing two runs tells you *what changed* and *whether it got worse*. - -This matters most in two situations: -- **Before merging a PR**: did the change improve the agent, or break it? -- **After deploying a new model version**: did quality hold, or did it regress? - -Without comparison, you're looking at absolute scores and hoping you remember what they were last time. With comparison, you get a structured diff that tells you exactly which metrics moved, which thresholds flipped, and which specific rows started failing. - -## Prerequisites - -- Python 3.11+ -- `pip install agentops-toolkit` -- A Foundry project with at least one model deployment (for model-direct) or a deployed agent (for agent evaluation) -- `az login` or equivalent Azure credentials -- Two completed evaluation runs, or the willingness to run two evaluations now - -## Part 1: Choosing your evaluation target - -Before you compare, you need to decide what you're evaluating. AgentOps supports two targets, and they produce meaningfully different results. - -### Model-direct (`target.type: model`) - -Sends your dataset prompts straight to a model deployment and evaluates the raw completions. There is no agent layer — no system instructions, no tools, no retrieval. The model sees each prompt in isolation and responds. - -This is useful when you want to: -- Benchmark a model deployment before building an agent on top of it -- Detect model-level regressions when Azure deploys a new model version -- Measure raw language capabilities (similarity, coherence, fluency) without agent complexity -- Establish a quality floor that your agent should at least match - -In practice, model-direct evaluations tend to produce **higher similarity scores** because the model responds concisely and closely to the expected answer. There is no agent personality reshaping the response. - -Run configuration: -```yaml -target: - type: model - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - model: gpt-5.1 -``` - -### Agent (`target.type: agent`) - -Routes each prompt through a deployed Foundry agent. The agent applies its system instructions, may call tools, may consult a knowledge base, and produces a response shaped by its configuration. - -This is useful when you want to: -- Evaluate the full end-to-end behavior your users actually experience -- Test whether agent instructions and tool configurations work correctly together -- Catch regressions caused by changes to agent settings, not just the underlying model -- Measure real latency including agent orchestration overhead - -Agent evaluations typically produce **lower similarity scores** than model-direct, even on the same questions. This is expected — the agent adds context, rephrases answers in its own style, and may include extra information from tools. A SimilarityEvaluator score of 5.0 on model-direct might become 3.4 on an agent for the same prompt. That does not necessarily mean the agent is worse; it means the agent is doing its job differently. - -Run configuration: -```yaml -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: my-agent:1 - model: gpt-5.1 +```bash +agentops eval run --baseline ``` -### When to compare model-direct vs agent +When `--baseline` is provided: -Comparing a model-direct run against an agent run is valid and sometimes valuable. It answers the question: *how much does the agent layer change the output quality?* +- Each metric is diffed against the baseline. +- The Markdown report grows a **Comparison vs Baseline** table with + 🟢 (improved) / 🔴 (regressed) / ⚪ (unchanged) markers. +- `results.json` includes a top-level `comparison:` block with the + per-metric deltas, machine-readable. -Expect to see: -- **Similarity drops** — the agent rephrases, which lowers textual similarity even when answers are correct -- **Latency increases** — agent orchestration adds overhead (thread creation, polling, tool calls) -- **Threshold flips** — thresholds set for model-direct may be too strict for agent responses +The exit code still follows the threshold contract (`0` / `2` / `1`). +The baseline does **not** by itself fail the run — your thresholds in +`agentops.yaml` do. -If you see a large similarity drop (say, from 5.0 to 1.0), that is worth investigating — the agent may be hallucinating, ignoring the question, or hitting an error in its tool chain. But a moderate drop (5.0 to 3.5) is usually the agent adding its own framing, which is fine. +## 1. Pick a baseline -For ongoing regression detection, compare **like against like**: model-direct against model-direct, or agent against agent. Cross-target comparisons are more diagnostic than gating. +Each `agentops eval run` writes to a timestamped folder under +`.agentops/results/` and refreshes `.agentops/results/latest/` with a +copy. So you have two options: -## Part 2: Running two evaluations +- **Local iteration** — point `--baseline` at + `.agentops/results/latest/results.json`. AgentOps loads the baseline + into memory before refreshing `latest/`, so it always means "the + run before this one". +- **CI / shared baseline** — commit a stable copy into the repo (or + publish it as a CI artifact). This is the path used by the + `agentops-pr.yml` workflow: -### Step 1: Run the baseline + ```bash + mkdir -p .agentops/baseline + cp .agentops/results/latest/results.json .agentops/baseline/results.json + git add .agentops/baseline/results.json + git commit -m "chore: capture AgentOps baseline" + ``` -Pick your target and run: - -```bash -# Model-direct baseline -agentops eval run -c .agentops/run.yaml +Use the first form while iterating; use the second when you want a +baseline that doesn't move every time someone runs `agentops eval run` +locally. -# Or agent baseline -agentops eval run -c .agentops/run-agent.yaml -``` - -This creates a timestamped directory: -``` -.agentops/results/2026-03-19_100000/ -├── results.json -├── report.md -└── backend_metrics.json -``` +## 2. Make your change -The run is also copied to `.agentops/results/latest/`. +Edit your prompt, swap the model deployment, change a tool — anything +you want to evaluate the impact of. -### Step 2: Make a change +## 3. Re-run with `--baseline` -Now change something you want to evaluate: -- Update the model deployment version -- Modify the agent's system instructions -- Add or remove a tool from the agent -- Update the evaluation dataset with new test cases -- Adjust a retrieval pipeline or knowledge base - -### Step 3: Run again +Local iteration (compares against the previous run): ```bash -agentops eval run -c .agentops/run.yaml +agentops eval run --baseline .agentops/results/latest/results.json ``` -You now have two runs under `.agentops/results/`. - -## Part 3: Comparing runs - -The compare command takes two run identifiers separated by a comma. The first is the baseline, the second is the current run. +Against a committed CI baseline: ```bash -# By timestamped folder name -agentops eval compare --runs 2026-03-19_100000,2026-03-19_140000 - -# Using 'latest' for the current run -agentops eval compare --runs 2026-03-19_100000,latest - -# Write output to a specific directory -agentops eval compare --runs 2026-03-19_100000,latest -o .agentops/results/my-comparison -``` - -Run identifiers can be: -- **Timestamped folder names** like `2026-03-19_100000` — resolved under `.agentops/results/` -- **`latest`** — points to the most recent run -- **Paths** — relative or absolute path to a `results.json` file or a directory containing one - -The command produces two files in the current run's output directory (or the `-o` directory): -- `comparison.json` — structured data for automation -- `comparison.md` — readable report for humans and PR reviews - -### Exit codes - -| Code | Meaning | -|---|---| -| `0` | No regressions detected — safe to proceed | -| `2` | Regressions detected — investigate before merging | -| `1` | Error — bad run ID, missing file, or other problem | - -These are the same exit codes used by `agentops eval run`, so CI pipelines handle them consistently. - -## Part 4: Reading the comparison report - -### How metric direction works - -AgentOps figures out whether "up" or "down" is good for each metric by looking at the threshold criteria in your results: - -- Metrics with `>=` or `>` thresholds are **higher-is-better** (e.g., SimilarityEvaluator). A decrease is flagged as a regression. -- Metrics with `<=` or `<` thresholds are **lower-is-better** (e.g., avg_latency_seconds). An increase is flagged as a regression. - -This means if your latency drops from 6s to 4s, the comparison correctly reports it as an **improvement**, not a regression. - -### The summary section - -The summary gives you the quick picture: - +agentops eval run --baseline .agentops/baseline/results.json ``` -Metrics improved: 1 -Metrics regressed: 1 -Thresholds flipped pass→fail: 1 -Items newly failing: 3 -``` - -If `has_regressions` is true (and exit code is 2), at least one of these is nonzero: metrics regressed, thresholds flipped to fail, or items started failing. -### Metric deltas table +Open `.agentops/results/latest/report.md` in VS Code (`code .agentops/results/latest/report.md`, then `Ctrl+Shift+V` for the rendered preview). The new section looks like: -Shows every metric that exists in both runs, with the delta and direction: +```markdown +## Comparison vs Baseline +| Metric | Baseline | Current | Δ | | +|---------------------|----------|---------|-------|-----| +| coherence | 4.20 | 4.45 | +0.25 | 🟢 | +| similarity | 4.10 | 3.85 | -0.25 | 🔴 | +| avg_latency_seconds | 1.94 | 2.71 | +0.77 | 🔴 | ``` -| SimilarityEvaluator | 5.00 | 1.80 | -3.20 | -64% | regressed | -| avg_latency_seconds | 5.69 | 4.59 | -1.10 | -19% | improved | -``` - -### Threshold changes table - -Only shows thresholds that **flipped** between runs. A stable threshold (pass→pass or fail→fail) is omitted for clarity. - -### Item changes table -Only shows rows that changed pass/fail status. If row 3 was passing in both runs, it is not listed. +`Δ` direction is metric-aware: higher latency is bad, higher +similarity is good. -## Part 5: Using comparison in CI +## 4. Wire into a PR check -A typical GitHub Actions pattern: +The `agentops-pr.yml` workflow shipped by `agentops workflow generate` +already supports this — drop a baseline file in your repo (e.g. +`.agentops/baseline/results.json`) and add this step: ```yaml -- name: Run evaluation - run: agentops eval run -o .agentops/results/current - -- name: Compare with baseline - run: agentops eval compare --runs baseline,current - # Exit code 2 fails the job if regressions are detected +- name: Run AgentOps eval against baseline + run: | + agentops eval run \ + --baseline .agentops/baseline/results.json ``` -### Choosing a baseline strategy - -There is no single right way to manage baselines. Pick the one that fits your workflow: - -**Committed baseline** — check a `results.json` into your repo under a stable name (e.g., `.agentops/results/baseline/`). Every PR compares against it. Update the baseline when you intentionally accept a quality change. This is simple and predictable, but requires manual baseline updates. - -**Artifact-based baseline** — download the baseline `results.json` from a previous CI run's artifacts. Each merge to `main` uploads the current results as the new baseline. This automates baseline drift but depends on your CI artifact retention. - -**Rolling latest** — always compare against the previous run. This catches run-over-run regressions but can miss gradual degradation over many runs. - -For most teams, the committed baseline approach works well. It acts as a quality contract: merge only if you match or exceed the baseline. - -## Part 6: Investigating regressions - -When the comparison says regressions were detected, work through these steps: - -1. **Read `comparison.md`** — start with the summary. How many metrics regressed? How many thresholds flipped? How many items are newly failing? +When a PR causes a metric to regress past your threshold, the run +exits `2` and the workflow fails, blocking merge until somebody +either fixes the regression or refreshes the baseline. -2. **Check concentration** — if 1 out of 50 items regressed, that might be a dataset edge case. If 40 out of 50 regressed, something fundamental changed. +## 5. Refresh the baseline -3. **Identify the variable** — what changed between the two runs? Only one thing should change at a time. If you changed the model *and* the dataset *and* the agent instructions simultaneously, you cannot attribute the regression to any single cause. +When a regression is intentional (e.g. you swapped models on +purpose), promote the new run to the baseline: -4. **Look at the actual responses** — read `backend.stdout.log` in the run output directory. It shows the expected and predicted text for each row. Often the root cause is obvious when you see the actual model/agent output. - -5. **Rerun with the previous configuration** — if you suspect the model deployment changed, rerun the baseline dataset against the current deployment. If scores still drop, the model is the cause. If scores hold, something else changed. - -### Typical regression patterns - -**Across-the-board similarity drop** — usually means the model deployment was updated or the agent's system instructions changed in a way that alters response style. Check whether the answers are still *correct* even if they are less *similar* to the expected text. - -**A few rows regressed, most are fine** — likely dataset-specific. Check whether the failing rows have unusual inputs, edge cases, or ambiguous expected answers. - -**Latency increased but quality held** — infrastructure issue, throttling, or the agent is now calling more tools. Check whether new tool calls were added to the agent configuration. - -**Threshold was borderline and flipped** — the metric is near the threshold value and normal variance pushed it over. Consider whether the threshold is set too tightly, or whether the metric genuinely degraded. +```bash +cp .agentops/results/latest/results.json .agentops/baseline/results.json +git add .agentops/baseline/results.json +git commit -m "chore: refresh AgentOps baseline after model upgrade" +``` -## Next steps +## See also -- [Model-Direct Evaluation Tutorial](tutorial-model-direct.md) — evaluate a model deployment without agents -- [RAG Evaluation Tutorial](tutorial-rag.md) — evaluate retrieval-augmented responses -- [Foundry Agent Evaluation Tutorial](tutorial-basic-foundry-agent.md) — evaluate an agent end-to-end -- [CI/CD Integration Guide](ci-github-actions.md) — set up automated evaluation in pipelines -- [CI/CD Integration Guide](ci-github-actions.md) +- [ci-github-actions.md](ci-github-actions.md) — full GenAIOps GitFlow with the four workflow templates +- [tutorial-quickstart.md](tutorial-quickstart.md) — the minimal AgentOps loop diff --git a/docs/tutorial-basic-foundry-agent.md b/docs/tutorial-basic-foundry-agent.md index 22f7ec8e..3504f3ce 100644 --- a/docs/tutorial-basic-foundry-agent.md +++ b/docs/tutorial-basic-foundry-agent.md @@ -104,40 +104,24 @@ agentops init ## Part 3: Configure the agent run -Open `.agentops/run-agent.yaml` and fill in your agent details: +Open `agentops.yaml` at your project root and point it at your agent: ```yaml version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: my-agent:1 # ← your agent name or asst_ ID - model: gpt-5.1 # ← used as judge model for evaluators - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: agent_workflow_baseline -dataset: - name: smoke-agent-tools -execution: - timeout_seconds: 1800 -output: - write_report: true +agent: "my-agent:1" # ← your agent name:version (or asst_ ID) +dataset: .agentops/data/smoke-agent-tools.jsonl +thresholds: + similarity: ">=3" + avg_latency_seconds: "<=20" ``` -Key differences from model-direct: -- `target.type: agent` — routes prompts through the agent instead of calling the model directly -- `target.endpoint.agent_id` — identifies which agent to invoke. Required for agent target. -- `target.endpoint.model` — still needed as the judge model for AI-assisted evaluators like SimilarityEvaluator. This is the model that *evaluates* the agent's responses, not the model the agent uses internally. - -### Why both `agent_id` and `model`? - -The `agent_id` determines *what* you are evaluating (the agent). The `model` determines *how* you evaluate it (the judge model that runs SimilarityEvaluator). They can be different deployments. In practice, most teams use the same deployment for both, but you could use a cheaper model as the judge if cost is a concern. +Key points: +- `agent` is a single string. AgentOps recognizes the `name:version` shape + and routes the run to the Foundry Agent Service automatically. +- The judge model used by AI-assisted evaluators (SimilarityEvaluator) is + taken from `AZURE_OPENAI_DEPLOYMENT` (set in Part 2). +- Evaluators are auto-selected from the dataset row shape — `input` + + `expected` triggers `SimilarityEvaluator`. No `bundle` to maintain. ## Part 4: Review the dataset @@ -157,7 +141,7 @@ For meaningful evaluation, your dataset should match what your agent is designed ## Part 5: Run the evaluation ```bash -agentops eval run -c .agentops/run-agent.yaml +agentops eval run ``` AgentOps will: @@ -166,7 +150,7 @@ AgentOps will: 3. Collect the agent's response 4. Run SimilarityEvaluator comparing the response to the expected answer 5. Measure latency per row -6. Write results under `.agentops/results/latest/` +6. Write results under `.agentops/results//` and mirror them to `.agentops/results/latest/` ### What to expect @@ -180,7 +164,13 @@ A 5-row agent evaluation typically takes 30–60 seconds in local mode, compared ### Reading the results -Open `.agentops/results/latest/report.md`. For an agent with the simple QA instructions above, expect: +Open the report in VS Code and press `Ctrl+Shift+V` to render the Markdown: + +```powershell +code .agentops/results/latest/report.md +``` + +For an agent with the simple QA instructions above, expect: - **SimilarityEvaluator** around 3–4 (the agent captures meaning but rephrases) - **avg_latency_seconds** around 5–15s per row (agent orchestration overhead) @@ -190,40 +180,30 @@ If most rows score 4–5, your agent is working well. If most score 1–2, check ## Part 6: Compare with a baseline -After you change the agent's instructions, add tools, or update the model deployment, run again and compare: - -```bash -agentops eval run -c .agentops/run-agent.yaml -agentops eval compare --runs ,latest -``` - -The comparison shows metric deltas, threshold flips, and per-row changes. See the [Baseline Comparison Tutorial](tutorial-baseline-comparison.md) for the full workflow. +You've only run a single evaluation, so `.agentops/results/` has one timestamped run plus the `latest/` mirror. To compare a future run against it, you don't need to copy anything — just point `--baseline` at the previous result. AgentOps loads the baseline into memory before refreshing `latest/`, so `latest/results.json` works as a shorthand for "the run before this one". -### Comparing agent vs model-direct +**1. Change something** — agent instructions, model deployment, an evaluator threshold, the dataset. -You can also compare your agent run against a model-direct run on the same dataset: +**2. Re-run with `--baseline`:** ```bash -agentops eval compare --runs model-direct-run,agent-run +agentops eval run --baseline .agentops/results/latest/results.json ``` -This tells you how much the agent layer changes the output quality. Expect: -- **Similarity drops** — the agent rephrases, which is normal -- **Latency increases** — agent orchestration adds overhead -- **Possible threshold flips** — thresholds set for model-direct may be too strict for agent responses +> Prefer a stable, named reference? Point at the specific timestamp folder you want to keep, e.g. `--baseline .agentops/results/2026-05-06T20-13-21Z/results.json`. -This comparison is useful for diagnostics but should not be used as a CI gate. Gate model-direct runs against model-direct baselines, and agent runs against agent baselines. +`report.md` now contains a **Comparison vs Baseline** table with per-metric deltas (🟢 improved / 🔴 regressed / ⚪ unchanged). See the [Baseline Comparison Tutorial](tutorial-baseline-comparison.md) for the full PR-gating workflow. ## Evaluation scenarios -AgentOps supports multiple scenarios, each with a different bundle: +AgentOps auto-selects evaluators from the dataset row shape: -| Scenario | Bundle | Target | Evaluators | Use case | -|---|---|---|---|---| -| **Model Quality** | `model_quality_baseline` | `model` | SimilarityEvaluator, CoherenceEvaluator, FluencyEvaluator, F1ScoreEvaluator | Benchmark raw model quality | -| **RAG Quality** | `rag_quality_baseline` | `agent` | GroundednessEvaluator, RelevanceEvaluator, RetrievalEvaluator | Evaluate grounding against context | -| **Conversational** | `conversational_agent_baseline` | `agent` | CoherenceEvaluator, FluencyEvaluator, RelevanceEvaluator, SimilarityEvaluator | Chatbots and Q&A agents | -| **Agent Workflow** | `agent_workflow_baseline` | `agent` | TaskCompletionEvaluator, ToolCallAccuracyEvaluator | Agents with tool calling | +| Scenario | Required row fields | Evaluators auto-selected | Use case | +|---|---|---|---| +| **Model Quality** | `input`, `expected` | SimilarityEvaluator, CoherenceEvaluator, FluencyEvaluator, F1ScoreEvaluator | Benchmark raw model quality | +| **RAG Quality** | `input`, `expected`, `context` | GroundednessEvaluator, RelevanceEvaluator, RetrievalEvaluator | Evaluate grounding against context | +| **Conversational** | `input`, `expected` | CoherenceEvaluator, FluencyEvaluator, RelevanceEvaluator, SimilarityEvaluator | Chatbots and Q&A agents | +| **Agent Workflow** | `input`, `expected`, `tool_definitions`, `tool_calls` | TaskCompletionEvaluator, ToolCallAccuracyEvaluator | Agents with tool calling | The RAG scenario uses GroundednessEvaluator instead of SimilarityEvaluator because the key question is whether the agent's response is grounded in the retrieved context, not whether it matches a specific expected answer. diff --git a/docs/tutorial-conversational-agent.md b/docs/tutorial-conversational-agent.md index 5b49583f..e68f970d 100644 --- a/docs/tutorial-conversational-agent.md +++ b/docs/tutorial-conversational-agent.md @@ -1,258 +1,98 @@ -# Tutorial: Evaluating a Conversational Agent (Agent Framework) +# Tutorial — conversational agent -This tutorial shows how to evaluate a **conversational agent** built with Microsoft Agent Framework using AgentOps. +Evaluate a multi-turn assistant or chatbot. The shape of a +*conversational* agent is identical to any other agent for AgentOps — +what makes it conversational is the **dataset**: the rows can include +prior turns the agent should consider. -Conversational agents — chatbots, Q&A assistants, multi-turn assistants — don't use tool calling or retrieval. The evaluation focuses on **response quality**: coherence, fluency, relevance, and similarity to expected answers. +## When to use this -## When to Use This Scenario +You have an assistant deployed as either: -Use the **conversational agent** evaluation when: +- A **Foundry prompt agent** (`name:version`) +- A **Foundry hosted endpoint** (`https://*.services.ai.azure.com/.../agents/`) +- A **plain HTTP service** (Container Apps, AKS, your own server) -- Your agent responds to open-ended user messages without calling external tools -- You want to measure response quality for a Q&A or chat assistant -- Your agent is built with Microsoft Agent Framework and runs as local Python code -- You want CI-friendly quality gates before deploying +…and you want to measure response **coherence**, **fluency**, +**similarity to a reference answer**, and **latency** across a curated +script of questions. -This tutorial uses the **callable adapter** to invoke the agent directly as a Python function — no subprocess, no HTTP server needed. - -## Prerequisites - -- Python 3.11+ -- AgentOps installed: `pip install agentops-toolkit` -- Microsoft Agent Framework SDK installed (for your agent code) -- An Azure OpenAI deployment for AI-assisted evaluators (CoherenceEvaluator, etc.) -- `az login` completed - -## Part 1: Initialize the Workspace +## 1. Bootstrap ```bash -cd your-project-root +pip install agentops-toolkit agentops init +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" ``` -This creates the `.agentops/` workspace with starter bundles, datasets, and templates. - -Confirm the conversational bundle and dataset exist: - -``` -.agentops/ -├── bundles/ -│ └── conversational_agent_baseline.yaml -├── datasets/ -│ └── smoke-conversational.yaml -├── data/ -│ └── smoke-conversational.jsonl -└── callable_adapter.py -``` - -## Part 2: Point to Your Agent Function - -The callable adapter lets you point AgentOps directly to a Python function in your project. Your function just needs to follow this contract: - -``` -(input_text: str, context: dict) -> dict returning {"response": "..."} -``` - -AgentOps calls it once per dataset row — no wrapper code, no subprocess, no HTTP server. - -### Option A: Point directly to your existing function +## 2. Edit `agentops.yaml` -If your project already has a function with the right signature, just reference it in `run.yaml`: +Pick the form that matches your agent: ```yaml -local: - callable: my_agent.app:chat -``` - -For example, if your Agent Framework code lives in `my_agent/app.py`: - -```python -# my_agent/app.py - -def chat(input_text: str, context: dict) -> dict: - """Entry point called by AgentOps for each dataset row.""" - result = agent.invoke(input_text) - return {"response": result.output} -``` - -### Option B: Use the starter template - -`agentops init` already creates `.agentops/callable_adapter.py` with the correct signature and placeholder code. Open it and replace the body with your agent call — typically 2-3 lines: - -```python -# .agentops/callable_adapter.py (created by agentops init) - -def run_evaluation(input_text: str, context: dict) -> dict: - from my_agent.app import agent - result = agent.invoke(input_text) - return {"response": result.output} +version: 1 +agent: "customer-support:3" # Foundry prompt agent (name:version) +dataset: .agentops/data/chat.jsonl ``` -The function must: -- Accept `(input_text: str, context: dict)` -- Return a dict with at least a `"response"` key -- Be importable from the project root - -## Part 3: Configure the Run - -Edit `.agentops/run.yaml` to point to your function and select the conversational bundle: - ```yaml version: 1 - -target: - type: agent - hosting: local - execution_mode: local - local: - # Point to your function: module.path:function_name - callable: my_agent.app:chat - -bundle: - name: conversational_agent_baseline - -dataset: - name: smoke-conversational - -execution: - timeout_seconds: 300 - -output: - write_report: true +agent: "https://api.example.com/chat" # any HTTP/JSON service +dataset: .agentops/data/chat.jsonl ``` -Key fields: -- `local.callable` — The `module:function` path to your agent function. Use your project's module path (e.g. `my_agent.app:chat`) or point to the starter template (`callable_adapter:run_evaluation`). -- `bundle.name: conversational_agent_baseline` — Evaluates coherence, fluency, relevance, and similarity. -- `dataset.name: smoke-conversational` — The conversational smoke dataset. - -## Part 4: Set Up AI-Assisted Evaluator Credentials - -The conversational evaluators (CoherenceEvaluator, FluencyEvaluator, etc.) are **AI-assisted** — they need an Azure OpenAI model to judge quality. - -Set the environment variables: +For HTTP targets, AgentOps POSTs `{"message": ""}` and reads +the response from the `text` field by default. If your service uses +different field names, override them: -```bash -export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://your-project.services.ai.azure.com" -export AZURE_OPENAI_ENDPOINT="https://your-openai.openai.azure.com/" -export AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o" -``` - -Or on Windows (PowerShell): - -```powershell -$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT = "https://your-project.services.ai.azure.com" -$env:AZURE_OPENAI_ENDPOINT = "https://your-openai.openai.azure.com/" -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME = "gpt-4o" +```yaml +version: 1 +agent: "https://api.example.com/chat" +http: + request_field: prompt + response_field: choices.0.message.content +dataset: .agentops/data/chat.jsonl ``` -## Part 5: Review the Dataset - -Check `.agentops/data/smoke-conversational.jsonl`: +## 3. Dataset shape (`chat.jsonl`) -```json -{"id":"1","input":"Hi, how are you doing today?","expected":"Hello! I'm doing well, thank you for asking. How can I help you today?"} -{"id":"2","input":"Can you explain what machine learning is in simple terms?","expected":"Machine learning is a type of artificial intelligence where computers learn patterns from data..."} +```jsonl +{"id":"1","input":"Hi, can you help me reset my password?","expected":"Sure — could you share the email on your account?"} +{"id":"2","input":"What's the SLA on a refund request?","expected":"Refunds are processed within 5 business days."} +{"id":"3","input":"My order #1234 hasn't arrived. Can you track it?","expected":"I can look that up. One moment, please."} ``` -Each row has: -- `input` — The user message sent to the agent -- `expected` — The reference response for similarity comparison +Rows have `input` and `expected`. With this shape AgentOps +auto-selects the **conversational baseline** evaluators: Coherence, +Fluency, Similarity, F1Score, average latency. -Replace these with real conversations from your agent's domain. +> Want to test multi-turn behaviour explicitly? Have your service +> accept a `history` field, then add `extra_fields: [history]` under +> `http:` and include a `history` array in each JSONL row. -## Part 6: Run the Evaluation - -```bash -agentops eval run --config .agentops/run.yaml -``` - -Or from the project root using default config: +## 4. Run ```bash agentops eval run ``` -### Output - -``` -AgentOps evaluation run - Config: .agentops/run.yaml - Bundle: conversational_agent_baseline - Dataset: smoke-conversational (5 rows) - Backend: local_adapter (callable) - -Processing row 1/5 -Processing row 2/5 -... - -Results: .agentops/results/latest/results.json -Report: .agentops/results/latest/report.md - -Summary: - Overall: PASSED - Thresholds: 4/4 passed - CoherenceEvaluator avg: 4.2 - FluencyEvaluator avg: 4.5 - RelevanceEvaluator avg: 3.8 - SimilarityEvaluator avg: 3.6 -``` - -### Exit Codes - -- `0` — All thresholds passed -- `2` — One or more thresholds failed -- `1` — Runtime or configuration error - -## Part 7: Review the Report +Open the report with `code .agentops/results/latest/report.md` and press `Ctrl+Shift+V` to render the Markdown — verdict, per-row +transcript, and aggregate scores. -Open `.agentops/results/latest/report.md` to see per-row scores and threshold results. +## 5. Lock in a baseline -To regenerate the report from existing results: +No extra step needed — `latest/results.json` is your previous run. +Diff your next run against it: ```bash -agentops report generate --in .agentops/results/latest/results.json -``` - -## Part 8: Compare Runs - -After improving your agent, run the evaluation again and compare: - -```bash -agentops eval run --output .agentops/results/after-improvement -agentops eval compare --runs .agentops/results/latest,.agentops/results/after-improvement -``` - -## Thresholds - -The `conversational_agent_baseline` bundle enforces: - -| Evaluator | Criteria | Threshold | -|---|---|---| -| CoherenceEvaluator | ≥ | 3.0 | -| FluencyEvaluator | ≥ | 3.0 | -| RelevanceEvaluator | ≥ | 3.0 | -| SimilarityEvaluator | ≥ | 3.0 | -| avg_latency_seconds | ≤ | 10.0 | - -Scores range from 1 to 5. Adjust thresholds in `.agentops/bundles/conversational_agent_baseline.yaml` for your quality bar. - -## CI/CD Integration - -Add to your GitHub Actions or Azure Pipelines workflow: - -```yaml -- name: Run conversational agent evaluation - run: | - pip install agentops-toolkit - agentops eval run --config .agentops/run.yaml +# … change a prompt / model / config … +agentops eval run --baseline .agentops/results/latest/results.json ``` -The exit code `2` fails the pipeline when thresholds are not met. +The next report adds *Comparison vs Baseline* with per-metric deltas. -## Notes +## See also -- **Callable vs subprocess**: The callable adapter is faster than subprocess because it avoids process spawning overhead and runs in-process. -- **Module resolution**: The callable path is resolved via `importlib.import_module()`. Ensure your module is importable from the project root (on `sys.path`). -- **AI-assisted evaluators**: CoherenceEvaluator, FluencyEvaluator, RelevanceEvaluator require an Azure OpenAI deployment. SimilarityEvaluator also requires a ground truth reference. -- **Local evaluator only**: If you want to skip AI-assisted evaluators, create a custom bundle with only `exact_match` and `avg_latency_seconds`. +- [tutorial-http-agent.md](tutorial-http-agent.md) — full HTTP-target walkthrough including auth headers +- [tutorial-agent-workflow.md](tutorial-agent-workflow.md) — same shape, plus tool calling +- [tutorial-baseline-comparison.md](tutorial-baseline-comparison.md) — regression detection diff --git a/docs/tutorial-copilot-skills.md b/docs/tutorial-copilot-skills.md deleted file mode 100644 index 6583fa82..00000000 --- a/docs/tutorial-copilot-skills.md +++ /dev/null @@ -1,213 +0,0 @@ -# Tutorial: Installing AgentOps Copilot Skills - -This tutorial explains how to install the AgentOps Copilot skills, what each skill does, and how to verify they are working correctly — including using AgentOps itself to evaluate skill quality. - -## Why install skills? - -When you ask GitHub Copilot a question about running evaluations or investigating a regression, it does its best with general knowledge. But Copilot does not know the specifics of AgentOps — what commands exist, what flags they accept, what outputs they produce, and which commands are still planned but not implemented. - -Skills close that gap. Each skill is a structured document that tells Copilot *exactly* how to help with a particular workflow. After installation, Copilot stops guessing and starts giving accurate, specific guidance grounded in the actual CLI behavior. - -The difference is noticeable. Without the skill, Copilot might suggest `agentops monitor dashboard` (which is planned but not implemented). With the skill, Copilot will tell you honestly that monitoring is planned, and pivot to what you *can* do today — inspect `results.json` and `report.md`. - -## The eight AgentOps skills - -| Skill | Purpose | When it activates | -|---|---|---| -| `agentops-eval` | Runs evaluations and comparisons. Covers `eval run` and `eval compare`. | You ask about running evaluations, starting an eval, comparing runs, or benchmarking. | -| `agentops-config` | Inspects the workspace to detect the evaluation scenario and endpoint, then generates `run.yaml`. | You ask about configuring an evaluation, which bundle to use, or setting up run.yaml. | -| `agentops-dataset` | Generates evaluation datasets (JSONL data + YAML config) tailored to your project. | You ask about creating test data, generating a dataset, or JSONL format. | -| `agentops-report` | Interprets evaluation reports and regenerates them from `results.json`. | You ask about understanding results, what scores mean, or regenerating a report. | -| `agentops-regression` | Guides regression investigation using run comparison. Structures findings into observations vs hypotheses with actionable next steps. | You mention score drops, threshold failures, comparing runs, or quality degradation. | -| `agentops-trace` | Provides guidance on tracing. Redirects to available artifacts while `trace init` is planned. | You ask about tracing, spans, telemetry, or execution details. | -| `agentops-monitor` | Provides guidance on monitoring. Redirects to comparison and CI gating while `monitor show`/`configure` are planned. | You ask about monitoring, dashboards, alerts, or quality trending. | -| `agentops-workflow` | Helps set up CI/CD pipelines with GitHub Actions for automated evaluations and PR gating. | You ask about CI/CD, GitHub Actions, pipelines, or `agentops workflow generate`. | - -The skills are composable: `agentops-config` → `agentops-dataset` → `agentops-eval` → `agentops-report`. Each works independently but integrates naturally in a workflow. `agentops-regression` helps when something goes wrong, `agentops-trace` and `agentops-monitor` set expectations about current vs planned capabilities, and `agentops-workflow` automates the pipeline. - -## Prerequisites - -- VS Code with the GitHub Copilot Chat extension -- The AgentOps CLI installed: `pip install agentops-toolkit` - -The skills reference CLI commands, so Copilot's guidance only works if the CLI is actually available in your environment. - -## Installation - -### Option 1: Install via CLI (recommended) - -The simplest way to install skills is via the AgentOps CLI: - -```bash -pip install agentops-toolkit -agentops skills install -``` - -This auto-detects your coding agent platform (GitHub Copilot, Claude Code) and copies the skills into the correct directory. If no platform is detected, it defaults to GitHub Copilot (`.github/skills/`). - -To install for a specific platform: - -```bash -agentops skills install --platform claude -agentops skills install --platform copilot --platform claude # both -``` - -To ask before installing when no platform is detected: - -```bash -agentops skills install --prompt -``` - -Skills are also installed automatically when you run `agentops init`. - -### Option 2: Install from GitHub - -The skills are distributed from the `Azure/agentops` repository, following the same pattern used by other Azure Copilot skills (like the ones in `microsoft/azure-skills`). - -In VS Code: - -1. Open **Copilot Chat**. -2. Use the skill install flow and point to this repository: - - **Source:** `Azure/agentops` - - **Skill path:** `plugins/agentops/skills/` -3. Select the skills you want to install. - -Once installed, the skills appear in `~/.agents/skills/` and a lock file (`~/.agents/.skill-lock.json`) tracks where they came from. Skills are available across all workspaces. - -### Option 3: Manual copy - -If you prefer to manage skills manually: - -**macOS / Linux:** -```bash -git clone https://github.com/Azure/agentops.git /tmp/agentops -cp -r /tmp/agentops/plugins/agentops/skills/* ~/.agents/skills/ -rm -rf /tmp/agentops -``` - -**Windows (PowerShell):** -```powershell -git clone https://github.com/Azure/agentops.git $env:TEMP\agentops -Copy-Item -Recurse "$env:TEMP\agentops\plugins\agentops\skills\*" "$env:USERPROFILE\.agents\skills\" -Remove-Item -Recurse -Force "$env:TEMP\agentops" -``` - -### Option 4: Project-scoped installation - -If you want the skills available only within a specific repository (useful for teams with different tool versions), copy them into the project: - -```bash -mkdir -p plugins/agentops/skills -cp -r /plugins/agentops/skills/* plugins/agentops/skills/ -``` - -This way the skills travel with the repo and every contributor gets them automatically. - -### Option 5: Agent Plugin Marketplace (cross-tool) - -The AgentOps plugin is published to the **Agent Plugin Marketplace**, which works -across VS Code Copilot, Copilot CLI, and Claude Code. - -**VS Code** — add the marketplace to your workspace or user settings: - -```json -{ - "chat.plugins.extraKnownMarketplaces": ["Azure/agentops"], - "chat.plugins.enabledPlugins": ["agentops-toolkit"] -} -``` - -**Claude Code** — register the marketplace from the CLI: - -```bash -claude plugin marketplace add Azure/agentops -``` - -The marketplace is defined in `.github/plugin/marketplace.json` (the canonical -location for VS Code and Copilot CLI) and `.claude-plugin/marketplace.json` -(the Claude Code discovery location). Both point to the same plugin at -`plugins/agentops/`. - -## Verifying the installation - -Check that the skill directories exist: - -```bash -ls ~/.agents/skills/ -# Expected: agentops-eval/ agentops-config/ agentops-dataset/ agentops-report/ agentops-regression/ agentops-trace/ agentops-monitor/ agentops-workflow/ -``` - -Each directory should contain a `SKILL.md` file with YAML frontmatter (the `name` and `description` fields that Copilot uses for skill matching). - -## Using the skills - -You do not need to invoke skills explicitly. Copilot matches your question to the right skill based on trigger phrases in the skill description. Just ask naturally. - -### Example: starting an evaluation - -> "How do I start running evaluations with AgentOps?" - -With the `agentops-eval` skill installed, Copilot will respond with the correct sequence: `agentops init` to scaffold the workspace, then `agentops eval run` to execute, then point you to `.agentops/results/latest/` for the outputs. It will not suggest commands that do not exist. - -### Example: investigating a regression - -> "My evaluation scores dropped after I switched model deployments. What should I do?" - -With `agentops-regression`, Copilot will suggest running `agentops eval compare --runs ,latest`, then walk you through interpreting the comparison report — which thresholds flipped, which metrics of the model or agent degraded, and whether the issue is broad or concentrated in specific rows. It separates factual observations from hypotheses and ends with concrete next steps. - -### Example: asking about monitoring - -> "Can I set up monitoring alerts for my evaluation quality?" - -With `agentops-monitor`, Copilot will tell you directly that `agentops monitor show` and `configure` commands are planned but not yet implemented. Instead of giving wrong instructions, it pivots to what works today: running evaluations periodically and comparing with `agentops eval compare --runs ,, -f html` to see quality trends. - -### Example: setting up CI/CD - -> "How do I run evals automatically on every PR?" - -With `agentops-workflow`, Copilot will guide you through `agentops workflow generate` to scaffold a GitHub Actions workflow, then help configure OIDC authentication and GitHub secrets. The workflow gates PRs on threshold pass/fail and posts the report as a PR comment. - -## Updating skills - -Pull the latest version from the repository and re-copy: - -```bash -git clone https://github.com/Azure/agentops.git /tmp/agentops -cp -r /tmp/agentops/plugins/agentops/skills/* ~/.agents/skills/ -rm -rf /tmp/agentops -``` - -If you installed via the VS Code skill install flow, the lock file tracks version hashes and will prompt for updates when the source repo changes. - -## Evaluating skill quality with AgentOps - -This is an advanced use case, but a natural one: you can use AgentOps to evaluate the quality of its own Copilot skills. - -The idea is to create a dataset where each row contains a user question paired with the skill content as context, along with an expected answer that reflects correct guidance. Then SimilarityEvaluator measures whether the model (acting as Copilot) produces responses that align with those expectations. - -For example, one row might be: -- **Input:** *"You are a Copilot assistant with this skill: [run-evals SKILL.md]. User asks: Is agentops eval compare available?"* -- **Expected:** *"Yes, agentops eval compare --runs is available. You can compare two runs by providing run IDs separated by a comma."* - -Run it the same way as any other evaluation: - -```bash -agentops eval run -c .agentops/run-skills.yaml -``` - -When we tested this against our three skills, the SimilarityEvaluator scored **4.2 out of 5** — the model consistently produced guidance aligned with what the skills intend. - -This approach is valuable when you are actively iterating on skill content. Before and after editing a skill, run the evaluation and compare: - -```bash -agentops eval compare --runs skill-baseline,latest -``` - -If the score drops, the skill change may have introduced inaccurate or confusing guidance. This is the same regression-detection pattern used for agents and models, applied to the skills themselves. - -## Next steps - -- [Baseline Comparison Tutorial](tutorial-baseline-comparison.md) — compare runs and detect regressions -- [Model-Direct Evaluation Tutorial](tutorial-model-direct.md) — evaluate a model deployment -- [RAG Evaluation Tutorial](tutorial-rag.md) — evaluate retrieval-augmented responses -- [CI/CD Integration Guide](ci-github-actions.md) — automate evaluation in pipelines diff --git a/docs/tutorial-end-to-end.md b/docs/tutorial-end-to-end.md new file mode 100644 index 00000000..d026789d --- /dev/null +++ b/docs/tutorial-end-to-end.md @@ -0,0 +1,680 @@ +# Tutorial — End-to-end with AgentOps + +This is the long-form, do-it-yourself tour of AgentOps. By the end you +will have a real Foundry hosted agent with **three function tools** +under evaluation, a baseline-vs-degraded comparison that demonstrates +tool-call regression detection, four GitFlow CI/CD workflows wired to +your own GitHub repo, and a watchdog report summarising your run +history. + +It takes around 60–90 minutes the first time. Every step is concrete: +you copy a command, you see an artefact, you keep moving. + +> **Why a tool-calling agent?** Production agents fail in interesting +> ways: they pick the wrong tool, fabricate arguments, or skip tool +> use entirely and answer from memory. AgentOps grades all of those +> behaviours — `tool_call_accuracy`, `intent_resolution`, +> `task_adherence` — alongside text quality. A trivia chatbot would +> only exercise the latter; this tutorial uses an agent where tool +> behaviour is the point. + +## What you will build + +- A Foundry hosted **support agent** with three function tools: + `lookup_order`, `refund_order`, `escalate_to_human`. +- A flat `agentops.yaml` pointing at that agent with thresholds on + both text-quality and tool-call metrics. +- A 5-row evaluation dataset of realistic support tickets, each + carrying `tool_definitions` and the expected `tool_calls`. +- Two evaluation runs (a tool-using **v1** baseline and a degraded + **v2** that answers from memory) compared side-by-side. The + baseline-vs-degraded delta shows tool-call accuracy collapse — + exactly the kind of regression CI is meant to catch. +- Four GitFlow workflows (`pr`, `dev`, `qa`, `prod`) wired to your + own GitHub repository, gated on threshold pass/fail. +- A watchdog report combining your run history with optional + Application Insights telemetry. + +## Prerequisites + +- Python 3.11 or later. +- Azure CLI (`az --version`) and `az login` working. +- An Azure AI Foundry project (`AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`). +- A model deployment in that project (`gpt-4o-mini` is enough). +- The **Azure AI User** RBAC role on the Foundry account + (data-plane access required to create agents and call them). +- A GitHub account and the `gh` CLI (or use the web UI for pushes). +- An existing or new GitHub repo — empty is fine; we will populate it. + +> **Verify your auth before running anything.** Most "this should +> have worked" failures in this tutorial come from a stale CLI token +> cache, being logged into the wrong tenant, or missing the role +> above. A 30-second sanity check: +> +> ```powershell +> az account show --query "{tenant:tenantId, user:user.name, sub:name}" -o table +> ``` +> +> If the tenant or subscription is wrong, run `az login --tenant ` +> and `az account set --subscription `. To grant the role +> to yourself (replace the placeholders with your account values): +> +> ```powershell +> az role assignment create ` +> --assignee "" ` +> --role "Azure AI User" ` +> --scope "/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/" +> ``` +> +> A 401 with `"Token not supported"` from +> `create_support_agent.py` almost always means one of: +> +> 1. **Stale CLI token cache** — most common when the script worked +> earlier today and now suddenly fails. Fix: +> ```powershell +> az account clear +> az login +> ``` +> 2. Wrong tenant (see above). +> 3. Missing **Azure AI User** role (see above). + +Set the project endpoint up front so every command picks it up. + +**PowerShell (Windows):** + +```powershell +$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT = "https://.services.ai.azure.com/api/projects/" +$env:AZURE_OPENAI_ENDPOINT = "https://.services.ai.azure.com" +$env:AZURE_OPENAI_DEPLOYMENT = "gpt-4o-mini" +``` + +**bash / zsh (Linux, macOS, WSL):** + +```bash +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +export AZURE_OPENAI_ENDPOINT="https://.services.ai.azure.com" +export AZURE_OPENAI_DEPLOYMENT="gpt-4o-mini" +``` + +> **Watch out for two endpoint shapes.** On a Foundry "AI Services" +> account, both env vars start with the same hostname but the +> project endpoint includes `/api/projects/` while +> `AZURE_OPENAI_ENDPOINT` is **only** the hostname (no path). If you +> paste the project URL into `AZURE_OPENAI_ENDPOINT` the evaluators +> fail with `BadRequest: API version not supported`. AgentOps +> defaults the API version to a release that works against both +> New Foundry and classic Azure OpenAI; override with +> `AZURE_OPENAI_API_VERSION` only if your resource needs a specific +> version. + +> The remaining shell snippets in this tutorial are written for +> **PowerShell** (the default on Windows). bash / zsh users can +> substitute `export VAR=value` for `$env:VAR = "value"`, `cat` +> for `Get-Content`, and `ls` for `Get-ChildItem`. + +## 1. Install AgentOps + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 # bash/zsh: source .venv/bin/activate +python -m pip install -U pip +python -m pip install agentops-toolkit +python -m pip install azure-ai-projects azure-identity azure-ai-evaluation +agentops --version +``` + +> `azure-ai-projects` and `azure-identity` are only needed by the +> helper script that creates the agent. `azure-ai-evaluation` is the +> SDK that runs the local evaluators (`ToolCallAccuracyEvaluator`, +> `IntentResolutionEvaluator`, `CoherenceEvaluator`, …) — without it +> `agentops eval run` exits with +> `Evaluators require the 'azure-ai-evaluation' package`. + +## 2. Create the Foundry hosted support agent + +The tutorial uses **three function tools** that a real support agent +would expose: + +| Tool | Purpose | Required arguments | +|---|---|---| +| `lookup_order` | Look up an order's status. | `order_id` | +| `refund_order` | Refund an order. | `order_id`, `reason` | +| `escalate_to_human` | Hand the conversation to a human agent. | `category` | + +Registering three tools through the portal is fiddly, so this +repository ships a small helper script, +[`scripts/create_support_agent.py`](../scripts/create_support_agent.py), +that does it in one command. **Just download the single file into the +root of your tutorial project** — there's no need to create a +`scripts/` folder, and the script has no AgentOps dependency (only +`azure-ai-projects` and `azure-identity`). Then run it from the same +folder: + +```powershell +python create_support_agent.py create --name support-bot +# stdout: support-bot:1 +``` + +The first line of stdout is the `name:version` identifier you paste +into `agentops.yaml` next. The script: + +- Creates a hosted prompt agent named `support-bot`. +- Registers the three function tools above with strict JSON Schema + parameters. +- Pins the system prompt to require tool use whenever the user asks + about an order, a refund, or talking to a human. +- Prints `support-bot:` on stdout and a friendly summary on + stderr (including a `Registered tools:` line so you can confirm + the attachment). + +> **Why don't I see the tools in the Playground?** The Foundry +> portal's Playground tab only lists tools you added through the +> portal's **Add** button. Tools registered through the SDK (like +> these) show up under the agent's **Code** / **YAML** tab and are +> invoked at runtime — `agentops eval run` exercises them either +> way. + +> **Prefer the portal?** Open +> [Azure AI Foundry](https://ai.azure.com) → your project → **Build → +> Agents → New agent**, register the three function tools manually +> (the script's source is the canonical schema), paste the system +> prompt from `INSTRUCTIONS_GOOD` in the script, save, and copy the +> resulting `name:version` string. + +## 3. Initialize the workspace + +In an empty folder (or the GitHub repo you want to use): + +```powershell +agentops init +``` + +You get: + +``` +.agentops/ +├── agentops.yaml +├── data/ +│ └── smoke.jsonl +├── datasets/ +│ └── smoke.yaml +└── results/ +.github/ +└── skills/ + └── agentops-*/SKILL.md +``` + +Open `.agentops/agentops.yaml` and configure it for the support +agent: + +```yaml +version: 1 +agent: "support-bot:1" + +dataset: ./data/tickets.jsonl + +thresholds: + # Tool-calling metrics (auto-inferred from tool_definitions / + # tool_calls in the dataset). + tool_call_accuracy: ">=0.8" + intent_resolution: ">=4" + task_adherence: ">=0.8" + # Text quality metrics. + coherence: ">=3" + fluency: ">=3" + similarity: ">=3" + # Latency budget. + avg_latency_seconds: "<=10" +``` + +The `agent: "name:version"` shape is recognised as a **Foundry hosted +agent**. AgentOps invokes it through the Foundry project endpoint +using your `az login` credentials. + +## 4. Author the support-ticket dataset + +Replace `.agentops/data/smoke.jsonl` with a new +`.agentops/data/tickets.jsonl` carrying five realistic support +tickets. Each row includes: + +- `input` — the customer message, +- `expected` — the expected outcome in plain prose, +- `tool_definitions` — every tool the agent has access to, +- `tool_calls` — the tool the agent **should** call (or an empty + list when the right behaviour is to answer with no tool). + +The variety of intents — order lookup, refund, escalation, an +ambiguous query that should resolve to a lookup, and a casual +greeting that should *not* trigger any tool — is what gives the +evaluators something interesting to grade. + +```jsonl +{"input": "Where is my order ORD-12345?", "expected": "Calls lookup_order with order_id='ORD-12345'.", "tool_definitions": [{"type": "function", "name": "lookup_order", "description": "Look up an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]}}, {"type": "function", "name": "refund_order", "description": "Refund an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}, "reason": {"type": "string"}}, "required": ["order_id", "reason"]}}, {"type": "function", "name": "escalate_to_human", "description": "Hand the conversation to a human.", "parameters": {"type": "object", "properties": {"category": {"type": "string"}}, "required": ["category"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "c1", "name": "lookup_order", "arguments": {"order_id": "ORD-12345"}}]} +{"input": "I want a refund for ORD-77821, it arrived broken.", "expected": "Calls refund_order with order_id='ORD-77821' and reason mentioning broken.", "tool_definitions": [{"type": "function", "name": "lookup_order", "description": "Look up an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]}}, {"type": "function", "name": "refund_order", "description": "Refund an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}, "reason": {"type": "string"}}, "required": ["order_id", "reason"]}}, {"type": "function", "name": "escalate_to_human", "description": "Hand the conversation to a human.", "parameters": {"type": "object", "properties": {"category": {"type": "string"}}, "required": ["category"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "c2", "name": "refund_order", "arguments": {"order_id": "ORD-77821", "reason": "arrived broken"}}]} +{"input": "Please connect me to a human about my refund — this has dragged on too long.", "expected": "Calls escalate_to_human with category='refund'.", "tool_definitions": [{"type": "function", "name": "lookup_order", "description": "Look up an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]}}, {"type": "function", "name": "refund_order", "description": "Refund an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}, "reason": {"type": "string"}}, "required": ["order_id", "reason"]}}, {"type": "function", "name": "escalate_to_human", "description": "Hand the conversation to a human.", "parameters": {"type": "object", "properties": {"category": {"type": "string"}}, "required": ["category"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "c3", "name": "escalate_to_human", "arguments": {"category": "refund"}}]} +{"input": "Did ORD-99001 ship yet?", "expected": "Calls lookup_order with order_id='ORD-99001'.", "tool_definitions": [{"type": "function", "name": "lookup_order", "description": "Look up an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]}}, {"type": "function", "name": "refund_order", "description": "Refund an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}, "reason": {"type": "string"}}, "required": ["order_id", "reason"]}}, {"type": "function", "name": "escalate_to_human", "description": "Hand the conversation to a human.", "parameters": {"type": "object", "properties": {"category": {"type": "string"}}, "required": ["category"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "c4", "name": "lookup_order", "arguments": {"order_id": "ORD-99001"}}]} +{"input": "Hi there!", "expected": "Replies with a brief greeting and does NOT call any tool.", "tool_definitions": [{"type": "function", "name": "lookup_order", "description": "Look up an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]}}, {"type": "function", "name": "refund_order", "description": "Refund an order.", "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}, "reason": {"type": "string"}}, "required": ["order_id", "reason"]}}, {"type": "function", "name": "escalate_to_human", "description": "Hand the conversation to a human.", "parameters": {"type": "object", "properties": {"category": {"type": "string"}}, "required": ["category"]}}], "tool_calls": []} +``` + +> **Why each row repeats the full `tool_definitions`?** Each dataset +> row is evaluated independently and the evaluators that check tool +> selection / argument accuracy need the **complete** tool catalogue +> per row. Repetition is the cost of row-level isolation; in real +> projects a small Python script can stamp the same definitions into +> every row at dataset-build time. + +The presence of `tool_definitions` and `tool_calls` is what auto- +selects the tool-calling evaluators on top of the standard text- +quality stack. When AgentOps loads the dataset it picks: + +| Evaluator | What it grades | +|---|---| +| `ToolCallAccuracyEvaluator` | Did the agent emit the expected tool calls (name + arguments)? | +| `IntentResolutionEvaluator` | Did the agent resolve the user's intent? | +| `TaskAdherenceEvaluator` | Did the agent stick to the system prompt's tool-use rules? | +| `CoherenceEvaluator` / `FluencyEvaluator` / `SimilarityEvaluator` / `F1ScoreEvaluator` | Standard text quality. | +| `avg_latency_seconds` | End-to-end latency budget. | + +## 5. Run your first evaluation + +```powershell +agentops eval run +``` + +The CLI: + +1. Resolves the target from `agentops.yaml`. +2. Calls the Foundry hosted agent once per row, capturing both the + final text response and the structured tool calls. +3. Runs evaluators using `AZURE_OPENAI_DEPLOYMENT`. +4. Writes a timestamped run under `.agentops/results//` and refreshes + `.agentops/results/latest/` with a copy of it. Pass `--output ` to write + the run only to that path instead. + +Open the report in VS Code (any OS, no extra tooling required) and press `Ctrl+Shift+V` to render the Markdown — tables and ✅/❌ display the same way they do on GitHub: + +```powershell +code .agentops/results/latest/report.md +``` + +> Tip: `Ctrl+K V` opens the rendered preview side-by-side with the source. + +The report has four sections you will revisit often: + +- **Verdict** — one line: pass or fail. +- **Per-row transcript** — input, expected, agent response, the + `tool_calls` the agent emitted, and every metric. The greeting + row's transcript shows an empty `tool_calls` block — useful when + debugging false-positive tool calls. +- **Aggregate metrics** — averages across rows. +- **Thresholds** — every rule from `agentops.yaml` with measured + value. With v1 you should see all the tool-calling thresholds in + the green. + +The exit code is `0` (all thresholds passed) or `2` (one or more +failed). `1` means a runtime error. + +## 6. Compare against a degraded baseline + +This is where the tutorial earns its keep. AgentOps writes every run to a +timestamped folder under `.agentops/results/` and refreshes +`.agentops/results/latest/` with a copy. The v1 run you just executed +is still on disk — you don't need to copy or re-run anything to use it +as the baseline. Just point `--baseline` at the previous run when you +execute v2: + +- `.agentops/results/latest/results.json` works as a shorthand for + "the run before this one" (AgentOps loads it into memory before + refreshing `latest/`). +- For a stable, named reference you can also point at a specific + timestamp folder, e.g. + `.agentops/results/2026-05-06T20-13-21Z/results.json`. + +Now create a **degraded** version of the agent — same model, no +tools, plain-text-only instructions — so the regression demo has +something to detect: + +```powershell +python create_support_agent.py create ` + --name support-bot ` + --variant v2-degraded +# stdout: support-bot:2 +``` + +Update `agentops.yaml`: + +```yaml +agent: "support-bot:2" +``` + +Re-run with the v1 result as the baseline: + +**PowerShell:** + +```powershell +agentops eval run --baseline .agentops/results/latest/results.json +``` + +**bash / zsh:** + +```bash +agentops eval run --baseline .agentops/results/latest/results.json +``` + +Then open the new report: + +```powershell +code .agentops/results/latest/report.md +``` + +Press `Ctrl+Shift+V` to render the Markdown. + +The new `report.md` adds a **Comparison vs Baseline** section with +per-metric deltas. Because v2 has **no tools attached at all**, the +agent literally cannot call `lookup_order`, `refund_order`, or +`escalate_to_human` — every order-specific row degrades to a +plain-text apology. You should see roughly: + +| Metric | Baseline (v1) | Current (v2) | Direction | +|---|---|---|---| +| `tool_call_accuracy` | high (≈ 5) | **collapses to `n/a` / floor** | 🔴 regressed | +| `intent_resolution` | high (≈ 4–5) | **drops noticeably** | 🔴 regressed | +| `task_adherence` | mid–high | **drops to floor (1.0)** | 🔴 regressed | +| `coherence` | ≈ 4 | ≈ 4 | ⚪ unchanged | +| `fluency` | ≈ 4 | ≈ 4 | ⚪ unchanged | +| `similarity` | ≈ 3 | ≈ 3 | ⚪ unchanged | + +Text quality barely moves — the degraded agent is still articulate +and on-topic — but the tool-related metrics collapse, the verdict +flips to fail, and the run exits `2`. **This is the regression-detection +loop you will wire into CI next.** + +> Exact numbers will jitter run-to-run because the evaluators +> themselves are model-graded, and metrics like `task_adherence` use +> an ordinal 1–5 scale (1.0 is the floor, not 0). What matters is the +> *shape* of the delta: tool/task metrics down, text-quality metrics +> flat. + +## 7. Generate the GitFlow workflows + +```powershell +agentops workflow generate +``` + +Four files appear under `.github/workflows/`: + +| Workflow | Trigger | Purpose | +|---|---|---| +| `agentops-pr.yml` | Pull request opened against `develop` or `main` | Runs `agentops eval run` against the baseline; comments the report on the PR; gates merge on threshold pass/fail. | +| `agentops-deploy-dev.yml` | Push to `develop` | Deploys to the **dev** environment after a passing eval. | +| `agentops-deploy-qa.yml` | Push to a `release/*` branch | Deploys to **qa**. | +| `agentops-deploy-prod.yml` | Push to `main` | Deploys to **prod** after a passing eval. | + +Read [`ci-github-actions.md`](ci-github-actions.md) for the full +reference. The defaults are sane: you do not need to edit them yet. + +## 8. Push to GitHub and watch it run + +Initialize the repo and push. Pick a unique suffix (your initials, a +date, anything) so the repo and the app registration you create later +don't collide with someone else running this same tutorial: + +```powershell +$suffix = "" # e.g. "pl-20260507" +git init -b main +git add . +git commit -m "feat: bootstrap AgentOps eval and CI/CD" +gh repo create "support-bot-$suffix" --public --source=. --push +git checkout -b develop +git push -u origin develop +``` + +> **Prefer the portal?** Create the repo at +> [github.com/new](https://github.com/new) named `support-bot-`, +> then push from your terminal: +> `git remote add origin https://github.com//support-bot-.git && git push -u origin main && git push -u origin develop`. + +### Wire the GitHub Environments + +The three workflows (`pr`, `deploy-dev`, `deploy-qa`, `deploy-prod`) +expect a GitHub **environment** per stage, each populated with the same +six variables and a federated credential so Azure trusts GitHub OIDC. + +The next four snippets create everything end-to-end. Run them in order +from the same PowerShell session you used above (so `$suffix` is still +in scope). + +#### 1. Create the app registration GitHub will impersonate + +```powershell +$app = az ad app create --display-name "support-bot-ci-$suffix" | ConvertFrom-Json +az ad sp create --id $app.appId | Out-Null +$client = $app.appId +$tenant = az account show --query tenantId -o tsv +$sub = az account show --query id -o tsv +Write-Host "AZURE_CLIENT_ID = $client" +Write-Host "AZURE_TENANT_ID = $tenant" +Write-Host "AZURE_SUBSCRIPTION_ID = $sub" +``` + +> **Notes** +> - **One app registration vs many.** This tutorial uses a single app +> registration shared across `dev`, `qa`, and `prod` to keep the +> walkthrough short. In production you typically create **one app +> registration per environment** so you can grant least-privilege +> roles per stage and rotate them independently. +> - **No CLI? Use the portal.** Create the app under **Microsoft Entra +> ID → App registrations → New registration**, then set +> `$client = ""` manually before running the +> next snippet. + +#### 2. Create the three environments and push the variables + +```powershell +$foundry = $env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT +$aoai = $env:AZURE_OPENAI_ENDPOINT +$deploy = "gpt-4o-mini" +$repo = gh repo view --json nameWithOwner -q .nameWithOwner + +foreach ($envName in @("dev","qa","prod")) { + gh api -X PUT "repos/$repo/environments/$envName" | Out-Null + gh variable set AZURE_TENANT_ID --env $envName --body $tenant + gh variable set AZURE_SUBSCRIPTION_ID --env $envName --body $sub + gh variable set AZURE_CLIENT_ID --env $envName --body $client + gh variable set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT --env $envName --body $foundry + gh variable set AZURE_OPENAI_ENDPOINT --env $envName --body $aoai + gh variable set AZURE_OPENAI_DEPLOYMENT --env $envName --body $deploy + Write-Host "Configured environment: $envName" +} +``` + +> **Prefer the portal?** Open your repo on github.com → **Settings → +> Environments → New environment** and create `dev`, `qa`, and `prod`. +> For each one, click **Add variable** and add the six rows from the +> table at the top of this section. + +#### 3. Add federated credentials so Azure trusts GitHub OIDC + +One credential per environment. The PR gate workflow runs **inside the +`dev` environment** (so it inherits the same `dev` variables and OIDC +subject) — no separate `pull_request` credential is needed. The JSON is +written to a temp file because `az` does not parse inline JSON reliably +under PowerShell: + +```powershell +$subjects = @{ + "dev" = "repo:${repo}:environment:dev" + "qa" = "repo:${repo}:environment:qa" + "prod" = "repo:${repo}:environment:prod" +} + +foreach ($name in $subjects.Keys) { + $payload = [ordered]@{ + name = "github-$name" + issuer = "https://token.actions.githubusercontent.com" + subject = $subjects[$name] + audiences = @("api://AzureADTokenExchange") + } + $tmp = New-TemporaryFile + $payload | ConvertTo-Json | Set-Content -Path $tmp -Encoding utf8 + + az ad app federated-credential create --id $client --parameters "@$tmp" | Out-Null + Remove-Item $tmp + Write-Host "Added federated credential: $name" +} +``` + +> **Prefer the portal?** Open **Microsoft Entra ID → App registrations +> → support-bot-ci-$suffix → Certificates & secrets → Federated +> credentials → Add credential**. Pick **GitHub Actions deploying Azure +> resources** as the scenario, then create one credential per subject +> in the table above (`environment:dev`, `environment:qa`, +> `environment:prod`). + +#### 4. Grant the app the roles it needs + +```powershell +$spId = az ad sp show --id $client --query id -o tsv + +# Resolve resource IDs from the endpoint URLs (no need to know the RG). +$foundryName = (($env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -split "//")[1] -split "\.")[0] +$aoaiName = (($env:AZURE_OPENAI_ENDPOINT -split "//")[1] -split "\.")[0] + +$foundryId = az resource list --name $foundryName ` + --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].id" -o tsv +$aoaiId = az resource list --name $aoaiName ` + --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].id" -o tsv + +if (-not $foundryId) { throw "Could not resolve Foundry resource id for '$foundryName'" } +if (-not $aoaiId) { throw "Could not resolve Azure OpenAI resource id for '$aoaiName'" } + +# Foundry project — read agents and runs +az role assignment create --assignee-object-id $spId ` + --assignee-principal-type ServicePrincipal ` + --role "Azure AI User" --scope $foundryId | Out-Null + +# Azure OpenAI — call the judge model +az role assignment create --assignee-object-id $spId ` + --assignee-principal-type ServicePrincipal ` + --role "Cognitive Services OpenAI User" --scope $aoaiId | Out-Null + +Write-Host "Roles granted on Foundry project and Azure OpenAI." +``` + +> **Prefer the portal?** Open your Foundry project resource → **Access +> control (IAM) → Add role assignment**, pick **Azure AI User**, and +> assign it to the `support-bot-ci-$suffix` app. Repeat on the Azure +> OpenAI resource with the **Cognitive Services OpenAI User** role. + +### Open a PR + +```powershell +git checkout -b feature/tweak-prompt +# make any small change, e.g. edit tickets.jsonl +git commit -am "test: refine ticket dataset" +git push -u origin feature/tweak-prompt +gh pr create --base develop --fill +``` + +The `agentops-pr.yml` workflow runs. When it finishes you will see: + +- A green or red check on the PR. +- A bot comment with the verdict, threshold table (including the + tool-call metrics), and a link to the full `report.md` artifact. + +Merge the PR. `agentops-deploy-dev.yml` triggers, runs an eval against +the dev environment, and deploys if it passes. + +## 9. Run the Watchdog + +The watchdog reads your accumulated run history and (optionally) +queries Application Insights and the Foundry control plane to flag +drifts that a single eval cannot see — repeated regressions, latency +trends, error spikes, safety findings. + +```powershell +pip install "agentops-toolkit[agent]" +agentops agent analyze +``` + +This produces `.agentops/agent/report.md`. With no `agent.yaml` +present, only the local results-history source is active and Azure +Monitor / Foundry control plane appear as `skipped` in the +diagnostics block. That is enough for the basic regression and +latency checks across all your previous runs. + +To pull production telemetry, drop a starter `agent.yaml` into the +workspace and edit it: + +```powershell +$tpl = python -c "import agentops, pathlib; print(pathlib.Path(agentops.__file__).parent / 'templates' / 'agent.yaml')" +Copy-Item $tpl .agentops/agent.yaml +``` + +```yaml +sources: + results_history: + enabled: true + azure_monitor: + enabled: true + app_insights_resource_id: /subscriptions//resourceGroups//providers/microsoft.insights/components/ + foundry_control: + enabled: true + project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT +``` + +Re-run `agentops agent analyze`. The findings table now mixes signals +from your eval history (including the v1 → v2 tool-call regression) +with live telemetry from the deployed agent. + +> **Optional — WAF-AI security audit.** The watchdog can also run a +> read-only audit of your Foundry resource group against the +> [Well-Architected Framework for AI workloads — Security pillar][waf-ai]. +> Enable the `azure_resources` source and the `posture` check in +> `agent.yaml` (commented stanzas are included), grant your identity +> `Reader` on the resource group, and re-run with +> `agentops agent analyze --categories security`. Full walkthrough: +> [`tutorial-agent-watchdog.md`](tutorial-agent-watchdog.md#2b-security-posture-audit-waf-ai). + +For deeper integration (Copilot Chat extension, ACA deploy), see +[`tutorial-agent-watchdog.md`](tutorial-agent-watchdog.md). + +[waf-ai]: https://learn.microsoft.com/azure/well-architected/ai/security + +## 10. Clean up + +The two agent versions live in your Foundry project until you delete +them. The helper script handles cleanup: + +```powershell +python create_support_agent.py delete --name support-bot +``` + +This removes every version (idempotent — ignores 404s). + +## 11. Where to go next + +You now have the full AgentOps loop running end-to-end with a real +tool-calling agent. From here: + +- **Per-scenario tutorials** — adapt the dataset shape to your own + agent: + - [`tutorial-rag.md`](tutorial-rag.md) — retrieval-augmented agents. + - [`tutorial-agent-workflow.md`](tutorial-agent-workflow.md) — + focused tool-calling reference (single-tool variants, HTTP-hosted + agents, dataset shape details). + - [`tutorial-conversational-agent.md`](tutorial-conversational-agent.md) + — multi-turn assistants. + - [`tutorial-http-agent.md`](tutorial-http-agent.md) — agents + deployed outside Foundry (ACA, AKS, custom). + - [`tutorial-model-direct.md`](tutorial-model-direct.md) — raw + model deployments without an agent layer. +- **Deeper baseline workflows** — + [`tutorial-baseline-comparison.md`](tutorial-baseline-comparison.md). +- **Watchdog as a Copilot extension** — + [`tutorial-agent-watchdog.md`](tutorial-agent-watchdog.md). +- **CI/CD reference** — + [`ci-github-actions.md`](ci-github-actions.md). +- **Architecture and concepts** — + [`how-it-works.md`](how-it-works.md), + [`concepts.md`](concepts.md). diff --git a/docs/tutorial-http-agent.md b/docs/tutorial-http-agent.md index 69353bd5..eefbc7f5 100644 --- a/docs/tutorial-http-agent.md +++ b/docs/tutorial-http-agent.md @@ -176,13 +176,13 @@ The backend: 4. Runs evaluators (`SimilarityEvaluator`, `avg_latency_seconds`). 5. Writes `backend_metrics.json`, then `results.json` and `report.md`. -Output lands in `.agentops/results//` and is also synced to `.agentops/results/latest/`. +Output lands in `.agentops/results//` and is mirrored to `.agentops/results/latest/`. Pass `--output ` to write the run only to that path instead. ## Part 5: Review results **Console:** AgentOps prints a summary with pass/fail per threshold. -**Report:** Open `.agentops/results/latest/report.md` for a human-readable summary. +**Report:** Open the report in VS Code with `code .agentops/results/latest/report.md` and press `Ctrl+Shift+V` to render the Markdown. **JSON:** Parse `.agentops/results/latest/results.json` for machine-readable scores. diff --git a/docs/tutorial-model-direct.md b/docs/tutorial-model-direct.md index a5bda1a4..d6ab4a7c 100644 --- a/docs/tutorial-model-direct.md +++ b/docs/tutorial-model-direct.md @@ -1,182 +1,88 @@ -# Tutorial: Model-Direct Evaluation +# Tutorial — model-direct evaluation -This tutorial runs an evaluation against a model deployment directly — no agent, no retrieval, no tools. The model receives each prompt in isolation and responds. You evaluate those responses using SimilarityEvaluator, which compares the model's answer against an expected reference on an ordinal scale of 1 to 5. - -Model-direct evaluation is the simplest starting point. It tells you what the raw model can do before you add the complexity of an agent layer, and it serves as a quality floor for anything you build on top. - -## When model-direct makes sense - -Use this when you want to: - -- **Benchmark a model deployment** before building an agent. If the model itself cannot answer basic QA correctly, no amount of agent instructions will fix that. -- **Detect model-level regressions** after Azure deploys a new model version or you switch deployments. Run the same dataset, compare results, and see if quality held. -- **Compare model deployments** side by side. Run the same dataset against `gpt-4o` and `gpt-5.1`, then use `agentops eval compare` to see which scores higher. -- **Establish a quality baseline** before investing in agent development. If model-direct scores 5.0 on your dataset and your agent scores 3.4, the gap tells you how much the agent layer is reshaping responses. - -Model-direct evaluations typically produce the **highest similarity scores** because the model responds concisely and directly. There is no agent personality rewriting the answer, no tool calls injecting extra context, and no system instructions shaping the tone. If your model-direct score is already low, the problem is either the dataset, the model, or the evaluator — not the agent. - -### What model-direct does *not* tell you - -Model-direct sends isolated prompts with no conversation history, no system instructions, and no memory of prior turns. It cannot evaluate: - -- Whether your agent handles multi-turn conversations correctly -- Whether tool calls execute and return useful results -- Whether retrieval augmentation improves groundedness -- Whether the agent's personality and guardrails work as intended - -For those, you need agent evaluation. See the [Foundry Agent Tutorial](tutorial-basic-foundry-agent.md). +Evaluate a Foundry **model deployment** (`gpt-4o`, `gpt-5.1`, …) with +no agent layer in between. Use this as your quality floor: if the raw +model can't answer your dataset, no agent prompt will save it. ## Prerequisites -- Python 3.11+ -- Azure CLI (`az login`) -- A Foundry project with at least one model deployment (e.g., `gpt-4o`, `gpt-5.1`) -- `pip install agentops-toolkit` +- Python 3.11+ and `pip install agentops-toolkit` +- A Foundry project with at least one model deployment +- `az login` (AgentOps uses `DefaultAzureCredential`) -## Part 1: Set up - -### 1) Azure login - -```bash -az login -``` - -AgentOps uses `DefaultAzureCredential` — no API keys, no manual token management. For local development, `az login` is all you need. In CI, use a service principal or managed identity. - -### 2) Set the project endpoint - -This is the only required environment variable. You can find it in the Foundry portal under your project settings. - -PowerShell: -```powershell -$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT = "https://.services.ai.azure.com/api/projects/" -``` - -Bash/zsh: -```bash -export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" -``` - -### 3) Initialize the workspace +## 1. Bootstrap ```bash agentops init +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" ``` -This creates `.agentops/` with starter configs, bundles, datasets, and sample data. The default `run.yaml` is already configured for model-direct evaluation. - -## Part 2: Configure the run - -Open `.agentops/run.yaml`. The only thing you need to change is the model deployment name: +## 2. Edit `agentops.yaml` ```yaml version: 1 -target: - type: model - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - model: gpt-5.1 # ← replace with your actual deployment name - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -execution: - timeout_seconds: 1800 -output: - write_report: true +agent: "model:gpt-4o" # <-- key part: the `model:` prefix +dataset: .agentops/data/smoke.jsonl ``` -The key fields: -- `target.type: model` — this is what makes it model-direct (as opposed to `target.type: agent`) -- `target.endpoint.model` — must match an existing deployment in your Foundry project. AgentOps will fail with a clear error if the deployment does not exist. -- No `agent_id` — not needed for model-direct +`agent: "model:"` is the model-direct shape — AgentOps +classifies it as `model_direct`, sends each row's `input` straight to +the deployment, and skips agent infrastructure entirely. -### What the bundle evaluates +## 3. Dataset shape -The `model_quality_baseline` bundle uses two evaluators: -- **SimilarityEvaluator** (source: foundry) — AI-assisted comparison of the model's response against the expected answer. Scores 1–5, threshold ≥ 3. -- **avg_latency_seconds** (source: local) — average response time per row, threshold ≤ 10 seconds. - -## Part 3: Review the dataset - -The sample dataset at `.agentops/data/smoke-model-direct.jsonl` contains five simple QA pairs: +`.agentops/data/smoke.jsonl` (one JSON object per line): ```jsonl {"id":"1","input":"What is the capital of France?","expected":"Paris is the capital of France."} -{"id":"2","input":"Which planet is known as the Red Planet?","expected":"Mars is known as the Red Planet."} +{"id":"2","input":"Which planet is known as the Red Planet?","expected":"Mars is the Red Planet."} ``` -Each row has: -- `input` — the prompt sent to the model -- `expected` — the reference answer that SimilarityEvaluator compares against - -For model-direct evaluation, these prompts are sent raw with no system instructions. The model sees only the `input` text. This is intentional — it isolates the model's capability from any agent configuration. - -### Writing your own dataset - -When you create your own dataset, keep the expected answers in the same style as the model. If the model tends to start with "The answer is..." but your expected answers are terse one-word responses, SimilarityEvaluator will penalize the style mismatch even though the content is correct. Match the level of detail you expect from the model. +The dataset has only `input` and `expected`, so AgentOps auto-selects +the **model quality** evaluators: Coherence, Fluency, Similarity, +F1Score, plus average latency. -## Part 4: Run the evaluation +## 4. Run ```bash agentops eval run ``` -By default this uses `.agentops/run.yaml`. If you want to point to a different config: +Outputs land in `.agentops/results//` and are mirrored to `.agentops/results/latest/`: -```bash -agentops eval run -c .agentops/run.yaml -``` - -AgentOps will: -1. Send each `input` to the model deployment via the Foundry Cloud Evaluation API -2. Run SimilarityEvaluator on each response against the `expected` answer -3. Check thresholds: SimilarityEvaluator ≥ 3 and avg_latency ≤ 10s -4. Write `results.json` and `report.md` under `.agentops/results/latest/` - -### Understanding the output - -Open `.agentops/results/latest/report.md` for the human-readable summary. You will see: +- `results.json` — machine-readable +- `report.md` — Markdown summary with thresholds, per-row metrics, + and aggregate scores. -- **Overall status** — PASS or FAIL based on all thresholds -- **Metrics** — aggregate SimilarityEvaluator score and average latency -- **Item verdicts** — per-row pass/fail showing which specific questions the model handled well or poorly -- **Threshold checks** — which thresholds passed and which failed, with item counts +Exit code `0` = all thresholds passed, `2` = at least one failed, +`1` = configuration / runtime error. -A SimilarityEvaluator score of 5.0 means the model's response is semantically equivalent to the expected answer. Scores of 3–4 mean the response captures the core meaning but may differ in phrasing or detail. Below 3 indicates a meaningful divergence — the model may have missed the point, hallucinated, or provided an unrelated answer. - -## Part 5: Compare against a future run - -After you change model deployments, update the dataset, or modify any configuration, run the evaluation again and compare: +## 5. Compare two model deployments ```bash +# Baseline run on gpt-4o agentops eval run -agentops eval compare --runs ,latest -``` -The comparison report shows exactly what changed — which metrics moved, which thresholds flipped, and which rows started failing. See the [Baseline Comparison Tutorial](tutorial-baseline-comparison.md) for the full workflow. - -## Transitioning to agent evaluation +# Switch agentops.yaml to agent: "model:gpt-5.1", run again, then: +agentops eval run --baseline .agentops/results/latest/results.json +``` -Once you are satisfied with model-direct quality, the next step is usually to build an agent and evaluate it. The transition is straightforward: +AgentOps loads the baseline before refreshing `latest/`, so +`latest/results.json` always means "the run before this one". For a +stable reference, point at a specific timestamp folder instead. -1. Create an agent in the Foundry portal with system instructions and (optionally) tools -2. Copy `run.yaml` to a new file and change `target: model` to `target: agent`, add the `agent_id` -3. Run the same dataset through the agent -4. Compare model-direct vs agent results with `agentops eval compare` +`report.md` now includes a *Comparison vs Baseline* table with +per-metric deltas (🟢 improved / 🔴 regressed / ⚪ unchanged). -Expect similarity scores to drop somewhat — the agent rephrases answers in its own style and may add contextual information. A drop from 5.0 to 3.5 is typical and usually acceptable. A drop to 1.0 suggests the agent is not functioning correctly. +## What model-direct does **not** evaluate -See the [Foundry Agent Tutorial](tutorial-basic-foundry-agent.md) for the full guide. +- Multi-turn conversation behaviour +- Tool calling +- Retrieval-augmented generation (RAG) -## Notes +For those, see: -- Cloud evaluation (default mode) runs the model and evaluators server-side in Foundry. Results appear in the Foundry portal under **Build > Evaluations**. -- Set `AGENTOPS_FOUNDRY_MODE=local` to run evaluators locally instead of via the cloud API. This requires `pip install azure-ai-evaluation`. -- Exit codes: `0` = all thresholds passed, `2` = one or more thresholds failed, `1` = error. +- [tutorial-basic-foundry-agent.md](tutorial-basic-foundry-agent.md) — Foundry prompt agent +- [tutorial-rag.md](tutorial-rag.md) — RAG agent (rows with `context`) +- [tutorial-http-agent.md](tutorial-http-agent.md) — agent deployed as an HTTP service +- [tutorial-agent-workflow.md](tutorial-agent-workflow.md) — agent with tool calling diff --git a/docs/tutorial-quickstart.md b/docs/tutorial-quickstart.md new file mode 100644 index 00000000..43659543 --- /dev/null +++ b/docs/tutorial-quickstart.md @@ -0,0 +1,134 @@ +# Tutorial: minimal quickstart + +This tutorial covers the simplest end-to-end AgentOps flow: bootstrap a workspace, point it at any agent, and run an evaluation. + +> Looking for the long-form, do-it-yourself tour that also covers +> a real tool-calling support agent, baseline comparison, GitFlow +> CI/CD, and the watchdog agent? See +> [tutorial-end-to-end.md](tutorial-end-to-end.md). + +## What you will build + +- A flat `agentops.yaml` at your project root. +- A small JSONL dataset. +- One `agentops eval run` execution producing `results.json` and `report.md`. + +The rest of the toolkit (legacy bundles, multi-file workspaces, custom adapters) still works, but is not required for the common case. + +## Prerequisites + +- Python 3.11 or later. +- Access to a target agent or model. Choose one: + - A **Foundry prompt agent** identified by `name:version` (for example `customer-support:3`). + - A **Foundry hosted endpoint** (`https://*.services.ai.azure.com/.../agents/`). + - A **generic HTTP/JSON agent** deployed anywhere (ACA, AKS, your own server). + - A **raw Foundry model deployment** (e.g. `gpt-4o`). +- For Foundry targets: `az login` (or a service principal) and `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` set. +- For AI-assisted evaluators (Coherence, Groundedness, etc.): `AZURE_OPENAI_ENDPOINT` and `AZURE_OPENAI_DEPLOYMENT` set. + +## 1. Install + +```bash +python -m venv .venv +python -m pip install -U pip +python -m pip install agentops-toolkit +``` + +## 2. Bootstrap the project + +```bash +agentops init +``` + +This creates two files: + +- `agentops.yaml` — your evaluation config (3 lines + comments). +- `.agentops/data/smoke.jsonl` — a 3-row seed dataset. + +## 3. Configure your agent + +Open `agentops.yaml` and set the `agent:` field. The classifier infers the target kind from the value: + +| Value | Resolves to | +| -------------------------------------------------------- | ------------------------------------ | +| `"customer-support:3"` | Foundry prompt agent (`name:version`) | +| `"https://.services.ai.azure.com/.../agents/"` | Foundry hosted endpoint | +| `"https://api.example.com/chat"` | Generic HTTP/JSON agent | +| `"model:gpt-4o"` | Raw Foundry model deployment | + +The full minimal config is just: + +```yaml +version: 1 +agent: "customer-support:3" +dataset: .agentops/data/smoke.jsonl +``` + +## 4. Run the evaluation + +Set credentials and run: + +```bash +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +agentops eval run +``` + +Outputs: + +``` +.agentops/results/ +├── 2026-05-06T14-30-22Z/ # Timestamped run (immutable history) +│ ├── results.json +│ └── report.md +└── latest/ # Mirror of the most recent run + ├── results.json + └── report.md +``` + +To view the report rendered (tables, ✅/❌), open it in VS Code and press `Ctrl+Shift+V`: + +```bash +code .agentops/results/latest/report.md +``` + +The CLI prints `Threshold status: PASSED` (exit code `0`) or `FAILED` (exit code `2`) so you can wire it into CI directly. + +## 5. Compare against a baseline + +Each `agentops eval run` writes to a timestamped folder and refreshes +`.agentops/results/latest/`. To diff a new run against the previous +one, just point `--baseline` at it — no copy needed: + +```bash +# ... change your prompt, model, or dataset ... +agentops eval run --baseline .agentops/results/latest/results.json +``` + +AgentOps loads the baseline into memory before refreshing `latest/`, +so `latest/results.json` is shorthand for "the run before this one". +For a stable reference (e.g. a CI baseline), point at a specific +timestamp folder instead. + +`report.md` now includes a `Comparison vs Baseline` section with per-metric deltas (🟢 improved / 🔴 regressed / ⚪ unchanged). + +## Where evaluators come from + +You did not pick evaluators — AgentOps inferred them: + +- **Always:** Coherence, Fluency, Similarity, F1Score, average latency. +- **If your dataset rows include `context`:** Groundedness, Relevance, Retrieval, ResponseCompleteness. +- **If your dataset rows include `tool_calls` or `tool_definitions`:** TaskCompletion, ToolCallAccuracy, IntentResolution, TaskAdherence. + +To override the auto-selection, list evaluator class names in `agentops.yaml`: + +```yaml +evaluators: + - GroundednessEvaluator + - CoherenceEvaluator +``` + +## Where to go next + +- [`docs/how-it-works.md`](how-it-works.md) — architecture and request flow. +- [`docs/ci-github-actions.md`](ci-github-actions.md) — wire AgentOps into PR checks with OIDC auth. +- The existing tutorials still apply if you stay on the legacy multi-file layout. diff --git a/docs/tutorial-rag.md b/docs/tutorial-rag.md index 72d56753..ce5ad259 100644 --- a/docs/tutorial-rag.md +++ b/docs/tutorial-rag.md @@ -73,40 +73,36 @@ agentops init ## Part 3: Configure the run -Update `.agentops/run-rag.yaml` for RAG evaluation: +Edit `agentops.yaml` at your project root for RAG evaluation: ```yaml version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: rag_quality_baseline -dataset: - name: smoke-rag -execution: - timeout_seconds: 1800 -output: - write_report: true +agent: ":" # e.g. "rag-helper:3" +dataset: .agentops/data/smoke.jsonl +thresholds: + groundedness: ">=3" + relevance: ">=3" + retrieval: ">=3" ``` -Key settings: -- `bundle.name: rag_quality_baseline` — uses `GroundednessEvaluator` -- `target.type: agent` — sends prompts to the Foundry agent -- `target.endpoint.agent_id` — your agent's ID +That is the entire config. AgentOps: + +- Classifies `:` as a Foundry **prompt** agent. +- Auto-selects the RAG evaluators (`Groundedness`, `Relevance`, + `Retrieval`, `ResponseCompleteness`) because dataset rows include a + `context` field (see [Part 4](#part-4-verify-the-dataset)). +- Reads the project endpoint from + `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` (set in [Part 2](#2-configure-the-project-endpoint)). +- Reads the judge-model deployment from + `AZURE_AI_MODEL_DEPLOYMENT_NAME` (set this if your project has more + than one deployment). ## Part 4: Verify the dataset -`agentops init` already created `.agentops/data/smoke-rag.jsonl` with sample data: +`agentops init` already created `.agentops/data/smoke.jsonl`. For RAG +you want each row to include a `context` column — that is what +triggers the auto-selection of `GroundednessEvaluator`. Replace the +seed file with something like: ```jsonl {"id":"1","input":"What is the capital of France?","expected":"Paris is the capital of France.","context":"France is a country in Western Europe. Its capital city is Paris, which is also the largest city in France."} @@ -119,16 +115,18 @@ Key settings: Each row has: - `input` — the question sent to the agent - `expected` — the reference answer -- `context` — the retrieved document context used by `GroundednessEvaluator` +- `context` — the retrieved document context that `GroundednessEvaluator` uses -The `GroundednessEvaluator` checks whether the agent's response is grounded in the `context` column. Set `format.context_field: context` in your dataset YAML so the evaluator maps it correctly. If `context_field` is not set, the evaluator falls back to `expected_field`. +When any row has a `context` field, the RAG evaluator set is added +automatically. -> **Tip**: For a real RAG scenario, populate the `context` field with actual retrieved passages from your knowledge base. +> **Tip**: For a real RAG scenario, populate the `context` field with +> actual retrieved passages from your knowledge base. ## Part 5: Run evaluation ```bash -agentops eval run --config .agentops/run-rag.yaml +agentops eval run ``` This will: @@ -138,6 +136,8 @@ This will: ### Check results +Under `.agentops/results/latest/` (mirrored from the timestamped run): + - `.agentops/results/latest/results.json` - `.agentops/results/latest/report.md` @@ -153,7 +153,10 @@ For model-only evaluation (no retrieval), see the [Model-Direct Tutorial](tutori ## Notes -- The `GroundednessEvaluator` is an AI-assisted evaluator — it uses a judge model to score groundedness. -- Set `backend.model` or `AZURE_AI_MODEL_DEPLOYMENT_NAME` to a deployment that exists in your Foundry project for the judge model. +- `Groundedness`, `Relevance`, `Retrieval`, and `ResponseCompleteness` + are AI-assisted evaluators — they use a judge model. +- Set `AZURE_AI_MODEL_DEPLOYMENT_NAME` to a deployment that exists in + your Foundry project for the judge model. If your project only has + one deployment, this is optional. - Authentication is automatic via `DefaultAzureCredential`. - For local development, `az login` is enough. diff --git a/examples/flat-quickstart/README.md b/examples/flat-quickstart/README.md new file mode 100644 index 00000000..f949eefa --- /dev/null +++ b/examples/flat-quickstart/README.md @@ -0,0 +1,15 @@ +# Flat quickstart example + +The smallest possible AgentOps 1.0 setup: an `agentops.yaml`, a JSONL dataset, and one CLI command. + +## Run + +```bash +cd examples/flat-quickstart +export AZURE_AI_FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +agentops eval run --config agentops.yaml --output ./out +``` + +Edit `agentops.yaml` first to point `agent:` at one of your real targets (Foundry prompt agent, Foundry hosted endpoint, generic HTTP/JSON agent, or `model:`). + +Outputs land in `./out/results.json` and `./out/report.md`. diff --git a/examples/flat-quickstart/agentops.yaml b/examples/flat-quickstart/agentops.yaml new file mode 100644 index 00000000..8efda340 --- /dev/null +++ b/examples/flat-quickstart/agentops.yaml @@ -0,0 +1,23 @@ +version: 1 + +# Pick one of these and remove the others: +# +# Foundry prompt agent (name:version): +agent: "my-rag:1" +# +# Foundry hosted endpoint: +# agent: "https://.services.ai.azure.com/api/projects//agents/" +# +# Generic HTTP/JSON agent (ACA, AKS, custom server): +# agent: "https://api.example.com/chat" +# +# Raw Foundry model deployment: +# agent: "model:gpt-4o" + +dataset: ./dataset.jsonl + +# Optional thresholds (override the auto-selected defaults): +# thresholds: +# coherence: ">=3" +# groundedness: ">=3" +# avg_latency_seconds: "<=10" diff --git a/examples/flat-quickstart/dataset.jsonl b/examples/flat-quickstart/dataset.jsonl new file mode 100644 index 00000000..5f37d6a1 --- /dev/null +++ b/examples/flat-quickstart/dataset.jsonl @@ -0,0 +1,3 @@ +{"input": "What is Microsoft Foundry?", "expected": "Foundry is Microsoft's enterprise AI platform for building, evaluating, and deploying agents."} +{"input": "What does AgentOps do?", "expected": "AgentOps standardizes evaluation workflows for Foundry agents and models."} +{"input": "How do I run an evaluation?", "expected": "Run `agentops eval run` after configuring agentops.yaml."} diff --git a/infra/e2e/agent-app/Dockerfile b/infra/e2e/agent-app/Dockerfile new file mode 100644 index 00000000..6be98c0a --- /dev/null +++ b/infra/e2e/agent-app/Dockerfile @@ -0,0 +1,24 @@ +# AgentOps E2E hello-agent +# +# Tiny FastAPI service exposing POST / -> {"text": "..."} backed by a +# Microsoft Agent Framework agent that calls Azure OpenAI (gpt-4o-mini). +# +# Built and pushed to the per-environment ACR by the e2e workflow's +# bootstrap-live job, then deployed as a per-run Azure Container App so +# the http-aca scenario exercises a real LLM (not just an echo). +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY app.py . + +EXPOSE 8080 + +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/infra/e2e/agent-app/app.py b/infra/e2e/agent-app/app.py new file mode 100644 index 00000000..9c6b979c --- /dev/null +++ b/infra/e2e/agent-app/app.py @@ -0,0 +1,175 @@ +"""AgentOps E2E hello-agent. + +A Microsoft Agent Framework chat agent with **tool calling** exposed over +HTTP so the ``http-aca`` AgentOps scenario can exercise the http-json +invocation path against a real LLM that actually uses tools. + +The agent is configured with one function tool, ``get_weather(location)``. +When the user asks about weather, the LLM picks the tool, the framework +executes it locally, the LLM observes the (canned) tool result and produces +a final natural-language answer. From AgentOps' perspective every request +is a single POST, but inside the agent there are multiple internal turns +(plan -> tool call -> tool result -> answer). This keeps the AgentOps +http-json contract simple while still exercising tool-call evaluation +metrics like ``tool_call_accuracy``. + +Endpoints: + GET / -> health check (``{"ok": true, "ready": }``) + POST / -> chat (``{"message": "..."}`` -> + ``{"text": "...", "tool_calls": [...]}``) + +Auth: + Azure OpenAI is reached via Microsoft Entra ID using + ``DefaultAzureCredential``. In Azure Container Apps this resolves to + the container's managed identity, which must be granted ``Cognitive + Services OpenAI User`` on the AI Services / Foundry account. + +Required environment: + AZURE_OPENAI_ENDPOINT e.g. https://.openai.azure.com/ + AZURE_OPENAI_DEPLOYMENT deployment name, e.g. ``gpt-4o-mini`` + +Optional: + AZURE_CLIENT_ID user-assigned managed identity client id + (DefaultAzureCredential picks it up + automatically when set). +""" + +from __future__ import annotations + +import logging +import os +from contextlib import asynccontextmanager +from typing import Any, Optional + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("hello-agent") + +INSTRUCTIONS = ( + "You are a concise factual assistant. " + "When the user asks about the weather in a location, you MUST call the " + "`get_weather` tool with that location instead of guessing. " + "After the tool returns, summarize the weather for the user in one short " + "sentence. For non-weather questions, answer directly in one short " + "sentence with no caveats or follow-ups." +) + +_agent = None +_credential = None + + +def _make_get_weather_tool(): + """Build the @tool-decorated get_weather function lazily. + + Defined inside a factory because the decorator import requires the + agent_framework package to be available, which we do at lifespan time + only (so the container can start even if deps are missing). + """ + from agent_framework import FunctionInvocationContext, tool + + @tool(approval_mode="never_require") + def get_weather(location: str, ctx: FunctionInvocationContext) -> str: + """Get the current weather for a given location.""" + result_text = ( + f"It's 72°F (22°C) and partly cloudy in {location}, with light winds." + ) + # Per-request list passed via function_invocation_kwargs so each + # POST captures only its own tool calls (no global state). + captured = ctx.kwargs.get("captured_calls") + if isinstance(captured, list): + captured.append({ + "type": "function_call", + "name": "get_weather", + "arguments": {"location": location}, + "result": result_text, + }) + return result_text + + return get_weather + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Initialize the agent once at startup, close credential at shutdown.""" + global _agent, _credential + from agent_framework import Agent + from agent_framework.openai import OpenAIChatCompletionClient + from azure.identity.aio import DefaultAzureCredential + + endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT") + deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT") + + if not endpoint or not deployment: + log.warning( + "AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_DEPLOYMENT not set — agent " + "will return 503 on POST until configured." + ) + yield + return + + _credential = DefaultAzureCredential() + _agent = Agent( + client=OpenAIChatCompletionClient( + model=deployment, + azure_endpoint=endpoint, + credential=_credential, + ), + instructions=INSTRUCTIONS, + tools=[_make_get_weather_tool()], + ) + log.info( + "Agent initialized (endpoint=%s, deployment=%s, tools=[get_weather])", + endpoint, deployment, + ) + try: + yield + finally: + if _credential is not None: + await _credential.close() + + +app = FastAPI(title="agentops-e2e-hello-agent", lifespan=lifespan) + + +class ChatRequest(BaseModel): + message: str + + +@app.get("/") +async def root(): + return { + "ok": True, + "agent": "agentops-e2e-hello-agent", + "ready": _agent is not None, + "tools": ["get_weather"] if _agent is not None else [], + } + + +def _extract_text(result: Any) -> str: + for attr in ("text", "content", "message"): + value = getattr(result, attr, None) + if isinstance(value, str) and value.strip(): + return value + return str(result) + + +@app.post("/") +async def chat(req: ChatRequest): + if _agent is None: + raise HTTPException( + status_code=503, + detail="agent not initialized; check AZURE_OPENAI_ENDPOINT/DEPLOYMENT and managed identity role", + ) + captured_calls: list[dict] = [] + try: + result = await _agent.run( + req.message, + function_invocation_kwargs={"captured_calls": captured_calls}, + ) + except Exception as exc: # noqa: BLE001 — surface real error to caller + log.exception("agent.run failed") + raise HTTPException(status_code=500, detail=f"agent.run failed: {exc}") from exc + + return {"text": _extract_text(result), "tool_calls": captured_calls} diff --git a/infra/e2e/agent-app/requirements.txt b/infra/e2e/agent-app/requirements.txt new file mode 100644 index 00000000..5df35411 --- /dev/null +++ b/infra/e2e/agent-app/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.115.6 +uvicorn[standard]==0.32.1 +agent-framework==1.2.1 +azure-identity==1.19.0 diff --git a/infra/e2e/bootstrap.bicep b/infra/e2e/bootstrap.bicep new file mode 100644 index 00000000..d1d1b1dd --- /dev/null +++ b/infra/e2e/bootstrap.bicep @@ -0,0 +1,211 @@ +// AgentOps E2E — long-lived shared infrastructure. +// +// Deploy once into an existing resource group. Creates the heavy/slow +// resources that the e2e workflow reuses across runs: AI Services account, +// Foundry project, gpt-4o-mini model deployment, Container Apps managed +// environment (with Log Analytics), and a Container Registry. +// +// Per-run ephemeral resources (ACA echo app, Foundry agents) are deployed +// separately by perrun.bicep + scripts/e2e_create_agents.py. +// +// Usage: +// az deployment group create \ +// -g \ +// -f infra/e2e/bootstrap.bicep \ +// -p prefix=agentops-e2e +// +// The deployment is idempotent — re-running it is a fast no-op when no +// resource shape changed. + +targetScope = 'resourceGroup' + +@description('Azure region. Must support AI Services + gpt-4o-mini + Container Apps.') +param location string = resourceGroup().location + +@description('Short prefix used for naming resources. Lowercase, max 12 chars.') +@maxLength(12) +param prefix string = 'agentopse2e' + +@description('Capacity (TPM, in thousands of tokens per minute) for the gpt-4o-mini deployment.') +@minValue(1) +@maxValue(500) +param modelCapacity int = 100 + +@description('Model deployment name surfaced to AgentOps as model:.') +param modelDeploymentName string = 'gpt-4o-mini' + +@description('Underlying model name to deploy.') +param modelName string = 'gpt-4o-mini' + +@description('Underlying model version. Pin to a specific version for reproducibility.') +param modelVersion string = '2024-07-18' + +var suffix = uniqueString(resourceGroup().id, prefix) +var aiServicesName = '${prefix}-ai-${suffix}' +var projectName = '${prefix}-proj' +var logAnalyticsName = '${prefix}-law-${suffix}' +var acaEnvName = '${prefix}-acaenv-${suffix}' +// ACR names must be alphanumeric, lowercase, 5-50 chars. +var acrName = toLower(replace('${prefix}acr${suffix}', '-', '')) + +// ---------- AI Services + Foundry project ---------- + +resource aiServices 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' = { + name: aiServicesName + location: location + identity: { + type: 'SystemAssigned' + } + kind: 'AIServices' + sku: { + name: 'S0' + } + properties: { + customSubDomainName: aiServicesName + publicNetworkAccess: 'Enabled' + disableLocalAuth: false + allowProjectManagement: true + } +} + +resource foundryProject 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' = { + parent: aiServices + name: projectName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + displayName: projectName + description: 'AgentOps e2e shared Foundry project.' + } +} + +resource gptDeployment 'Microsoft.CognitiveServices/accounts/deployments@2025-04-01-preview' = { + parent: aiServices + name: modelDeploymentName + sku: { + name: 'GlobalStandard' + capacity: modelCapacity + } + properties: { + model: { + format: 'OpenAI' + name: modelName + version: modelVersion + } + versionUpgradeOption: 'OnceNewDefaultVersionAvailable' + } +} + +// ---------- Log Analytics + Container Apps env ---------- + +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsName + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + } +} + +resource acaEnv 'Microsoft.App/managedEnvironments@2024-03-01' = { + name: acaEnvName + location: location + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalytics.properties.customerId + sharedKey: logAnalytics.listKeys().primarySharedKey + } + } + zoneRedundant: false + } +} + +// ---------- Container Registry ---------- + +resource acr 'Microsoft.ContainerRegistry/registries@2023-11-01-preview' = { + name: acrName + location: location + sku: { + name: 'Basic' + } + properties: { + adminUserEnabled: false + publicNetworkAccess: 'Enabled' + } +} + +// ---------- Long-lived UAMI for per-run ACA hello-agent apps ---------- +// +// The hello-agent ACA app (deployed by perrun.bicep on every workflow run) +// pulls its image from the ACR and calls Azure OpenAI. We give it a *single*, +// long-lived User-Assigned Managed Identity here — instead of creating a new +// UAMI per run — because Entra ID role assignments take several minutes to +// propagate to issued tokens, and a freshly-created UAMI will see 401s from +// Azure OpenAI for the entire duration of a typical e2e run. Reusing the same +// UAMI across runs sidesteps that propagation delay entirely. + +var acaUamiName = '${prefix}-aca-uami-${suffix}' + +// Built-in role definition ids (subscription-scoped). +var acrPullRoleId = '7f951dda-4ed3-4680-a7ca-43fe172d538d' +var openAiUserRoleId = '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' // Cognitive Services OpenAI User + +resource acaUami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: acaUamiName + location: location +} + +resource acaUamiAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(acr.id, acaUami.id, 'AcrPull') + scope: acr + properties: { + principalId: acaUami.properties.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId) + } +} + +resource acaUamiOpenAiUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(aiServices.id, acaUami.id, 'CognitiveServicesOpenAIUser') + scope: aiServices + properties: { + principalId: acaUami.properties.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', openAiUserRoleId) + } +} + +// ---------- Outputs (capture into GitHub Actions Variables) ---------- + +@description('Foundry project endpoint URL — set as AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT.') +output foundryProjectEndpoint string = 'https://${aiServices.name}.services.ai.azure.com/api/projects/${projectName}' + +@description('Azure OpenAI endpoint of the AI Services account.') +output azureOpenAiEndpoint string = aiServices.properties.endpoint + +@description('Model deployment name — set as AZURE_E2E_MODEL_DEPLOYMENT.') +output modelDeployment string = modelDeploymentName + +@description('Container Apps managed environment resource id — set as AZURE_E2E_ACA_ENV_ID.') +output acaEnvironmentId string = acaEnv.id + +@description('ACR login server — set as AZURE_E2E_ACR_LOGIN_SERVER.') +output acrLoginServer string = acr.properties.loginServer + +@description('AI Services account name (for diagnostics).') +output aiServicesName string = aiServices.name + +@description('Resource id of the long-lived UAMI used by per-run ACA hello-agent apps. Has AcrPull on the ACR and Cognitive Services OpenAI User on AI Services.') +output acaUamiResourceId string = acaUami.id + +@description('Client id (appId) of the long-lived UAMI — set as AZURE_CLIENT_ID inside the ACA container so DefaultAzureCredential picks the right identity.') +output acaUamiClientId string = acaUami.properties.clientId + +@description('Name of the long-lived UAMI (for diagnostics).') +output acaUamiName string = acaUami.name diff --git a/infra/e2e/bootstrap.parameters.example.json b/infra/e2e/bootstrap.parameters.example.json new file mode 100644 index 00000000..79b98847 --- /dev/null +++ b/infra/e2e/bootstrap.parameters.example.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "location": { "value": "eastus2" }, + "prefix": { "value": "agentopse2e" }, + "modelCapacity": { "value": 10 }, + "modelDeploymentName": { "value": "gpt-4o-mini" }, + "modelName": { "value": "gpt-4o-mini" }, + "modelVersion": { "value": "2024-07-18" } + } +} diff --git a/infra/e2e/perrun.bicep b/infra/e2e/perrun.bicep new file mode 100644 index 00000000..e0a31981 --- /dev/null +++ b/infra/e2e/perrun.bicep @@ -0,0 +1,121 @@ +// AgentOps E2E — per-run ephemeral resources. +// +// Deploys an Azure Container Apps app running the AgentOps E2E hello-agent +// (Microsoft Agent Framework + Azure OpenAI) so the http-aca scenario can +// exercise AgentOps' http-json invocation path against a real LLM. +// +// Auth flow (no secrets): +// The long-lived UAMI created by bootstrap.bicep already has AcrPull on +// the ACR and Cognitive Services OpenAI User on the AI Services account, +// so all this template does is attach that UAMI to the ACA app. +// +// Per-run resources are named with a unique suffix derived from +// github.run_id so multiple workflow runs do not collide and teardown is a +// straight `az containerapp delete`. + +targetScope = 'resourceGroup' + +@description('Azure region for the ACA app.') +param location string = resourceGroup().location + +@description('Resource id of the long-lived Container Apps managed environment from bootstrap.bicep.') +param acaEnvironmentId string + +@description('Unique suffix for this workflow run (e.g. github.run_id).') +param suffix string + +@description('Fully qualified container image (e.g. .azurecr.io/agentops-e2e/hello-agent:run123).') +param image string + +@description('Login server of the long-lived ACR (e.g. .azurecr.io) — used by the registry config.') +param acrLoginServer string + +@description('Resource id of the long-lived UAMI (created by bootstrap.bicep) that already has AcrPull + Cognitive Services OpenAI User. Reusing a long-lived UAMI avoids the multi-minute Entra ID propagation delay a fresh per-run UAMI would suffer.') +param uamiResourceId string + +@description('Client id of the long-lived UAMI — set as AZURE_CLIENT_ID in the container so DefaultAzureCredential picks it.') +param uamiClientId string + +@description('Azure OpenAI endpoint URL (https://.cognitiveservices.azure.com/ or .openai.azure.com/).') +param azureOpenAiEndpoint string + +@description('Azure OpenAI deployment name (e.g. gpt-4o-mini).') +param azureOpenAiDeployment string + +@description('Container target port. The hello-agent listens on 8080 by default.') +param targetPort int = 8080 + +var appName = 'aca-agent-${suffix}' + +resource agentApp 'Microsoft.App/containerApps@2024-03-01' = { + name: appName + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${uamiResourceId}': {} + } + } + properties: { + managedEnvironmentId: acaEnvironmentId + configuration: { + activeRevisionsMode: 'Single' + ingress: { + external: true + targetPort: targetPort + transport: 'auto' + allowInsecure: false + traffic: [ + { + latestRevision: true + weight: 100 + } + ] + } + registries: [ + { + server: acrLoginServer + identity: uamiResourceId + } + ] + } + template: { + containers: [ + { + name: 'agent' + image: image + resources: { + cpu: json('0.5') + memory: '1Gi' + } + env: [ + { + name: 'AZURE_OPENAI_ENDPOINT' + value: azureOpenAiEndpoint + } + { + name: 'AZURE_OPENAI_DEPLOYMENT' + value: azureOpenAiDeployment + } + { + // DefaultAzureCredential needs the UAMI client id to disambiguate + // when more than one identity could be picked up. + name: 'AZURE_CLIENT_ID' + value: uamiClientId + } + ] + } + ] + scale: { + minReplicas: 1 + maxReplicas: 1 + } + } + } +} + +@description('Public ingress URL for the hello-agent — used by the http-aca scenario.') +output agentUrl string = 'https://${agentApp.properties.configuration.ingress.fqdn}' + +@description('App name (for teardown).') +output appName string = appName diff --git a/plugins/agentops/skills/agentops-agent/SKILL.md b/plugins/agentops/skills/agentops-agent/SKILL.md new file mode 100644 index 00000000..974d49dc --- /dev/null +++ b/plugins/agentops/skills/agentops-agent/SKILL.md @@ -0,0 +1,101 @@ +--- +name: agentops-agent +description: AgentOps Watchdog — surface regressions, latency spikes, error rates, and safety hits across AgentOps eval history, Azure Monitor traces, and Foundry control plane. +--- + +# `agentops-agent` — Watchdog skill + +Use this skill when the user asks any of: + +- *"Are my agents healthy in production?"* +- *"Run the watchdog"* +- *"Anything regressed in our last evals?"* +- *"Show latency / error spikes from Azure Monitor"* +- *"Open the AgentOps watchdog report"* + +This skill is the front door to `agentops agent analyze` and the +`agentops agent serve` Copilot Extension. It does **not** invent +findings — it shells out to the CLI which reads real data from: + +1. `.agentops/results/*/results.json` (eval history) +2. Application Insights traces emitted by Foundry agents +3. Foundry control plane (`azure-ai-projects`) + +## Workflow + +### 1. Validate the workspace + +Look for `.agentops/agent.yaml`. If absent, copy the template: + +```bash +mkdir -p .agentops +cp $(python -c "import agentops, os, pathlib; +print(pathlib.Path(agentops.__file__).parent / 'templates' / 'agent.yaml')") .agentops/agent.yaml +``` + +Edit `app_insights_resource_id` and `project_endpoint_env` if the user +wants the Azure Monitor / Foundry sources to be live. Without those +values the sources skip gracefully. + +### 2. Run the analyzer + +```bash +agentops agent analyze --severity-fail critical +``` + +The command writes `.agentops/agent/report.md`. Exit codes: + +- `0` — no findings at or above the configured severity floor +- `2` — at least one finding meets the severity floor (use this in CI) +- `1` — runtime / configuration error + +### 3. Read and summarize + +Open `.agentops/agent/report.md`. The report has: + +- **Verdict banner** — overall pass / warning / critical +- **Summary** — counts by severity +- **Sources** — which sources ran, which were skipped and why +- **Findings** — sorted by severity, each with a recommendation +- **Recent runs** — appendix of the last `lookback_runs` evals + +When summarising for the user, lead with the verdict, then the top +3 findings, each with the recommendation. Always cite the finding `id` +so the user can grep them later. + +### 4. Drive remediation, do not invent it + +For each finding the report includes a `Recommendation`. Follow it +verbatim — for example, if the finding says "compare the latest run +against the baseline runs in `.agentops/results/`", actually open +those folders. + +## Copilot Extension server + +If the user wants the watchdog inside Copilot Chat, they can: + +```bash +pip install agentops-toolkit[agent] +agentops agent serve --no-verify # local dev +``` + +For production, point them at: + +- `src/agentops/templates/agent-server/Dockerfile` +- `src/agentops/templates/agent-server/main.bicep` +- `src/agentops/templates/agent-server/README.md` + +These are the deploy scaffold for hosting the watchdog as a Copilot +Extension on Azure Container Apps. + +## Guardrails + +- Do **not** fabricate findings, metric values, or recommendations. +- Do **not** invent CLI flags. The contract is exactly: + - `agentops agent analyze [--workspace] [--config] [--out] [--lookback-days] [--severity-fail]` + - `agentops agent serve [--host] [--port] [--config] [--no-verify] [--workers]` +- If a source is `skipped` or `error`, surface that as the *first* + thing in the user-facing summary so they know the analyzer ran with + partial data. +- Never suggest disabling content-safety checks — recommend filtering + the offending row or tightening the system prompt instead. diff --git a/plugins/agentops/skills/agentops-config/SKILL.md b/plugins/agentops/skills/agentops-config/SKILL.md index 06845854..309df268 100644 --- a/plugins/agentops/skills/agentops-config/SKILL.md +++ b/plugins/agentops/skills/agentops-config/SKILL.md @@ -1,258 +1,87 @@ --- name: agentops-config -description: Infer evaluation scenario from codebase and generate run.yaml. Trigger when users ask to configure an evaluation, create a run config, detect the evaluation scenario, or choose a bundle. Common phrases include "configure", "run.yaml", "which bundle", "set up eval", "scenario", "endpoint", "agentops config", "create run config", "what should I evaluate". Install agentops-toolkit via pip. +description: Generate or update agentops.yaml (flat 1.0 schema) by inspecting the workspace. Trigger on "configure agentops", "agentops.yaml", "set up evaluation", "what should I evaluate". Infer the agent target and dataset from the codebase; ask only when nothing can be found. --- # AgentOps Config -Generate a complete `.agentops/run.yaml` by inspecting the workspace. Infer everything possible — ask only for values that cannot be found. +Generate `agentops.yaml` at the project root. The flat schema has only a +handful of fields — most projects need just `version`, `agent`, and +`dataset`. ## Step 0 — Prerequisites -1. Run `pip install agentops-toolkit` if `agentops` command is not available. -2. Run `agentops init` if `.agentops/` directory does not exist. +1. `pip install agentops-toolkit` if `agentops` is missing. +2. `agentops init` if `agentops.yaml` does not exist. -## Step 1 — Detect scenario +## Step 1 — Detect the agent target -Analyze the codebase holistically to understand the agent's **primary purpose**: +Search the codebase for the strongest signal and pick one: -1. Read the README, system prompt, main entry point, and tool/function definitions. -2. Identify which patterns are present: - - **Tool use**: `@tool`, `tool_definitions`, `function_call`, MCP tools, tool schemas - - **Retrieval**: search client, vector store, retriever, embeddings, index references, context fetching - - **Conversation**: chat history, multi-turn, session management, assistant persona - - **Direct model call**: completion API, no orchestration logic - -3. Pick the scenario that best matches the agent's **primary job** — not just the first signal found: - -| Primary purpose | `bundle.name` | +| Signal | `agent:` value | |---|---| -| Agent that orchestrates tools to complete tasks | `agent_workflow_baseline` | -| Agent that retrieves context to answer questions | `rag_quality_baseline` | -| Conversational assistant (chat, Q&A, persona) | `conversational_agent_baseline` | -| Direct model call with no agent logic | `model_quality_baseline` | - -> A RAG agent that uses a search tool is still primarily RAG — pick `rag_quality_baseline`, not `agent_workflow_baseline`. The test is: *what is the agent's main job?* - -4. State what you found: *"Detected RAG scenario — the agent's primary purpose is answering questions using retrieved context (found retriever logic in retriever.py)."* - -5. **Responsible AI (optional)**: Ask *"Do you also want to include safety evaluators (violence, hate/unfairness, self-harm, protected material)?"* If yes, add the safety evaluators from `safe_agent_baseline` to the selected bundle. - -## Step 2 — Detect endpoint type - -| Search for | `endpoint.kind` | `hosting` | `execution_mode` | -|---|---|---|---| -| `AIProjectClient`, `azure-ai-projects`, Foundry URL | `foundry_agent` | `foundry` | `remote` | -| FastAPI, Flask, Django, Express — JSON POST/response | `http` | `containerapps` / `aks` / `local` | `remote` | -| SSE/streaming, non-standard body, custom auth, no server | — | `local` / `containerapps` / `aks` | `local` (callable) | - -Also check: `agent_id` references, Dockerfile, bicep, ACA manifests, `.env` files. - -**Discover the endpoint URL** — search in this order, stop when found: -1. Env vars: `$env:AGENT_HTTP_URL`, `$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` -2. `.env` / `.env.local` in project root -3. `.azure//.env` files -4. Azure CLI (if hosting is `containerapps` or ACA-deployed): - ```bash - az containerapp list -g $RG --subscription $SUB --query "[].{name:name, url:properties.configuration.ingress.fqdn}" -o json - ``` -5. Azure CLI (if hosting is App Service / webapp): - ```bash - az webapp list -g $RG --subscription $SUB --query "[].{name:name, url:defaultHostName}" -o json - ``` - -**Detect auth pattern** — search the codebase: -- `dapr-api-token` / `APP_API_TOKEN` → Dapr auth -- `X-API-KEY` / `api_key` / `API_KEY` → API key auth -- `Authorization` / `Bearer` → Bearer token auth -- Nothing found → assume no auth needed - -## Step 3 — Discover Azure values - -Search these locations **in order** — stop as soon as each value is found: - -1. Shell environment variables (`$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`, etc.) -2. `.env`, `.env.local` in project root -3. `.azure//.env` files (azd environments) — also read `AZURE_RESOURCE_GROUP`, `AZURE_SUBSCRIPTION_ID` -4. `.azure/config.json` for `defaultEnvironment` to pick the right env folder - -### Validate azd environment (if using `.azure//.env`) - -Before trusting values from `.azure//.env`, verify the environment is still valid: - -1. **Check the environment is current** — run `azd env list` and confirm the selected environment appears. If multiple environments exist, list them and ask the user which to use. -2. **Verify the resource group exists**: - ```bash - az group exists --name $RG --subscription $SUB - ``` - If this returns `false`, warn: *"Resource group '$RG' no longer exists. Your azd environment may be outdated."* -3. **If validation fails**, ask the user for correct values or to select a different environment. - -If values are **not found** in any file, run Azure CLI discovery: -```bash -# 1. Confirm auth and get subscription -az account show --query "{sub:id, tenant:tenantId}" -o json - -# 2. Find AI Services / Foundry accounts and endpoints -az cognitiveservices account list -o json --query "[].{name:name, rg:resourceGroup, endpoint:properties.endpoint, kind:kind}" - -# 3. Find model deployments -az cognitiveservices account deployment list --name $ACCOUNT -g $RG --subscription $SUB --query "[].{name:name, model:properties.model.name, version:properties.model.version}" -o json - -# 4. Find Foundry projects -az resource list -g $RG --subscription $SUB --resource-type "Microsoft.CognitiveServices/accounts/projects" --query "[].name" -o tsv - -# 5. Build endpoints from discovered names -# Foundry: https://.services.ai.azure.com/api/projects/ -# OpenAI: https://.openai.azure.com/ -``` +| `AIProjectClient(...)` + agent ID literal `name:N` | `":"` | +| Foundry hosted agent URL `https://...services.ai.azure.com/...agents/...` | the full URL | +| Any other HTTP endpoint your agent serves (FastAPI, Express, ACA, AKS) | the full URL | +| Direct model use (`openai.chat.completions.create(model=...)`) with no orchestration | `"model:"` | -**Pre-warm Azure token** (prevents intermittent `AzureCliCredential.get_token failed` errors): -```bash -az account get-access-token --resource "https://cognitiveservices.azure.com" --query accessToken -o tsv -``` -If this fails, Azure CLI auth is not active — ask the user to run `az login`. - -**Only ask the user** if no `.azure/` dir exists AND no env vars are set. - -## Step 4 — Pick evaluator model - -Read the bundle YAML from `.agentops/bundles/.yaml`. If it contains **any** evaluator with `source: foundry`, then an evaluator model is required. - -Pick from available deployments (discovered in Step 3): `gpt-4.1-mini` > `gpt-4o-mini` > `gpt-4o` > `gpt-4.1`. **Never** use reasoning models (`o1`, `o3`, `o4`, `gpt-5`, `gpt-5-nano`). - -If no suitable deployment was found, ask: *"Which model deployment should score your agent's responses? (e.g. gpt-4o-mini)"* +Look in: `README.md`, `main.py`/`server.py`/`app.ts`, `.env`/`.env.local`, +`.azure//.env`, `infra/`, IaC outputs. If nothing is found, ask the +user once. -## Step 4.5 — Evaluator compatibility check (optional) +## Step 2 — Detect the dataset -This step is **optional** — skip it if the bundle only uses widely available evaluators. +If a JSONL with rows that include `input` already exists in the repo, use +its path. Otherwise leave the default `.agentops/data/smoke.jsonl` and +hand off to the `agentops-dataset` skill before the first run. -**Key facts:** -- `SimilarityEvaluator`, `CoherenceEvaluator`, `FluencyEvaluator`, `RelevanceEvaluator`, `GroundednessEvaluator` → **widely available**, no check needed. -- `F1ScoreEvaluator`, `BleuScoreEvaluator`, `RougeScoreEvaluator`, `GleuScoreEvaluator` → **local text-overlap**, no Azure credentials needed, widely available. -- `TaskCompletionEvaluator`, `ToolCallAccuracyEvaluator`, `IntentResolutionEvaluator`, `TaskAdherenceEvaluator`, `ToolSelectionEvaluator`, `ToolInputAccuracyEvaluator`, `ResponseCompletenessEvaluator` → **SDK version dependent**, verify before using. +## Step 3 — Write agentops.yaml -If the bundle uses SDK-version-dependent evaluators, verify they exist. You may check the SDK version, read release notes, or try any efficient approach. Do **not** get stuck in environment path issues — if a quick check fails, proceed and let the evaluation surface any errors. +Minimal example: -If an evaluator is missing: set `enabled: false` in the bundle, remove its threshold, and tell the user. - -## Step 5 — Write run.yaml - -Write `.agentops/run.yaml` using the exact structure below. Fill **every** value — no placeholders. - -**Remote (Foundry agent):** ```yaml version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: -dataset: - name: dataset -output: - write_report: true +agent: "my-rag:3" +dataset: .agentops/data/smoke.jsonl ``` -**Remote (HTTP):** -```yaml -version: 1 -target: - type: agent - hosting: containerapps - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - request_field: message - response_field: text -bundle: - name: -dataset: - name: dataset -output: - write_report: true -``` +HTTP/JSON example: -**Local (callable adapter):** ```yaml version: 1 -target: - type: agent - hosting: local - execution_mode: local - local: - callable: callable_adapter:run_evaluation -bundle: - name: -dataset: - name: dataset -output: - write_report: true +agent: "https://my-aca-app.eastus2.azurecontainerapps.io/chat" +dataset: .agentops/data/smoke.jsonl +request_field: message # default is "message" +response_field: text # dot-path; default is "text" +auth_header_env: MY_API_TOKEN ``` -## Step 6 — Write callable adapter (if execution_mode is local) - -Create `callable_adapter.py` at the **project root**. Use ONLY stdlib (`urllib.request`, `json`, `os`). - -```python -import json -import os -import urllib.request - -ENDPOINT = os.environ["AGENT_HTTP_URL"] -# Auth: set APP_API_TOKEN, API_KEY, or remove the auth lines below. -AUTH_TOKEN = os.environ.get("APP_API_TOKEN", "") - -def run_evaluation(input_text: str, context: dict) -> dict: - body = json.dumps({"message": input_text}).encode() - headers = {"Content-Type": "application/json"} - if AUTH_TOKEN: - headers["dapr-api-token"] = AUTH_TOKEN # Change header name if using API_KEY or Bearer - req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - return {"response": data.get("text", data.get("response", ""))} -``` +Optional extras (only add when the user asks for them): -After writing the file, run: `python -c "from callable_adapter import run_evaluation; print('OK')"` - -**Auth detection:** Search codebase for `dapr-api-token`/`APP_API_TOKEN` → Dapr header. `X-API-KEY`/`api_key`/`API_KEY` → API key header. `Authorization`/`Bearer` → recommend HTTP backend with `auth_header_env` instead. Nothing found → remove auth lines. - -## Step 7 — Present and confirm - -Present a **confirmation table** with all discovered values (do not ask each one separately): -``` -┌─────────────────────────┬──────────────────────────────────────────┬────────┐ -│ Setting │ Value │ Source │ -├─────────────────────────┼──────────────────────────────────────────┼────────┤ -│ Scenario │ RAG │ code │ -│ Bundle │ rag_quality_baseline │ auto │ -│ Endpoint kind │ http │ code │ -│ Endpoint URL │ https://myapp.azurecontainerapps.io/chat │ .env │ -│ Auth │ dapr-api-token (APP_API_TOKEN) │ code │ -│ Evaluator model │ gpt-4o-mini │ Azure │ -│ Project endpoint │ https://acct.services.ai.azure.com/... │ .env │ -└─────────────────────────┴──────────────────────────────────────────┴────────┘ +```yaml +thresholds: + coherence: ">=3" + groundedness: ">=3" + avg_latency_seconds: "<=10" + +publish: foundry # Classic Foundry panel (works for any target) +# publish: foundry_cloud # New Foundry panel (preview; name:version agents only) +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" + +evaluators: # rare - AgentOps auto-selects from agent + dataset + - name: similarity + threshold: ">=4" ``` -Ask: *"Everything look correct? (yes / edit)"* +## Step 4 — Validate -Explain: scenario detected, endpoint type, evaluator model chosen, and any assumptions made. +Run `agentops eval run` once. If the config is malformed AgentOps prints a +clear error pointing at the offending key. Adjust and re-run. -## Rules +## Guardrails -- **NEVER** include `backend:` key in run.yaml — it causes a runtime error. -- **NEVER** leave `` placeholders in run.yaml. -- **NEVER** fabricate `agent_id`, model names, or endpoint URLs. -- **NEVER** use dotted import paths like `.agentops.callable_adapter` — they fail. -- If a bundle uses SDK-version-dependent evaluators, verify availability before running (Step 4.5). Don't block on this — if verification is hard, proceed and fix on failure. -- Do not generate datasets — delegate to `/agentops-dataset`. -- Do not run evaluations — delegate to `/agentops-eval`. -- Always state what you detected and what you assumed. \ No newline at end of file +- Do **not** add legacy keys (`bundle`, `target`, `execution`, `output`, + `backend`). The 1.0 schema rejects them. +- Do **not** fabricate agent IDs, endpoint URLs, or model deployment + names. Ask the user when uncertain. +- Keep the file small. Auto-selection covers most metrics. diff --git a/plugins/agentops/skills/agentops-dataset/SKILL.md b/plugins/agentops/skills/agentops-dataset/SKILL.md index 602b334e..625d9acc 100644 --- a/plugins/agentops/skills/agentops-dataset/SKILL.md +++ b/plugins/agentops/skills/agentops-dataset/SKILL.md @@ -1,128 +1,69 @@ --- name: agentops-dataset -description: Generate evaluation datasets (JSONL data + YAML config) tailored to the project. Trigger when users ask to create test data, generate a dataset, or prepare evaluation data. Common phrases include "dataset", "test data", "evaluation data", "JSONL", "generate data", "create dataset", "sample data". Install agentops-toolkit via pip. +description: Create or extend a JSONL evaluation dataset for AgentOps. Trigger on "create dataset", "generate test data", "JSONL", "more eval rows". Infer the agent's domain from the codebase and produce realistic rows; never fabricate data when the domain is unclear. --- # AgentOps Dataset -Generate a custom evaluation dataset from the codebase. Never offer starter datasets — always create project-specific data. +Generate a small, realistic JSONL dataset for the agent under +evaluation. Default location: `.agentops/data/smoke.jsonl` (referenced +from `agentops.yaml`). ## Step 0 — Prerequisites -1. Run `pip install agentops-toolkit` if `agentops` command is not available. -2. Run `agentops init` if `.agentops/` directory does not exist. - -## Step 1 — Understand the domain - -Read the codebase: system prompt, tool definitions, README, sample inputs/outputs, test fixtures. Understand the agent's **primary purpose** and identify the scenario: - -| Primary purpose | Scenario | -|---|---| -| Agent that orchestrates tools to complete tasks | Agent with tools | -| Agent that retrieves context to answer questions | RAG | -| Conversational assistant (chat, Q&A, persona) | Conversational | -| Direct model call with no agent logic | Model quality | - -> A RAG agent that uses a search tool is still primarily RAG. The test is: *what is the agent's main job?* - -## Step 2 — Confirm topics and count - -1. Ask: *"What topics should the test data cover?"* -2. Ask: *"How many rows? (suggest 5–10)"* - -## Step 3 — Generate JSONL rows - -Use the correct fields for the scenario: - -| Scenario | JSONL fields | -|---|---| -| Model quality | `input`, `expected` | -| Conversational | `input`, `expected` | -| RAG | `input`, `expected`, `context` | -| Agent with tools | `input`, `expected`, `tool_definitions`, `tool_calls` | -| Content safety | `input`, `expected` | - -Write `.agentops/data/data.jsonl` — one JSON object per line. Rows must: -- Cover distinct use cases from the codebase -- Include realistic, domain-specific content -- Have at least one edge case -- Reflect actual tool schemas and system prompt - -## Step 4 — Write dataset YAML config - -Write `.agentops/datasets/dataset.yaml` using this **exact** structure — no alternatives: -```yaml -version: 1 -name: dataset -description: -source: - type: file - path: ../data/data.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: - size_hint: -``` +1. `pip install agentops-toolkit` if `agentops` is missing. +2. `agentops init` if `agentops.yaml` does not exist. -**NEVER** use `path:` or `fields:` at the top level — the correct keys are `source:` and `format:`. If unsure, read an existing starter config from `.agentops/datasets/` as a reference template. +## Step 1 — Pick the columns -For RAG scenarios, add `context_field: context` under `format:`: -```yaml -format: - type: jsonl - input_field: input - expected_field: expected - context_field: context -``` +Read `agentops.yaml` (and the agent code) to figure out the agent type, +then choose the row schema: + +| Agent type | Required columns | Optional columns | +|---|---|---| +| Direct model / Q&A | `input`, `expected` | — | +| RAG | `input`, `expected`, `context` | — | +| Conversational | `input`, `expected` | — | +| Tool-using agent | `input`, `expected`, `tool_calls` | `tool_definitions` | -## Step 4.5 — RAG context enrichment +`input` is always the user prompt. `expected` is the gold answer. +`context` is the retrieved passage(s). `tool_calls` is a list of +`{name, arguments}` describing the expected tool invocations. -If the scenario is **RAG** and the generated JSONL has no `context` field: +## Step 2 — Ground the rows in the codebase -1. **Find the project's retrieval logic** — search the codebase for how it fetches context today: - - Look for search/retrieval client initialization, index or collection names, embedding calls - - Check `.env` files and code for endpoint URLs, API keys, index names used by the retriever - - The project may use Azure AI Search, Cosmos DB vector search, FAISS, Pinecone, or any other store — read the code to find out +- Read the README, system prompt, tool definitions, and any sample + fixtures. +- Generate **5–10 rows** that exercise the agent's actual capabilities. +- If the domain is unclear, generate a tiny generic draft and clearly + flag it as a placeholder. -2. **Build a retrieval script** at `.agentops/rag_context.py` (**never** in `src/`) that: - - Reads the project's own retrieval config (env vars, endpoint, index name) from whatever the project uses - - For each row in the JSONL, queries the retrieval backend with `row["input"]` and writes the result into `row["context"]` - - Uses only stdlib (`urllib.request`, `json`, `os`, `subprocess`, `sys`, `shutil`) — no third-party dependencies - - Accepts the JSONL file path as a CLI argument: `python .agentops/rag_context.py .agentops/data/data.jsonl` - - **Must be cross-platform** (Windows + Linux/macOS) — when calling external CLIs (e.g. `az`), use: - ```python - import shutil, subprocess, sys - def _run_cli(args: list[str], **kwargs) -> subprocess.CompletedProcess: - exe = shutil.which(args[0]) - if exe is None: - raise FileNotFoundError(f"'{args[0]}' not found in PATH.") - return subprocess.run([exe] + args[1:], **kwargs, shell=(sys.platform == "win32")) - ``` +## Step 3 — Write the JSONL -3. Verify: each JSONL row now has a `context` field. -4. Update dataset YAML to include `context_field: context` under `format:`. +One JSON object per line, no trailing commas, UTF-8: -If no retrieval backend can be identified, state: *"RAG context cannot be populated automatically — either add `context` manually to each row or switch to `model_quality_baseline` bundle which does not require it."* +```json +{"input": "What is the refund policy?", "expected": "Refunds within 30 days...", "context": "Refund policy: ..."} +``` + +Save to the path referenced by `dataset:` in `agentops.yaml` (default +`.agentops/data/smoke.jsonl`). -## Step 5 — Present for review +## Step 4 — Sanity-check -Show the generated rows and say: *"These are starter rows for validation. For production evaluations, use real user queries or domain expert–curated data."* +Run a quick eval and confirm rows are picked up: -## Outputs +```bash +agentops eval run +``` -- `.agentops/data/data.jsonl` — JSONL rows -- `.agentops/datasets/dataset.yaml` — dataset config +Open `.agentops/results/latest/report.md` and confirm the row count +matches. -## Rules +## Guardrails -- **NEVER** offer starter datasets (`smoke-model-direct.jsonl`, etc.) — always generate custom data. -- **NEVER** leave `` placeholders in JSONL or YAML. -- **NEVER** use `path:` or `fields:` at the dataset config top level — the correct structure uses `source:` and `format:`. Read a starter config from `.agentops/datasets/` if unsure. -- Use generic file names: `data.jsonl`, `dataset.yaml` — not project-specific prefixes. -- State the scenario assumption: *"Generating dataset for RAG scenario (detected retriever)"*. -- Mark generated data as draft — not production-grade. -- Do not run evaluations — delegate to `/agentops-eval`. -- Do not generate run.yaml — delegate to `/agentops-config`. +- Do not invent customer data, real names, or sensitive content. +- Keep rows short — datasets are meant to be quick gates, not full QA + suites. +- If the user already has a domain dataset, prefer pointing + `agentops.yaml` at that file rather than generating new rows. diff --git a/plugins/agentops/skills/agentops-eval/SKILL.md b/plugins/agentops/skills/agentops-eval/SKILL.md index 2463a46c..a3d58f5a 100644 --- a/plugins/agentops/skills/agentops-eval/SKILL.md +++ b/plugins/agentops/skills/agentops-eval/SKILL.md @@ -1,610 +1,105 @@ --- name: agentops-eval -description: Guide users through running AgentOps evaluations end to end — codebase analysis, dataset generation, config creation, single runs, multi-model benchmarks, and N-run comparisons. Trigger when users ask to run an evaluation, compare runs, benchmark models, create eval config, generate datasets, or summarize results. Common phrases include "run eval", "evaluate", "start agentops", "compare models", "benchmark agents", "run.yaml", "report", "evaluation results", "which model is best", "set up eval", "create dataset". Install agentops-toolkit via pip. Commands are agentops init, agentops eval run, agentops eval compare, and agentops report generate. +description: Run AgentOps evaluations end-to-end against any agent (Foundry hosted/prompt agent, HTTP/JSON endpoint, or raw model deployment). Trigger on phrases like "run eval", "evaluate my agent", "benchmark", "agentops eval", "compare runs". Uses the flat agentops.yaml schema. --- # AgentOps Eval -End-to-end evaluation workflow: analyze codebase → generate dataset → configure run → validate → execute → summarize. +End-to-end workflow: install → init → configure → run → read report. -## Step 0 — Verify setup +## Step 0 — Setup -1. Run `pip install agentops-toolkit` if `agentops` command is not available. -2. Run `agentops init` if `.agentops/` directory does not exist. +1. Install if missing: `pip install agentops-toolkit`. +2. If `agentops.yaml` does not exist at the project root, run `agentops init`. -Then proceed to analyze the codebase. Only ask questions about things you cannot find in the code. +## Step 1 — Identify the agent target -## Step 1 — Detect evaluation scenario +Read the codebase (README, entry point, env vars) and pick the right value +for the `agent:` field of `agentops.yaml`: -Analyze the codebase holistically to understand the agent's **primary purpose**: - -1. Read the README, system prompt, main entry point, and tool/function definitions. -2. Identify which patterns are present: - - **Tool use**: `@tool`, `tool_definitions`, `function_call`, MCP tools, tool schemas - - **Retrieval**: search client, vector store, retriever, embeddings, index references, context fetching - - **Conversation**: chat history, multi-turn, session management, assistant persona - - **Direct model call**: completion API, no orchestration logic - -3. Pick the scenario that best matches the agent's **primary job** — not just the first signal found: - -| Primary purpose | `bundle.name` | +| Pattern in code / env | `agent:` value | |---|---| -| Agent that orchestrates tools to complete tasks | `agent_workflow_baseline` | -| Agent that retrieves context to answer questions | `rag_quality_baseline` | -| Conversational assistant (chat, Q&A, persona) | `conversational_agent_baseline` | -| Direct model call with no agent logic | `model_quality_baseline` | - -> A RAG agent that uses a search tool is still primarily RAG — pick `rag_quality_baseline`, not `agent_workflow_baseline`. The test is: *what is the agent's main job?* - -4. State your reasoning: *"Detected RAG scenario — the agent's primary purpose is answering questions using retrieved context (found retriever logic in retriever.py)."* - -5. **Responsible AI (optional)**: Ask *"Do you also want to include safety evaluators (violence, hate/unfairness, self-harm, protected material)? These can be added alongside your main bundle."* If yes, add the safety evaluators from `safe_agent_baseline` to the selected bundle. - -6. **Unit tests (optional)**: Only ask this if **all** of the following are true: (a) the codebase has testable agent code in Python, JavaScript, or TypeScript (endpoint handlers, tool definitions, orchestration logic), (b) no existing test directory or test files are detected (e.g., `tests/`, `test_*.py`, `*_test.py`, `*.test.ts`, `*.test.js`, `__tests__/`). If both conditions are met, ask: *"Would you also like me to generate unit tests for your agent code? (e.g., mocked HTTP calls, response parsing, error handling)"*. If the user declines or if conditions are not met, skip silently. See the **Unit Test Generation** section at the end of this skill for details. - -## Step 2 — Detect endpoint type - -| Search for | `endpoint.kind` | `hosting` | `execution_mode` | -|---|---|---|---| -| `AIProjectClient`, `azure-ai-projects`, Foundry URL | `foundry_agent` | `foundry` | `remote` | -| FastAPI/Flask/Django — JSON POST → JSON response | `http` | `containerapps`/`aks`/`local` | `remote` | -| SSE/streaming, custom auth, non-standard body, no server | — | `local`/`containerapps`/`aks` | `local` (callable) | - -**Discover the endpoint URL** — search in this order, stop when found: -1. Env vars: `$env:AGENT_HTTP_URL`, `$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` -2. `.env` / `.env.local` in project root -3. `.azure//.env` files -4. Azure CLI (if hosting is `containerapps` or ACA-deployed): - ```bash - az containerapp list -g $RG --subscription $SUB --query "[].{name:name, url:properties.configuration.ingress.fqdn}" -o json - ``` -5. Azure CLI (if hosting is App Service / webapp): - ```bash - az webapp list -g $RG --subscription $SUB --query "[].{name:name, url:defaultHostName}" -o json - ``` - -**Detect auth pattern** — search the codebase for auth headers used in requests: -- `dapr-api-token` / `APP_API_TOKEN` → Dapr auth (use in callable adapter) -- `X-API-KEY` / `api_key` / `API_KEY` → API key auth (set `auth_header_env`) -- `Authorization` / `Bearer` → Bearer token (set `auth_header_env`) -- No auth headers found → assume no auth needed - -Only ask *"What is the URL where your agent is running?"* if discovery finds nothing. - -## Step 3 — Generate dataset - -**Never offer starter datasets** — always generate a custom one. - -1. Read the codebase: system prompt, tools, domain, README. -2. Ask the user what topics the test data should cover. -3. Ask how many rows (suggest 5–10). -4. Write `.agentops/data/data.jsonl` with the correct fields: - -| Scenario | JSONL fields | -|---|---| -| Model quality | `input`, `expected` | -| Conversational | `input`, `expected` | -| RAG | `input`, `expected`, `context` | -| Agent with tools | `input`, `expected`, `tool_definitions`, `tool_calls` | - -5. Write `.agentops/datasets/dataset.yaml` using this **exact** structure (no alternatives): -```yaml -version: 1 -name: dataset -description: -source: - type: file - path: ../data/data.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: - size_hint: -``` -**NEVER** use `path:` or `fields:` at the top level — the correct keys are `source:` and `format:`. If unsure, read an existing starter config from `.agentops/datasets/` as a reference template first. - -6. Show the generated rows to the user for review. - -### RAG context enrichment - -If the scenario is **RAG** and the dataset has no `context` field: - -1. **Find the project's retrieval logic** — search the codebase for how it fetches context today: - - Look for search/retrieval client initialization, index or collection names, embedding calls - - Check `.env` files and code for endpoint URLs, API keys, index names used by the retriever - - The project may use Azure AI Search, Cosmos DB vector search, FAISS, Pinecone, or any other store — read the code to find out - -2. **Build a retrieval script** at `.agentops/rag_context.py` (**never** in `src/`) that: - - Reads the project's own retrieval config (env vars, endpoint, index name) from whatever the project uses - - For each row in the JSONL, queries the retrieval backend with `row["input"]` and writes the result into `row["context"]` - - Uses only stdlib (`urllib.request`, `json`, `os`, `subprocess`, `sys`, `shutil`) — no third-party dependencies - - Accepts the JSONL file path as a CLI argument: `python .agentops/rag_context.py .agentops/data/data.jsonl` - - **Must be cross-platform** (Windows + Linux/macOS) — when calling external CLIs (e.g. `az`), use the following pattern: - ```python - import shutil - import subprocess - import sys - - def _run_cli(args: list[str], **kwargs) -> subprocess.CompletedProcess: - """Run an external CLI command, cross-platform.""" - exe = shutil.which(args[0]) - if exe is None: - raise FileNotFoundError( - f"'{args[0]}' not found in PATH. " - "Make sure it is installed and available." - ) - return subprocess.run( - [exe] + args[1:], - **kwargs, - shell=(sys.platform == "win32"), - ) - ``` - - This avoids `FileNotFoundError` on Windows where `subprocess.run(["az", ...])` fails without `shell=True` - -3. Update dataset YAML to include `context_field: context` under `format:`. -4. Now `rag_quality_baseline` with GroundednessEvaluator and RetrievalEvaluator can be used. - -If no retrieval backend can be identified, fall back to `model_quality_baseline` and explain why. - -## Step 4 — Discover Azure values - -Search these locations in order — stop as soon as each value is found: - -1. Shell env vars (`$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`, `$env:AZURE_OPENAI_ENDPOINT`, `$env:AZURE_OPENAI_DEPLOYMENT`) -2. `.env` / `.env.local` in project root -3. `.azure//.env` (azd environments) — also read `AZURE_RESOURCE_GROUP`, `AZURE_SUBSCRIPTION_ID` -4. `.azure/config.json` for `defaultEnvironment` to pick the right env folder +| `AIProjectClient`, `azure-ai-projects`, Foundry agent ID like `name:1` | `":"` (Foundry prompt agent) | +| Foundry hosted agent endpoint URL ending in `/agents/...` | `"https://.services.ai.azure.com/api/projects/

/agents/..."` | +| Plain HTTP/JSON endpoint (FastAPI, Express, ACA, AKS) | `"https:///"` | +| Raw Foundry/Azure OpenAI model deployment | `"model:"` | -### Validate azd environment (if using `.azure//.env`) +If nothing is found, ask the user once for the agent identifier. -Before trusting values from `.azure//.env`, verify the environment is still valid: +## Step 2 — Make sure the dataset exists -1. **Check if the environment is current** — run `azd env list` and confirm the selected environment appears in the output. If multiple environments exist, list them and ask the user which one to use. -2. **Verify the resource group exists** — after reading `AZURE_RESOURCE_GROUP` and `AZURE_SUBSCRIPTION_ID` from the env file, run: - ```bash - az group exists --name $RG --subscription $SUB - ``` - If this returns `false`, the environment is stale (resources were deleted). Warn the user: *"The resource group '$RG' no longer exists. Your azd environment may be outdated. Please re-run `azd up` or provide current Azure values."* -3. **If validation fails**, do not silently proceed with stale values — ask the user for correct values or to select a different environment. +`agentops.yaml` points to a JSONL file (default +`.agentops/data/smoke.jsonl`). Each row needs at least `input` and a label +that maps to the metric you care about (`expected`, `context`, +`tool_calls`...). If the dataset is empty or unrelated, run the +`agentops-dataset` skill before running the eval. -If values are **not found** in files, use Azure CLI to discover them: +## Step 3 — Run the evaluation ```bash -# 1. Confirm auth and get subscription -az account show --query "{sub:id, tenant:tenantId}" -o json +agentops eval run +``` -# 2. Find AI Services / Foundry accounts and endpoints -az cognitiveservices account list -o json --query "[].{name:name, rg:resourceGroup, endpoint:properties.endpoint, kind:kind}" -# Or scope to a known RG: -az cognitiveservices account list -g $RG --subscription $SUB --query "[].{name:name, endpoint:properties.endpoint}" -o json +Optional flags: -# 3. Find model deployments (chat, embedding) -az cognitiveservices account deployment list --name $ACCOUNT -g $RG --subscription $SUB --query "[].{name:name, model:properties.model.name, version:properties.model.version}" -o json +- `--config ` — point at a different `agentops.yaml`. +- `--output

` — choose where to write `results.json` and `report.md` + (defaults to `.agentops/results//`). -# 4. Find Foundry projects -az resource list -g $RG --subscription $SUB --resource-type "Microsoft.CognitiveServices/accounts/projects" --query "[].name" -o tsv +Exit codes: -# 5. Build endpoints from discovered names -# Foundry: https://.services.ai.azure.com/api/projects/ -# OpenAI: https://.openai.azure.com/ -``` +- `0` — succeeded and all thresholds passed +- `2` — succeeded but at least one threshold failed (gate-friendly) +- `1` — runtime/configuration error -For evaluator model, pick from available deployments: `gpt-4.1-mini` > `gpt-4o-mini` > `gpt-4o` > `gpt-4.1`. **Never** reasoning models (`o1`, `o3`, `o4`, `gpt-5`, `gpt-5-nano`). +## Step 4 — Inspect results -**Pre-warm Azure token** (prevents intermittent `AzureCliCredential.get_token failed` errors): ```bash -az account get-access-token --resource "https://cognitiveservices.azure.com" --query accessToken -o tsv +agentops report generate # regenerate report.md from latest results.json +agentops report generate --in ``` -If this fails, Azure CLI auth is not active — ask the user to run `az login`. - -Check Azure auth: `az account show`. If not logged in, ask the user to run `az login` or set API key. - -## Step 4.5 — Evaluator compatibility check (optional) - -This step is **optional** — skip it if you are confident the bundle evaluators match the installed SDK. If the evaluation fails later due to a missing evaluator, come back here. - -Use the reference table below to decide whether the selected bundle is safe to use **without running any probes**. Evaluators marked "Widely available" work on all recent `azure-ai-evaluation` versions. Only the SDK-version-dependent ones need caution. - -### Evaluator compatibility reference - -| Evaluator | Category | Needs credentials | Availability | -|---|---|---|---| -| `SimilarityEvaluator` | AI-assisted | Yes | Widely available | -| `CoherenceEvaluator` | AI-assisted | Yes | Widely available | -| `FluencyEvaluator` | AI-assisted | Yes | Widely available | -| `RelevanceEvaluator` | AI-assisted | Yes | Widely available | -| `GroundednessEvaluator` | AI-assisted | Yes | Widely available | -| `F1ScoreEvaluator` | Local text-overlap | No | Widely available | -| `BleuScoreEvaluator` | Local text-overlap | No | Widely available | -| `RougeScoreEvaluator` | Local text-overlap | No | Widely available | -| `GleuScoreEvaluator` | Local text-overlap | No | Widely available | -| `TaskCompletionEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ToolCallAccuracyEvaluator` | AI-assisted | Yes | SDK version dependent | -| `IntentResolutionEvaluator` | AI-assisted | Yes | SDK version dependent | -| `TaskAdherenceEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ToolSelectionEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ToolInputAccuracyEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ResponseCompletenessEvaluator` | AI-assisted | Yes | SDK version dependent | -### When to verify +Open `.agentops/results/latest/report.md`. To compare two runs, hand both +`results.json` files to the user and walk them through metric deltas; +AgentOps does not ship a separate `eval compare` command. -- If the bundle only uses **widely available** evaluators → proceed directly, no verification needed. -- If the bundle uses **SDK-version-dependent** evaluators → verify they exist before running. You may check `pip show azure-ai-evaluation` for version, read SDK release notes, or use any approach you find efficient. Do **not** get stuck in environment path issues — if a quick check fails, just proceed and let the evaluation surface any import errors. +## Step 5 — (Optional) Publish to Foundry Evaluations -### If an evaluator is missing +Two modes are supported. Both write a deep-link into +`.agentops/results/latest/cloud_evaluation.json` and require +`AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` (or the inline `project_endpoint`). -- Disable it in the bundle (`enabled: false`) and remove its threshold. -- Tell the user: *"Disabled [X] — not available in your SDK version."* +**Classic Foundry Evaluations panel** (default — works for any target +kind, uploads metrics that AgentOps already computed locally): -## Step 5 — Write run.yaml - -Update `.agentops/run.yaml` (the default config). Do **not** create a custom-named file. - -**Remote Foundry agent:** ```yaml -version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: -dataset: - name: dataset -output: - write_report: true +publish: foundry +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" ``` -**Remote HTTP:** -```yaml -version: 1 -target: - type: agent - hosting: containerapps - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - request_field: message - response_field: text -bundle: - name: -dataset: - name: dataset -output: - write_report: true -``` +**New Foundry Evaluations panel** (preview — re-runs the agent + builtin +evaluators server-side via the OpenAI Evals API; only works for +`name:version` Foundry agents): -**Local callable adapter:** ```yaml -version: 1 -target: - type: agent - hosting: local - execution_mode: local - local: - callable: callable_adapter:run_evaluation -bundle: - name: -dataset: - name: dataset -output: - write_report: true -``` - -Fill **every** `` with a real discovered value. If any value cannot be found, ask the user for just that value. - -## Step 5.5 — Write callable adapter (if execution_mode is local) - -Create `.agentops/callable_adapter.py`. Use ONLY stdlib. All generated files must live inside `.agentops/` to avoid polluting the project root. - -First, examine the agent's response format by reading the endpoint handler code: -- Look for `yield`, `StreamingResponse`, `EventSourceResponse` → SSE/streaming -- Look for `JSONResponse`, `return {"text": ...}` → standard JSON -- Look for conversation ID prefixes, UUID patterns in responses - -**Standard JSON adapter:** -```python -import json -import os -import urllib.request - -ENDPOINT = os.environ["AGENT_HTTP_URL"] -# Auth: set AGENT_AUTH_HEADER and AGENT_AUTH_TOKEN env vars if your endpoint requires auth. -# Example: AGENT_AUTH_HEADER=dapr-api-token AGENT_AUTH_TOKEN=dev-token -# AGENT_AUTH_HEADER=X-API-KEY AGENT_AUTH_TOKEN=my-key -AUTH_HEADER = os.environ.get("AGENT_AUTH_HEADER", "") -AUTH_TOKEN = os.environ.get("AGENT_AUTH_TOKEN", "") - -def run_evaluation(input_text: str, context: dict) -> dict: - body = json.dumps({"message": input_text}).encode() - headers = {"Content-Type": "application/json"} - if AUTH_HEADER and AUTH_TOKEN: - headers[AUTH_HEADER] = AUTH_TOKEN - req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=120) as resp: - data = json.loads(resp.read()) - return {"response": data.get("text", data.get("response", ""))} -``` - -**SSE/streaming adapter** (use when agent uses `StreamingResponse`, `yield`, or SSE): -```python -import json -import os -import urllib.request - -ENDPOINT = os.environ["AGENT_HTTP_URL"] -# Auth: set AGENT_AUTH_HEADER and AGENT_AUTH_TOKEN env vars if your endpoint requires auth. -AUTH_HEADER = os.environ.get("AGENT_AUTH_HEADER", "") -AUTH_TOKEN = os.environ.get("AGENT_AUTH_TOKEN", "") - -def run_evaluation(input_text: str, context: dict) -> dict: - body = json.dumps({"message": input_text}).encode() - headers = {"Content-Type": "application/json"} - if AUTH_HEADER and AUTH_TOKEN: - headers[AUTH_HEADER] = AUTH_TOKEN - req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") - chunks = [] - try: - with urllib.request.urlopen(req, timeout=120) as resp: - for raw_line in resp: - line = raw_line.decode("utf-8", errors="replace").strip() - if not line or line.startswith(":"): # SSE comment or keep-alive - continue - if line.startswith("event:"): # SSE event type — skip - continue - if line.startswith("data: "): - payload = line[6:] - if payload == "[DONE]": - break - try: - event = json.loads(payload) - # Adapt field extraction to match the project's SSE format - chunk = event.get("content", event.get("text", "")) - if chunk: - chunks.append(chunk) - except json.JSONDecodeError: - chunks.append(payload) # plain text SSE - else: - chunks.append(line) # raw text line - except Exception as e: - return {"response": f"ERROR: {e}"} - response_text = "".join(chunks).strip() - return {"response": response_text} -``` - -Customize the adapter: -- **Apply the auth pattern detected in Step 2.** Use the table below to wire the correct header and env var into the adapter: - -| Auth detected in Step 2 | Adapter env var | Header line in adapter | -|---|---|---| -| `dapr-api-token` / `APP_API_TOKEN` | `AGENT_AUTH_TOKEN` (tell user to set it to their Dapr token) | `headers["dapr-api-token"] = AUTH_TOKEN` | -| `X-API-KEY` / `api_key` / `API_KEY` | `AGENT_AUTH_TOKEN` (tell user to set it to their API key) | `headers["X-API-KEY"] = AUTH_TOKEN` | -| `Authorization: Bearer` | Recommend HTTP backend with `auth_header_env` instead of callable adapter | N/A | -| No auth detected | Remove `AUTH_TOKEN` and auth header lines entirely | N/A | - - **Important**: Do NOT generate the adapter with auth lines commented out or using hardcoded tokens. If auth was detected, the adapter must include the correct header from the start — otherwise the smoke test will fail with 401. - -- **Choose the right template:** If the agent code uses `yield`, `StreamingResponse`, `EventSourceResponse`, or `text/event-stream` content type, use the **SSE/streaming adapter** template. Otherwise use the **standard JSON adapter**. -- **Customize the request field:** If the agent expects a different key than `"message"` (e.g. `"ask"`, `"question"`, `"input"`), change the `json.dumps({"message": ...})` line to match. -- **Customize the response extraction:** If the agent returns a different key than `"text"` or `"response"`, update the `.get()` call accordingly. - -### Context sanitization (RAG scenarios) - -If the dataset has a `context` field populated from Azure AI Search or similar document stores, the raw content often includes HTML comments (``), document source tags (`[Copy 002 ...]`), and OCR artifacts. Add this helper to the adapter and call it when enriching context: - -```python -import re - -_HTML_COMMENT_RE = re.compile(r"", re.DOTALL) -_MULTI_BLANK_RE = re.compile(r"\n{3,}") - -def _sanitize_context(text: str) -> str: - """Strip HTML comments, document metadata, and collapse blank lines.""" - text = _HTML_COMMENT_RE.sub("", text) - text = re.sub(r"^\[.*?\]\s*$", "", text, flags=re.MULTILINE) - text = _MULTI_BLANK_RE.sub("\n\n", text) - return text.strip() -``` - -Apply it to the `context` field in JSONL rows before writing or in the adapter before returning: -```python -ctx = context.get("context", "") -if ctx: - context["context"] = _sanitize_context(ctx) -``` - -After writing the file: `python -c "import sys; sys.path.insert(0, '.agentops'); from callable_adapter import run_evaluation; print('OK')"` - -## Step 6 — Pre-flight validation - -Check **all** of these **before** running. Fix any failures first. Do NOT run-fail-fix iteratively. - -- [ ] run.yaml has no `backend:` key (causes runtime error) -- [ ] No `` placeholders in run.yaml -- [ ] Bundle file exists: `.agentops/bundles/.yaml` -- [ ] Dataset file exists: `.agentops/datasets/dataset.yaml` -- [ ] Dataset YAML has `source:` and `format:` keys (NOT `path:` or `fields:` at top level) -- [ ] JSONL file exists: `.agentops/data/data.jsonl` -- [ ] If RAG: JSONL rows have `context` field; dataset YAML has `context_field: context` -- [ ] If bundle uses SDK-version-dependent evaluators: verified availability (see Step 4.5) -- [ ] If callable: `python -c "import sys; sys.path.insert(0, '.agentops'); from callable_adapter import run_evaluation; print('OK')"` succeeds -- [ ] If callable: `AGENT_HTTP_URL` env var is set -- [ ] If callable with auth: auth token env var is set (`APP_API_TOKEN`, `API_KEY`, etc.) -- [ ] **Callable smoke test**: one real call succeeds (see subsection below) -- [ ] If Foundry: `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` env var is set -- [ ] If bundle has `source: foundry` evaluators: evaluator model is configured (`endpoint.model` or `AZURE_OPENAI_ENDPOINT` + `AZURE_OPENAI_DEPLOYMENT`) -- [ ] Azure auth: `az account show` succeeds OR `AZURE_OPENAI_API_KEY` is set -- [ ] Endpoint reachable: `curl -s -o /dev/null -w "%{http_code}" ` returns 200/401/405 (not connection refused) -- [ ] Evaluator model responds: `az cognitiveservices account deployment list --name -g ` confirms deployment exists - -Present a **confirmation table** with all discovered values (do not ask each one separately): -``` -┌─────────────────────────┬──────────────────────────────────────────┬────────┐ -│ Setting │ Value │ Source │ -├─────────────────────────┼──────────────────────────────────────────┼────────┤ -│ Scenario │ RAG │ code │ -│ Bundle │ rag_quality_baseline │ auto │ -│ Endpoint URL │ https://myapp.azurecontainerapps.io/chat │ .env │ -│ Auth │ dapr-api-token (APP_API_TOKEN) │ code │ -│ Evaluator model │ gpt-4o-mini │ Azure │ -│ Project endpoint │ https://acct.services.ai.azure.com/... │ .env │ -│ Azure auth │ az login active │ CLI │ -│ Endpoint reachable │ ✔ (200) │ check │ -│ Dataset rows │ 8 │ file │ -└─────────────────────────┴──────────────────────────────────────────┴────────┘ -``` - -Ask: *"Everything look correct? (yes / edit)"* - -### Callable smoke test - -A single real end-to-end call catches auth issues (401), wrong request body fields (400/422), and response parsing problems BEFORE wasting an entire evaluation run. - -```bash -python -c " -import sys; sys.path.insert(0, '.agentops') -from callable_adapter import run_evaluation -result = run_evaluation('hello', {}) -assert 'response' in result, f'Missing response key: {result}' -resp = result['response'] -assert not resp.startswith('ERROR:'), f'Adapter error: {resp}' -assert len(resp.strip()) > 0, 'Empty response — check endpoint and request format' -print('Smoke test PASSED') -print(f'Response length: {len(resp)} chars') -print('Response preview:', resp[:200]) -" -``` - -If the smoke test fails: -- **Connection refused** → the agent endpoint is not running. Start it first. -- **401 Unauthorized** → auth token is missing or wrong. Check `AGENT_AUTH_HEADER` and `AGENT_AUTH_TOKEN` env vars. -- **400/422** → the request body format doesn't match the endpoint. Check the `json.dumps({"message": ...})` field name in the adapter — the endpoint may expect a different key (e.g. `"ask"`, `"question"`, `"input"`). -- **Response starts with `ERROR:`** → the adapter caught an exception. Read the error message. -- **Empty response** → the endpoint returned successfully but the adapter extracted no text. Check `response_field` / `.get()` key in the adapter. -- **Response contains unexpected prefix** (UUID, metadata, HTML) → add a post-processing step to the adapter to strip it. Common pattern: `re.sub(r'^[0-9a-f-]{36}\s*', '', response_text)` for UUID prefixes. - -### Smoke test response format verification - -After the basic smoke test passes, verify the response format matches expectations: -1. If the response contains HTML tags (``, `

`, etc.) but the adapter expects plain text → the endpoint may be returning an error page, not agent output. -2. If the response is very short (< 10 chars) for a conversational prompt like "hello" → warn the user: *"Response seems unusually short. Verify the endpoint is returning the full agent response."* -3. If the response starts with `data:` or contains SSE markers but the adapter uses the standard JSON template → switch to the SSE/streaming adapter template. - -Do NOT proceed to Step 7 until the smoke test passes. - -## Step 7 — Execute - -Ask the user: *"Ready to run the evaluation?"* - -If yes: -```bash -agentops eval run -f all -``` - -After it completes, read `.agentops/results/latest/report.md` and summarize the results. - -## Comparing Runs - -For multi-model benchmarks, create one run.yaml per model: -```bash -agentops eval run -c .agentops/run-modelA.yaml -agentops eval run -c .agentops/run-modelB.yaml -agentops eval compare --runs , -f html -``` - -For agent version comparison, change `agent_id` per run. - -## Commands Reference - -```bash -agentops init # Scaffold workspace -agentops eval run [-c run.yaml] [-f md|html|all] # Run evaluation -agentops eval compare --runs id1,id2 [-f md|html|all] # Compare runs -agentops report generate [--in results.json] # Regenerate report -``` - -## Exit Codes - -- `0` — all thresholds passed -- `2` — threshold(s) failed -- `1` — runtime or configuration error - -## Rules - -- **NEVER** include `backend:` key in run.yaml — it causes a runtime error. -- **NEVER** leave `` placeholders in any generated file. -- **NEVER** fabricate `agent_id`, model names, or endpoint URLs. -- **NEVER** edit `.agentops/` template files (`run-callable.yaml`, `run-http-rag.yaml`, etc.) — always update `.agentops/run.yaml`. -- **NEVER** use dotted import paths like `.agentops.callable_adapter` — they fail. -- **NEVER** create files outside `.agentops/` — all generated artifacts (adapters, datasets, configs, scripts) belong in `.agentops/`. Exception: unit tests go in the project's existing test directory. -- **NEVER** try `az login` automatically — ask the user to authenticate. -- **NEVER** use `requests` or `httpx` in callable adapters — use only stdlib (`urllib.request`, `json`, `os`). -- If a bundle uses SDK-version-dependent evaluators, verify availability before running (Step 4.5). Don't block on this — if verification is hard, proceed and fix on failure. -- Always update `.agentops/run.yaml` — do not create custom-named files except for multi-model benchmarks. -- Use generic file names: `dataset.yaml`, `data.jsonl` — not project-specific prefixes. -- Use plain language in questions — not technical jargon ("callable adapter", "SSE", "POST"). -- Always run pre-flight (Step 6) before executing. Fix all issues first. - -## Unit Test Generation (Optional) - -This section is only executed if the user accepted the unit test offer in Step 1. - -### When to generate - -- The codebase has Python, JavaScript, or TypeScript agent code with testable logic (endpoint handlers, tool definitions, response parsing, orchestration). -- No existing test files or test directories were detected. - -### What to generate - -Create tests in the project's conventional test directory (e.g. `tests/test_agent.py` for Python, `__tests__/agent.test.ts` for TypeScript). Use only standard testing libraries — no extra dependencies. - -**For Python agents**, generate `pytest` tests using `unittest.mock`: - -1. **Endpoint handler test** — mock the HTTP framework (FastAPI `TestClient`, Flask `test_client`) and verify the handler returns expected response format. -2. **Response parsing test** — if the agent has response parsing logic (JSON extraction, SSE chunk assembly, UUID stripping), test it with known inputs/outputs. -3. **Error handling test** — verify the agent handles timeouts, 4xx/5xx from downstream services, and malformed inputs gracefully. -4. **Tool schema test** (if applicable) — if the agent defines tools with schemas, validate the schema structure is correct (required fields, types). - -**Template pattern** (adapt to the detected code): -```python -"""Unit tests for agent endpoint — generated by AgentOps.""" -import json -from unittest.mock import MagicMock, patch - -import pytest - - -class TestAgentEndpoint: - """Tests for the agent's HTTP endpoint handler.""" - - def test_returns_valid_response_format(self): - # Mock the downstream model/service call - # Call the endpoint handler directly - # Assert response has expected keys and types - ... - - def test_handles_empty_input(self): - # Verify the agent handles empty or whitespace-only input - ... - - def test_handles_downstream_timeout(self): - # Mock the downstream call to raise a timeout - # Assert the agent returns an error response (not a crash) - ... -``` - -### Rules for generated tests - -- Tests must run **without** Azure credentials or live services — all external calls must be mocked. -- Do not generate tests that duplicate what AgentOps evaluations already cover (response quality, groundedness, coherence). -- Focus on **functional correctness**: does the code do what it's supposed to do? -- Place tests in the project's existing test directory structure, not in `.agentops/`. -- If the project uses a specific test runner or framework (detected via `pyproject.toml`, `package.json`, `conftest.py`), follow its conventions. +publish: foundry_cloud +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" +``` + +Foundry-side latency and judges replace the local view in this mode; +`results.json` from the local run remains the canonical record. + +## Tips + +- Evaluators are auto-selected from the agent type and dataset columns. + Override only when needed via the `evaluators:` block — most users do + not need it. +- Set thresholds in `thresholds:` to gate CI: + ```yaml + thresholds: + coherence: ">=3" + avg_latency_seconds: "<=10" + ``` +- For HTTP/JSON agents that need auth, set + `auth_header_env: MY_TOKEN_VAR` and AgentOps adds + `Authorization: Bearer $MY_TOKEN_VAR`. diff --git a/plugins/agentops/skills/agentops-monitor/SKILL.md b/plugins/agentops/skills/agentops-monitor/SKILL.md deleted file mode 100644 index 67afa4c8..00000000 --- a/plugins/agentops/skills/agentops-monitor/SKILL.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: agentops-monitor -description: Guidance on monitoring evaluation quality over time. Trigger when users ask about tracking scores, setting up dashboards, or configuring quality alerts. Common phrases include "monitoring", "dashboards", "alerts", "monitor setup", "quality over time", "trending", "track scores", "evaluation health". Install agentops-toolkit via pip. ---- - -# AgentOps Monitor - -## Purpose - -Provide guidance on monitoring evaluation quality over time. The `agentops monitor` commands are **planned but not yet implemented**. - -## Before You Start - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. - -## Status - -🚧 **Not yet implemented.** The CLI stubs exist but have no runtime behavior. - -## Current Alternatives - -Until `agentops monitor` is available: - -| Approach | How | -|---|---| -| Manual trending | Compare `results.json` across timestamped runs in `.agentops/results/` | -| CI gating | Use exit code `2` in GitHub Actions to block PRs on quality regressions | -| Foundry portal | View evaluation history in the Foundry Experience dashboard | -| Run comparison | `agentops eval compare --runs ,` for side-by-side delta | - -## What Will Be Available - -When implemented: -- `agentops monitor show` — Display evaluation quality dashboard -- `agentops monitor configure` — Set up alerts and quality thresholds - -## Guardrails - -- Do not pretend monitoring features exist — clearly state they are planned. -- For quality tracking today, recommend `agentops eval compare` and CI exit codes. -- For production monitoring, recommend Azure Monitor and Foundry portal. diff --git a/plugins/agentops/skills/agentops-regression/SKILL.md b/plugins/agentops/skills/agentops-regression/SKILL.md deleted file mode 100644 index 6a8d295b..00000000 --- a/plugins/agentops/skills/agentops-regression/SKILL.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -name: agentops-regression -description: Investigate evaluation regressions — compare runs, analyze per-row scores, identify root causes. Trigger when users report score drops, threshold failures, or quality degradation between runs. Common phrases include "regression", "score dropped", "threshold failed", "compare runs", "why worse", "which rows failed", "debug evaluation", "quality degradation". Install agentops-toolkit via pip. ---- - -# AgentOps Regression - -## Purpose - -Investigate evaluation score drops and threshold failures. Compare runs side-by-side, identify which rows regressed, and guide root-cause analysis. - -## When to Use - -- Exit code `2` — thresholds failed. -- Scores dropped between two runs. -- User asks "why did this eval get worse" or "which rows failed". - -## Before You Start - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. -4. **Two runs available?** Need a baseline and a current run. Check `.agentops/results/` for timestamped directories. -5. **Results exist?** Each run must have `results.json`. - -## Steps - -### Step 1 — Identify the regression - -```bash -agentops eval compare --runs , -``` - -Review the comparison output for ↓ indicators and delta values. - -### Step 2 — Analyze per-row scores - -Open `results.json` for both runs. Compare `row_metrics` to find rows where scores dropped. Look for: -- Rows with the largest negative delta -- Rows that went from pass → fail -- Clusters of failures in specific evaluators - -### Step 3 — Check what changed - -Common regression causes: -| Cause | What to check | -|---|---| -| Model update | Deployment version, model name change | -| Prompt drift | System prompt or instructions changed | -| Data drift | New dataset rows, different distribution | -| Tool schema change | Tool definitions modified | -| Context quality | RAG retriever returning different passages | -| Threshold tightened | Bundle threshold values changed | - -### Step 4 — Act on findings - -| Finding | Action | -|---|---| -| Model regression | Pin model version or switch deployment | -| Prompt issue | Revert or iterate on prompt changes | -| Bad test rows | Fix dataset and re-run | -| Threshold too strict | Adjust thresholds in bundle (use `/agentops-config`) | -| Retriever degraded | Debug retrieval pipeline separately | - -### Step 5 — Verify fix - -Re-run the evaluation after the fix: -```bash -agentops eval run -agentops eval compare --runs ,latest -``` - -## Guardrails - -- Work with actual scores — never guess what caused a regression. -- Do not modify `results.json` — it is immutable. -- Do not adjust thresholds to hide real regressions. -- Delegate execution to `/agentops-eval` and config changes to `/agentops-config`. diff --git a/plugins/agentops/skills/agentops-report/SKILL.md b/plugins/agentops/skills/agentops-report/SKILL.md index dc10fd8f..72ed2bd4 100644 --- a/plugins/agentops/skills/agentops-report/SKILL.md +++ b/plugins/agentops/skills/agentops-report/SKILL.md @@ -1,92 +1,69 @@ --- name: agentops-report -description: Interpret evaluation reports, explain indicators, and regenerate reports. Trigger when users ask to understand results, explain scores, or regenerate a report. Common phrases include "report", "interpret results", "what does this mean", "explain scores", "report generate", "results.json", "pass rate", "threshold". Install agentops-toolkit via pip. +description: Read, regenerate, and explain AgentOps evaluation reports. Trigger on "show report", "explain scores", "regenerate report", "what do these metrics mean". Operates on results.json and report.md produced by `agentops eval run`. --- # AgentOps Report -## Purpose +Help the user understand a finished AgentOps run. -Help users understand evaluation results, explain report indicators, and regenerate reports from existing `results.json` files. +## Step 0 — Locate the run -## When to Use +Latest run: `.agentops/results/latest/`. Each run produces: -- User asks what an evaluation result means. -- User wants to regenerate a report after manual edits. -- User needs to compare report sections between runs. -- User asks about pass rates, thresholds, or score meanings. +- `results.json` — machine-readable metrics, per-row scores, thresholds. +- `report.md` — human-readable summary suitable for PR comments. +- `cloud_evaluation.json` (only when `publish:` was set) — deep-link to + the Foundry Evaluations panel. `mode: classic` for `publish: foundry`, + `mode: cloud` for `publish: foundry_cloud` (preview, server-side run + via the OpenAI Evals API). -## Before You Start +## Step 1 — Regenerate report.md if needed -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Results exist?** Check for `.agentops/results/latest/results.json`. If missing, run `/agentops-eval` first. -4. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. - -## Commands - -| Command | Purpose | -|---|---| -| `agentops report generate --in [--out ]` | Regenerate report from results | - -## Report Indicators - -| Symbol | Meaning | -|---|---| -| `●` (green) | Score meets or exceeds threshold | -| `●` (red) | Score below threshold | -| `↑` | Score improved vs. baseline | -| `↓` | Score regressed vs. baseline | -| `—` | No baseline available | - -## Key Metrics - -| Metric | Description | -|---|---| -| `run_pass` | `true` if all thresholds passed | -| `threshold_pass_rate` | Fraction of thresholds met | -| `items_pass_rate` | Fraction of rows passing all evaluators | -| per-evaluator avg | Mean score across all rows for one evaluator | -| per-evaluator stddev | Standard deviation (high = inconsistent) | +```bash +agentops report generate # uses .agentops/results/latest/results.json +agentops report generate --in --out +``` -## Report Sections +`report generate` always reads the flat 1.0 results schema and emits +Markdown. There is no HTML format. -### Single Run (`report.md`) -- **Summary**: overall pass/fail, item counts -- **Threshold Results**: per-evaluator threshold vs. actual score -- **Row Details**: per-row scores for each evaluator +## Step 2 — Explain the metrics -### Comparison (`agentops eval compare`) -- **Side-by-side**: baseline vs. current scores -- **Delta**: absolute change per evaluator -- **Direction**: ↑ improved, ↓ regressed, — unchanged +Common metrics and their meaning: -## Steps +| Metric | Range | Higher is better? | Notes | +|---|---|---|---| +| `similarity` | 1-5 | yes | LLM-judged similarity to `expected`. | +| `coherence` | 1-5 | yes | Answer is internally consistent. | +| `fluency` | 1-5 | yes | Natural language quality. | +| `groundedness` | 1-5 | yes | Answer is supported by `context` (RAG). | +| `relevance` | 1-5 | yes | Answer is on-topic for `input`. | +| `f1_score` | 0-1 | yes | Token overlap with `expected`. | +| `tool_call_accuracy` | 0-1 | yes | Predicted tool calls match `tool_calls`. | +| `intent_resolution` | 0-1 | yes | User intent was resolved. | +| `task_completion` | 0-1 | yes | Multi-step task finished. | +| `avg_latency_seconds` | seconds | no | Wall-clock latency per row. | -### Interpreting results -1. Open `.agentops/results/latest/report.md`. -2. Check the summary — is `run_pass: true`? -3. If false, find which thresholds failed (red dots). -4. Look at per-row scores to identify weak rows. -5. For AI evaluators (coherence, groundedness), scores are 1–5. -6. For content safety evaluators, lower is better (0 = safe). +Pass/fail rows are derived from `thresholds:` in `agentops.yaml`. The +exit code of the original run reflects the gate: -### Regenerating a report -```bash -agentops report generate --in .agentops/results/latest/results.json -``` +- `0` → all thresholds passed +- `2` → one or more thresholds failed +- `1` → runtime error -## Exit Codes +## Step 3 — Help the user act on results -| Code | Meaning | -|---|---| -| `0` | Success and all thresholds passed | -| `2` | Success but threshold(s) failed | -| `1` | Runtime or configuration error | +- For low scores on a specific metric, point at the lowest-scoring rows + in `results.json` (`row_metrics[]` and `item_evaluations[]`) and + suggest concrete prompt or retrieval changes. +- For latency regressions, look at `run_metrics.avg_latency_seconds` and + per-row latency. +- To compare two runs, diff the two `results.json` files at the metric + level and surface the deltas; AgentOps does not ship a separate + comparison CLI. ## Guardrails -- Use actual scores from `results.json` — never guess or estimate. -- Do not run evaluations — delegate to `/agentops-eval`. -- Do not modify `results.json` — it is an immutable run artifact. -- If the user needs different thresholds, delegate to `/agentops-config` to update the bundle. +- Never invent metric values. If a metric is absent, say so. +- Do not edit `results.json` by hand — re-run the eval. diff --git a/plugins/agentops/skills/agentops-trace/SKILL.md b/plugins/agentops/skills/agentops-trace/SKILL.md deleted file mode 100644 index 33435e9e..00000000 --- a/plugins/agentops/skills/agentops-trace/SKILL.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: agentops-trace -description: Guidance on tracing for AgentOps evaluations. Trigger when users ask about tracing agent execution, setting up telemetry, or inspecting spans. Common phrases include "tracing", "trace init", "trace setup", "distributed tracing", "span", "telemetry", "trace evaluation", "trace agent". Install agentops-toolkit via pip. ---- - -# AgentOps Trace - -## Purpose - -Provide guidance on tracing agent execution. The `agentops trace` command is **planned but not yet implemented**. - -## Before You Start - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. - -## Status - -🚧 **Not yet implemented.** The CLI stub exists but has no runtime behavior. - -## Current Alternatives - -Until `agentops trace` is available, use these tools directly: - -| Tool | Use case | -|---|---| -| Azure Monitor / Application Insights | Production tracing for Foundry agents | -| OpenTelemetry SDK | Custom span instrumentation | -| Foundry portal | Built-in agent execution traces | -| `results.json` row metrics | Per-row latency via `avg_latency_seconds` | - -## What Will Be Available - -When implemented, `agentops trace init` will: -- Configure OpenTelemetry export for AgentOps evaluation runs -- Capture per-row agent execution spans -- Link traces to evaluation results for debugging - -## Guardrails - -- Do not pretend tracing features exist — clearly state they are planned. -- For latency analysis, point users to `avg_latency_seconds` in evaluation bundles. -- For production tracing, recommend Azure Monitor or OpenTelemetry directly. diff --git a/plugins/agentops/skills/agentops-workflow/SKILL.md b/plugins/agentops/skills/agentops-workflow/SKILL.md index 79d70bfa..b81a98a1 100644 --- a/plugins/agentops/skills/agentops-workflow/SKILL.md +++ b/plugins/agentops/skills/agentops-workflow/SKILL.md @@ -1,165 +1,152 @@ --- name: agentops-workflow -description: Generate CI/CD pipelines tailored to the project — PR gating, post-merge CI evaluation, and CD with safety QA + deploy placeholder. Trigger when users ask to automate evaluations in CI, set up PR gating, generate workflow files, or create pipelines for their project. Common phrases include "CI/CD", "GitHub Actions", "pipeline", "workflow", "PR gating", "continuous evaluation", "automate evals", "workflow generate", "CI setup", "generate pipelines", "create pipelines for my project". Install agentops-toolkit via pip. +description: Set up the full GenAIOps GitFlow CI/CD scaffold for an AgentOps project. Generates four GitHub Actions workflows (PR gate + Deploy DEV / QA / PROD) wired to GitHub Environments, OIDC auth, and AgentOps eval gating. Trigger on "CI", "CD", "pipeline", "workflow", "GitHub Actions", "PR gate", "deploy", "environments", "GitFlow", "release branch", "promote to prod", "DevOps", "GenAIOps pipeline". --- # AgentOps Workflow -Generate a complete CI/CD pipeline suite for AgentOps evaluations — tailored to the project's evaluation scenarios, bundles, and Foundry configuration. +Help the user wire AgentOps into a real GenAIOps GitFlow CI/CD setup with +three environments (`dev`, `qa`, `production`) and an automatic eval gate +on every change. -## Pipeline Types +This skill produces four workflow files via `agentops workflow generate` +and then walks the user through the GitHub-side configuration (OIDC, +environments, branch protection, deploy step). -`agentops workflow generate` auto-detects which pipelines to create: +## Branch model assumed -| Pipeline | File | When generated | Purpose | -|---|---|---|---| -| **PR Evaluation** | `agentops-eval.yml` | Always | Fast evaluation gate on pull requests | -| **CI Evaluation** | `agentops-eval-ci.yml` | Multiple bundles or run configs detected | Full evaluation on merge to develop/main | -| **CD Pipeline** | `agentops-eval-cd.yml` | Multiple bundles or run configs detected | Safety QA gate + deploy placeholder on merge to main | - -### Pipeline Flow (GenAIOps-inspired) - -``` -feature/* → PR to develop → agentops-eval.yml (PR gate) - merge to develop → agentops-eval-ci.yml (CI evaluation) - release/* → PR to main → agentops-eval.yml (PR gate) - merge to main → agentops-eval-cd.yml (safety QA → deploy) -``` - -## Step 0 — Prerequisites - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`, `.azure//.env`. If not found, ask the user for the endpoint URL. -4. **run.yaml ready?** A valid run config is required. If missing, delegate to `/agentops-config`. - -## Step 1 — Workspace Inspection - -Before generating, inspect the workspace to understand what pipelines are needed: - -1. **List bundles**: Read `.agentops/bundles/` — identify which evaluation scenarios are configured. -2. **List run configs**: Check `.agentops/` for `run*.yaml` files — if multiple configs exist, CI and CD pipelines are appropriate. -3. **Check Foundry endpoint**: Look for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` or `project_endpoint` in run.yaml and env vars. -4. **Detect branches**: Run `git branch -a` to list local and remote branches. - - If `main` and `develop` exist → use them (default convention, no question needed). - - If branches don't exist yet → use `main`/`develop` convention (no question needed). - - If the repo uses different names (e.g. `master` instead of `main`, or no `develop`) → ask the user to confirm which branches to use for PR targets and push triggers. - -Present a summary: -``` -Detected: - Bundles: model_quality_baseline, rag_quality_baseline - Run configs: run.yaml - Foundry endpoint: ✓ (from .env) - Branches: main, develop - Pipelines: PR (always), CI + CD (multiple bundles detected) ``` - -## Step 2 — Ask Only What Cannot Be Inferred - -Only ask critical questions that workspace inspection cannot answer: - -1. If no Foundry endpoint found: *"What is your Azure AI Foundry project endpoint URL?"* -2. If branches differ from the `main`/`develop` convention: *"Your repo uses `master` instead of `main`. Should the pipelines target `master`, or do you plan to rename it to `main`?"* - -**DO NOT ask about**: -- Bundle selection (inferred from workspace) -- Evaluation scenarios (inferred from bundles) -- Authentication method (always OIDC / Workload Identity Federation) -- Workflow file locations (standard `.github/workflows/` paths) -- Which pipelines to generate (auto-detected) - -## Step 3 — Generate Workflows - -```bash -agentops workflow generate [--force] [--dir ] +feature/* ── PR ──▶ develop [agentops-pr.yml] gate + │ + └── merge ─▶ develop [agentops-deploy-dev.yml] build + eval + deploy DEV +release/* ── push [agentops-deploy-qa.yml] build + eval + deploy QA +release/* ── PR ──▶ main [agentops-pr.yml] gate + │ + └── merge ─▶ main [agentops-deploy-prod.yml] safety eval + build + deploy PROD ``` -Flags: -- `--force` — Overwrite existing workflow files. -- `--dir` — Target directory (default: current directory). - -After generation, explain what was created and why: -- `agentops-eval.yml` — Runs on PRs to main/develop. Gates merges on evaluation thresholds. -- `agentops-eval-ci.yml` — (if generated) Runs on push to develop/main when `.agentops/`, `src/`, or `pyproject.toml` change. Comprehensive post-merge evaluation with commented-out matrix strategy and baseline comparison. -- `agentops-eval-cd.yml` — (if generated) Runs on push to main. Two-job pipeline: safety QA evaluation gate → deploy placeholder. The deploy job is a TODO for the team to fill in with their deployment commands. - -## Step 4 — Configure Authentication - -All pipelines use **Workload Identity Federation (OIDC)** — no client secrets to manage or rotate. - -### Azure Setup (one-time) - -1. **Create or reuse an App Registration** in Microsoft Entra ID (Azure AD). -2. **Add a Federated Credential**: - - Go to App Registration → Certificates & secrets → Federated credentials → Add credential - - Organization: your GitHub org/user - - Repository: your repo name - - Entity type: select **Pull Request** (for PR pipeline) AND **Branch** (for CI and CD pipelines) - - Name: e.g. `github-agentops-eval` -3. **Grant the app required roles** on the Foundry project resource group: - - `Cognitive Services User` — invoke agents and evaluator models - - `Azure AI Developer` — access evaluation APIs and Foundry features +If the user is on trunk-based development, omit `qa` and `release/**` +and have them generate `--kinds pr,dev,prod`. -### GitHub Setup - -Set these as **repository variables** (Settings → Secrets and variables → Actions → Variables tab): - -| Variable | Value | -|---|---| -| `AZURE_CLIENT_ID` | Application (client) ID from App Registration | -| `AZURE_TENANT_ID` | Directory (tenant) ID | -| `AZURE_SUBSCRIPTION_ID` | Azure subscription ID | - -Set this as a **repository secret** (Secrets tab): +## Step 0 — Prerequisites -| Secret | Value | -|---|---| -| `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Foundry project endpoint URL | +1. `pip install agentops-toolkit` if `agentops` is missing. +2. `.agentops/run.yaml` exists and `agentops eval run` works locally. +3. The user's repo follows GitFlow (or is willing to). If not, ask which + branches map to dev/qa/prod and adjust the `on:` triggers after + generation. -### Verify Auth Locally +## Step 1 — Generate the workflows ```bash -az login -az account show --query "{sub:id, tenant:tenantId}" -o json -az account get-access-token --resource "https://cognitiveservices.azure.com" --query accessToken -o tsv +agentops workflow generate ``` -## Step 5 — Verify Pipelines - -1. **PR pipeline**: Push a branch and open a PR → check the Actions tab for `AgentOps Evaluation`. -2. **CI pipeline**: Merge to develop → check Actions tab for `AgentOps CI Evaluation`. -3. **CD pipeline**: Merge to main → check Actions tab for `AgentOps CD Pipeline`. The safety-qa job runs evaluation; the deploy job prints a placeholder notice. -4. **Check results**: Download artifacts, review PR comments, inspect job summaries. - -If any pipeline fails with authentication errors: -- Verify federated credential entity types match (Pull Request for PRs, Branch for push) -- Confirm the App Registration has `Cognitive Services User` role on the Foundry resource -- Check that variables and secrets are set at the repository level (not organization) - -## Exit Code Gating - -All pipelines use the same exit code contract: +This writes **four** files into `.github/workflows/`: -| Exit code | CI result | Meaning | +| File | Trigger | Environment | |---|---|---| -| `0` | ✅ Pass | All thresholds met | -| `2` | ❌ Fail | Threshold(s) failed — blocks merge / blocks deploy | -| `1` | ❌ Fail | Runtime or configuration error | - -## Customisation After Generation - -- **Change branch triggers**: Edit `on.pull_request.branches` or `on.push.branches` in the workflow files. -- **Enable matrix strategy**: Uncomment the `strategy.matrix` block in `agentops-eval-ci.yml` and list your run configs. -- **Enable baseline comparison**: Uncomment the comparison step in `agentops-eval-ci.yml`. -- **Add deployment steps**: Edit the `deploy` job in `agentops-eval-cd.yml` — replace the placeholder with your actual deployment commands. -- **Add environment approval**: Uncomment `environment: production` in the deploy job for manual approval gates. - -## Rules - -- Do not modify generated workflow files beyond user-requested customisation. -- Always recommend OIDC / Workload Identity Federation over client secrets. -- Delegate evaluation configuration to `/agentops-config`. -- Delegate dataset creation to `/agentops-dataset`. -- Do not fabricate endpoint URLs, agent IDs, or deployment names. -- Do not ask about bundle/scenario selection if it can be inferred from the workspace. +| `agentops-pr.yml` | PRs to `develop`, `release/**`, `main` | (none) | +| `agentops-deploy-dev.yml` | push to `develop` | `dev` | +| `agentops-deploy-qa.yml` | push to `release/**` | `qa` | +| `agentops-deploy-prod.yml` | push to `main` | `production` | + +Useful flags: + +- `--force` — overwrite existing workflow files. +- `--kinds pr,dev,qa,prod` — generate a subset (e.g. `--kinds pr,dev,prod` + for trunk-based teams). +- `--dir ` — non-default repo root. + +## Step 2 — Configure GitHub Environments + +Walk the user through Settings → Environments and create three: + +1. **`dev`** — no extra protection. Set any DEV-specific variables here + (e.g. `ACA_APP_NAME`, `AZURE_RESOURCE_GROUP` pointing at the dev RG). +2. **`qa`** — usually no required reviewers, but isolated variables for + the QA environment. +3. **`production`** — set: + - **Required reviewers**: at least one (deploys to PROD will pause + here until approved). + - (Optional) **Wait timer** for an extra delay. + - (Optional) **Deployment branches**: restrict to `main`. + - PROD-specific variables (e.g. production resource group). + +Tell the user that env-specific variables on the `production` environment +will override repo-level ones automatically inside the prod workflow. + +## Step 3 — Configure repository variables for OIDC + +At repository level (Settings → Secrets and variables → Actions → +**Variables** tab), set: + +- `AZURE_CLIENT_ID` — App registration / managed identity used for OIDC. +- `AZURE_TENANT_ID` +- `AZURE_SUBSCRIPTION_ID` +- `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` — Foundry project URL used by the + eval step. + +Then configure Workload Identity Federation on the Azure side +(`federated-credentials` on the app registration) for **each branch / +environment** the workflows will run from. See +`docs/ci-github-actions.md` for the exact `az` commands. + +## Step 4 — Fill in the Build and Deploy placeholders + +Each `agentops-deploy-*.yml` has a `Build (placeholder)` and a +`Deploy (placeholder)` step. The dev template includes commented +example snippets for the most common stacks. Replace them based on +the user's stack: + +- **Container Apps** — replace Build with `az acr build` and Deploy + with `az containerapp update --image ...`. +- **App Service** — replace Build with the package step, Deploy with + `azure/webapps-deploy@v3`. +- **Foundry hosted agent** — Build is typically empty; Deploy publishes + a new agent version (project-specific tooling). +- **azd-managed app** — replace Build with `azd package` and Deploy + with `azd deploy --no-prompt` (set `AZURE_ENV_NAME` per environment). + +Don't invent commands you can't see in the user's repo. If the stack +isn't obvious, ask. + +## Step 5 — Branch protection + +In Settings → Branches, add a rule for both `develop` and `main`: + +- Require a pull request before merging. +- Require status checks to pass: select **`AgentOps PR / Eval (PR gate)`** + (the job name from `agentops-pr.yml`). +- Optional: require linear history. + +This makes the eval gate a hard merge requirement. + +## Step 6 — Iterate + +Common follow-ups: + +- **Tighten thresholds for QA/PROD** — copy `.agentops/run.yaml` to + `.agentops/run-qa.yaml` / `.agentops/run-prod.yaml` and tighten the + bundle thresholds. Point each workflow at its own config via the + `inputs.config` default. +- **Scheduled runs** — add a `schedule:` entry in `agentops-pr.yml` (or a + new `agentops-nightly.yml`) to evaluate against `main` nightly. +- **Matrix per scenario** — if the user has multiple `runs/*.yaml` files, + extend the eval job with `strategy.matrix.config:` and reference + `${{ matrix.config }}`. +- **Regression baseline** — wire the deploy templates to download the + previous run's `results.json` artifact and call + `agentops eval compare`. + +## Guardrails + +- Do **not** invent CLI flags. The supported `workflow generate` flags + are `--force`, `--dir`, `--kinds`. +- Do **not** create parallel workflow files. Prefer editing the + generated ones. +- Do **not** auto-fill Build/Deploy with steps you can't justify from + the user's existing code. Ask before guessing. +- The four workflow names (`agentops-pr`, `agentops-deploy-dev`, + `agentops-deploy-qa`, `agentops-deploy-prod`) are fixed — don't rename + them or branch-protection wiring will break. diff --git a/pyproject.toml b/pyproject.toml index d3cad410..e0215266 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,19 @@ dependencies = [ ] license = { file = "LICENSE" } +[project.optional-dependencies] +mcp = ["mcp>=1.0,<2"] +agent = [ + "fastapi>=0.110", + "uvicorn[standard]>=0.30", + "httpx>=0.27", + "cryptography>=42", + "azure-monitor-query>=1.3", + "azure-identity>=1.17", + "azure-mgmt-cognitiveservices>=13.5", + "azure-mgmt-monitor>=6.0", +] + [project.scripts] agentops = "agentops.cli.app:main" @@ -27,25 +40,19 @@ where = ["src"] [tool.setuptools.package-data] "agentops.templates" = [ - "config.yaml", - "run.yaml", - "run-rag.yaml", - "run-agent.yaml", - "run-http-model.yaml", - "run-http-rag.yaml", - "run-http-agent-tools.yaml", - "run-callable.yaml", - "callable_adapter.py", + "agentops.yaml", + "agent.yaml", + "smoke.jsonl", ".gitignore", - "bundles/*.yaml", - "datasets/*.yaml", - "data/*.jsonl", + "project.gitignore", "workflows/*.yml", "skills/*/SKILL.md", + "agent-server/*", ] [dependency-groups] dev = [ + "azure-ai-evaluation>=1.0", "mypy>=1.19.1", "pre-commit>=4.0", "pytest>=8.0", @@ -56,3 +63,6 @@ dev = [ [tool.setuptools_scm] local_scheme = "no-local-version" + +[tool.mypy] +plugins = ["pydantic.mypy"] diff --git a/scripts/create_support_agent.py b/scripts/create_support_agent.py new file mode 100644 index 00000000..2ddfb82a --- /dev/null +++ b/scripts/create_support_agent.py @@ -0,0 +1,351 @@ +"""Create or delete a hosted Foundry support agent for the end-to-end tutorial. + +The tutorial in ``docs/tutorial-end-to-end.md`` walks the user through a +realistic agent-with-tools evaluation. This helper avoids forcing the user to +click through the Foundry portal: it registers three function tools +(``lookup_order``, ``refund_order``, ``escalate_to_human``) on a fresh hosted +prompt agent in one command. + +Usage:: + + # Create the agent. Prints the ``name:version`` identifier you paste into + # ``agentops.yaml``. + python scripts/create_support_agent.py create --name support-bot + + # Create a degraded version that omits the tool-call instruction. Used to + # demonstrate baseline regression detection. + python scripts/create_support_agent.py create --name support-bot --variant v2-degraded + + # Delete every version of the agent (idempotent — ignores 404s). + python scripts/create_support_agent.py delete --name support-bot + +Authentication uses ``DefaultAzureCredential``; the project endpoint is read +from ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT``. The user must hold an Azure AI +data-plane role (``Azure AI User`` is enough) on the Foundry account. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time + +INSTRUCTIONS_GOOD = ( + "You are a customer support assistant. You MUST use the provided tools to " + "answer the user. Choose exactly one tool per message and supply complete, " + "correct arguments. Use:\n" + "- lookup_order when the user asks about an order's status, location, or " + "delivery details.\n" + "- refund_order when the user explicitly asks for a refund or to return " + "an item.\n" + "- escalate_to_human when the user asks to speak with a human, manager, " + "or representative, or expresses serious frustration.\n" + "If the user is just greeting you or making small talk, respond briefly " + "in plain text without calling any tool. Never invent data and never " + "answer order-specific questions from memory." +) + +INSTRUCTIONS_DEGRADED = ( + "You are a friendly customer support assistant. Answer the user in a " + "warm, conversational tone. Reassure them and apologize for any " + "inconvenience. Do not use any tools — just reply in plain text." +) + + +LOOKUP_ORDER_PARAMETERS = { + "type": "object", + "properties": { + "order_id": { + "type": "string", + "description": "The order identifier the user mentioned, e.g. 'ORD-12345'.", + } + }, + "required": ["order_id"], + "additionalProperties": False, +} + +REFUND_ORDER_PARAMETERS = { + "type": "object", + "properties": { + "order_id": { + "type": "string", + "description": "The order identifier to refund.", + }, + "reason": { + "type": "string", + "description": "Short reason text the user gave (e.g. 'arrived broken').", + }, + }, + "required": ["order_id", "reason"], + "additionalProperties": False, +} + +ESCALATE_TO_HUMAN_PARAMETERS = { + "type": "object", + "properties": { + "category": { + "type": "string", + "description": "Short topic the user wants to discuss (e.g. 'refund', 'billing').", + } + }, + "required": ["category"], + "additionalProperties": False, +} + + +TOOL_SPECS = [ + ( + "lookup_order", + "Look up the current status and shipping details of a customer order.", + LOOKUP_ORDER_PARAMETERS, + ), + ( + "refund_order", + "Issue a refund for a customer order, given the order id and a short reason.", + REFUND_ORDER_PARAMETERS, + ), + ( + "escalate_to_human", + "Hand the conversation over to a human agent for the given topic.", + ESCALATE_TO_HUMAN_PARAMETERS, + ), +] + + +def _client(): + import logging + + from azure.ai.projects import AIProjectClient + from azure.core.exceptions import ClientAuthenticationError + from azure.identity import DefaultAzureCredential + + # Silence azure-identity's verbose credential-chain logging so the + # friendly "Run `az login`" message below isn't drowned out. + logging.getLogger("azure.identity").setLevel(logging.ERROR) + logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel( + logging.ERROR + ) + + endpoint = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise SystemExit( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT is required. " + "Set it to your Foundry project URL, e.g. " + "'https://.services.ai.azure.com/api/projects/'." + ) + cred = DefaultAzureCredential(exclude_developer_cli_credential=True) + # Preflight: fail fast with a clear message if no Azure identity is available. + # Without this, the SDK would otherwise dump a 30+ line credential-chain + # error five times (once per retry) before bailing out. + try: + cred.get_token("https://ai.azure.com/.default") + except ClientAuthenticationError: + raise SystemExit( + "Unable to acquire an Azure access token. Run `az login` " + "(or set AZURE_CLIENT_ID/AZURE_TENANT_ID/AZURE_CLIENT_SECRET for " + "service-principal auth) and try again." + ) + return AIProjectClient(endpoint=endpoint, credential=cred) + + +def cmd_create(args: argparse.Namespace) -> int: + from azure.ai.projects.models import FunctionTool, PromptAgentDefinition + from azure.core.exceptions import HttpResponseError, ServiceResponseError + + instructions = ( + INSTRUCTIONS_DEGRADED if args.variant == "v2-degraded" else INSTRUCTIONS_GOOD + ) + if args.variant == "v2-degraded": + # The whole point of the degraded variant is to demonstrate a tool-quality + # regression. We strip the tools entirely so the agent literally cannot + # call lookup_order / refund_order / escalate_to_human — forcing + # tool_call_accuracy and task_adherence to collapse. + tools: list = [] + else: + tools = [ + FunctionTool(name=name, description=desc, parameters=params, strict=True) + for name, desc, params in TOOL_SPECS + ] + + definition = PromptAgentDefinition( + model=args.model, + instructions=instructions, + tools=tools, + ) + + client = _client() + description = ( + "AgentOps tutorial support agent (degraded baseline)." + if args.variant == "v2-degraded" + else "AgentOps tutorial support agent (lookup_order, refund_order, escalate_to_human)." + ) + + last_exc: Exception | None = None + version = None + for attempt in range(1, 6): + try: + version = client.agents.create_version( + agent_name=args.name, + definition=definition, + description=description, + ) + break + except (HttpResponseError, ServiceResponseError) as exc: + status = getattr(exc, "status_code", None) + transient = status is None or status >= 500 or status == 429 + print( + f"create_version attempt {attempt}/5 failed (status={status}): {exc}", + file=sys.stderr, + ) + if status in (401, 403): + # Don't retry on auth errors and surface a concrete fix. + raise SystemExit( + "Foundry rejected the access token (HTTP " + f"{status}). Likely causes, in order of frequency:\n" + " 1. Stale Azure CLI token cache (very common when " + "this script worked earlier today and now suddenly " + "fails). Refresh with:\n" + " az account clear\n" + " az login\n" + " If you have multiple tenants, add `--tenant `.\n" + " 2. You're logged into the wrong Azure tenant. " + "Verify with `az account show` and re-login with " + "`az login --tenant ` if needed.\n" + " 3. Your account is missing the 'Azure AI User' " + "role on the Foundry account. Ask an admin (or run " + "yourself if you have permissions):\n" + " az role assignment create \\\n" + " --assignee \\\n" + " --role 'Azure AI User' \\\n" + " --scope \n" + " 4. AZURE_AI_FOUNDRY_PROJECT_ENDPOINT points at a " + "different project than the one where your role " + "assignment lives." + ) from exc + if not transient or attempt == 5: + raise + last_exc = exc + time.sleep(min(2**attempt, 30)) + if version is None: + raise SystemExit(f"create_version failed after retries: {last_exc!r}") + + version_id = getattr(version, "version", None) or getattr(version, "id", None) + if not version_id: + raise SystemExit(f"Could not determine version id from response: {version!r}") + + print(f"{args.name}:{version_id}") + print( + f"Created hosted agent {args.name}:{version_id} " + f"(variant={args.variant}, model={args.model}).", + file=sys.stderr, + ) + + # Read back the registered tools so the user can confirm they are + # attached. The Foundry portal's Playground tab only surfaces tools + # added through the portal's "Add" button — SDK-registered tools are + # invisible there but DO get used at runtime. Printing them here + # avoids the "I created the agent but Tools is empty" confusion. + try: + fetched = client.agents.get_version( + agent_name=args.name, version=str(version_id) + ) + definition = getattr(fetched, "definition", None) + raw_tools = ( + getattr(definition, "tools", None) + if definition is not None + else None + ) or [] + tool_names = [] + for t in raw_tools: + n = getattr(t, "name", None) + if n is None and isinstance(t, dict): + n = t.get("name") + if n: + tool_names.append(n) + if tool_names: + print( + f"Registered tools: {', '.join(tool_names)}", + file=sys.stderr, + ) + except Exception: # noqa: BLE001 — read-back is best-effort + pass + + print( + "Paste the identifier above into the 'agent:' field of agentops.yaml.", + file=sys.stderr, + ) + print( + "Note: the Foundry portal's Playground 'Tools' panel only lists " + "tools added via the portal UI. SDK-registered tools (like these) " + "show up under the agent's 'Code' / 'YAML' tab and ARE invoked " + "at runtime — `agentops eval run` will exercise them.", + file=sys.stderr, + ) + return 0 + + +def cmd_delete(args: argparse.Namespace) -> int: + from azure.core.exceptions import ResourceNotFoundError + + client = _client() + deleted = 0 + try: + for v in client.agents.list_versions(agent_name=args.name): + ver_id = getattr(v, "version", None) or getattr(v, "id", None) + if ver_id is None: + continue + try: + client.agents.delete_version(agent_name=args.name, version=str(ver_id)) + deleted += 1 + except ResourceNotFoundError: + pass + except ResourceNotFoundError: + print(f"Agent {args.name} not found (already deleted).", file=sys.stderr) + return 0 + + try: + client.agents.delete(agent_name=args.name) + except ResourceNotFoundError: + pass + + print( + f"Deleted hosted agent {args.name} ({deleted} version(s)).", + file=sys.stderr, + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + + p_create = sub.add_parser("create", help="Create the support agent.") + p_create.add_argument("--name", required=True, help="Agent name, e.g. 'support-bot'.") + p_create.add_argument( + "--model", + default="gpt-4o-mini", + help="Model deployment to bind the agent to (default: gpt-4o-mini).", + ) + p_create.add_argument( + "--variant", + choices=["v1-good", "v2-degraded"], + default="v1-good", + help=( + "Which system prompt variant to register. v1-good is the " + "tool-calling support assistant; v2-degraded is the friendly " + "chatbot used to demonstrate regression detection." + ), + ) + p_create.set_defaults(func=cmd_create) + + p_delete = sub.add_parser("delete", help="Delete every version of the agent.") + p_delete.add_argument("--name", required=True) + p_delete.set_defaults(func=cmd_delete) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/e2e_aggregate_summary.py b/scripts/e2e_aggregate_summary.py new file mode 100644 index 00000000..71942046 --- /dev/null +++ b/scripts/e2e_aggregate_summary.py @@ -0,0 +1,134 @@ +"""Aggregate per-scenario E2E artifacts into a single Markdown summary. + +Reads downloaded GitHub Actions artifacts from ``artifacts//`` and +emits a single Markdown summary table to stdout (or ``--out`` if provided) +covering both the offline smoke scenarios and every live-* scenario, so the +run page shows one consolidated picture instead of just the offline summary. + +Inputs it understands: + * ``artifacts/offline-smoke/SUMMARY.md`` (already rendered by e2e_demo.py) + * ``artifacts/live-*/.agentops/results/latest/results.json`` (live scenarios) + * ``artifacts/live-*/HEADER.md`` (optional context line) + +The script never raises on missing fields; if a scenario's results are +unparseable it is shown with a ``?`` so the summary is still useful. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +ARTIFACT_ROOT = Path("artifacts") + + +def _read_results(job_dir: Path) -> dict | None: + candidates = sorted(job_dir.glob("**/.agentops/results/latest/results.json")) + if not candidates: + candidates = sorted(job_dir.glob("**/results.json")) + if not candidates: + return None + try: + return json.loads(candidates[0].read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def _headline_metric(results: dict) -> tuple[str, str]: + """Return (metric_name, formatted_value) for the most informative metric.""" + summary = results.get("summary") or {} + for preferred in ("items_pass_rate", "threshold_pass_rate"): + if preferred in summary: + try: + return preferred, f"{float(summary[preferred]):.3f}" + except (TypeError, ValueError): + return preferred, str(summary[preferred]) + agg = results.get("aggregate_metrics") or results.get("metrics") or {} + if agg: + k = next(iter(agg)) + v = agg[k] + try: + return k, f"{float(v):.3f}" + except (TypeError, ValueError): + return k, str(v) + return "—", "—" + + +def _row_from_live(job_name: str, job_dir: Path) -> str: + results = _read_results(job_dir) + if not results: + return f"| `{job_name}` | ? | ❓ | — |" + summary = results.get("summary") or {} + passed = summary.get("overall_passed") + # AgentOps exit code contract: 0 = passed, 2 = thresholds failed. + if passed is True: + exit_code = 0 + icon = "✅" + elif passed is False: + exit_code = 2 + icon = "❌" + else: + exit_code = "?" + icon = "❓" + metric_name, metric_value = _headline_metric(results) + return f"| `{job_name}` | {exit_code} | {icon} | {metric_name} = {metric_value} |" + + +def _offline_block(job_dir: Path) -> str: + summary = next(job_dir.glob("**/SUMMARY.md"), None) + if not summary: + return "" + return summary.read_text(encoding="utf-8").strip() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", default=str(ARTIFACT_ROOT), + help="Directory containing per-job artifact folders.") + parser.add_argument("--out", default="-", + help="Output file (default: stdout).") + args = parser.parse_args() + + root = Path(args.root) + lines: list[str] = [] + lines.append("# AgentOps E2E run summary") + lines.append("") + lines.append("Aggregated outcome of every job in this workflow run.") + lines.append("") + + # Live scenarios table. + live_jobs = sorted(p for p in root.glob("live-*") if p.is_dir()) + if live_jobs: + lines.append("## Live scenarios") + lines.append("") + lines.append("| Job | Exit code | Overall passed | Headline metric |") + lines.append("|---|---|---|---|") + for job_dir in live_jobs: + lines.append(_row_from_live(job_dir.name, job_dir)) + lines.append("") + + # Offline smoke (already a self-contained markdown block). + offline_dir = root / "offline-smoke" + if offline_dir.is_dir(): + lines.append("## Offline smoke (`offline-smoke`)") + lines.append("") + block = _offline_block(offline_dir) + if block: + lines.append(block) + else: + lines.append("_No SUMMARY.md found in offline-smoke artifact._") + lines.append("") + + output = "\n".join(lines).rstrip() + "\n" + + if args.out == "-": + print(output, end="") + else: + Path(args.out).write_text(output, encoding="utf-8") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/e2e_data/basic.jsonl b/scripts/e2e_data/basic.jsonl new file mode 100644 index 00000000..92001fc4 --- /dev/null +++ b/scripts/e2e_data/basic.jsonl @@ -0,0 +1,3 @@ +{"input": "What is 2+2?", "expected": "4"} +{"input": "Capital of France?", "expected": "Paris"} +{"input": "Color of the sky on a clear day?", "expected": "blue"} diff --git a/scripts/e2e_data/rag.jsonl b/scripts/e2e_data/rag.jsonl new file mode 100644 index 00000000..a896cde7 --- /dev/null +++ b/scripts/e2e_data/rag.jsonl @@ -0,0 +1,2 @@ +{"input": "What is the capital of France?", "expected": "Paris", "context": "France is a country in Western Europe. Its capital is Paris."} +{"input": "What language is spoken in Brazil?", "expected": "Portuguese", "context": "Brazil is a South American country. The official language is Portuguese."} diff --git a/scripts/e2e_data/tools.jsonl b/scripts/e2e_data/tools.jsonl new file mode 100644 index 00000000..4f1e0fd2 --- /dev/null +++ b/scripts/e2e_data/tools.jsonl @@ -0,0 +1,3 @@ +{"input": "What's the weather in Paris, France?", "expected": "Calls get_weather with location='Paris, France'.", "tool_definitions": [{"type": "function", "name": "get_weather", "description": "Get the current weather for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "call_1", "name": "get_weather", "arguments": {"location": "Paris, France"}}]} +{"input": "How is the weather right now in Tokyo, Japan?", "expected": "Calls get_weather with location='Tokyo, Japan'.", "tool_definitions": [{"type": "function", "name": "get_weather", "description": "Get the current weather for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "call_2", "name": "get_weather", "arguments": {"location": "Tokyo, Japan"}}]} +{"input": "Tell me the current weather in Sao Paulo, Brazil.", "expected": "Calls get_weather with location='Sao Paulo, Brazil'.", "tool_definitions": [{"type": "function", "name": "get_weather", "description": "Get the current weather for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}], "tool_calls": [{"type": "tool_call", "tool_call_id": "call_3", "name": "get_weather", "arguments": {"location": "Sao Paulo, Brazil"}}]} diff --git a/scripts/e2e_demo.py b/scripts/e2e_demo.py new file mode 100644 index 00000000..001ea2ec --- /dev/null +++ b/scripts/e2e_demo.py @@ -0,0 +1,252 @@ +"""End-to-end demo runner for AgentOps. + +Exercises the full CLI surface against an in-process HTTP echo agent and +produces a self-contained ``evidence/`` folder suitable for pull-request +reviews and GitHub Actions artifact uploads. + +The script is offline by design: it does not contact Azure, Foundry, or any +real model provider. It validates the parts of the pipeline that are most +prone to regression: + +* ``agentops init`` creates ``agentops.yaml`` and a seed dataset. +* ``agentops eval run`` invokes the agent, runs the inferred evaluators, + writes ``results.json`` and ``report.md``, and exits with the documented + exit-code contract. +* ``agentops eval run --baseline`` produces the comparison block. +* ``agentops report generate`` regenerates ``report.md`` from results. + +Each scenario writes its artifacts under ``evidence///`` +and a final ``SUMMARY.md`` aggregates the outcomes. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import shutil +import subprocess +import sys +import threading +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parent.parent +DATASET_BODY = ( + '{"input": "What is 2+2?", "expected": "4"}\n' + '{"input": "Capital of France?", "expected": "Paris"}\n' + '{"input": "Color of the sky?", "expected": "blue"}\n' +) + +logger = logging.getLogger("agentops.e2e_demo") + + +class _EchoHandler(BaseHTTPRequestHandler): + """Minimal HTTP/JSON agent: echoes the ``message`` field as ``text``.""" + + def do_POST(self) -> None: # noqa: N802 (BaseHTTPRequestHandler API) + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length).decode("utf-8") if length else "{}" + try: + payload = json.loads(body) + except json.JSONDecodeError: + payload = {} + message = str(payload.get("message", "")) + # Echo back a "smart" response that exact-matches the expected answer + # for the seed dataset, so evaluators score positively. + canned = { + "What is 2+2?": "4", + "Capital of France?": "Paris", + "Color of the sky?": "blue", + } + text = canned.get(message, message) + response = json.dumps({"text": text}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + def log_message(self, format: str, *args) -> None: # noqa: A002 + return # silence + + +def _start_echo_server() -> tuple[HTTPServer, threading.Thread, str]: + server = HTTPServer(("127.0.0.1", 0), _EchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + host, port = server.server_address + return server, thread, f"http://{host}:{port}/" + + +def _write_agentops_yaml(target_dir: Path, agent_url: str) -> Path: + yaml_path = target_dir / "agentops.yaml" + yaml_path.write_text( + "version: 1\n" + f'agent: "{agent_url}"\n' + "dataset: ./dataset.jsonl\n" + "evaluators:\n" + " - name: F1ScoreEvaluator\n", + encoding="utf-8", + ) + (target_dir / "dataset.jsonl").write_text(DATASET_BODY, encoding="utf-8") + return yaml_path + + +def _run_cli(*args: str, cwd: Path) -> subprocess.CompletedProcess: + cmd = [sys.executable, "-m", "agentops", *args] + logger.info("$ %s (cwd=%s)", " ".join(cmd), cwd) + return subprocess.run( # noqa: S603 + cmd, + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def _capture_artifacts( + *, + label: str, + proc: subprocess.CompletedProcess, + project_dir: Path, + evidence_dir: Path, +) -> dict: + bucket = evidence_dir / label + bucket.mkdir(parents=True, exist_ok=True) + (bucket / "stdout.log").write_text(proc.stdout, encoding="utf-8") + (bucket / "stderr.log").write_text(proc.stderr, encoding="utf-8") + (bucket / "exit_code.txt").write_text(str(proc.returncode), encoding="utf-8") + + results_dir = project_dir / ".agentops" / "results" / "latest" + summary_metric: Optional[float] = None + overall_passed: Optional[bool] = None + if results_dir.exists(): + for name in ("results.json", "report.md", "cloud_evaluation.json"): + src = results_dir / name + if src.exists(): + shutil.copy2(src, bucket / name) + results_path = bucket / "results.json" + if results_path.exists(): + data = json.loads(results_path.read_text(encoding="utf-8")) + overall_passed = data.get("summary", {}).get("overall_passed") + agg = data.get("aggregate_metrics", {}) + summary_metric = agg.get("f1_score") or next(iter(agg.values()), None) + + return { + "label": label, + "exit_code": proc.returncode, + "summary_metric": summary_metric, + "overall_passed": overall_passed, + } + + +def _render_summary(records: list[dict], evidence_dir: Path) -> Path: + lines = ["# AgentOps E2E demo summary", ""] + lines.append(f"Generated: {datetime.now(timezone.utc).isoformat()}Z") + lines.append("") + lines.append("| Scenario | Exit code | Overall passed | Headline metric |") + lines.append("|---|---|---|---|") + for record in records: + metric = ( + f"{record['summary_metric']:.3f}" + if record["summary_metric"] is not None + else "—" + ) + passed = ( + "✅" if record["overall_passed"] else + ("❌" if record["overall_passed"] is False else "—") + ) + lines.append( + f"| {record['label']} | {record['exit_code']} | {passed} | {metric} |" + ) + summary_path = evidence_dir / "SUMMARY.md" + summary_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return summary_path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--evidence-dir", + type=Path, + default=REPO_ROOT / "evidence", + help="Where to write the evidence// folder.", + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + evidence_dir = args.evidence_dir / timestamp + evidence_dir.mkdir(parents=True, exist_ok=True) + logger.info("evidence -> %s", evidence_dir) + + server, _thread, agent_url = _start_echo_server() + logger.info("echo agent listening on %s", agent_url) + + records: list[dict] = [] + project_dir = evidence_dir / "_workspace" + project_dir.mkdir() + + try: + _write_agentops_yaml(project_dir, agent_url) + + # 1. Initial evaluation. Should pass cleanly against the canned echo. + proc = _run_cli("eval", "run", cwd=project_dir) + records.append( + _capture_artifacts( + label="01-initial-run", + proc=proc, + project_dir=project_dir, + evidence_dir=evidence_dir, + ) + ) + + # 2. Baseline comparison. Re-run pointed at the previous results.json. + baseline_path = ( + project_dir / ".agentops" / "results" / "latest" / "results.json" + ) + if baseline_path.exists(): + stash = evidence_dir / "_baseline.json" + shutil.copy2(baseline_path, stash) + proc = _run_cli( + "eval", + "run", + "--baseline", + str(stash), + cwd=project_dir, + ) + records.append( + _capture_artifacts( + label="02-baseline-comparison", + proc=proc, + project_dir=project_dir, + evidence_dir=evidence_dir, + ) + ) + + # 3. Report regeneration from existing results. + proc = _run_cli("report", "generate", cwd=project_dir) + records.append( + _capture_artifacts( + label="03-report-regenerate", + proc=proc, + project_dir=project_dir, + evidence_dir=evidence_dir, + ) + ) + finally: + server.shutdown() + + summary_path = _render_summary(records, evidence_dir) + logger.info("summary -> %s", summary_path) + + failed = [r for r in records if r["exit_code"] not in (0, 2)] + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/e2e_hosted_agent.py b/scripts/e2e_hosted_agent.py new file mode 100644 index 00000000..2c1d78ff --- /dev/null +++ b/scripts/e2e_hosted_agent.py @@ -0,0 +1,199 @@ +"""Create or delete a transient Foundry hosted agent for the E2E pipeline. + +The agent is a prompt agent with a single ``get_weather(location)`` function +tool, used to exercise the agent-with-tools evaluators (TaskCompletion, +ToolCallAccuracy, IntentResolution, ToolSelection, ToolInputAccuracy). + +Subcommands: + create --name [--model ] + Creates the agent and prints two GitHub Actions output lines: + agent_id=: + agent_name= + Also writes ``e2e-runs/foundry-hosted/agent-info.json`` with the + agent metadata so the transcript script can render an informative + header. + + delete --name + Deletes every version of the named agent, then deletes the agent + itself. Idempotent: ignores 404s. + +Authentication uses ``DefaultAzureCredential``. The Foundry project endpoint +is read from ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT``. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCENARIO_DIR = ROOT / "e2e-runs" / "foundry-hosted" + +INSTRUCTIONS = ( + "You are a concise weather assistant. When the user asks about the " + "weather, always call the get_weather function with the location they " + "mention. Do not invent weather data and do not answer without calling " + "the tool." +) + +WEATHER_TOOL_PARAMETERS = { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country, e.g. 'Paris, France'.", + } + }, + "required": ["location"], + "additionalProperties": False, +} + + +def _client(): + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + + endpoint = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise SystemExit("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT is required") + cred = DefaultAzureCredential(exclude_developer_cli_credential=True) + return AIProjectClient(endpoint=endpoint, credential=cred) + + +def _emit_output(key: str, value: str) -> None: + out = os.environ.get("GITHUB_OUTPUT") + line = f"{key}={value}" + if out: + with open(out, "a", encoding="utf-8") as fh: + fh.write(line + "\n") + print(line) + + +def cmd_create(args: argparse.Namespace) -> int: + from azure.ai.projects.models import ( + FunctionTool, + PromptAgentDefinition, + ) + + client = _client() + tool = FunctionTool( + name="get_weather", + description="Get the current weather for a given location.", + parameters=WEATHER_TOOL_PARAMETERS, + strict=True, + ) + definition = PromptAgentDefinition( + model=args.model, + instructions=INSTRUCTIONS, + tools=[tool], + ) + + from azure.core.exceptions import HttpResponseError, ServiceResponseError + + last_exc: Exception | None = None + version = None + for attempt in range(1, 6): + try: + version = client.agents.create_version( + agent_name=args.name, + definition=definition, + description="AgentOps E2E transient hosted agent (weather + get_weather tool).", + ) + break + except (HttpResponseError, ServiceResponseError) as exc: + status = getattr(exc, "status_code", None) + transient = status is None or status >= 500 or status == 429 + print( + f"create_version attempt {attempt}/5 failed (status={status}): {exc}", + file=sys.stderr, + ) + if not transient or attempt == 5: + raise + last_exc = exc + time.sleep(min(2 ** attempt, 30)) + if version is None: + raise SystemExit(f"create_version failed after retries: {last_exc!r}") + + version_id = getattr(version, "version", None) or getattr(version, "id", None) + if not version_id: + raise SystemExit(f"Could not determine version id from response: {version!r}") + + SCENARIO_DIR.mkdir(parents=True, exist_ok=True) + info = { + "name": args.name, + "version": str(version_id), + "model": args.model, + "instructions": INSTRUCTIONS, + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": tool.description, + "parameters": WEATHER_TOOL_PARAMETERS, + } + ], + } + (SCENARIO_DIR / "agent-info.json").write_text( + json.dumps(info, indent=2), encoding="utf-8" + ) + + _emit_output("agent_id", f"{args.name}:{version_id}") + _emit_output("agent_name", args.name) + print(f"Created hosted agent: {args.name}:{version_id}", file=sys.stderr) + return 0 + + +def cmd_delete(args: argparse.Namespace) -> int: + from azure.core.exceptions import ResourceNotFoundError + + client = _client() + deleted_versions = 0 + try: + for v in client.agents.list_versions(agent_name=args.name): + ver_id = getattr(v, "version", None) or getattr(v, "id", None) + if ver_id is None: + continue + try: + client.agents.delete_version(agent_name=args.name, version=str(ver_id)) + deleted_versions += 1 + except ResourceNotFoundError: + pass + except ResourceNotFoundError: + print(f"Agent {args.name} not found (already deleted)", file=sys.stderr) + return 0 + + try: + client.agents.delete(agent_name=args.name) + except ResourceNotFoundError: + pass + + print( + f"Deleted hosted agent {args.name} ({deleted_versions} version(s))", + file=sys.stderr, + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + + p_create = sub.add_parser("create", help="Create the transient hosted agent") + p_create.add_argument("--name", required=True) + p_create.add_argument("--model", default="gpt-4o-mini") + p_create.set_defaults(func=cmd_create) + + p_delete = sub.add_parser("delete", help="Delete the transient hosted agent") + p_delete.add_argument("--name", required=True) + p_delete.set_defaults(func=cmd_delete) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/e2e_make_transcript.py b/scripts/e2e_make_transcript.py new file mode 100644 index 00000000..8e181800 --- /dev/null +++ b/scripts/e2e_make_transcript.py @@ -0,0 +1,211 @@ +"""Render a Markdown transcript for one E2E scenario. + +Reads ``/HEADER.md`` (rendered by ``e2e_render_config.py``) and +``/.agentops/results/latest/results.json`` (produced by +``agentops eval run``) and writes ``/transcript.md``. + +The transcript is meant to be a single self-contained markdown document +that explains what was being evaluated, what the agent answered for each +row, which evaluators ran and their per-row scores, and the final +pass/fail verdict. Markdown so it renders nicely in the GitHub Actions +artifact viewer and PR reviews. + +Usage: + python scripts/e2e_make_transcript.py +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any, Dict, List + + +def _fmt_number(value: Any) -> str: + if isinstance(value, float): + return f"{value:.4f}" + return str(value) + + +def _fenced(value: Any, lang: str = "") -> str: + if value is None or value == "": + return "_(none)_" + if isinstance(value, str): + return f"```{lang}\n{value.rstrip()}\n```" + return f"```{lang or 'json'}\n{json.dumps(value, ensure_ascii=False, indent=2)}\n```" + + +def _render_row(idx: int, row: Dict[str, Any]) -> str: + parts: List[str] = [] + parts.append(f"### Row {idx}") + parts.append("") + parts.append("**Input**") + parts.append("") + parts.append(_fenced(row.get("input"))) + parts.append("") + + if row.get("context"): + parts.append("**Context**") + parts.append("") + parts.append(_fenced(row["context"])) + parts.append("") + + if row.get("expected") is not None: + parts.append("**Expected**") + parts.append("") + parts.append(_fenced(row["expected"])) + parts.append("") + + parts.append("**Response**") + parts.append("") + parts.append(_fenced(row.get("response", ""))) + parts.append("") + + tool_calls = row.get("tool_calls") + if tool_calls: + parts.append("**Tool calls**") + parts.append("") + parts.append(_fenced(tool_calls, "json")) + parts.append("") + + latency = row.get("latency_seconds") + if latency is not None: + parts.append(f"**Latency:** `{latency:.3f}s`") + parts.append("") + + metrics = row.get("metrics") or [] + if metrics: + parts.append("**Metrics**") + parts.append("") + parts.append("| Metric | Value |") + parts.append("|---|---|") + for m in metrics: + name = m.get("name", "?") + value = m.get("value") + err = m.get("error") + if err: + parts.append(f"| `{name}` | ⚠️ ERROR: {err} |") + else: + parts.append(f"| `{name}` | {_fmt_number(value)} |") + parts.append("") + + err = row.get("error") + if err: + parts.append(f"> ❌ **Row error:** {err}") + parts.append("") + + return "\n".join(parts) + + +def main() -> int: + if len(sys.argv) != 2: + print("Usage: e2e_make_transcript.py ", file=sys.stderr) + return 2 + + scenario_dir = Path(sys.argv[1]).resolve() + if not scenario_dir.is_dir(): + print(f"Not a directory: {scenario_dir}", file=sys.stderr) + return 2 + + header_path = scenario_dir / "HEADER.md" + results_path = scenario_dir / ".agentops" / "results" / "latest" / "results.json" + out_path = scenario_dir / "transcript.md" + + header = ( + header_path.read_text(encoding="utf-8") + if header_path.exists() + else f"# Scenario: {scenario_dir.name}\n" + ).rstrip() + + if not results_path.exists(): + out_path.write_text( + header + + "\n\n---\n\n" + + "## Verdict: ⚠️ NO RESULTS\n\n" + + f"`results.json` not found at `{results_path}`.\n\n" + + "The evaluation run did not complete successfully. Check the\n" + + "job logs (Run AgentOps eval step) for the underlying error.\n", + encoding="utf-8", + ) + print(f"Wrote {out_path} (no results)") + return 0 + + results = json.loads(results_path.read_text(encoding="utf-8")) + summary = results.get("summary") or {} + target = results.get("target") or {} + metrics_aggregate: Dict[str, float] = results.get("aggregate_metrics") or {} + threshold_results = results.get("thresholds") or [] + rows = results.get("rows") or [] + + lines: List[str] = [header, "", "---", "", "## Target", ""] + lines.append("| Field | Value |") + lines.append("|---|---|") + for k, v in target.items(): + lines.append(f"| `{k}` | `{v}` |") + lines.append("") + + lines.append("## Per-row transcript") + lines.append("") + for i, row in enumerate(rows, start=1): + lines.append(_render_row(i, row)) + + lines.append("---") + lines.append("") + lines.append("## Aggregate metrics") + lines.append("") + if metrics_aggregate: + lines.append("| Metric | Value |") + lines.append("|---|---|") + for name, value in sorted(metrics_aggregate.items()): + lines.append(f"| `{name}` | {_fmt_number(value)} |") + else: + lines.append("_(none)_") + lines.append("") + + lines.append("## Thresholds") + lines.append("") + if threshold_results: + lines.append("| Result | Metric | Criteria | Expected | Actual |") + lines.append("|---|---|---|---|---|") + for t in threshold_results: + name = t.get("metric", "?") + criteria = t.get("criteria", "") + expected = t.get("expected", "?") + actual = t.get("actual", "?") + passed = t.get("passed") + verdict = "✅ PASS" if passed else "❌ FAIL" + lines.append(f"| {verdict} | `{name}` | `{criteria}` | `{expected}` | `{actual}` |") + else: + lines.append("_(none)_") + lines.append("") + + overall = summary.get("overall_passed") + if overall: + verdict = "✅ PASS" + elif overall is False: + verdict = "❌ FAIL" + else: + verdict = "⚠️ UNKNOWN" + lines.append("---") + lines.append("") + lines.append(f"## Verdict: {verdict}") + lines.append("") + if summary: + items_total = summary.get("items_total", "?") + items_passed = summary.get("items_passed_all", "?") + items_rate = summary.get("items_pass_rate", 0) + thr_total = summary.get("thresholds_total", "?") + thr_passed = summary.get("thresholds_passed", "?") + thr_rate = summary.get("threshold_pass_rate", 0) + lines.append(f"- **Items:** {items_passed}/{items_total} passed ({items_rate:.2%})") + lines.append(f"- **Thresholds:** {thr_passed}/{thr_total} passed ({thr_rate:.2%})") + lines.append("") + + out_path.write_text("\n".join(lines), encoding="utf-8") + print(f"Wrote {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/e2e_render_config.py b/scripts/e2e_render_config.py new file mode 100644 index 00000000..0680f2a4 --- /dev/null +++ b/scripts/e2e_render_config.py @@ -0,0 +1,335 @@ +"""Render scenario-specific agentops.yaml files for the e2e workflow. + +Reads target identifiers from environment variables (set by the GitHub +Actions workflow from repo Actions Variables + Bicep outputs) and writes +one agentops.yaml per scenario into ``./e2e-runs//``. + +Scenarios: + - foundry-prompt: AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT (e.g. ``e2e-prompt:1``) + - foundry-hosted: AGENTOPS_E2E_FOUNDRY_HOSTED_URL (https URL) + - http-aca: AGENTOPS_E2E_ACA_URL (https URL of echo app) + - model-direct: AGENTOPS_E2E_MODEL_DEPLOYMENT (deployment name) + +A scenario is skipped (no file written) when its env var is unset, which +lets the workflow run partial scenarios via ``workflow_dispatch.inputs``. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DATASET_BASIC = ROOT / "scripts" / "e2e_data" / "basic.jsonl" +DATASET_RAG = ROOT / "scripts" / "e2e_data" / "rag.jsonl" +DATASET_TOOLS = ROOT / "scripts" / "e2e_data" / "tools.jsonl" + + +def _ensure_datasets() -> None: + DATASET_BASIC.parent.mkdir(parents=True, exist_ok=True) + if not DATASET_BASIC.exists(): + rows = [ + {"input": "What is 2+2?", "expected": "4"}, + {"input": "Capital of France?", "expected": "Paris"}, + {"input": "Color of the sky on a clear day?", "expected": "blue"}, + ] + DATASET_BASIC.write_text( + "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" + ) + if not DATASET_RAG.exists(): + rows = [ + { + "input": "What is the capital of France?", + "expected": "Paris", + "context": "France is a country in Western Europe. Its capital is Paris.", + }, + { + "input": "What language is spoken in Brazil?", + "expected": "Portuguese", + "context": "Brazil is a South American country. The official language is Portuguese.", + }, + ] + DATASET_RAG.write_text( + "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" + ) + if not DATASET_TOOLS.exists(): + weather_tool = { + "type": "function", + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + } + rows = [ + { + "input": f"What's the weather in {city}?", + "expected": f"Calls get_weather with location='{city}'.", + "tool_definitions": [weather_tool], + "tool_calls": [ + { + "type": "tool_call", + "tool_call_id": f"call_{i}", + "name": "get_weather", + "arguments": {"location": city}, + } + ], + } + for i, city in enumerate( + ["Paris, France", "Tokyo, Japan", "Sao Paulo, Brazil"], start=1 + ) + ] + DATASET_TOOLS.write_text( + "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" + ) + + +def _write(scenario: str, body: str, header: str | None = None) -> Path: + out_dir = ROOT / "e2e-runs" / scenario + out_dir.mkdir(parents=True, exist_ok=True) + cfg = out_dir / "agentops.yaml" + cfg.write_text(body, encoding="utf-8") + if header is not None: + (out_dir / "HEADER.md").write_text(header, encoding="utf-8") + return cfg + + +def render() -> list[str]: + _ensure_datasets() + written: list[str] = [] + rel_basic = DATASET_BASIC.relative_to(ROOT).as_posix() + rel_rag = DATASET_RAG.relative_to(ROOT).as_posix() + rel_tools = DATASET_TOOLS.relative_to(ROOT).as_posix() + # Pulled from repo Actions Variables by the workflow. We only use it for + # human-readable HEADER text — the actual deployment that's exercised is + # always whatever the workflow has configured in `AZURE_OPENAI_DEPLOYMENT`, + # so this is purely cosmetic and falls back to a generic placeholder. + model = os.environ.get("AGENTOPS_E2E_MODEL_DEPLOYMENT") or "the configured Azure OpenAI deployment" + + prompt_agent = os.environ.get("AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT") + if prompt_agent: + _write( + "foundry-prompt", + f"""version: 1 +agent: {prompt_agent} +dataset: ../../{rel_basic} +# Permissive thresholds: e2e is a smoke test for the pipeline, not a quality gate. +thresholds: + coherence: ">=1" + fluency: ">=1" + similarity: ">=1" + f1_score: ">=0" + avg_latency_seconds: "<=60" +""", + header=f"""# Scenario: foundry-prompt + +**Target:** Foundry prompt agent `{prompt_agent}` (created manually in the Foundry portal). + +**What it does:** A general-purpose prompt agent backed by `{model}`. It +answers short factual questions with the canonical short answer +(e.g. "What is 2+2?" → "4"). No tools, no retrieval. + +**Why this scenario exists:** It exercises the AgentOps invocation path for +agents referenced via `agent_reference` (the OpenAI Responses API integration +exposed by the Foundry Agent Service). + +**Dataset:** `{rel_basic}` (3 short factual rows). + +**Evaluators (auto-inferred from dataset shape):** `coherence`, `fluency`, +`similarity`, `f1_score`, plus `avg_latency_seconds`. Thresholds are very +permissive — this is a pipeline smoke test, not a quality gate. +""", + ) + written.append("foundry-prompt") + + hosted_agent = os.environ.get("AGENTOPS_E2E_FOUNDRY_HOSTED_AGENT") + if hosted_agent: + _write( + "foundry-hosted", + f"""version: 1 +agent: {hosted_agent} +dataset: ../../{rel_tools} +thresholds: + tool_call_accuracy: ">=0" + intent_resolution: ">=0" + task_adherence: ">=0" + f1_score: ">=0" + coherence: ">=0" + fluency: ">=0" + similarity: ">=0" + avg_latency_seconds: "<=60" +""", + header=f"""# Scenario: foundry-hosted (agent with tools) + +**Target:** Foundry hosted agent `{hosted_agent}` — created dynamically by +this workflow run via `scripts/e2e_hosted_agent.py create` and deleted in +`teardown-live`. + +**What it does:** A weather assistant backed by `{model}` with a single +function tool `get_weather(location)`. The agent's instructions tell it to +*always* invoke `get_weather` when the user asks about the weather, instead +of fabricating an answer. + +**Tool registered on the agent:** + +```json +{{ + "type": "function", + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": {{ + "type": "object", + "properties": {{ "location": {{ "type": "string" }} }}, + "required": ["location"] + }} +}} +``` + +**Dataset:** `{rel_tools}` (3 weather questions, each with the expected +`get_weather` tool call as ground truth). + +**Evaluators (auto-inferred from `tool_definitions` + `tool_calls`):** +`tool_call_accuracy`, `intent_resolution`, `task_adherence`, plus +`f1_score` and `avg_latency_seconds`. Thresholds are very permissive — +this is a pipeline smoke test, not a quality gate. + +> **Note on `intent_resolution` / `task_adherence` low scores:** these are +> AI-judge evaluators that grade the *natural-language* portion of the +> response. This eval is single-turn — the agent stops at the +> `function_call` and we never execute the tool, so the model never gets +> to produce a final natural-language answer. The judges therefore see +> only the synthetic `[Called get_weather(...)]` summary and score it +> low. `tool_call_accuracy` (which judges the structured tool call +> itself) is the meaningful metric for this scenario. +""", + ) + written.append("foundry-hosted") + + aca_url = os.environ.get("AGENTOPS_E2E_ACA_URL") + if aca_url: + # The hello-agent ACA app is a real LLM-backed agent with one tool + # (`get_weather`). The http-aca scenario exercises *both* the http-json + # invocation path AND tool-call evaluation — the dataset asks weather + # questions in three different cities, the agent picks the tool, the + # framework runs it, and the agent produces a final natural-language + # answer. AgentOps captures the structured tool calls via + # `tool_calls_field` for `tool_call_accuracy` while quality evaluators + # grade the final text. + _write( + "http-aca", + f"""version: 1 +agent: {aca_url} +dataset: ../../{rel_tools} +protocol: http-json +request_field: message +response_field: text +tool_calls_field: tool_calls +# Permissive thresholds: e2e smoke test of the http-json + tool-calling +# invocation path against a real LLM, not a quality gate for the model. +thresholds: + tool_call_accuracy: ">=0" + intent_resolution: ">=0" + task_adherence: ">=0" + coherence: ">=0" + fluency: ">=0" + similarity: ">=0" + f1_score: ">=0" + avg_latency_seconds: "<=60" +""", + header=f"""# Scenario: http-aca (HTTP agent with tool calling) + +**Target:** A *real* Microsoft Agent Framework chat agent +(`agent_framework.Agent` + `OpenAIChatCompletionClient` against Azure +OpenAI `{model}`) deployed as an Azure Container App per workflow run by +`infra/e2e/perrun.bicep` at `{aca_url}`. + +**What it does:** The agent (see `infra/e2e/agent-app/app.py`) is a small +FastAPI service that exposes `POST /` accepting `{{"message": "..."}}` +and returning `{{"text": "...", "tool_calls": [...]}}`. The agent is +configured with one function tool, `get_weather(location)`, and these +instructions: + +> *You are a concise factual assistant. When the user asks about the +> weather in a location, you MUST call the `get_weather` tool with that +> location instead of guessing. After the tool returns, summarize the +> weather for the user in one short sentence...* + +Each POST is a single AgentOps invocation, but **inside** the agent +there are multiple internal turns: the model picks the tool, the +framework executes it locally, the model observes the canned tool +result, and produces a final natural-language answer. AgentOps captures +the structured tool calls (via `tool_calls_field: tool_calls`) for +`tool_call_accuracy` while the quality evaluators grade the final text. + +The container authenticates to Azure OpenAI via a User-Assigned Managed +Identity (no API keys) granted `Cognitive Services OpenAI User` on the +shared AI Services account. + +**Why this scenario exists:** It exercises AgentOps' `http-json` +invocation path *plus* tool-call evaluation against a freshly-deployed +Azure resource the workflow itself owns end to end (image built +server-side via `az acr build`, deployed via Bicep, pulled with managed +identity, torn down by the teardown job). + +**Dataset:** `{rel_tools}` (3 weather questions across Paris, Tokyo, +São Paulo, each with the expected `get_weather` tool call). + +**Evaluators (auto-inferred from dataset shape):** `tool_call_accuracy`, +`coherence`, `fluency`, `f1_score`, plus `avg_latency_seconds`. +Thresholds are intentionally permissive (`>=0`) because the goal is to +validate connectivity and the eval pipeline, not to gate on `{model}` +quality. +""", + ) + written.append("http-aca") + + model_deployment = os.environ.get("AGENTOPS_E2E_MODEL_DEPLOYMENT") + if model_deployment: + _write( + "model-direct", + f"""version: 1 +agent: model:{model_deployment} +dataset: ../../{rel_basic} +thresholds: + coherence: ">=1" + fluency: ">=1" + similarity: ">=1" + f1_score: ">=0" + avg_latency_seconds: "<=60" +""", + header=f"""# Scenario: model-direct + +**Target:** Azure OpenAI model deployment `{model_deployment}` invoked via +`chat.completions` (no agent layer between AgentOps and the model). + +**What it does:** Sends each dataset row's `input` straight to the model as a +single user message; the model's reply is taken as the response. + +**Dataset:** `{rel_basic}` (3 short factual rows). + +**Evaluators (auto-inferred from dataset shape):** `coherence`, `fluency`, +`similarity`, `f1_score`, plus `avg_latency_seconds`. Thresholds are very +permissive — this is a pipeline smoke test, not a quality gate. +""", + ) + written.append("model-direct") + + return written + + +def main() -> int: + written = render() + if not written: + print("ERROR: no scenario env vars set; nothing to render.", file=sys.stderr) + return 1 + for s in written: + print(f"rendered: e2e-runs/{s}/agentops.yaml") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/setup-e2e-new-tenant.ps1 b/scripts/setup-e2e-new-tenant.ps1 new file mode 100644 index 00000000..78463487 --- /dev/null +++ b/scripts/setup-e2e-new-tenant.ps1 @@ -0,0 +1,204 @@ +# One-shot bootstrap script for the E2E pipeline against a NEW Azure tenant. +# +# Run this from the repo root after `az login --tenant `. It will: +# +# 1. Deploy the shared E2E infra (AI Services + Foundry project + gpt-4o-mini +# deployment + Container Apps env + ACR + UAMI) via `infra/e2e/bootstrap.bicep`. +# 2. Create an Entra app registration in the current tenant, grant it Contributor +# + User Access Administrator on the resource group, and add a federated +# credential bound to the current branch of this repo. +# 3. Update every GitHub Actions Variable that the E2E workflow consumes +# (subscription, tenant, client id, endpoints, ACA env id, ACR, model). +# 4. Print the manual remaining steps (creating the `e2e-prompt:1` Foundry +# agent) and the `gh workflow run` command to trigger the pipeline. +# +# Idempotent: re-running is safe. Each step checks for the existing resource +# before creating. +# +# Required tools on PATH: az, gh, jq is NOT required. +# Required CLI auth: az login (correct tenant) AND gh auth status. + +#Requires -Version 7.0 +[CmdletBinding()] +param( + [string]$SubscriptionId = '9788a92c-2f71-4629-8173-7ad449cb50e1', + [string]$TenantId = '16b3c013-d300-468d-ac64-7eda0820b6d3', + [string]$ResourceGroup = 'rg-agentops-e2e', + [string]$Repo = 'Azure/agentops', + [string]$AppName = 'agentops-e2e', + [string]$Branch = (& git rev-parse --abbrev-ref HEAD).Trim(), + [string]$BicepParams = 'infra/e2e/bootstrap.parameters.example.json', + [string]$DeploymentName = 'agentops-e2e-bootstrap' +) + +$ErrorActionPreference = 'Stop' +function Header($msg) { Write-Host ""; Write-Host "==> $msg" -ForegroundColor Cyan } + +# --------------------------------------------------------------------------- +# 0. Pre-flight +# --------------------------------------------------------------------------- +Header "Pre-flight" + +$current = (az account show --query "{sub:id, tenant:tenantId}" -o json | ConvertFrom-Json) +if ($current.sub -ne $SubscriptionId -or $current.tenant -ne $TenantId) { + Write-Host "Switching az context to $SubscriptionId in tenant $TenantId..." + az account set --subscription $SubscriptionId | Out-Null + $current = (az account show --query "{sub:id, tenant:tenantId}" -o json | ConvertFrom-Json) + if ($current.tenant -ne $TenantId) { + throw "az is logged into tenant $($current.tenant) but expected $TenantId. Run: az login --tenant $TenantId" + } +} +Write-Host "OK: subscription=$($current.sub) tenant=$($current.tenant) branch=$Branch" + +if (-not (az group exists --name $ResourceGroup)) { + Write-Host "Resource group $ResourceGroup does not exist; creating in eastus2..." + az group create --name $ResourceGroup --location eastus2 | Out-Null +} + +# --------------------------------------------------------------------------- +# 1. Deploy bootstrap.bicep +# --------------------------------------------------------------------------- +Header "Deploying bootstrap.bicep (this can take 5-10 minutes)" + +az deployment group create ` + --resource-group $ResourceGroup ` + --name $DeploymentName ` + --template-file infra/e2e/bootstrap.bicep ` + --parameters "@$BicepParams" ` + --output none + +$outputs = az deployment group show ` + -g $ResourceGroup -n $DeploymentName ` + --query properties.outputs -o json | ConvertFrom-Json + +$foundryEndpoint = $outputs.foundryProjectEndpoint.value +$openAiEndpoint = $outputs.azureOpenAiEndpoint.value +$modelDeployment = $outputs.modelDeployment.value +$acaEnvId = $outputs.acaEnvironmentId.value +$acrLoginServer = $outputs.acrLoginServer.value + +Write-Host " foundry = $foundryEndpoint" +Write-Host " openai = $openAiEndpoint" +Write-Host " modelDeployment= $modelDeployment" +Write-Host " acaEnv = $acaEnvId" +Write-Host " acrLoginServer = $acrLoginServer" + +# --------------------------------------------------------------------------- +# 2. Entra app + federated credential +# --------------------------------------------------------------------------- +Header "Entra app + federated credential ($AppName)" + +$appId = (az ad app list --display-name $AppName --query "[0].appId" -o tsv) +if (-not $appId) { + Write-Host "Creating app registration $AppName..." + $appId = (az ad app create --display-name $AppName --query appId -o tsv) +} +$spId = (az ad sp list --filter "appId eq '$appId'" --query "[0].id" -o tsv) +if (-not $spId) { + $spId = (az ad sp create --id $appId --query id -o tsv) +} +Write-Host " appId=$appId spId=$spId" + +foreach ($role in @('Contributor', 'User Access Administrator')) { + $existing = az role assignment list ` + --assignee $appId ` + --role "$role" ` + --scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroup" ` + --query "[0].id" -o tsv 2>$null + if (-not $existing) { + Write-Host " granting $role on RG..." + az role assignment create ` + --assignee-object-id $spId ` + --assignee-principal-type ServicePrincipal ` + --role "$role" ` + --scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroup" | Out-Null + } else { + Write-Host " already has $role" + } +} + +$ficName = "agentops-e2e-$($Branch -replace '[^a-zA-Z0-9-]', '-')" +$ficSubject = "repo:${Repo}:ref:refs/heads/$Branch" +$existingFic = az ad app federated-credential list --id $appId ` + --query "[?subject=='$ficSubject'].name" -o tsv +if (-not $existingFic) { + $ficJson = @{ + name = $ficName + issuer = 'https://token.actions.githubusercontent.com' + subject = $ficSubject + audiences = @('api://AzureADTokenExchange') + } | ConvertTo-Json -Compress + $tmp = New-TemporaryFile + Set-Content -Path $tmp -Value $ficJson -Encoding UTF8 + az ad app federated-credential create --id $appId --parameters "@$tmp" | Out-Null + Remove-Item $tmp + Write-Host " federated cred created: $ficName -> $ficSubject" +} else { + Write-Host " federated cred already exists for $ficSubject" +} + +# Also add a federated cred bound to the `e2e` GitHub Environment so the +# environment-protected jobs can mint OIDC tokens. +$envSubject = "repo:${Repo}:environment:e2e" +$existingEnvFic = az ad app federated-credential list --id $appId ` + --query "[?subject=='$envSubject'].name" -o tsv +if (-not $existingEnvFic) { + $ficJson2 = @{ + name = 'agentops-e2e-environment' + issuer = 'https://token.actions.githubusercontent.com' + subject = $envSubject + audiences = @('api://AzureADTokenExchange') + } | ConvertTo-Json -Compress + $tmp2 = New-TemporaryFile + Set-Content -Path $tmp2 -Value $ficJson2 -Encoding UTF8 + az ad app federated-credential create --id $appId --parameters "@$tmp2" | Out-Null + Remove-Item $tmp2 + Write-Host " federated cred created: e2e environment -> $envSubject" +} else { + Write-Host " federated cred already exists for $envSubject" +} + +# --------------------------------------------------------------------------- +# 3. Update GitHub Actions Variables +# --------------------------------------------------------------------------- +Header "Updating GitHub Actions Variables on $Repo" + +$vars = [ordered]@{ + AZURE_SUBSCRIPTION_ID = $SubscriptionId + AZURE_TENANT_ID = $TenantId + AZURE_CLIENT_ID = $appId + AZURE_E2E_RESOURCE_GROUP = $ResourceGroup + AZURE_E2E_FOUNDRY_PROJECT_ENDPOINT = $foundryEndpoint + AZURE_E2E_OPENAI_ENDPOINT = $openAiEndpoint + AZURE_E2E_MODEL_DEPLOYMENT = $modelDeployment + AZURE_E2E_ACA_ENV_ID = $acaEnvId + AZURE_E2E_ACR_LOGIN_SERVER = $acrLoginServer +} +foreach ($kv in $vars.GetEnumerator()) { + gh variable set $kv.Key --repo $Repo --body "$($kv.Value)" | Out-Null + Write-Host " set $($kv.Key)" +} + +# --------------------------------------------------------------------------- +# 4. Manual steps + workflow trigger +# --------------------------------------------------------------------------- +Header "Manual step still required" +Write-Host "Foundry agents cannot yet be created via Bicep. Open the AI Foundry portal:" +Write-Host " https://ai.azure.com/" +Write-Host "" +Write-Host "1. Open the project at: $foundryEndpoint" +Write-Host "2. Create a prompt-based agent named 'e2e-prompt' using model '$modelDeployment'." +Write-Host "3. Publish it (note the version, usually 1)." +Write-Host "4. Set the AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT GitHub variable:" +Write-Host "" +Write-Host " gh variable set AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT --repo $Repo --body 'e2e-prompt:1'" +Write-Host "" +Write-Host "Then trigger the workflow:" +Write-Host "" +Write-Host " gh workflow run e2e.yml --repo $Repo --ref $Branch -f scenarios=foundry-prompt" +Write-Host "" +Write-Host "Or run all scenarios:" +Write-Host "" +Write-Host " gh workflow run e2e.yml --repo $Repo --ref $Branch -f scenarios=all" +Write-Host "" +Write-Host "Done." diff --git a/src/agentops/agent/__init__.py b/src/agentops/agent/__init__.py new file mode 100644 index 00000000..02819d8b --- /dev/null +++ b/src/agentops/agent/__init__.py @@ -0,0 +1,12 @@ +"""AgentOps Watchdog Agent. + +The watchdog agent reads three signal sources (AgentOps eval history, +Azure Monitor / App Insights traces, Foundry control plane), runs a +set of checks over the gathered data, and produces a Markdown findings +report. It is exposed both as a CLI (``agentops agent analyze``) and as +a Copilot Extension HTTP server (``agentops agent serve``). +""" + +from agentops.agent.findings import Finding, Severity + +__all__ = ["Finding", "Severity"] diff --git a/src/agentops/agent/analyzer.py b/src/agentops/agent/analyzer.py new file mode 100644 index 00000000..720cf453 --- /dev/null +++ b/src/agentops/agent/analyzer.py @@ -0,0 +1,117 @@ +"""Analyzer orchestration for the watchdog agent.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set + +from agentops.agent.checks.errors import run_errors_check +from agentops.agent.checks.latency import run_latency_check +from agentops.agent.checks.posture import run_posture_check +from agentops.agent.checks.regression import run_regression_check +from agentops.agent.checks.safety import run_safety_check +from agentops.agent.config import AgentConfig +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_monitor import ( + AzureMonitorPayload, + collect_azure_monitor, +) +from agentops.agent.sources.azure_resources import ( + AzureResourcesPayload, + collect_azure_resources, +) +from agentops.agent.sources.foundry_control import ( + FoundryControlPayload, + collect_foundry_control, +) +from agentops.agent.sources.results_history import ( + ResultsHistory, + collect_results_history, +) + + +@dataclass +class AnalysisResult: + findings: List[Finding] = field(default_factory=list) + history: Optional[ResultsHistory] = None + monitor: Optional[AzureMonitorPayload] = None + foundry: Optional[FoundryControlPayload] = None + resources: Optional[AzureResourcesPayload] = None + diagnostics: Dict[str, Any] = field(default_factory=dict) + + @property + def max_severity(self) -> Optional[Severity]: + if not self.findings: + return None + return max(f.severity for f in self.findings) + + +def _normalize_categories( + categories: Optional[Iterable[str]], +) -> Optional[Set[Category]]: + if categories is None: + return None + out: Set[Category] = set() + for c in categories: + if not c: + continue + try: + out.add(Category(c.strip().lower())) + except ValueError: + continue + return out or None + + +def analyze( + workspace: Path, + config: AgentConfig, + *, + categories: Optional[Iterable[str]] = None, + exclude_rules: Optional[Iterable[str]] = None, +) -> AnalysisResult: + """Run every configured source + check and return the merged result. + + ``categories`` (when provided) limits the findings to the listed + :class:`Category` values. ``exclude_rules`` is forwarded to the + posture check to skip individual WAF rule ids on top of any + exclusions configured in ``agent.yaml``. + """ + history = collect_results_history(workspace, config.sources.results_history) + monitor = collect_azure_monitor(config.sources.azure_monitor, config.lookback_days) + foundry = collect_foundry_control(config.sources.foundry_control) + resources = collect_azure_resources(config.sources.azure_resources) + + posture_config = config.checks.posture + if exclude_rules: + merged = list(posture_config.exclude_rules) + [ + r.strip() for r in exclude_rules if r and r.strip() + ] + posture_config = posture_config.model_copy(update={"exclude_rules": merged}) + + findings: List[Finding] = [] + findings.extend(run_regression_check(history, config.checks.regression)) + findings.extend(run_latency_check(history, monitor, config.checks.latency)) + findings.extend(run_errors_check(monitor, foundry, config.checks.errors)) + findings.extend(run_safety_check(history, config.checks.safety)) + findings.extend(run_posture_check(resources, posture_config)) + + allowed = _normalize_categories(categories) + if allowed is not None: + findings = [f for f in findings if f.category in allowed] + + findings.sort(key=lambda f: (-f.severity.rank, f.category.value, f.id)) + + return AnalysisResult( + findings=findings, + history=history, + monitor=monitor, + foundry=foundry, + resources=resources, + diagnostics={ + "results_history": history.diagnostics, + "azure_monitor": monitor.diagnostics, + "foundry_control": foundry.diagnostics, + "azure_resources": resources.diagnostics, + }, + ) diff --git a/src/agentops/agent/checks/__init__.py b/src/agentops/agent/checks/__init__.py new file mode 100644 index 00000000..f4ebbd80 --- /dev/null +++ b/src/agentops/agent/checks/__init__.py @@ -0,0 +1 @@ +"""Watchdog agent checks.""" diff --git a/src/agentops/agent/checks/errors.py b/src/agentops/agent/checks/errors.py new file mode 100644 index 00000000..44e591db --- /dev/null +++ b/src/agentops/agent/checks/errors.py @@ -0,0 +1,86 @@ +"""Errors / failure rate check.""" + +from __future__ import annotations + +from typing import List, Optional + +from agentops.agent.config import ErrorsCheckConfig +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_monitor import AzureMonitorPayload +from agentops.agent.sources.foundry_control import FoundryControlPayload + + +def run_errors_check( + monitor: Optional[AzureMonitorPayload], + foundry: Optional[FoundryControlPayload], + config: ErrorsCheckConfig, +) -> List[Finding]: + findings: List[Finding] = [] + + if ( + monitor + and monitor.error_rate is not None + and monitor.error_rate > config.rate_threshold + ): + severity = ( + Severity.CRITICAL + if monitor.error_rate >= config.rate_threshold * 2 + else Severity.WARNING + ) + findings.append( + Finding( + id="errors.production_rate", + severity=severity, + category=Category.RELIABILITY, + title="Production error rate above threshold", + summary=( + f"App Insights reports {monitor.error_count} failed " + f"requests over {monitor.request_count} total " + f"({monitor.error_rate * 100:.2f}%), above the " + f"{config.rate_threshold * 100:.2f}% threshold." + ), + recommendation=( + "Open the App Insights resource, group failures by " + "operation, and inspect the most common exception " + "type." + ), + source="azure_monitor", + evidence={ + "error_count": monitor.error_count, + "request_count": monitor.request_count, + "error_rate": monitor.error_rate, + "threshold": config.rate_threshold, + }, + ) + ) + + if ( + foundry + and foundry.failure_rate is not None + and foundry.failure_rate > config.rate_threshold + ): + findings.append( + Finding( + id="errors.foundry_runs", + severity=Severity.WARNING, + category=Category.RELIABILITY, + title="Foundry agent run failure rate elevated", + summary=( + f"Foundry control plane reports " + f"{foundry.failed_runs}/{foundry.total_runs} failed " + f"runs ({foundry.failure_rate * 100:.2f}%)." + ), + recommendation=( + "Review recent Foundry runs, paying attention to " + "tool-call errors and rate limits." + ), + source="foundry_control", + evidence={ + "failed_runs": foundry.failed_runs, + "total_runs": foundry.total_runs, + "failure_rate": foundry.failure_rate, + }, + ) + ) + + return findings diff --git a/src/agentops/agent/checks/latency.py b/src/agentops/agent/checks/latency.py new file mode 100644 index 00000000..6b7fbb46 --- /dev/null +++ b/src/agentops/agent/checks/latency.py @@ -0,0 +1,84 @@ +"""Latency check based on Azure Monitor and AgentOps results history.""" + +from __future__ import annotations + +from typing import List, Optional + +from agentops.agent.config import LatencyCheckConfig +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_monitor import AzureMonitorPayload +from agentops.agent.sources.results_history import ResultsHistory + + +def run_latency_check( + history: ResultsHistory, + monitor: Optional[AzureMonitorPayload], + config: LatencyCheckConfig, +) -> List[Finding]: + findings: List[Finding] = [] + threshold = config.p95_threshold_seconds + + if monitor and monitor.p95_duration_seconds is not None: + p95 = monitor.p95_duration_seconds + if p95 > threshold: + severity = ( + Severity.CRITICAL if p95 >= threshold * 2 else Severity.WARNING + ) + findings.append( + Finding( + id="latency.p95_production", + severity=severity, + category=Category.PERFORMANCE, + title="Production p95 latency exceeds threshold", + summary=( + f"Application Insights reports p95 latency of " + f"{p95:.2f}s, above the configured threshold of " + f"{threshold:.2f}s." + ), + recommendation=( + "Review recent deployments for tool-call loops or " + "long-running RAG retrievals, and consider scaling " + "out the agent runtime." + ), + source="azure_monitor", + evidence={ + "p95_seconds": p95, + "threshold_seconds": threshold, + "request_count": monitor.request_count, + }, + ) + ) + + if history.runs: + latest = history.runs[-1] + avg_latency = latest.metrics.get("avg_latency_seconds") + if avg_latency is not None and avg_latency > threshold: + severity = ( + Severity.CRITICAL + if avg_latency >= threshold * 2 + else Severity.WARNING + ) + findings.append( + Finding( + id="latency.eval_avg", + severity=severity, + category=Category.PERFORMANCE, + title="Evaluation average latency above threshold", + summary=( + f"Run `{latest.run_id}` averaged " + f"{avg_latency:.2f}s per item, above the " + f"{threshold:.2f}s threshold." + ), + recommendation=( + "Profile the slowest dataset rows and inspect tool " + "calls; re-run evals after addressing the regression." + ), + source="results_history", + evidence={ + "run_id": latest.run_id, + "avg_latency_seconds": avg_latency, + "threshold_seconds": threshold, + }, + ) + ) + return findings diff --git a/src/agentops/agent/checks/posture.py b/src/agentops/agent/checks/posture.py new file mode 100644 index 00000000..498cfa36 --- /dev/null +++ b/src/agentops/agent/checks/posture.py @@ -0,0 +1,36 @@ +"""Posture check — runs the WAF-AI rule registry against the resource snapshot.""" + +from __future__ import annotations + +from typing import List + +from agentops.agent.checks.posture_rules import RULE_REGISTRY +from agentops.agent.config import PostureCheckConfig +from agentops.agent.findings import Finding +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +SOURCE_NAME = "azure_resources" + + +def run_posture_check( + resources: AzureResourcesPayload, + config: PostureCheckConfig, +) -> List[Finding]: + if not config.enabled: + return [] + + diag = resources.diagnostics or {} + if diag.get("status") != "ok" or resources.account is None: + return [] + + excluded = {rid.strip() for rid in config.exclude_rules if rid and rid.strip()} + + findings: List[Finding] = [] + for rule_id, rule_fn in RULE_REGISTRY.items(): + if rule_id in excluded: + continue + try: + findings.extend(rule_fn(resources, SOURCE_NAME)) + except Exception: # pragma: no cover - rules must be defensive + continue + return findings diff --git a/src/agentops/agent/checks/posture_rules/__init__.py b/src/agentops/agent/checks/posture_rules/__init__.py new file mode 100644 index 00000000..b04b2f31 --- /dev/null +++ b/src/agentops/agent/checks/posture_rules/__init__.py @@ -0,0 +1,54 @@ +"""Rule registry for the WAF-AI posture check. + +Each rule is a small callable that receives the +:class:`AzureResourcesPayload` and the source name, and returns a list +of :class:`Finding`s (zero, one, or many). Rules are independent and +pure. + +The ``posture`` check (see :mod:`agentops.agent.checks.posture`) +iterates the rules registered here and aggregates the findings. + +To add a new rule: + +* Add a module under this package. +* Implement ``def evaluate(payload, source_name) -> list[Finding]``. +* Register it in :data:`RULE_REGISTRY` below. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List + +from agentops.agent.findings import Finding +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +RuleFn = Callable[[AzureResourcesPayload, str], List[Finding]] + + +def _build_registry() -> Dict[str, RuleFn]: + from agentops.agent.checks.posture_rules.content_filter import ( + evaluate as content_filter_rule, + ) + from agentops.agent.checks.posture_rules.diagnostics import ( + evaluate as diagnostics_rule, + ) + from agentops.agent.checks.posture_rules.local_auth import ( + evaluate as local_auth_rule, + ) + from agentops.agent.checks.posture_rules.managed_identity import ( + evaluate as managed_identity_rule, + ) + from agentops.agent.checks.posture_rules.network import ( + evaluate as network_rule, + ) + + return { + "waf.security.local_auth_disabled": local_auth_rule, + "waf.security.public_network_access": network_rule, + "waf.security.managed_identity": managed_identity_rule, + "waf.security.diagnostic_settings": diagnostics_rule, + "waf.security.content_filter": content_filter_rule, + } + + +RULE_REGISTRY: Dict[str, RuleFn] = _build_registry() diff --git a/src/agentops/agent/checks/posture_rules/content_filter.py b/src/agentops/agent/checks/posture_rules/content_filter.py new file mode 100644 index 00000000..237c42fa --- /dev/null +++ b/src/agentops/agent/checks/posture_rules/content_filter.py @@ -0,0 +1,59 @@ +"""WAF-AI Security: every model deployment needs a content filter (RAI policy). + +The WAF-AI Security pillar (Responsible AI subsection) requires that +each Azure OpenAI / AI Foundry model deployment have a content filter +applied. The default ``Microsoft.Default`` policy is acceptable; a +deployment with no policy at all is not. + +This rule fires for **each** deployment that has no +``rai_policy_name``. +""" + +from __future__ import annotations + +from typing import List + +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +RULE_ID = "waf.security.content_filter" + + +def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: + account = payload.account + if account is None or not payload.deployments: + return [] + + missing = [d for d in payload.deployments if not d.rai_policy_name] + if not missing: + return [] + + return [ + Finding( + id=RULE_ID, + severity=Severity.CRITICAL, + category=Category.SECURITY, + title="One or more deployments have no content filter applied", + summary=( + f"{len(missing)} of {len(payload.deployments)} " + f"deployment(s) on account `{account.name}` have no " + "RAI / content-filter policy. The WAF-AI Security " + "pillar requires Responsible AI policies on every " + "model deployment." + ), + recommendation=( + "Apply a content-filter policy (start with " + "`Microsoft.Default`, then tune severity thresholds " + "for your workload) to every deployment listed below. " + "See https://learn.microsoft.com/azure/ai-services/openai/concepts/content-filter" + ), + source=source_name, + evidence={ + "account": account.name, + "deployments_missing_filter": [ + {"name": d.name, "model": d.model} for d in missing + ], + "deployments_total": len(payload.deployments), + }, + ) + ] diff --git a/src/agentops/agent/checks/posture_rules/diagnostics.py b/src/agentops/agent/checks/posture_rules/diagnostics.py new file mode 100644 index 00000000..3a0dbc56 --- /dev/null +++ b/src/agentops/agent/checks/posture_rules/diagnostics.py @@ -0,0 +1,74 @@ +"""WAF-AI Security: diagnostic settings must be configured. + +Without diagnostic settings, audit logs and request traces from the +Cognitive Services account never reach a Log Analytics workspace, +storage account, or event hub — making incident investigation and +content-safety auditing effectively impossible. + +The WAF-AI Security pillar recommends streaming diagnostic logs to +Log Analytics for every AI account in production. + +This rule fires when **none** of the diagnostic settings on the +account ship logs to a destination (workspace / storage / event hub). +""" + +from __future__ import annotations + +from typing import List + +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +RULE_ID = "waf.security.diagnostic_settings" + + +def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: + account = payload.account + if account is None: + return [] + + has_destination = any( + s.workspace_id or s.storage_account_id or s.event_hub_authorization_rule_id + for s in payload.diagnostic_settings + ) + has_categories = any(s.enabled_log_categories for s in payload.diagnostic_settings) + + if has_destination and has_categories: + return [] + + return [ + Finding( + id=RULE_ID, + severity=Severity.WARNING, + category=Category.SECURITY, + title="Diagnostic settings are missing or incomplete", + summary=( + f"Cognitive Services account `{account.name}` has " + f"{len(payload.diagnostic_settings)} diagnostic " + "setting(s), but none route enabled log categories to " + "a Log Analytics workspace, storage account, or event " + "hub. Audit and content-safety logs are not being " + "captured." + ), + recommendation=( + "Create a diagnostic setting that ships the " + "`Audit`, `RequestResponse`, and `Trace` log categories " + "to a Log Analytics workspace. See " + "https://learn.microsoft.com/azure/ai-services/diagnostic-logging" + ), + source=source_name, + evidence={ + "account": account.name, + "diagnostic_settings": [ + { + "name": s.name, + "workspace_id": s.workspace_id, + "storage_account_id": s.storage_account_id, + "event_hub_authorization_rule_id": s.event_hub_authorization_rule_id, + "enabled_log_categories": s.enabled_log_categories, + } + for s in payload.diagnostic_settings + ], + }, + ) + ] diff --git a/src/agentops/agent/checks/posture_rules/local_auth.py b/src/agentops/agent/checks/posture_rules/local_auth.py new file mode 100644 index 00000000..5345f329 --- /dev/null +++ b/src/agentops/agent/checks/posture_rules/local_auth.py @@ -0,0 +1,55 @@ +"""WAF-AI Security: local (key-based) authentication must be disabled. + +Cognitive Services / Azure OpenAI accounts ship with key-based auth +enabled by default. Microsoft Entra ID is the recommended path for +production AI workloads — keys can be exfiltrated, hard to rotate, and +bypass conditional access policies. + +WAF-AI Security pillar reference: +https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity +""" + +from __future__ import annotations + +from typing import List + +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +RULE_ID = "waf.security.local_auth_disabled" + + +def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: + account = payload.account + if account is None: + return [] + if account.disable_local_auth is True: + return [] + + return [ + Finding( + id=RULE_ID, + severity=Severity.CRITICAL, + category=Category.SECURITY, + title="Local (API key) authentication is enabled", + summary=( + f"Cognitive Services account `{account.name}` has " + f"`disableLocalAuth={account.disable_local_auth}`. " + "Key-based authentication is enabled, which contradicts " + "the WAF-AI Security pillar guidance to use Microsoft " + "Entra ID exclusively." + ), + recommendation=( + "Set `disableLocalAuth: true` on the account, grant the " + "agent runtime the `Cognitive Services OpenAI User` " + "role via managed identity, and rotate any keys that " + "may have leaked. See " + "https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity" + ), + source=source_name, + evidence={ + "account": account.name, + "disable_local_auth": account.disable_local_auth, + }, + ) + ] diff --git a/src/agentops/agent/checks/posture_rules/managed_identity.py b/src/agentops/agent/checks/posture_rules/managed_identity.py new file mode 100644 index 00000000..e481242e --- /dev/null +++ b/src/agentops/agent/checks/posture_rules/managed_identity.py @@ -0,0 +1,59 @@ +"""WAF-AI Security: account must have a managed identity assigned. + +Cognitive Services / Azure OpenAI accounts call downstream Azure +resources (Storage for fine-tuning data, Key Vault for customer keys, +etc.). The WAF-AI Security pillar recommends using a managed identity +for those calls instead of connection strings or keys. + +This rule fires when the account ``identity.type`` is missing or +``None`` — i.e. neither system-assigned nor user-assigned managed +identity is configured. +""" + +from __future__ import annotations + +from typing import List + +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +RULE_ID = "waf.security.managed_identity" + +_NO_IDENTITY_VALUES = {"", "none", "null"} + + +def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: + account = payload.account + if account is None: + return [] + type_ = (account.identity_type or "").strip().lower() + if type_ and type_ not in _NO_IDENTITY_VALUES: + return [] + + return [ + Finding( + id=RULE_ID, + severity=Severity.WARNING, + category=Category.SECURITY, + title="Account has no managed identity assigned", + summary=( + f"Cognitive Services account `{account.name}` has no " + "managed identity. The WAF-AI Security pillar " + "recommends assigning a system- or user-assigned MI so " + "downstream calls (Storage, Key Vault, Search) avoid " + "connection strings." + ), + recommendation=( + "Enable a system-assigned managed identity (or attach " + "a user-assigned one) on the account, and grant it the " + "minimum role it needs on each downstream resource. " + "See https://learn.microsoft.com/azure/ai-services/authentication" + ), + source=source_name, + evidence={ + "account": account.name, + "identity_type": account.identity_type, + "user_assigned_identities": account.user_assigned_identities, + }, + ) + ] diff --git a/src/agentops/agent/checks/posture_rules/network.py b/src/agentops/agent/checks/posture_rules/network.py new file mode 100644 index 00000000..b326b805 --- /dev/null +++ b/src/agentops/agent/checks/posture_rules/network.py @@ -0,0 +1,68 @@ +"""WAF-AI Security: restrict public network access to the AI account. + +Cognitive Services / Azure OpenAI accounts default to ``Enabled`` +public network access for convenience. For production AI workloads the +WAF-AI Security pillar recommends restricting network access via +private endpoints or a strict network ACL. + +This rule fires unless ONE of the following is true: + +* ``publicNetworkAccess == 'Disabled'`` +* At least one private endpoint connection is attached +* Network ACLs default action is ``Deny`` +""" + +from __future__ import annotations + +from typing import List + +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.azure_resources import AzureResourcesPayload + +RULE_ID = "waf.security.public_network_access" + + +def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: + account = payload.account + if account is None: + return [] + + pna = (account.public_network_access or "").lower() + has_private_endpoint = account.private_endpoint_count > 0 + acl_default = (account.network_acls_default_action or "").lower() + + if ( + pna == "disabled" + or has_private_endpoint + or acl_default == "deny" + ): + return [] + + return [ + Finding( + id=RULE_ID, + severity=Severity.WARNING, + category=Category.SECURITY, + title="Public network access is open and unrestricted", + summary=( + f"Cognitive Services account `{account.name}` allows " + "public network access without a deny-by-default ACL or " + "a private endpoint. The WAF-AI Security pillar " + "recommends restricting network access for production " + "AI workloads." + ), + recommendation=( + "Either set `publicNetworkAccess: Disabled` and attach " + "a private endpoint, or configure network ACLs with " + "`defaultAction: Deny` and an explicit allow list. See " + "https://learn.microsoft.com/azure/ai-services/cognitive-services-virtual-networks" + ), + source=source_name, + evidence={ + "account": account.name, + "public_network_access": account.public_network_access, + "private_endpoint_count": account.private_endpoint_count, + "network_acls_default_action": account.network_acls_default_action, + }, + ) + ] diff --git a/src/agentops/agent/checks/regression.py b/src/agentops/agent/checks/regression.py new file mode 100644 index 00000000..3e2a6f23 --- /dev/null +++ b/src/agentops/agent/checks/regression.py @@ -0,0 +1,77 @@ +"""Regression check: detect metric drops vs a rolling baseline.""" + +from __future__ import annotations + +from statistics import mean +from typing import List + +from agentops.agent.config import RegressionCheckConfig +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.results_history import ResultsHistory + + +def run_regression_check( + history: ResultsHistory, config: RegressionCheckConfig +) -> List[Finding]: + runs = history.runs + if len(runs) < config.min_runs: + return [] + + latest = runs[-1] + baseline_runs = runs[:-1] + if not baseline_runs: + return [] + + findings: List[Finding] = [] + for metric in config.metrics: + baseline_values = [ + r.metrics[metric] for r in baseline_runs if metric in r.metrics + ] + if not baseline_values: + continue + if metric not in latest.metrics: + continue + + baseline = mean(baseline_values) + current = latest.metrics[metric] + if baseline <= 0: + continue + + drop = (baseline - current) / baseline + if drop < config.threshold_drop: + continue + + severity = ( + Severity.CRITICAL + if drop >= max(config.threshold_drop * 2, 0.20) + else Severity.WARNING + ) + + findings.append( + Finding( + id=f"regression.{metric}", + severity=severity, + category=Category.QUALITY, + title=f"Regression detected on `{metric}`", + summary=( + f"`{metric}` dropped {drop * 100:.1f}% in run " + f"`{latest.run_id}` (current={current:.4f}, " + f"baseline={baseline:.4f} over {len(baseline_values)} runs)." + ), + recommendation=( + "Compare the latest run against the baseline runs in " + "`.agentops/results/`, inspect prompt/model/dataset " + "changes, and re-run the evaluation after the fix." + ), + source="results_history", + evidence={ + "metric": metric, + "current": current, + "baseline_avg": baseline, + "drop_ratio": drop, + "baseline_runs": len(baseline_values), + "latest_run_id": latest.run_id, + }, + ) + ) + return findings diff --git a/src/agentops/agent/checks/safety.py b/src/agentops/agent/checks/safety.py new file mode 100644 index 00000000..2f921c7c --- /dev/null +++ b/src/agentops/agent/checks/safety.py @@ -0,0 +1,92 @@ +"""Safety check: scans the latest evaluation for content-safety hits.""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from agentops.agent.config import SafetyCheckConfig +from agentops.agent.findings import Category, Finding, Severity +from agentops.agent.sources.results_history import ResultsHistory + +_SEVERITY_RANK = {"low": 0, "medium": 1, "high": 2} + +_SAFETY_METRICS = ( + "violence", + "self_harm", + "sexual", + "hate_unfairness", + "protected_material", +) + + +def _severity_value(raw: Any) -> int: + if isinstance(raw, str): + return _SEVERITY_RANK.get(raw.strip().lower(), -1) + if isinstance(raw, (int, float)): + return int(raw) + return -1 + + +def run_safety_check( + history: ResultsHistory, config: SafetyCheckConfig +) -> List[Finding]: + if not history.runs: + return [] + + floor = _SEVERITY_RANK.get(config.severity_floor.strip().lower(), 1) + latest = history.runs[-1] + + hits: Dict[str, Dict[str, Any]] = {} + for item in latest.item_evaluations: + if not isinstance(item, dict): + continue + metrics = item.get("metrics") or item.get("scores") or {} + if not isinstance(metrics, dict): + continue + for key, value in metrics.items(): + metric = str(key).lower() + if not any(m in metric for m in _SAFETY_METRICS): + continue + level = _severity_value(value) + if level < floor: + continue + existing = hits.get(metric) + if existing is None or level > existing.get("level", -1): + hits[metric] = { + "level": level, + "value": value, + "row": item.get("input") + or item.get("question") + or item.get("id"), + } + + findings: List[Finding] = [] + for metric, info in hits.items(): + severity = ( + Severity.CRITICAL if info["level"] >= 2 else Severity.WARNING + ) + findings.append( + Finding( + id=f"safety.{metric}", + severity=severity, + category=Category.QUALITY, + title=f"Content-safety hit on `{metric}`", + summary=( + f"Run `{latest.run_id}` produced a `{metric}` rating " + f"of `{info['value']}` on at least one row." + ), + recommendation=( + "Inspect the offending dataset row and the model " + "response, tighten the system prompt or add a safety " + "filter, and re-evaluate." + ), + source="results_history", + evidence={ + "metric": metric, + "value": info["value"], + "row": info.get("row"), + "run_id": latest.run_id, + }, + ) + ) + return findings diff --git a/src/agentops/agent/config.py b/src/agentops/agent/config.py new file mode 100644 index 00000000..4b3c22bb --- /dev/null +++ b/src/agentops/agent/config.py @@ -0,0 +1,146 @@ +"""Pydantic configuration model for the watchdog agent.""" + +from __future__ import annotations + +from pathlib import Path +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class ResultsHistorySourceConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + enabled: bool = True + path: str = ".agentops/results" + lookback_runs: int = Field(10, ge=2) + + +class AzureMonitorSourceConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + enabled: bool = True + app_insights_resource_id: Optional[str] = None + log_analytics_workspace_id: Optional[str] = None + + +class FoundryControlSourceConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + enabled: bool = True + project_endpoint: Optional[str] = None + project_endpoint_env: str = "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT" + agent_ids: List[str] = Field(default_factory=list) + + +class AzureResourcesSourceConfig(BaseModel): + """Read-only management-plane source for Azure resource posture audits. + + Requires ``Reader`` (or stronger) RBAC on the resource group, and the + ``[agent]`` extra (which pulls in ``azure-mgmt-cognitiveservices`` and + ``azure-mgmt-monitor``). + """ + + model_config = ConfigDict(extra="forbid") + enabled: bool = False + subscription_id: Optional[str] = None + subscription_id_env: str = "AZURE_SUBSCRIPTION_ID" + resource_group: Optional[str] = None + cognitive_services_account: Optional[str] = None + + +class SourcesConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + results_history: ResultsHistorySourceConfig = Field( + default_factory=ResultsHistorySourceConfig + ) + azure_monitor: AzureMonitorSourceConfig = Field( + default_factory=AzureMonitorSourceConfig + ) + foundry_control: FoundryControlSourceConfig = Field( + default_factory=FoundryControlSourceConfig + ) + azure_resources: AzureResourcesSourceConfig = Field( + default_factory=AzureResourcesSourceConfig + ) + + +class RegressionCheckConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + metrics: List[str] = Field( + default_factory=lambda: [ + "coherence", + "fluency", + "similarity", + "f1_score", + "groundedness", + "tool_call_accuracy", + ] + ) + threshold_drop: float = Field(0.10, ge=0.0, le=1.0) + min_runs: int = Field(3, ge=2) + + +class LatencyCheckConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + p95_threshold_seconds: float = Field(5.0, gt=0) + + +class ErrorsCheckConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + rate_threshold: float = Field(0.05, ge=0.0, le=1.0) + + +class SafetyCheckConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + severity_floor: str = "Medium" # Low | Medium | High + + +class PostureCheckConfig(BaseModel): + """WAF-AI posture audit configuration. + + The MVP rule set targets the **Security** pillar of the + Microsoft Well-Architected Framework for AI workloads. + + The check is opt-in: ``enabled`` defaults to ``False`` because it + requires the ``azure_resources`` source to be configured and an + Azure Reader role on the target resource group. + """ + + model_config = ConfigDict(extra="forbid") + enabled: bool = False + pillar: str = "security" + exclude_rules: List[str] = Field(default_factory=list) + + +class ChecksConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + regression: RegressionCheckConfig = Field(default_factory=RegressionCheckConfig) + latency: LatencyCheckConfig = Field(default_factory=LatencyCheckConfig) + errors: ErrorsCheckConfig = Field(default_factory=ErrorsCheckConfig) + safety: SafetyCheckConfig = Field(default_factory=SafetyCheckConfig) + posture: PostureCheckConfig = Field(default_factory=PostureCheckConfig) + + +class ServerConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + github_app_client_id: Optional[str] = None + + +class AgentConfig(BaseModel): + """Root config for ``.agentops/agent.yaml``.""" + + model_config = ConfigDict(extra="forbid") + version: int = 1 + sources: SourcesConfig = Field(default_factory=SourcesConfig) + checks: ChecksConfig = Field(default_factory=ChecksConfig) + server: ServerConfig = Field(default_factory=ServerConfig) + lookback_days: int = Field(7, ge=1) + + +def load_agent_config(path: Optional[Path]) -> AgentConfig: + """Load an :class:`AgentConfig` from a YAML file (or return defaults).""" + if path is None or not path.exists(): + return AgentConfig() + + from agentops.utils.yaml import load_yaml + + raw = load_yaml(path) + return AgentConfig.model_validate(raw) diff --git a/src/agentops/agent/findings.py b/src/agentops/agent/findings.py new file mode 100644 index 00000000..62031bb9 --- /dev/null +++ b/src/agentops/agent/findings.py @@ -0,0 +1,102 @@ +"""Severity-ranked findings produced by the watchdog agent.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict + + +class Category(str, Enum): + """High-level grouping for a finding. + + Categories are stable user-facing buckets used for filtering and for + grouping the watchdog report. They are independent of severity: + a `quality` finding can be `critical`, `warning`, or `info`. + + * ``quality`` — eval-driven signals (regression, content-safety) + * ``performance`` — latency / throughput signals + * ``reliability`` — error / failure signals + * ``security`` — Azure resource posture audits (WAF-AI Security pillar) + """ + + QUALITY = "quality" + PERFORMANCE = "performance" + RELIABILITY = "reliability" + SECURITY = "security" + + +class Severity(str, Enum): + """Severity level for a finding.""" + + INFO = "info" + WARNING = "warning" + CRITICAL = "critical" + + @property + def rank(self) -> int: + return _SEVERITY_RANK[self] + + def __lt__(self, other: object) -> bool: # type: ignore[override] + if not isinstance(other, Severity): + return NotImplemented + return self.rank < other.rank + + def __le__(self, other: object) -> bool: # type: ignore[override] + if not isinstance(other, Severity): + return NotImplemented + return self.rank <= other.rank + + def __gt__(self, other: object) -> bool: # type: ignore[override] + if not isinstance(other, Severity): + return NotImplemented + return self.rank > other.rank + + def __ge__(self, other: object) -> bool: # type: ignore[override] + if not isinstance(other, Severity): + return NotImplemented + return self.rank >= other.rank + + +_SEVERITY_RANK = { + Severity.INFO: 0, + Severity.WARNING: 1, + Severity.CRITICAL: 2, +} + + +_SEVERITY_EMOJI = { + Severity.INFO: "ℹ️", + Severity.WARNING: "⚠️", + Severity.CRITICAL: "🚨", +} + + +def severity_emoji(severity: Severity) -> str: + return _SEVERITY_EMOJI[severity] + + +@dataclass +class Finding: + """A single observation the watchdog agent surfaces.""" + + id: str + severity: Severity + title: str + summary: str + recommendation: str + source: str + category: Category = Category.QUALITY + evidence: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "severity": self.severity.value, + "category": self.category.value, + "title": self.title, + "summary": self.summary, + "recommendation": self.recommendation, + "source": self.source, + "evidence": self.evidence, + } diff --git a/src/agentops/agent/report.py b/src/agentops/agent/report.py new file mode 100644 index 00000000..ad627a84 --- /dev/null +++ b/src/agentops/agent/report.py @@ -0,0 +1,197 @@ +"""Markdown renderer for watchdog agent findings.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Dict, List + +from agentops.agent.analyzer import AnalysisResult +from agentops.agent.findings import Category, Finding, Severity, severity_emoji + +_CATEGORY_ORDER: List[Category] = [ + Category.SECURITY, + Category.RELIABILITY, + Category.PERFORMANCE, + Category.QUALITY, +] + +_CATEGORY_LABEL: Dict[Category, str] = { + Category.SECURITY: "Security posture (WAF-AI — Security pillar)", + Category.RELIABILITY: "Reliability", + Category.PERFORMANCE: "Performance", + Category.QUALITY: "Quality", +} + +_CATEGORY_FOOTER: Dict[Category, str] = { + Category.SECURITY: ( + "_Audit reference: Microsoft Well-Architected Framework for AI " + "workloads — Security pillar — " + "https://learn.microsoft.com/azure/well-architected/ai/security_" + ), +} + + +def _format_diagnostics_row(name: str, diagnostics: dict) -> str: + status = diagnostics.get("status", "unknown") + detail = diagnostics.get("reason") or diagnostics.get("runs_loaded") or "" + return f"| `{name}` | `{status}` | {detail} |" + + +def _format_finding_row(finding: Finding) -> str: + return ( + f"| {severity_emoji(finding.severity)} `{finding.severity.value}` " + f"| `{finding.id}` | {finding.title} | `{finding.source}` |" + ) + + +def _verdict_banner(result: AnalysisResult) -> str: + if not result.findings: + return "## Verdict: ✅ No issues detected" + max_sev = result.max_severity + if max_sev == Severity.CRITICAL: + return "## Verdict: 🚨 CRITICAL issues found" + if max_sev == Severity.WARNING: + return "## Verdict: ⚠️ Warnings found" + return "## Verdict: ℹ️ Informational findings" + + +def _group_by_category(findings: List[Finding]) -> Dict[Category, List[Finding]]: + grouped: Dict[Category, List[Finding]] = {} + for f in findings: + grouped.setdefault(f.category, []).append(f) + return grouped + + +def _render_finding_detail(lines: List[str], finding: Finding) -> None: + lines.append( + f"#### {severity_emoji(finding.severity)} `{finding.id}` — {finding.title}" + ) + lines.append("") + lines.append(f"- **Severity:** `{finding.severity.value}`") + lines.append(f"- **Category:** `{finding.category.value}`") + lines.append(f"- **Source:** `{finding.source}`") + lines.append("") + lines.append(finding.summary) + lines.append("") + lines.append(f"**Recommendation:** {finding.recommendation}") + lines.append("") + if finding.evidence: + lines.append("**Evidence:**") + lines.append("") + lines.append("```json") + lines.append(json.dumps(finding.evidence, indent=2, default=str)) + lines.append("```") + lines.append("") + + +def render_report(result: AnalysisResult) -> str: + lines: List[str] = [] + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + lines.append("# AgentOps Watchdog Report") + lines.append("") + lines.append(f"_Generated: {now}_") + lines.append("") + lines.append(_verdict_banner(result)) + lines.append("") + + # Summary counts + sev_counts = {Severity.CRITICAL: 0, Severity.WARNING: 0, Severity.INFO: 0} + cat_counts: Dict[Category, int] = {c: 0 for c in _CATEGORY_ORDER} + for f in result.findings: + sev_counts[f.severity] += 1 + cat_counts[f.category] = cat_counts.get(f.category, 0) + 1 + + lines.append("## Summary") + lines.append("") + lines.append("| Severity | Count |") + lines.append("|---|---|") + lines.append(f"| 🚨 Critical | {sev_counts[Severity.CRITICAL]} |") + lines.append(f"| ⚠️ Warning | {sev_counts[Severity.WARNING]} |") + lines.append(f"| ℹ️ Info | {sev_counts[Severity.INFO]} |") + lines.append("") + lines.append("| Category | Count |") + lines.append("|---|---|") + for cat in _CATEGORY_ORDER: + lines.append(f"| {_CATEGORY_LABEL[cat]} | {cat_counts.get(cat, 0)} |") + lines.append("") + + # Sources + lines.append("## Sources") + lines.append("") + lines.append("| Source | Status | Detail |") + lines.append("|---|---|---|") + for name, diag in result.diagnostics.items(): + lines.append(_format_diagnostics_row(name, diag)) + lines.append("") + + # Findings grouped by category + if result.findings: + grouped = _group_by_category(result.findings) + lines.append("## Findings") + lines.append("") + for cat in _CATEGORY_ORDER: + bucket = grouped.get(cat) + if not bucket: + continue + lines.append(f"### {_CATEGORY_LABEL[cat]}") + lines.append("") + lines.append("| Severity | ID | Title | Source |") + lines.append("|---|---|---|---|") + for f in bucket: + lines.append(_format_finding_row(f)) + lines.append("") + for f in bucket: + _render_finding_detail(lines, f) + footer = _CATEGORY_FOOTER.get(cat) + if footer: + lines.append(footer) + lines.append("") + else: + lines.append("## Findings") + lines.append("") + lines.append("_No findings — all configured checks passed._") + lines.append("") + + # History appendix + if result.history and result.history.runs: + lines.append("## Recent runs") + lines.append("") + lines.append("| Run ID | Timestamp | Items pass | Run pass |") + lines.append("|---|---|---|---|") + for run in result.history.runs[-10:]: + ts = run.timestamp.strftime("%Y-%m-%d %H:%M") if run.timestamp else "-" + items = ( + f"{run.items_passed_all}/{run.items_total}" + if run.items_total + else "-" + ) + run_pass = ( + "✅" if run.run_pass else "❌" if run.run_pass is False else "-" + ) + lines.append(f"| `{run.run_id}` | {ts} | {items} | {run_pass} |") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def short_chat_summary(result: AnalysisResult) -> str: + """Compact one-screen summary used by the Copilot Extension server.""" + if not result.findings: + return "✅ No issues detected by the AgentOps watchdog." + counts = {Severity.CRITICAL: 0, Severity.WARNING: 0, Severity.INFO: 0} + for f in result.findings: + counts[f.severity] += 1 + parts = [ + f"AgentOps watchdog found {len(result.findings)} finding(s): " + f"🚨 {counts[Severity.CRITICAL]} critical, " + f"⚠️ {counts[Severity.WARNING]} warning, " + f"ℹ️ {counts[Severity.INFO]} info." + ] + parts.append("") + parts.append("Top items:") + for f in result.findings[:5]: + parts.append( + f"- {severity_emoji(f.severity)} **{f.id}** — `{f.category.value}` — {f.title}" + ) + return "\n".join(parts) diff --git a/src/agentops/agent/server/__init__.py b/src/agentops/agent/server/__init__.py new file mode 100644 index 00000000..3d216685 --- /dev/null +++ b/src/agentops/agent/server/__init__.py @@ -0,0 +1 @@ +"""FastAPI Copilot Extension server for the watchdog agent.""" diff --git a/src/agentops/agent/server/app.py b/src/agentops/agent/server/app.py new file mode 100644 index 00000000..a880377d --- /dev/null +++ b/src/agentops/agent/server/app.py @@ -0,0 +1,84 @@ +"""FastAPI factory for the Copilot Extension server. + +Exposes: +- ``POST /agents/messages`` — Copilot Extensions protocol (SSE response) +- ``GET /healthz`` — liveness check +- ``GET /`` — small HTML index +""" + +import logging +from pathlib import Path + +from agentops.agent.config import AgentConfig + +log = logging.getLogger(__name__) + + +def create_app( + workspace: Path, + config: AgentConfig, + verify_signature: bool = True, +): + """Build a FastAPI app for the watchdog Copilot Extension server.""" + try: + from fastapi import FastAPI, HTTPException, Request + from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse + except ImportError as exc: # pragma: no cover - import guard + raise RuntimeError( + "FastAPI is not installed. Install agent extras with " + "'pip install agentops-toolkit[agent]'." + ) from exc + + from agentops.agent.server.chat import stream_reply + from agentops.agent.server.protocol import parse_copilot_request + + app = FastAPI(title="AgentOps Watchdog", version="1.0") + + @app.get("/healthz") + def healthz() -> JSONResponse: + return JSONResponse({"status": "ok"}) + + @app.get("/", response_class=HTMLResponse) + def index() -> str: + return ( + "" + "

AgentOps Watchdog

" + "

Copilot Extension endpoint: POST /agents/messages

" + "

Health: /healthz

" + "" + ) + + @app.post("/agents/messages") + async def messages(request: Request): + body_bytes = await request.body() + + if verify_signature: + from agentops.agent.server.auth import verify_signature as _verify + + try: + _verify( + body_bytes, + request.headers.get("x-github-public-key-identifier"), + request.headers.get("x-github-public-key-signature"), + ) + except ValueError as exc: + raise HTTPException(status_code=401, detail=str(exc)) from exc + + try: + payload = await request.json() + except Exception as exc: + raise HTTPException( + status_code=400, detail=f"invalid JSON body: {exc}" + ) from exc + + copilot_request = parse_copilot_request( + payload if isinstance(payload, dict) else {} + ) + + def _generator(): + for chunk in stream_reply(workspace, config, copilot_request): + yield chunk + + return StreamingResponse(_generator(), media_type="text/event-stream") + + return app diff --git a/src/agentops/agent/server/auth.py b/src/agentops/agent/server/auth.py new file mode 100644 index 00000000..e1ae9e67 --- /dev/null +++ b/src/agentops/agent/server/auth.py @@ -0,0 +1,94 @@ +"""Copilot Extension request signature validation. + +Validates the ``X-GitHub-Public-Key-Identifier`` and +``X-GitHub-Public-Key-Signature`` headers against GitHub's published +public keys. The validation can be disabled (``--no-verify``) for local +development and tests. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field +from typing import Dict, Optional + +log = logging.getLogger(__name__) + +GITHUB_KEYS_URL = "https://api.github.com/meta/public_keys/copilot_api" +KEY_CACHE_TTL_SECONDS = 60 * 30 # 30 minutes + + +@dataclass +class _KeyCache: + keys: Dict[str, str] = field(default_factory=dict) + fetched_at: float = 0.0 + + +_cache = _KeyCache() + + +def _fetch_keys() -> Dict[str, str]: + import httpx # local import keeps base CLI lean + + with httpx.Client(timeout=10.0) as client: + response = client.get(GITHUB_KEYS_URL) + response.raise_for_status() + payload = response.json() + keys = {} + for entry in payload.get("public_keys", []): + identifier = entry.get("key_identifier") + key = entry.get("key") + if identifier and key: + keys[identifier] = key + return keys + + +def _get_keys(force_refresh: bool = False) -> Dict[str, str]: + now = time.time() + if ( + force_refresh + or not _cache.keys + or now - _cache.fetched_at > KEY_CACHE_TTL_SECONDS + ): + _cache.keys = _fetch_keys() + _cache.fetched_at = now + return _cache.keys + + +def verify_signature( + body: bytes, + key_identifier: Optional[str], + signature_b64: Optional[str], +) -> None: + """Raise ``ValueError`` if the request signature is invalid.""" + if not key_identifier or not signature_b64: + raise ValueError("missing Copilot signature headers") + + import base64 + + from cryptography.exceptions import InvalidSignature + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + + keys = _get_keys() + pem = keys.get(key_identifier) + if pem is None: + keys = _get_keys(force_refresh=True) + pem = keys.get(key_identifier) + if pem is None: + raise ValueError(f"unknown key identifier {key_identifier!r}") + + public_key = serialization.load_pem_public_key(pem.encode("utf-8")) + if not isinstance(public_key, ec.EllipticCurvePublicKey): + raise ValueError("Copilot public key is not an EC key") + + try: + signature = base64.b64decode(signature_b64) + except Exception as exc: # pragma: no cover - malformed inputs + raise ValueError(f"invalid signature encoding: {exc}") from exc + + try: + public_key.verify(signature, body, ec.ECDSA(hashes.SHA256())) + except InvalidSignature as exc: + raise ValueError("signature verification failed") from exc diff --git a/src/agentops/agent/server/chat.py b/src/agentops/agent/server/chat.py new file mode 100644 index 00000000..dfb791a6 --- /dev/null +++ b/src/agentops/agent/server/chat.py @@ -0,0 +1,44 @@ +"""Chat orchestration: turns a Copilot user message into an SSE reply.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Iterable + +from agentops.agent.analyzer import analyze +from agentops.agent.config import AgentConfig +from agentops.agent.report import render_report, short_chat_summary +from agentops.agent.server.protocol import CopilotRequest, stream_markdown + + +def _intro_for(message: str) -> str: + msg = (message or "").lower() + if any(word in msg for word in ("regress", "drop", "score")): + focus = "regressions" + elif any(word in msg for word in ("latency", "slow", "p95")): + focus = "latency" + elif any(word in msg for word in ("error", "fail", "exception")): + focus = "production errors" + elif any(word in msg for word in ("safety", "harm", "violen")): + focus = "content safety" + else: + focus = "agent health" + return ( + f"I scanned your AgentOps eval history, Azure Monitor, and Foundry " + f"control plane focused on **{focus}**.\n\n" + ) + + +def build_reply(workspace: Path, config: AgentConfig, request: CopilotRequest) -> str: + user_message = request.last_user_message or "" + result = analyze(workspace, config) + intro = _intro_for(user_message) + summary = short_chat_summary(result) + report = render_report(result) + return f"{intro}{summary}\n\n---\n\n{report}" + + +def stream_reply( + workspace: Path, config: AgentConfig, request: CopilotRequest +) -> Iterable[bytes]: + return stream_markdown(build_reply(workspace, config, request)) diff --git a/src/agentops/agent/server/protocol.py b/src/agentops/agent/server/protocol.py new file mode 100644 index 00000000..63eb14dd --- /dev/null +++ b/src/agentops/agent/server/protocol.py @@ -0,0 +1,72 @@ +"""GitHub Copilot Extensions request/response protocol helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional + +import json + + +@dataclass +class CopilotMessage: + role: str + content: str + + +@dataclass +class CopilotRequest: + messages: List[CopilotMessage] + raw: Dict[str, Any] + + @property + def last_user_message(self) -> Optional[str]: + for message in reversed(self.messages): + if message.role == "user" and message.content: + return message.content + return None + + +def parse_copilot_request(body: Dict[str, Any]) -> CopilotRequest: + raw_messages = body.get("messages") or [] + messages: List[CopilotMessage] = [] + if isinstance(raw_messages, list): + for entry in raw_messages: + if not isinstance(entry, dict): + continue + role = str(entry.get("role") or "user") + content = entry.get("content") or "" + if isinstance(content, list): + # Multipart content -> concatenate text parts. + parts: List[str] = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + parts.append(str(part.get("text", ""))) + content = "".join(parts) + messages.append(CopilotMessage(role=role, content=str(content))) + return CopilotRequest(messages=messages, raw=body) + + +def sse_text_chunk(text: str) -> bytes: + payload = { + "choices": [ + { + "delta": {"role": "assistant", "content": text}, + "index": 0, + } + ] + } + return f"data: {json.dumps(payload)}\n\n".encode("utf-8") + + +def sse_done() -> bytes: + return b"data: [DONE]\n\n" + + +def stream_markdown(markdown: str, chunk_size: int = 512) -> Iterable[bytes]: + """Yield SSE chunks for a Markdown reply, then a [DONE] sentinel.""" + if not markdown: + markdown = "_(empty reply)_" + for start in range(0, len(markdown), chunk_size): + yield sse_text_chunk(markdown[start : start + chunk_size]) + yield sse_done() diff --git a/src/agentops/agent/sources/__init__.py b/src/agentops/agent/sources/__init__.py new file mode 100644 index 00000000..74376e0f --- /dev/null +++ b/src/agentops/agent/sources/__init__.py @@ -0,0 +1 @@ +"""Watchdog agent signal sources.""" diff --git a/src/agentops/agent/sources/azure_monitor.py b/src/agentops/agent/sources/azure_monitor.py new file mode 100644 index 00000000..0a5e757f --- /dev/null +++ b/src/agentops/agent/sources/azure_monitor.py @@ -0,0 +1,135 @@ +"""Azure Monitor / Application Insights source. + +Lazy-imports ``azure.monitor.query`` at call time so the base CLI does +not require the SDK. When the source is not configured or the SDK is +not installed, returns an empty payload with a diagnostic note. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from agentops.agent.config import AzureMonitorSourceConfig + +log = logging.getLogger(__name__) + + +@dataclass +class AzureMonitorPayload: + request_count: int = 0 + error_count: int = 0 + p95_duration_seconds: Optional[float] = None + avg_duration_seconds: Optional[float] = None + error_rate: Optional[float] = None + safety_violations: List[Dict[str, Any]] = field(default_factory=list) + diagnostics: Dict[str, Any] = field(default_factory=dict) + + +_REQUESTS_KQL = """ +requests +| where timestamp > ago({lookback_days}d) +| summarize + request_count = count(), + error_count = countif(success == false), + avg_duration_ms = avg(duration), + p95_duration_ms = percentile(duration, 95) +""" + + +def collect_azure_monitor( + config: AzureMonitorSourceConfig, + lookback_days: int, +) -> AzureMonitorPayload: + """Run KQL queries against Application Insights for the lookback window.""" + diagnostics: Dict[str, Any] = {"enabled": config.enabled} + + if not config.enabled: + diagnostics["status"] = "disabled" + return AzureMonitorPayload(diagnostics=diagnostics) + + if not config.app_insights_resource_id and not config.log_analytics_workspace_id: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "neither app_insights_resource_id nor log_analytics_workspace_id " + "is configured" + ) + return AzureMonitorPayload(diagnostics=diagnostics) + + try: + from azure.identity import DefaultAzureCredential + from azure.monitor.query import LogsQueryClient, LogsQueryStatus + except ImportError as exc: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "azure-monitor-query / azure-identity not installed " + "(install agentops-toolkit[agent])" + ) + log.info("azure-monitor-query unavailable: %s", exc) + return AzureMonitorPayload(diagnostics=diagnostics) + + workspace_or_resource = ( + config.log_analytics_workspace_id or config.app_insights_resource_id + ) + diagnostics["target"] = workspace_or_resource + + try: + credential = DefaultAzureCredential(exclude_developer_cli_credential=True) + client = LogsQueryClient(credential) + kql = _REQUESTS_KQL.format(lookback_days=int(lookback_days)) + if config.log_analytics_workspace_id: + response = client.query_workspace( + workspace_id=config.log_analytics_workspace_id, + query=kql, + timespan=None, + ) + else: + # query_resource is available on newer SDKs. + query_resource = getattr(client, "query_resource", None) + if query_resource is None: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "Installed azure-monitor-query does not support " + "query_resource; upgrade to >=1.3.0 or use " + "log_analytics_workspace_id." + ) + return AzureMonitorPayload(diagnostics=diagnostics) + response = query_resource( + resource_id=config.app_insights_resource_id, + query=kql, + timespan=None, + ) + except Exception as exc: # pragma: no cover - network / auth errors + diagnostics["status"] = "error" + diagnostics["reason"] = str(exc) + log.warning("Azure Monitor query failed: %s", exc) + return AzureMonitorPayload(diagnostics=diagnostics) + + if getattr(response, "status", None) == LogsQueryStatus.FAILURE: + diagnostics["status"] = "error" + diagnostics["reason"] = "query failed" + return AzureMonitorPayload(diagnostics=diagnostics) + + payload = AzureMonitorPayload(diagnostics=diagnostics) + diagnostics["status"] = "ok" + + tables = getattr(response, "tables", []) or [] + if tables: + rows = list(tables[0].rows) + if rows: + row = rows[0] + columns = [c.name if hasattr(c, "name") else str(c) for c in tables[0].columns] + data = dict(zip(columns, row)) + payload.request_count = int(data.get("request_count", 0) or 0) + payload.error_count = int(data.get("error_count", 0) or 0) + avg_ms = data.get("avg_duration_ms") + p95_ms = data.get("p95_duration_ms") + if avg_ms is not None: + payload.avg_duration_seconds = float(avg_ms) / 1000.0 + if p95_ms is not None: + payload.p95_duration_seconds = float(p95_ms) / 1000.0 + if payload.request_count > 0: + payload.error_rate = payload.error_count / payload.request_count + + return payload diff --git a/src/agentops/agent/sources/azure_resources.py b/src/agentops/agent/sources/azure_resources.py new file mode 100644 index 00000000..9ab6b448 --- /dev/null +++ b/src/agentops/agent/sources/azure_resources.py @@ -0,0 +1,232 @@ +"""Azure management-plane source for security posture audits. + +Reads the configuration of a Cognitive Services / Azure OpenAI account +and the diagnostic settings attached to it. This is a **read-only** +source intended for the WAF-AI Security pillar checklist. + +The source lazy-imports ``azure-mgmt-cognitiveservices`` and +``azure-mgmt-monitor`` so the base CLI does not require the management +SDKs. When the source is disabled, not configured, or the SDK is +missing, returns an empty payload with a diagnostic note (same fail-open +pattern as ``azure_monitor`` and ``foundry_control``). + +Required RBAC: ``Reader`` on the resource group (or on each individual +resource), granted to whoever runs ``agentops agent analyze``. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from agentops.agent.config import AzureResourcesSourceConfig + +log = logging.getLogger(__name__) + + +@dataclass +class CognitiveAccountSnapshot: + """Subset of cognitive-services account properties relevant to posture.""" + + name: str + location: Optional[str] = None + sku: Optional[str] = None + kind: Optional[str] = None + disable_local_auth: Optional[bool] = None + public_network_access: Optional[str] = None + private_endpoint_count: int = 0 + network_acls_default_action: Optional[str] = None + identity_type: Optional[str] = None + user_assigned_identities: List[str] = field(default_factory=list) + custom_subdomain_name: Optional[str] = None + + +@dataclass +class DeploymentSnapshot: + name: str + model: Optional[str] = None + rai_policy_name: Optional[str] = None + + +@dataclass +class DiagnosticSettingSnapshot: + name: str + workspace_id: Optional[str] = None + storage_account_id: Optional[str] = None + event_hub_authorization_rule_id: Optional[str] = None + enabled_log_categories: List[str] = field(default_factory=list) + + +@dataclass +class AzureResourcesPayload: + account: Optional[CognitiveAccountSnapshot] = None + deployments: List[DeploymentSnapshot] = field(default_factory=list) + diagnostic_settings: List[DiagnosticSettingSnapshot] = field(default_factory=list) + diagnostics: Dict[str, Any] = field(default_factory=dict) + + +def _resolve_subscription_id(config: AzureResourcesSourceConfig) -> Optional[str]: + if config.subscription_id: + return config.subscription_id + if config.subscription_id_env: + return os.environ.get(config.subscription_id_env) + return None + + +def _network_rule_set_to_default_action(rule_set: Any) -> Optional[str]: + if rule_set is None: + return None + action = getattr(rule_set, "default_action", None) + if action is None: + return None + return getattr(action, "value", str(action)) + + +def _identity_to_snapshot(identity: Any) -> tuple[Optional[str], List[str]]: + if identity is None: + return None, [] + type_ = getattr(identity, "type", None) + type_str = getattr(type_, "value", None) or (str(type_) if type_ else None) + user_assigned = getattr(identity, "user_assigned_identities", None) or {} + if isinstance(user_assigned, dict): + ids = list(user_assigned.keys()) + else: + ids = [] + return type_str, ids + + +def collect_azure_resources( + config: AzureResourcesSourceConfig, +) -> AzureResourcesPayload: + """Read the cognitive-services account, deployments, and diagnostic settings.""" + diagnostics: Dict[str, Any] = {"enabled": config.enabled} + + if not config.enabled: + diagnostics["status"] = "disabled" + return AzureResourcesPayload(diagnostics=diagnostics) + + subscription_id = _resolve_subscription_id(config) + if not subscription_id or not config.resource_group or not config.cognitive_services_account: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "azure_resources requires subscription_id (or " + "subscription_id_env), resource_group, and " + "cognitive_services_account." + ) + return AzureResourcesPayload(diagnostics=diagnostics) + + diagnostics["target"] = ( + f"/subscriptions/{subscription_id}/resourceGroups/" + f"{config.resource_group}/providers/Microsoft.CognitiveServices/" + f"accounts/{config.cognitive_services_account}" + ) + + try: + from azure.identity import DefaultAzureCredential + from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient + from azure.mgmt.monitor import MonitorManagementClient + except ImportError as exc: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "azure-mgmt-cognitiveservices / azure-mgmt-monitor not installed " + "(install agentops-toolkit[agent])" + ) + log.info("azure-mgmt-* unavailable: %s", exc) + return AzureResourcesPayload(diagnostics=diagnostics) + + payload = AzureResourcesPayload(diagnostics=diagnostics) + + try: + credential = DefaultAzureCredential(exclude_developer_cli_credential=True) + cs_client = CognitiveServicesManagementClient(credential, subscription_id) + monitor_client = MonitorManagementClient(credential, subscription_id) + + account = cs_client.accounts.get( + resource_group_name=config.resource_group, + account_name=config.cognitive_services_account, + ) + props = getattr(account, "properties", None) + sku = getattr(account, "sku", None) + identity_type, user_assigned = _identity_to_snapshot( + getattr(account, "identity", None) + ) + payload.account = CognitiveAccountSnapshot( + name=config.cognitive_services_account, + location=getattr(account, "location", None), + sku=getattr(sku, "name", None) if sku else None, + kind=getattr(account, "kind", None), + disable_local_auth=getattr(props, "disable_local_auth", None) if props else None, + public_network_access=( + getattr(getattr(props, "public_network_access", None), "value", None) + or (str(props.public_network_access) if props and props.public_network_access else None) + ), + private_endpoint_count=len( + getattr(props, "private_endpoint_connections", []) or [] + ) if props else 0, + network_acls_default_action=_network_rule_set_to_default_action( + getattr(props, "network_acls", None) if props else None + ), + identity_type=identity_type, + user_assigned_identities=user_assigned, + custom_subdomain_name=getattr(props, "custom_sub_domain_name", None) if props else None, + ) + + # Deployments and content-filter (RAI) policies. + try: + deployments = list( + cs_client.deployments.list( + resource_group_name=config.resource_group, + account_name=config.cognitive_services_account, + ) + ) + for d in deployments: + d_props = getattr(d, "properties", None) + model = None + if d_props and getattr(d_props, "model", None): + model = getattr(d_props.model, "name", None) + payload.deployments.append( + DeploymentSnapshot( + name=getattr(d, "name", "") or "", + model=model, + rai_policy_name=getattr(d_props, "rai_policy_name", None) if d_props else None, + ) + ) + except Exception as exc: # pragma: no cover - tolerate per-call failures + diagnostics["deployments_warning"] = str(exc) + + # Diagnostic settings. + try: + settings = list( + monitor_client.diagnostic_settings.list(resource_uri=diagnostics["target"]) + ) + for s in settings: + logs = getattr(s, "logs", []) or [] + enabled = [ + getattr(log_, "category", None) or getattr(log_, "category_group", None) + for log_ in logs + if getattr(log_, "enabled", False) + ] + payload.diagnostic_settings.append( + DiagnosticSettingSnapshot( + name=getattr(s, "name", "") or "", + workspace_id=getattr(s, "workspace_id", None), + storage_account_id=getattr(s, "storage_account_id", None), + event_hub_authorization_rule_id=getattr( + s, "event_hub_authorization_rule_id", None + ), + enabled_log_categories=[c for c in enabled if c], + ) + ) + except Exception as exc: # pragma: no cover + diagnostics["diagnostic_settings_warning"] = str(exc) + + except Exception as exc: # pragma: no cover + diagnostics["status"] = "error" + diagnostics["reason"] = str(exc) + log.warning("Azure resources read failed: %s", exc) + return payload + + diagnostics["status"] = "ok" + return payload diff --git a/src/agentops/agent/sources/foundry_control.py b/src/agentops/agent/sources/foundry_control.py new file mode 100644 index 00000000..8a394dc1 --- /dev/null +++ b/src/agentops/agent/sources/foundry_control.py @@ -0,0 +1,117 @@ +"""Foundry control-plane source. + +Lazy-imports ``azure.ai.projects`` to read agent metadata and recent +runs. Fails open: missing config or SDK is reported via diagnostics. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from agentops.agent.config import FoundryControlSourceConfig + +log = logging.getLogger(__name__) + + +@dataclass +class FoundryAgentSummary: + agent_id: str + name: Optional[str] = None + model: Optional[str] = None + updated_at: Optional[str] = None + + +@dataclass +class FoundryControlPayload: + agents: List[FoundryAgentSummary] = field(default_factory=list) + failed_runs: int = 0 + total_runs: int = 0 + diagnostics: Dict[str, Any] = field(default_factory=dict) + + @property + def failure_rate(self) -> Optional[float]: + if self.total_runs <= 0: + return None + return self.failed_runs / self.total_runs + + +def _resolve_endpoint(config: FoundryControlSourceConfig) -> Optional[str]: + if config.project_endpoint: + return config.project_endpoint + if config.project_endpoint_env: + return os.environ.get(config.project_endpoint_env) + return None + + +def collect_foundry_control( + config: FoundryControlSourceConfig, +) -> FoundryControlPayload: + diagnostics: Dict[str, Any] = {"enabled": config.enabled} + + if not config.enabled: + diagnostics["status"] = "disabled" + return FoundryControlPayload(diagnostics=diagnostics) + + endpoint = _resolve_endpoint(config) + if not endpoint: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "no project_endpoint configured " + f"(env var: {config.project_endpoint_env})" + ) + return FoundryControlPayload(diagnostics=diagnostics) + + diagnostics["endpoint"] = endpoint + + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + except ImportError as exc: + diagnostics["status"] = "skipped" + diagnostics["reason"] = ( + "azure-ai-projects / azure-identity not installed " + "(install agentops-toolkit[foundry])" + ) + log.info("azure-ai-projects unavailable: %s", exc) + return FoundryControlPayload(diagnostics=diagnostics) + + payload = FoundryControlPayload(diagnostics=diagnostics) + + try: + credential = DefaultAzureCredential(exclude_developer_cli_credential=True) + client = AIProjectClient(endpoint=endpoint, credential=credential) + except Exception as exc: # pragma: no cover + diagnostics["status"] = "error" + diagnostics["reason"] = f"client init failed: {exc}" + return payload + + try: + agents_iter = getattr(client, "agents", None) + if agents_iter is not None: + list_agents = getattr(agents_iter, "list_agents", None) or getattr( + agents_iter, "list", None + ) + if list_agents: + for raw in list_agents(): + aid = str(getattr(raw, "id", "") or getattr(raw, "name", "")) + if config.agent_ids and aid not in config.agent_ids: + continue + payload.agents.append( + FoundryAgentSummary( + agent_id=aid, + name=getattr(raw, "name", None), + model=getattr(raw, "model", None), + updated_at=str(getattr(raw, "updated_at", "") or "") + or None, + ) + ) + except Exception as exc: # pragma: no cover + log.warning("Foundry agents listing failed: %s", exc) + diagnostics["agents_error"] = str(exc) + + diagnostics["status"] = "ok" + diagnostics["agents_count"] = len(payload.agents) + return payload diff --git a/src/agentops/agent/sources/results_history.py b/src/agentops/agent/sources/results_history.py new file mode 100644 index 00000000..275ac4cd --- /dev/null +++ b/src/agentops/agent/sources/results_history.py @@ -0,0 +1,177 @@ +"""AgentOps results-history source. + +Reads ``.agentops/results/*/results.json`` and produces a normalized +list of run summaries ordered oldest-to-newest. This source is offline +and always available — it is the foundation of the regression and +safety checks. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from agentops.agent.config import ResultsHistorySourceConfig + +log = logging.getLogger(__name__) + + +@dataclass +class RunSummary: + """One historical AgentOps run.""" + + run_id: str + timestamp: Optional[datetime] + metrics: Dict[str, float] + run_pass: Optional[bool] + items_total: int + items_passed_all: int + raw_path: Path + item_evaluations: List[Dict[str, Any]] = field(default_factory=list) + + +@dataclass +class ResultsHistory: + """Aggregated results-history payload.""" + + runs: List[RunSummary] + diagnostics: Dict[str, Any] = field(default_factory=dict) + + +def _coerce_timestamp(raw: Any) -> Optional[datetime]: + if raw is None: + return None + if isinstance(raw, datetime): + return raw if raw.tzinfo else raw.replace(tzinfo=timezone.utc) + if isinstance(raw, (int, float)): + try: + return datetime.fromtimestamp(float(raw), tz=timezone.utc) + except (OSError, OverflowError, ValueError): + return None + if isinstance(raw, str): + candidate = raw.replace("Z", "+00:00") + try: + ts = datetime.fromisoformat(candidate) + except ValueError: + return None + return ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc) + return None + + +def _summarize(path: Path) -> Optional[RunSummary]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + log.warning("Skipping unreadable results.json at %s: %s", path, exc) + return None + + if not isinstance(data, dict): + return None + + metrics_raw = data.get("metrics") or data.get("run_metrics") or {} + metrics: Dict[str, float] = {} + if isinstance(metrics_raw, dict): + for key, value in metrics_raw.items(): + try: + metrics[str(key)] = float(value) + except (TypeError, ValueError): + continue + + summary = data.get("summary") or {} + run_pass: Optional[bool] = None + if isinstance(summary, dict) and "run_pass" in summary: + run_pass = bool(summary["run_pass"]) + elif "run_pass" in metrics_raw: + try: + run_pass = bool(float(metrics_raw["run_pass"])) + except (TypeError, ValueError): + run_pass = None + + items_total = 0 + items_passed_all = 0 + if isinstance(summary, dict): + items_total = int(summary.get("items_total", 0) or 0) + items_passed_all = int(summary.get("items_passed_all", 0) or 0) + + item_evaluations = data.get("item_evaluations") or [] + if not isinstance(item_evaluations, list): + item_evaluations = [] + + timestamp_raw = ( + data.get("timestamp") + or data.get("created_at") + or (summary.get("timestamp") if isinstance(summary, dict) else None) + ) + timestamp = _coerce_timestamp(timestamp_raw) + if timestamp is None: + # Fall back to file mtime so ordering still works. + try: + timestamp = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc) + except OSError: + timestamp = None + + run_id = str(data.get("run_id") or path.parent.name) + + return RunSummary( + run_id=run_id, + timestamp=timestamp, + metrics=metrics, + run_pass=run_pass, + items_total=items_total, + items_passed_all=items_passed_all, + raw_path=path, + item_evaluations=item_evaluations, + ) + + +def collect_results_history( + workspace: Path, + config: ResultsHistorySourceConfig, +) -> ResultsHistory: + """Walk the configured results directory and build an ordered history.""" + diagnostics: Dict[str, Any] = { + "enabled": config.enabled, + "path": str(config.path), + } + if not config.enabled: + diagnostics["status"] = "disabled" + return ResultsHistory(runs=[], diagnostics=diagnostics) + + base = (workspace / config.path).resolve() + diagnostics["resolved_path"] = str(base) + + if not base.exists(): + diagnostics["status"] = "missing" + diagnostics["reason"] = f"results directory not found at {base}" + return ResultsHistory(runs=[], diagnostics=diagnostics) + + candidates: List[Path] = [] + for child in base.iterdir(): + if not child.is_dir(): + continue + if child.name == "latest": + continue + target = child / "results.json" + if target.is_file(): + candidates.append(target) + + summaries: List[RunSummary] = [] + for path in candidates: + summary = _summarize(path) + if summary is not None: + summaries.append(summary) + + summaries.sort( + key=lambda s: s.timestamp or datetime.fromtimestamp(0, tz=timezone.utc) + ) + + if config.lookback_runs > 0: + summaries = summaries[-config.lookback_runs :] + + diagnostics["status"] = "ok" + diagnostics["runs_loaded"] = len(summaries) + return ResultsHistory(runs=summaries, diagnostics=diagnostics) diff --git a/src/agentops/backends/__init__.py b/src/agentops/backends/__init__.py deleted file mode 100644 index 7413f478..00000000 --- a/src/agentops/backends/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Execution backends package.""" diff --git a/src/agentops/backends/base.py b/src/agentops/backends/base.py deleted file mode 100644 index 9bf4258a..00000000 --- a/src/agentops/backends/base.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Backend protocol and shared execution models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Protocol - -from agentops.core.models import RunConfig - - -@dataclass(frozen=True) -class BackendRunContext: - run_config: RunConfig - bundle_path: Path - dataset_path: Path - backend_output_dir: Path - - -@dataclass(frozen=True) -class BackendExecutionResult: - backend: str - command: str - started_at: str - finished_at: str - duration_seconds: float - exit_code: int - stdout_file: Path - stderr_file: Path - - -class Backend(Protocol): - def execute(self, context: BackendRunContext) -> BackendExecutionResult: - """Execute backend work and return normalized execution metadata.""" diff --git a/src/agentops/backends/eval_engine.py b/src/agentops/backends/eval_engine.py deleted file mode 100644 index fbe5e80c..00000000 --- a/src/agentops/backends/eval_engine.py +++ /dev/null @@ -1,961 +0,0 @@ -"""Shared evaluation engine used by all AgentOps backends. - -This module contains evaluator loading, instantiation, execution, scoring, -dataset utilities, and cloud-evaluator mapping helpers. Every backend -(Foundry, HTTP, Local Adapter) imports from here instead of coupling to -a specific backend implementation. -""" - -from __future__ import annotations - -import importlib -import inspect -import json -import logging -import os -import re -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from agentops.core.models import EvaluatorConfig - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Suppress noisy SDK warnings for single-turn evaluation inputs -# --------------------------------------------------------------------------- - -class _ConversationHistoryFilter(logging.Filter): - """Suppress 'Conversation history could not be parsed' from azure-ai-evaluation. - - This warning fires on every single-turn evaluation row because plain-text - inputs are not in conversation-list format. It is expected and harmless. - """ - - def filter(self, record: logging.LogRecord) -> bool: - return "Conversation history could not be parsed" not in record.getMessage() - - -# Apply filter to SDK loggers that emit the warning. -# Each evaluator module passes its own logger to reformat_conversation_history(). -for _sdk_logger_name in ( - "azure.ai.evaluation._common.utils", - "azure.ai.evaluation._evaluators._task_adherence._task_adherence", - "azure.ai.evaluation._evaluators._intent_resolution._intent_resolution", - "azure.ai.evaluation._evaluators._task_completion._task_completion", - "azure.ai.evaluation._evaluators._tool_call_accuracy._tool_call_accuracy", - "azure.ai.evaluation", -): - logging.getLogger(_sdk_logger_name).addFilter(_ConversationHistoryFilter()) - - -# --------------------------------------------------------------------------- -# Cloud-only evaluator sentinel -# --------------------------------------------------------------------------- - - -class _CloudOnlyEvaluatorError(Exception): - """Raised when an evaluator is only available via Foundry Cloud Evaluation.""" - - -# --------------------------------------------------------------------------- -# Credential help (shared by _default_credential and _acquire_token) -# --------------------------------------------------------------------------- - -_CREDENTIAL_HELP_MESSAGE = ( - "Azure authentication failed. To fix this, do one of the following:\n" - "\n" - " 1. Run 'az login' (Azure CLI) to authenticate interactively.\n" - " 2. Set AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_CLIENT_SECRET \n" - " environment variables for service-principal authentication.\n" - " 3. If running on Azure, ensure a managed identity is configured.\n" - "\n" - "Docs: https://aka.ms/azsdk/python/identity/defaultazurecredential/troubleshoot" -) - -# --------------------------------------------------------------------------- -# Evaluator classification constants -# --------------------------------------------------------------------------- - -_NLP_ONLY_EVALUATORS = frozenset( - { - "f1_score", - "bleu_score", - "rouge_score", - "meteor_score", - "gleu_score", - } -) - -_EVALUATORS_NEEDING_GROUND_TRUTH = frozenset( - { - "similarity", - "response_completeness", - "f1_score", - "bleu_score", - "rouge_score", - "meteor_score", - "gleu_score", - } -) - -_EVALUATORS_NEEDING_CONTEXT = frozenset( - { - "groundedness", - "groundedness_pro", - "relevance", - "retrieval", - } -) - -_EVALUATORS_NEEDING_TOOL_CALLS = frozenset( - { - "tool_call_accuracy", - "tool_selection", - } -) - -_EVALUATORS_NEEDING_TOOL_DEFS_ONLY = frozenset( - { - "tool_input_accuracy", - "tool_output_utilization", - "tool_call_success", - } -) - -_EVALUATORS_NEEDING_OUTPUT_ITEMS = frozenset( - { - "task_adherence", - } -) - -_SAFETY_EVALUATORS = frozenset( - { - "violence", - "sexual", - "self_harm", - "hate_unfairness", - "content_safety", - "protected_material", - "code_vulnerability", - "ungrounded_attributes", - "indirect_attack", - } -) - -_AI_ASSISTED_EVALUATORS = { - "GroundednessEvaluator", - "RelevanceEvaluator", - "CoherenceEvaluator", - "FluencyEvaluator", - "SimilarityEvaluator", - "RetrievalEvaluator", - "ResponseCompletenessEvaluator", - "QAEvaluator", - "IntentResolutionEvaluator", - "TaskAdherenceEvaluator", - "ToolCallAccuracyEvaluator", - "TaskCompletionEvaluator", - "TaskNavigationEfficiencyEvaluator", - "ToolSelectionEvaluator", - "ToolInputAccuracyEvaluator", - "ToolOutputUtilizationEvaluator", - "ToolCallSuccessEvaluator", -} - -_SAFETY_EVALUATOR_CLASSES = frozenset( - { - "ViolenceEvaluator", - "SexualEvaluator", - "SelfHarmEvaluator", - "HateUnfairnessEvaluator", - "ContentSafetyEvaluator", - "ProtectedMaterialEvaluator", - "CodeVulnerabilityEvaluator", - "UngroundedAttributesEvaluator", - "IndirectAttackEvaluator", - "GroundednessProEvaluator", - } -) - -_SUPPORTED_LOCAL_EVALUATORS = { - "exact_match", - "latency_seconds", - "avg_latency_seconds", -} - -# --------------------------------------------------------------------------- -# Runtime dataclass -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class FoundryEvaluatorRuntime: - name: str - evaluator: Callable[..., dict[str, Any]] - input_mapping: dict[str, str] - score_keys: list[str] - - -# --------------------------------------------------------------------------- -# Dataset utilities -# --------------------------------------------------------------------------- - - -def _resolve_dataset_source_path(dataset_config_path: Path, source_path: Path) -> Path: - if source_path.is_absolute(): - return source_path - - candidate = (dataset_config_path.parent / source_path).resolve() - if candidate.exists(): - return candidate - - fallback = (Path.cwd() / source_path).resolve() - if fallback.exists(): - return fallback - - return candidate - - -def _load_jsonl(path: Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if not stripped: - continue - payload = json.loads(stripped) - if not isinstance(payload, dict): - raise ValueError("Dataset JSONL rows must be objects") - rows.append(payload) - if not rows: - raise ValueError(f"Dataset is empty: {path}") - return rows - - -def _normalize_text(value: Any) -> str: - if value is None: - return "" - return str(value).strip() - - -# --------------------------------------------------------------------------- -# Evaluator name / mapping helpers -# --------------------------------------------------------------------------- - - -def _to_builtin_evaluator_name(evaluator_name: str) -> str: - """Convert 'SimilarityEvaluator' → 'similarity'.""" - normalized = evaluator_name.strip() - normalized = normalized.removesuffix("Evaluator") - snake = re.sub(r"(? str: - return re.sub(r"(? dict[str, str]: - """Build ``data_mapping`` for an ``azure_ai_evaluator`` testing criterion.""" - item_input = "{{item." + input_field + "}}" - item_expected = "{{item." + expected_field + "}}" - sample_response = "{{sample.output_text}}" - - mapping: dict[str, str] = {} - if builtin_name in _SAFETY_EVALUATORS: - mapping["query"] = item_input - mapping["response"] = sample_response - return mapping - if builtin_name not in _NLP_ONLY_EVALUATORS: - mapping["query"] = item_input - if builtin_name in _EVALUATORS_NEEDING_OUTPUT_ITEMS: - mapping["response"] = "{{sample.output_items}}" - else: - mapping["response"] = sample_response - if builtin_name in _EVALUATORS_NEEDING_GROUND_TRUTH: - mapping["ground_truth"] = item_expected - elif builtin_name in _EVALUATORS_NEEDING_CONTEXT: - context_item = "{{item." + (context_field or expected_field) + "}}" - mapping["context"] = context_item - elif builtin_name in _EVALUATORS_NEEDING_TOOL_CALLS: - mapping["tool_calls"] = "{{sample.tool_calls}}" - mapping["tool_definitions"] = "{{item.tool_definitions}}" - elif builtin_name in _EVALUATORS_NEEDING_TOOL_DEFS_ONLY: - mapping["tool_definitions"] = "{{item.tool_definitions}}" - return mapping - - -def _cloud_evaluator_needs_model(builtin_name: str) -> bool: - """Return True if the evaluator is AI-assisted and needs a deployment_name.""" - if builtin_name in _SAFETY_EVALUATORS: - return False - return builtin_name not in _NLP_ONLY_EVALUATORS - - -# Default initialization_parameters for evaluators that require them but are -# not AI-assisted (so they don't get deployment_name automatically). -_NLP_DEFAULT_INIT_PARAMS: dict[str, dict[str, Any]] = { - "rouge_score": {"rouge_type": "rouge1"}, -} - - -def _parse_agent_name_version(agent_id: str) -> tuple[str, str | None]: - """Parse 'my-agent:3' into ('my-agent', '3').""" - if ":" in agent_id: - name, version = agent_id.split(":", 1) - return name.strip(), version.strip() or None - return agent_id.strip(), None - - -# --------------------------------------------------------------------------- -# Evaluator input mapping defaults -# --------------------------------------------------------------------------- - - -def _default_foundry_input_mapping(name: str) -> dict[str, str]: - if name == "SimilarityEvaluator": - return { - "query": "$prompt", - "response": "$prediction", - "ground_truth": "$expected", - } - if name == "GroundednessEvaluator": - return { - "query": "$prompt", - "response": "$prediction", - "context": "$row.context", - } - if name in ("CoherenceEvaluator", "FluencyEvaluator"): - return { - "query": "$prompt", - "response": "$prediction", - } - if name == "F1ScoreEvaluator": - return { - "response": "$prediction", - "ground_truth": "$expected", - } - if name in ("RelevanceEvaluator", "RetrievalEvaluator"): - return { - "query": "$prompt", - "response": "$prediction", - "context": "$row.context", - } - if name == "ResponseCompletenessEvaluator": - return { - "response": "$prediction", - "ground_truth": "$expected", - } - if name in ( - "TaskCompletionEvaluator", - "IntentResolutionEvaluator", - "TaskAdherenceEvaluator", - ): - return { - "query": "$prompt", - "response": "$prediction", - } - if name == "ToolCallAccuracyEvaluator": - return { - "query": "$prompt", - "response": "$prediction", - "tool_calls": "$row.tool_calls", - "tool_definitions": "$row.tool_definitions", - } - if name in ("ToolSelectionEvaluator",): - return { - "query": "$prompt", - "response": "$prediction", - "tool_calls": "$row.tool_calls", - "tool_definitions": "$row.tool_definitions", - } - if name in ( - "ToolInputAccuracyEvaluator", - "ToolOutputUtilizationEvaluator", - "ToolCallSuccessEvaluator", - ): - return { - "query": "$prompt", - "response": "$prediction", - "tool_definitions": "$row.tool_definitions", - } - if name in ( - "ViolenceEvaluator", - "SexualEvaluator", - "SelfHarmEvaluator", - "HateUnfairnessEvaluator", - "ContentSafetyEvaluator", - "ProtectedMaterialEvaluator", - "CodeVulnerabilityEvaluator", - "UngroundedAttributesEvaluator", - "IndirectAttackEvaluator", - "GroundednessProEvaluator", - ): - return { - "query": "$prompt", - "response": "$prediction", - } - return {} - - -def _default_score_keys(name: str) -> list[str]: - snake_name = _to_snake_case(name) - bare_name = snake_name.replace("_evaluator", "") - keys = [ - bare_name, - snake_name, - f"{bare_name}_score", - f"gpt_{bare_name}", - "score", - "value", - ] - seen: set[str] = set() - ordered: list[str] = [] - for key in keys: - if key not in seen: - seen.add(key) - ordered.append(key) - return ordered - - -# --------------------------------------------------------------------------- -# Validation -# --------------------------------------------------------------------------- - - -def _validate_supported_local_evaluators(evaluators: list[EvaluatorConfig]) -> None: - unsupported = sorted( - evaluator.name - for evaluator in evaluators - if evaluator.enabled - and evaluator.source == "local" - and evaluator.name not in _SUPPORTED_LOCAL_EVALUATORS - ) - if unsupported: - raise ValueError( - "Unsupported local evaluator(s): " - + ", ".join(unsupported) - + ". Supported local evaluators are: " - + ", ".join(sorted(_SUPPORTED_LOCAL_EVALUATORS)) - ) - - -# --------------------------------------------------------------------------- -# Azure credential helpers (lazy imports) -# --------------------------------------------------------------------------- - - -def _default_credential() -> Any: - try: - from azure.identity import DefaultAzureCredential # noqa: WPS433 - except ImportError as exc: - raise ImportError( - "Foundry evaluators require 'azure-identity'. " - "Install with: pip install azure-identity" - ) from exc - - try: - return DefaultAzureCredential(exclude_developer_cli_credential=True) - except Exception as exc: - raise RuntimeError(_CREDENTIAL_HELP_MESSAGE) from exc - - -def _azure_ai_project_config() -> str: - """Return the Foundry project endpoint for safety/RAI evaluators.""" - project_endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") - if not project_endpoint: - raise ValueError( - "Safety evaluators require an Azure AI Foundry project endpoint. " - "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT or provide " - "config.init.azure_ai_project in the bundle evaluator config." - ) - return project_endpoint - - -def _azure_openai_model_config( - *, - fallback_endpoint: str | None = None, - fallback_deployment: str | None = None, -) -> dict[str, str]: - endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") or fallback_endpoint - deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") or fallback_deployment - api_version = os.getenv("AZURE_OPENAI_API_VERSION") - - missing: list[str] = [] - if not endpoint: - missing.append("AZURE_OPENAI_ENDPOINT") - if not deployment: - missing.append("AZURE_OPENAI_DEPLOYMENT") - - if missing: - raise ValueError( - "Foundry evaluator requires Azure OpenAI evaluator model settings. " - "Missing: " + ", ".join(missing) - ) - - assert endpoint is not None - assert deployment is not None - - model_config: dict[str, str] = { - "azure_endpoint": endpoint, - "azure_deployment": deployment, - } - if api_version: - model_config["api_version"] = api_version - return model_config - - -# --------------------------------------------------------------------------- -# Evaluator instantiation helpers -# --------------------------------------------------------------------------- - - -def _is_reasoning_like_deployment_name(name: str) -> bool: - normalized = name.strip().lower() - if not normalized: - return False - return ( - normalized.startswith("o1") - or normalized.startswith("o3") - or normalized.startswith("o4") - or normalized.startswith("gpt-5") - ) - - -def _should_enable_reasoning_mode( - *, - evaluator_name: str, - init_kwargs: dict[str, Any], -) -> bool: - if evaluator_name not in _AI_ASSISTED_EVALUATORS: - return False - if "is_reasoning_model" in init_kwargs: - return False - - model_config = init_kwargs.get("model_config") - if not isinstance(model_config, dict): - return False - - deployment = model_config.get("azure_deployment") or model_config.get("model") - if not isinstance(deployment, str): - return False - - return _is_reasoning_like_deployment_name(deployment) - - -def _instantiate_evaluator_symbol( - evaluator_symbol: Any, - *, - evaluator_name: str, - init_kwargs: dict[str, Any], -) -> Callable[..., dict[str, Any]]: - if not inspect.isclass(evaluator_symbol): - if callable(evaluator_symbol): - if init_kwargs: - raise ValueError( - f"Evaluator '{evaluator_name}' resolved to callable and does not support config.init" - ) - return evaluator_symbol - raise ValueError(f"Evaluator '{evaluator_name}' is not callable") - - try: - return evaluator_symbol(**init_kwargs) - except TypeError as exc: - if "is_reasoning_model" in init_kwargs: - fallback_kwargs = dict(init_kwargs) - fallback_kwargs.pop("is_reasoning_model", None) - return evaluator_symbol(**fallback_kwargs) - raise exc - - -def _interpolate_env_values(value: Any) -> Any: - if isinstance(value, str): - match = re.fullmatch(r"\$\{env:([A-Za-z_][A-Za-z0-9_]*)\}", value) - if not match: - return value - env_name = match.group(1) - env_value = os.getenv(env_name) - if env_value is None: - raise ValueError( - f"Missing environment variable required by evaluator config: {env_name}" - ) - return env_value - if isinstance(value, dict): - return {key: _interpolate_env_values(item) for key, item in value.items()} - if isinstance(value, list): - return [_interpolate_env_values(item) for item in value] - return value - - -def _load_foundry_evaluator_callable( - *, - evaluator_name: str, - evaluator_config: dict[str, Any], - fallback_endpoint: str | None = None, - fallback_deployment: str | None = None, -) -> Callable[..., dict[str, Any]]: - kind = str(evaluator_config.get("kind", "builtin")).strip().lower() - init_kwargs_raw = evaluator_config.get("init", {}) - if init_kwargs_raw is None: - init_kwargs_raw = {} - if not isinstance(init_kwargs_raw, dict): - raise ValueError(f"Evaluator '{evaluator_name}' config.init must be an object") - init_kwargs = _interpolate_env_values(init_kwargs_raw) - - if kind == "builtin": - class_name = str(evaluator_config.get("class_name") or evaluator_name).strip() - if not class_name: - raise ValueError( - f"Evaluator '{evaluator_name}' class_name must be non-empty" - ) - - if class_name in _AI_ASSISTED_EVALUATORS and "model_config" not in init_kwargs: - init_kwargs["model_config"] = _azure_openai_model_config( - fallback_endpoint=fallback_endpoint, - fallback_deployment=fallback_deployment, - ) - - if ( - class_name in _SAFETY_EVALUATOR_CLASSES - and "azure_ai_project" not in init_kwargs - ): - init_kwargs["azure_ai_project"] = _azure_ai_project_config() - - if "credential" not in init_kwargs: - init_kwargs["credential"] = _default_credential() - - if _should_enable_reasoning_mode( - evaluator_name=class_name, - init_kwargs=init_kwargs, - ): - init_kwargs["is_reasoning_model"] = True - - try: - module = importlib.import_module("azure.ai.evaluation") - evaluator_symbol = getattr(module, class_name) - except ImportError as exc: - raise ImportError( - "Foundry evaluators require 'azure-ai-evaluation'. " - "Install with: pip install azure-ai-evaluation" - ) from exc - except AttributeError as exc: - raise _CloudOnlyEvaluatorError( - f"Evaluator '{class_name}' is not available in the local " - f"azure-ai-evaluation SDK. It may only be available via " - f"Foundry Cloud Evaluation (builtin.{_to_builtin_evaluator_name(class_name)}). " - f"Use 'hosting: foundry' with 'execution_mode: remote' to " - f"run this evaluator, or disable it for local runs." - ) from exc - - return _instantiate_evaluator_symbol( - evaluator_symbol, - evaluator_name=evaluator_name, - init_kwargs=init_kwargs, - ) - - if kind == "custom": - callable_path = evaluator_config.get("callable_path") - if not isinstance(callable_path, str) or not callable_path.strip(): - raise ValueError( - f"Evaluator '{evaluator_name}' with kind=custom requires config.callable_path" - ) - - module_name, separator, symbol_name = callable_path.partition(":") - if not separator or not module_name.strip() or not symbol_name.strip(): - raise ValueError( - f"Evaluator '{evaluator_name}' callable_path must be ':'" - ) - - module = importlib.import_module(module_name.strip()) - evaluator_symbol = getattr(module, symbol_name.strip()) - - return _instantiate_evaluator_symbol( - evaluator_symbol, - evaluator_name=evaluator_name, - init_kwargs=init_kwargs, - ) - - raise ValueError( - f"Evaluator '{evaluator_name}' has unsupported config.kind '{kind}'. " - "Use 'builtin' or 'custom'." - ) - - -# --------------------------------------------------------------------------- -# Build evaluator runtimes from bundle config -# --------------------------------------------------------------------------- - - -def _build_foundry_evaluator_runtimes( - evaluators: list[EvaluatorConfig], - *, - fallback_endpoint: str | None = None, - fallback_deployment: str | None = None, -) -> list[FoundryEvaluatorRuntime]: - runtimes: list[FoundryEvaluatorRuntime] = [] - for evaluator in evaluators: - if not evaluator.enabled or evaluator.source != "foundry": - continue - - config = evaluator.config or {} - if not isinstance(config, dict): - raise ValueError(f"Evaluator '{evaluator.name}' config must be an object") - - input_mapping_raw = config.get("input_mapping") - if input_mapping_raw is None: - input_mapping = _default_foundry_input_mapping(evaluator.name) - else: - if not isinstance(input_mapping_raw, dict): - raise ValueError( - f"Evaluator '{evaluator.name}' config.input_mapping must be an object" - ) - input_mapping = { - str(key): str(value) for key, value in input_mapping_raw.items() - } - - score_keys_raw = config.get("score_keys") - if score_keys_raw is None: - score_keys = _default_score_keys(evaluator.name) - else: - if not isinstance(score_keys_raw, list) or not all( - isinstance(item, str) for item in score_keys_raw - ): - raise ValueError( - f"Evaluator '{evaluator.name}' config.score_keys must be a list of strings" - ) - score_keys = score_keys_raw - - try: - evaluator_callable = _load_foundry_evaluator_callable( - evaluator_name=evaluator.name, - evaluator_config=config, - fallback_endpoint=fallback_endpoint, - fallback_deployment=fallback_deployment, - ) - except _CloudOnlyEvaluatorError: - logger.warning( - "Skipping evaluator '%s' — not available in the local " - "azure-ai-evaluation SDK. This evaluator is only supported " - "via Foundry Cloud Evaluation (hosting: foundry, " - "execution_mode: remote). It will be ignored for this " - "local run.", - evaluator.name, - ) - continue - - runtimes.append( - FoundryEvaluatorRuntime( - name=evaluator.name, - evaluator=evaluator_callable, - input_mapping=input_mapping, - score_keys=score_keys, - ) - ) - return runtimes - - -# --------------------------------------------------------------------------- -# Evaluator score extraction -# --------------------------------------------------------------------------- - - -def _as_number(value: Any) -> float | None: - if isinstance(value, bool): - return None - if isinstance(value, (int, float)): - return float(value) - return None - - -def _find_numeric_value(payload: Any) -> float | None: - direct = _as_number(payload) - if direct is not None: - return direct - - if isinstance(payload, dict): - for item in payload.values(): - found = _find_numeric_value(item) - if found is not None: - return found - elif isinstance(payload, list): - for item in payload: - found = _find_numeric_value(item) - if found is not None: - return found - - return None - - -def _extract_evaluator_score( - payload: dict[str, Any], preferred_keys: list[str], evaluator_name: str -) -> float: - for key in preferred_keys: - if key in payload: - numeric = _find_numeric_value(payload[key]) - if numeric is not None: - return numeric - - for value in payload.values(): - numeric = _find_numeric_value(value) - if numeric is not None: - return numeric - - raise ValueError(f"Foundry evaluator '{evaluator_name}' returned no numeric score") - - -# --------------------------------------------------------------------------- -# Evaluator mapping resolution and execution -# --------------------------------------------------------------------------- - - -def _resolve_mapping_value( - expression: Any, - *, - prompt: str, - prediction: str, - expected: str, - row: dict[str, Any], -) -> Any: - if not isinstance(expression, str): - return expression - - env_match = re.fullmatch(r"\$\{env:([A-Za-z_][A-Za-z0-9_]*)\}", expression) - if env_match: - env_name = env_match.group(1) - env_value = os.getenv(env_name) - if env_value is None: - raise ValueError( - f"Missing environment variable required by evaluator mapping: {env_name}" - ) - return env_value - - if expression.startswith("$row."): - row_key = expression[5:] - if row_key not in row: - raise ValueError( - f"Missing row field referenced by evaluator mapping: {row_key}" - ) - return row[row_key] - - if expression.startswith("$"): - token = expression[1:] - aliases: dict[str, Any] = { - "prompt": prompt, - "query": prompt, - "input": prompt, - "prediction": prediction, - "response": prediction, - "output_text": prediction, - "expected": expected, - "ground_truth": expected, - "reference": expected, - "context": expected, - } - if token in aliases: - return aliases[token] - if token in row: - return row[token] - raise ValueError(f"Unknown evaluator mapping token: {expression}") - - return expression - - -def _build_evaluator_kwargs( - runtime: FoundryEvaluatorRuntime, - *, - prompt: str, - prediction: str, - expected: str, - row: dict[str, Any], -) -> dict[str, Any]: - if runtime.input_mapping: - return { - key: _resolve_mapping_value( - value, - prompt=prompt, - prediction=prediction, - expected=expected, - row=row, - ) - for key, value in runtime.input_mapping.items() - } - - base_context: dict[str, Any] = { - "prompt": prompt, - "query": prompt, - "input": prompt, - "response": prediction, - "prediction": prediction, - "output_text": prediction, - "expected": expected, - "ground_truth": expected, - "reference": expected, - "context": expected, - } - - signature = inspect.signature(runtime.evaluator) - accepts_kwargs = any( - param.kind == inspect.Parameter.VAR_KEYWORD - for param in signature.parameters.values() - ) - - if accepts_kwargs: - merged = dict(base_context) - merged.update(row) - return merged - - kwargs: dict[str, Any] = {} - for name, param in signature.parameters.items(): - if param.kind not in { - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - }: - continue - if name in row: - kwargs[name] = row[name] - continue - if name in base_context: - kwargs[name] = base_context[name] - continue - if param.default is inspect.Parameter.empty: - raise ValueError( - f"Evaluator '{runtime.name}' requires argument '{name}'. " - "Provide evaluators[].config.input_mapping in bundle config." - ) - return kwargs - - -def _run_foundry_evaluator( - runtime: FoundryEvaluatorRuntime, - *, - prompt: str, - prediction: str, - expected: str, - row: dict[str, Any], -) -> float: - kwargs = _build_evaluator_kwargs( - runtime, - prompt=prompt, - prediction=prediction, - expected=expected, - row=row, - ) - payload = runtime.evaluator(**kwargs) - if not isinstance(payload, dict): - raise ValueError(f"Evaluator '{runtime.name}' returned invalid payload") - - score = _extract_evaluator_score( - payload, - preferred_keys=runtime.score_keys, - evaluator_name=runtime.name, - ) - return round(score, 6) diff --git a/src/agentops/backends/foundry_backend.py b/src/agentops/backends/foundry_backend.py deleted file mode 100644 index 1ceead82..00000000 --- a/src/agentops/backends/foundry_backend.py +++ /dev/null @@ -1,1232 +0,0 @@ -"""Native Microsoft Foundry Agent Service backend implementation for AgentOps.""" - -from __future__ import annotations - -import json -import logging -import os -import re -import time -import urllib.error -import urllib.request -import uuid -from dataclasses import dataclass, replace -from datetime import UTC, datetime -from pathlib import Path -from time import perf_counter -from typing import Any - -from agentops.backends.base import BackendExecutionResult, BackendRunContext -from agentops.backends.eval_engine import ( - _CREDENTIAL_HELP_MESSAGE, - _NLP_DEFAULT_INIT_PARAMS, - _build_foundry_evaluator_runtimes, - _cloud_evaluator_data_mapping, - _cloud_evaluator_needs_model, - _load_jsonl, - _normalize_text, - _parse_agent_name_version, - _resolve_dataset_source_path, - _run_foundry_evaluator, - _to_builtin_evaluator_name, - _validate_supported_local_evaluators, -) -from agentops.core.config_loader import load_bundle_config, load_dataset_config -from agentops.utils.telemetry import agent_invoke_span, set_agent_invoke_result - -logger = logging.getLogger(__name__) - - -def _to_utc_timestamp(value: datetime) -> str: - return value.astimezone(UTC).isoformat().replace("+00:00", "Z") - - -# --------------------------------------------------------------------------- -# Cloud evaluation routing -# --------------------------------------------------------------------------- - - -def _should_use_cloud_evaluation(project_endpoint: str) -> bool: - """Return True when cloud evaluation should be used (New Foundry Experience).""" - mode = os.getenv("AGENTOPS_FOUNDRY_MODE", "cloud").strip().lower() - if mode in {"local", "legacy"}: - return False - if "example.services.ai.azure.com" in project_endpoint: - return False - return True - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _acquire_token(scope: str) -> str: - """Acquire a bearer token for the Foundry Agent Service. - - Uses ``DefaultAzureCredential`` which supports: - - Local dev: ``az login`` / VS Code credential - - CI/CD: service principal (``AZURE_CLIENT_ID``, ``AZURE_TENANT_ID``, - ``AZURE_CLIENT_SECRET``) - - Azure hosted: managed identity (zero config) - """ - try: - from azure.identity import DefaultAzureCredential # noqa: WPS433 - except ImportError as exc: - raise ImportError( - "Foundry backend requires 'azure-identity'. " - "Install with: pip install azure-identity" - ) from exc - - try: - credential = DefaultAzureCredential(exclude_developer_cli_credential=True) - token = credential.get_token(scope) - return token.token - except Exception as exc: - # Catch ClientAuthenticationError and any other credential failures - # and re-raise with a clean, actionable message. - raise RuntimeError(_CREDENTIAL_HELP_MESSAGE) from exc - - -def _preferred_scope_for_agent_id(agent_id: str) -> str: - if agent_id.startswith("asst_"): - return "https://cognitiveservices.azure.com/.default" - return "https://ai.azure.com/.default" - - -def _alternate_scope(scope: str) -> str: - if scope == "https://ai.azure.com/.default": - return "https://cognitiveservices.azure.com/.default" - return "https://ai.azure.com/.default" - - -def _is_audience_mismatch(details: str) -> bool: - lowered = details.lower() - if "audience is incorrect" in lowered: - return True - return bool(re.search(r"audience.*(incorrect|invalid)", lowered)) - - -@dataclass(frozen=True) -class FoundrySettings: - project_endpoint: str - agent_id: str | None - model: str | None - api_version: str - agent_token: str - token_scope: str - poll_interval_seconds: float - max_poll_attempts: int - target: str = "agent" # 'agent' or 'model' - - -def _derive_openai_endpoint_from_project(project_endpoint: str) -> str: - """Derive the Azure OpenAI base endpoint from a Foundry project endpoint. - - ``https://account.services.ai.azure.com/api/projects/proj`` - → ``https://account.services.ai.azure.com/`` - """ - from urllib.parse import urlparse # noqa: WPS433 - - parsed = urlparse(project_endpoint) - return f"{parsed.scheme}://{parsed.netloc}/" - - -class FoundryBackend: - def _read_settings(self, context: BackendRunContext) -> FoundrySettings: - target_cfg = context.run_config.target - endpoint = target_cfg.endpoint - assert endpoint is not None, "Foundry backend requires target.endpoint" - - project_endpoint_env = ( - endpoint.project_endpoint_env or "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT" - ) - - project_endpoint = endpoint.project_endpoint or os.getenv(project_endpoint_env) - agent_id = endpoint.agent_id - target = target_cfg.type # "agent" or "model" - model = endpoint.model or os.getenv("AZURE_AI_MODEL_DEPLOYMENT_NAME") - api_version = endpoint.api_version or "2025-05-01" - - if not project_endpoint: - raise ValueError( - f"Foundry backend requires a project endpoint. Set it via:\n" - f"\n" - f" 1. 'target.endpoint.project_endpoint' in your run.yaml, or\n" - f" 2. Environment variable {project_endpoint_env}:\n" - f"\n" - f" PowerShell:\n" - f' $env:{project_endpoint_env} = "https://.services.ai.azure.com/api/projects/"\n' - f"\n" - f" Bash/zsh:\n" - f' export {project_endpoint_env}="https://.services.ai.azure.com/api/projects/"\n' - f"\n" - f"You can find this URL in the Azure AI Foundry portal under your project settings." - ) - if target == "agent" and not agent_id: - raise ValueError( - "Foundry backend requires target.endpoint.agent_id when target type is 'agent'" - ) - if target == "model" and not model: - raise ValueError( - "Foundry backend requires a model deployment name when target type is 'model'. " - "Set 'target.endpoint.model' in run.yaml or AZURE_AI_MODEL_DEPLOYMENT_NAME." - ) - - if target == "model": - # Model-direct: use cognitive services scope - token_scope = "https://cognitiveservices.azure.com/.default" - else: - assert agent_id is not None - token_scope = _preferred_scope_for_agent_id(agent_id) - logger.info("Acquiring token via DefaultAzureCredential…") - agent_token = _acquire_token(token_scope) - - return FoundrySettings( - project_endpoint=project_endpoint.rstrip("/"), - agent_id=agent_id, - model=model, - api_version=api_version, - agent_token=agent_token, - token_scope=token_scope, - poll_interval_seconds=endpoint.poll_interval_seconds or 2.0, - max_poll_attempts=endpoint.max_poll_attempts or 120, - target=target, - ) - - def _request_json( - self, - *, - method: str, - url: str, - headers: dict[str, str], - timeout_seconds: int | None, - body: dict[str, Any] | None = None, - ) -> dict[str, Any]: - request_body = json.dumps(body).encode("utf-8") if body is not None else None - request = urllib.request.Request( - url=url, - method=method, - data=request_body, - headers=headers, - ) - - with urllib.request.urlopen(request, timeout=timeout_seconds) as response: - payload = json.loads(response.read().decode("utf-8")) - - if not isinstance(payload, dict): - raise ValueError( - "Invalid Foundry Agent Service response: expected JSON object" - ) - return payload - - def _extract_agent_message_text(self, messages_payload: dict[str, Any]) -> str: - entries = messages_payload.get("data") - if not isinstance(entries, list): - raise ValueError( - "Invalid Foundry Agent Service response: missing messages data" - ) - - for message in entries: - if not isinstance(message, dict) or message.get("role") != "assistant": - continue - - content = message.get("content") - if isinstance(content, str): - return content.strip() - - if isinstance(content, list): - parts: list[str] = [] - for item in content: - if not isinstance(item, dict): - continue - text_payload = item.get("text") - if isinstance(text_payload, dict): - value = text_payload.get("value") - if isinstance(value, str): - parts.append(value) - elif isinstance(item.get("value"), str): - parts.append(item["value"]) - if parts: - return "\n".join(parts).strip() - - raise ValueError( - "Invalid Foundry Agent Service response: no assistant message found" - ) - - def _extract_response_output_text(self, response_payload: dict[str, Any]) -> str: - output = response_payload.get("output") - if not isinstance(output, list): - raise ValueError("Invalid Foundry response payload: missing output array") - - for item in output: - if not isinstance(item, dict) or item.get("type") != "message": - continue - - content = item.get("content") - if not isinstance(content, list): - continue - - parts: list[str] = [] - for part in content: - if not isinstance(part, dict): - continue - if part.get("type") == "output_text" and isinstance( - part.get("text"), str - ): - parts.append(part["text"]) - - if parts: - return "\n".join(parts).strip() - - raise ValueError( - "Invalid Foundry response payload: no assistant output text found" - ) - - def _invoke_agent_reference( - self, - settings: FoundrySettings, - prompt: str, - timeout_seconds: int | None, - ) -> str: - if not settings.model: - raise ValueError( - "Foundry agent reference mode requires a model deployment name. " - "Set 'backend.model' in run.yaml or AZURE_AI_MODEL_DEPLOYMENT_NAME." - ) - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {settings.agent_token}", - } - - assert settings.agent_id is not None - agent_name, agent_version = (settings.agent_id, None) - if ":" in settings.agent_id: - split_name, split_version = settings.agent_id.split(":", 1) - agent_name = split_name.strip() - agent_version = split_version.strip() or None - - agent_reference: dict[str, Any] = { - "type": "agent_reference", - "name": agent_name, - } - if agent_version: - agent_reference["version"] = agent_version - - response_payload = self._request_json( - method="POST", - url=f"{settings.project_endpoint}/openai/v1/responses", - headers=headers, - timeout_seconds=timeout_seconds, - body={ - "model": settings.model, - "input": [{"role": "user", "content": prompt}], - "agent_reference": agent_reference, - }, - ) - - return self._extract_response_output_text(response_payload) - - def _invoke_agent_service( - self, settings: FoundrySettings, prompt: str, timeout_seconds: int | None - ) -> str: - assert settings.agent_id is not None - if not settings.agent_id.startswith("asst_"): - return self._invoke_agent_reference(settings, prompt, timeout_seconds) - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {settings.agent_token}", - } - - thread_url = ( - f"{settings.project_endpoint}/threads?api-version={settings.api_version}" - ) - thread_payload = self._request_json( - method="POST", - url=thread_url, - headers=headers, - timeout_seconds=timeout_seconds, - body={}, - ) - thread_id = thread_payload.get("id") - if not isinstance(thread_id, str) or not thread_id: - raise ValueError( - "Invalid Foundry Agent Service response: missing thread id" - ) - - message_url = f"{settings.project_endpoint}/threads/{thread_id}/messages?api-version={settings.api_version}" - self._request_json( - method="POST", - url=message_url, - headers=headers, - timeout_seconds=timeout_seconds, - body={"role": "user", "content": prompt}, - ) - - run_url = f"{settings.project_endpoint}/threads/{thread_id}/runs?api-version={settings.api_version}" - run_payload = self._request_json( - method="POST", - url=run_url, - headers=headers, - timeout_seconds=timeout_seconds, - body={"assistant_id": settings.agent_id}, - ) - run_id = run_payload.get("id") - if not isinstance(run_id, str) or not run_id: - raise ValueError("Invalid Foundry Agent Service response: missing run id") - - status_url = ( - f"{settings.project_endpoint}/threads/{thread_id}/runs/{run_id}" - f"?api-version={settings.api_version}" - ) - - terminal_success = {"completed"} - terminal_failure = {"failed", "cancelled", "expired", "requires_action"} - - for _ in range(settings.max_poll_attempts): - status_payload = self._request_json( - method="GET", - url=status_url, - headers=headers, - timeout_seconds=timeout_seconds, - ) - status = status_payload.get("status") - if isinstance(status, str): - if status in terminal_success: - break - if status in terminal_failure: - raise RuntimeError( - f"Foundry agent run ended with status '{status}'" - ) - time.sleep(settings.poll_interval_seconds) - else: - raise TimeoutError("Timed out waiting for Foundry agent run completion") - - messages_url = f"{settings.project_endpoint}/threads/{thread_id}/messages?api-version={settings.api_version}" - messages_payload = self._request_json( - method="GET", - url=messages_url, - headers=headers, - timeout_seconds=timeout_seconds, - ) - - return self._extract_agent_message_text(messages_payload) - - def _invoke_model_direct(self, settings: FoundrySettings, prompt: str) -> str: - """Call the model deployment directly via the OpenAI chat completions API. - - Used when ``target=model`` — no agent is involved. The Foundry project - endpoint is used to derive the Azure OpenAI base URL, and the model - deployment name comes from ``settings.model``. - """ - try: - from azure.ai.projects import AIProjectClient # noqa: WPS433 - from azure.identity import DefaultAzureCredential # noqa: WPS433 - except ImportError as exc: - raise ImportError( - "Model-direct evaluation requires 'azure-ai-projects>=2.0.1' " - "and 'azure-identity'. " - "Install with: pip install 'azure-ai-projects>=2.0.1' azure-identity openai" - ) from exc - - credential = DefaultAzureCredential(exclude_developer_cli_credential=True) - project_client = AIProjectClient( - endpoint=settings.project_endpoint, - credential=credential, - ) - openai_client = project_client.get_openai_client() - - assert settings.model is not None - response = openai_client.chat.completions.create( - model=settings.model, - messages=[{"role": "user", "content": prompt}], - ) - - if response.choices: - message = response.choices[0].message - if message and message.content: - return message.content.strip() - - raise ValueError("Model-direct invocation returned no content") - - def _execute_cloud_evaluation( - self, - *, - context: BackendRunContext, - settings: FoundrySettings, - bundle_config: Any, - dataset_config: Any, - dataset_source_path: Path, - started: datetime, - started_perf: float, - stdout_path: Path, - stderr_path: Path, - metrics_path: Path, - ) -> BackendExecutionResult: - """Run evaluation via the Foundry Project Evals API (New Experience). - - Uses the Foundry Project REST endpoint - ``{project_endpoint}/openai/evals?api-version=2025-11-15-preview`` - with ``azure_ai_evaluator`` testing criteria so results appear in the - Foundry Evaluations page. - - Reference: https://learn.microsoft.com/azure/foundry/how-to/develop/cloud-evaluation - """ - # The Foundry Project Evals API version that supports azure_ai_evaluator. - _EVALS_API_VERSION = "2025-11-15-preview" - - rows = _load_jsonl(dataset_source_path) - total_rows = len(rows) - input_field = dataset_config.format.input_field - expected_field = dataset_config.format.expected_field - - enabled_evaluators = [ - evaluator for evaluator in bundle_config.evaluators if evaluator.enabled - ] - _validate_supported_local_evaluators(enabled_evaluators) - enabled_evaluator_order = [evaluator.name for evaluator in enabled_evaluators] - - foundry_evaluators = [ - evaluator - for evaluator in enabled_evaluators - if evaluator.source == "foundry" - ] - if not foundry_evaluators: - raise ValueError( - "Foundry Cloud Evaluation requires at least one enabled evaluator with source='foundry'" - ) - - logger.info( - "Starting Foundry Cloud Evaluation for %d dataset row(s) " - "(target=%s, agent=%s, model=%s, evaluators=%s)", - total_rows, - settings.target, - settings.agent_id or "(none)", - settings.model, - [e.name for e in foundry_evaluators], - ) - - # --- Build testing criteria (azure_ai_evaluator) --------------------- - testing_criteria: list[dict[str, Any]] = [] - for evaluator in foundry_evaluators: - builtin_name = _to_builtin_evaluator_name(evaluator.name) - criterion: dict[str, Any] = { - "type": "azure_ai_evaluator", - "name": evaluator.name, - "evaluator_name": f"builtin.{builtin_name}", - "data_mapping": _cloud_evaluator_data_mapping( - builtin_name, - input_field, - expected_field, - context_field=dataset_config.format.context_field, - ), - } - if _cloud_evaluator_needs_model(builtin_name): - if not settings.model: - raise ValueError( - f"Evaluator '{evaluator.name}' requires a model deployment name. " - "Set 'backend.model' in run.yaml or AZURE_AI_MODEL_DEPLOYMENT_NAME." - ) - criterion["initialization_parameters"] = { - "deployment_name": settings.model, - } - elif builtin_name in _NLP_DEFAULT_INIT_PARAMS: - criterion["initialization_parameters"] = dict( - _NLP_DEFAULT_INIT_PARAMS[builtin_name] - ) - testing_criteria.append(criterion) - - # --- Acquire token for Foundry Project Evals API -------------------- - try: - evals_token = _acquire_token("https://ai.azure.com/.default") - except Exception as exc: - raise RuntimeError(_CREDENTIAL_HELP_MESSAGE) from exc - - evals_base_url = settings.project_endpoint.rstrip("/") - evals_headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {evals_token}", - } - - def _evals_post(path: str, body: dict[str, Any]) -> dict[str, Any]: - url = ( - f"{evals_base_url}/openai/evals{path}?api-version={_EVALS_API_VERSION}" - ) - return self._request_json( - method="POST", - url=url, - headers=evals_headers, - timeout_seconds=60, - body=body, - ) - - def _evals_get(path: str, extra_params: str = "") -> dict[str, Any]: - params = f"api-version={_EVALS_API_VERSION}" - if extra_params: - params = f"{params}&{extra_params}" - url = f"{evals_base_url}/openai/evals{path}?{params}" - return self._request_json( - method="GET", - url=url, - headers=evals_headers, - timeout_seconds=60, - ) - - # --- Data schema ---------------------------------------------------- - item_schema: dict[str, Any] = { - "type": "object", - "properties": { - input_field: {"type": "string"}, - expected_field: {"type": "string"}, - }, - "required": [input_field, expected_field], - } - - eval_name = f"agentops-eval-{uuid.uuid4().hex[:8]}" - eval_object = _evals_post( - "", - { - "name": eval_name, - "data_source_config": { - "type": "custom", - "item_schema": item_schema, - "include_sample_schema": True, - }, - "testing_criteria": testing_criteria, - }, - ) - eval_id = eval_object["id"] - logger.info("Cloud evaluation created: %s", eval_id) - - # --- Target + input messages ---------------------------------------- - input_messages: dict[str, Any] = { - "type": "template", - "template": [ - { - "type": "message", - "role": "user", - "content": { - "type": "input_text", - "text": "{{item." + input_field + "}}", - }, - } - ], - } - - run_name = f"agentops-run-{uuid.uuid4().hex[:8]}" - - if settings.target == "model": - # Model-direct: use completions data source (no agent) - eval_run = _evals_post( - f"/{eval_id}/runs", - { - "name": run_name, - "data_source": { - "type": "completions", - "source": { - "type": "file_content", - "content": [{"item": row} for row in rows], - }, - "input_messages": input_messages, - "model": settings.model, - }, - }, - ) - else: - # Agent target - assert settings.agent_id is not None - agent_name, agent_version = _parse_agent_name_version(settings.agent_id) - target: dict[str, Any] = { - "type": "azure_ai_agent", - "name": agent_name, - } - if agent_version: - target["version"] = agent_version - - eval_run = _evals_post( - f"/{eval_id}/runs", - { - "name": run_name, - "data_source": { - "type": "azure_ai_target_completions", - "source": { - "type": "file_content", - "content": [{"item": row} for row in rows], - }, - "input_messages": input_messages, - "target": target, - }, - }, - ) - - run_id = eval_run["id"] - logger.info( - "Cloud evaluation run started: %s (polling every %.0fs, timeout %.0fs)", - run_id, - settings.poll_interval_seconds, - settings.poll_interval_seconds * settings.max_poll_attempts, - ) - - # --- Poll until completion ------------------------------------------ - terminal_success = {"completed", "succeeded"} - terminal_failure = {"failed", "cancelled", "canceled", "expired", "error"} - poll_start = perf_counter() - last_logged_status: str | None = None - latest_run: dict[str, Any] = eval_run - - for attempt in range(1, settings.max_poll_attempts + 1): - latest_run = _evals_get(f"/{eval_id}/runs/{run_id}") - run_status = str(latest_run.get("status", "unknown")).lower() - - # Only log when the status changes to avoid flooding the console. - if run_status != last_logged_status: - elapsed = perf_counter() - poll_start - logger.info( - "Cloud eval status: %s (%.0fs elapsed)", - run_status, - elapsed, - ) - last_logged_status = run_status - - if run_status in terminal_success: - break - if run_status in terminal_failure: - raise RuntimeError( - f"Foundry cloud evaluation run ended with status '{run_status}'. " - "Check the Foundry portal for details." - ) - time.sleep(settings.poll_interval_seconds) - else: - elapsed = perf_counter() - poll_start - raise TimeoutError( - f"Timed out after {elapsed:.0f}s waiting for Foundry cloud evaluation" - ) - - # --- Collect output items ------------------------------------------- - output_items_resp = _evals_get( - f"/{eval_id}/runs/{run_id}/output_items", - extra_params="order=asc&limit=100", - ) - output_items: list[dict[str, Any]] = output_items_resp.get("data", []) - if not output_items: - raise RuntimeError( - "Foundry cloud evaluation completed with no output items" - ) - - evaluator_aggregate_values: dict[str, list[float]] = { - name: [] for name in enabled_evaluator_order - } - # Track which local evaluators the bundle actually requests. - enabled_local_names = frozenset( - e.name for e in enabled_evaluators if e.source == "local" - ) - - # Approximate per-row latency from total cloud eval duration. - eval_elapsed = perf_counter() - poll_start - approx_latency_per_row = eval_elapsed / len(output_items) - if {"latency_seconds", "avg_latency_seconds"} & enabled_local_names: - logger.info( - "Latency in cloud evaluation is estimated from total eval duration " - "(%.1fs / %d rows ≈ %.2fs per row)", - eval_elapsed, - len(output_items), - approx_latency_per_row, - ) - - row_metrics_payload: list[dict[str, Any]] = [] - stdout_lines: list[str] = [] - stderr_lines: list[str] = [] - - for index, item in enumerate(output_items, start=1): - datasource_item = item.get("datasource_item", {}) or {} - row_data = ( - datasource_item.get("item", datasource_item) - if isinstance(datasource_item, dict) - else {} - ) - - prompt = _normalize_text(row_data.get(input_field)) - expected = _normalize_text(row_data.get(expected_field)) - - # Extract prediction from sample - sample = item.get("sample", None) - prediction = "" - if isinstance(sample, dict): - prediction = _normalize_text(sample.get("output_text", "")) - - row_metric_entries: list[dict[str, Any]] = [] - for result in item.get("results", []) or []: - metric_name = result.get("name", "") if isinstance(result, dict) else "" - metric_score = ( - result.get("score", None) if isinstance(result, dict) else None - ) - if isinstance(metric_name, str) and isinstance( - metric_score, (int, float) - ): - # Normalize names like "SimilarityEvaluator-" → "SimilarityEvaluator" - for eval_name in enabled_evaluator_order: - if metric_name == eval_name or metric_name.startswith( - eval_name + "-" - ): - metric_name = eval_name - break - value = float(metric_score) - row_metric_entries.append({"name": metric_name, "value": value}) - - # Only emit local evaluator metrics if they are configured in the bundle. - if "exact_match" in enabled_local_names: - passed = prediction.lower() == expected.lower() if expected else False - row_metric_entries.append( - { - "name": "exact_match", - "value": 1.0 if passed else 0.0, - } - ) - if "latency_seconds" in enabled_local_names: - row_metric_entries.append( - { - "name": "latency_seconds", - "value": approx_latency_per_row, - } - ) - if "avg_latency_seconds" in enabled_local_names: - row_metric_entries.append( - { - "name": "avg_latency_seconds", - "value": approx_latency_per_row, - } - ) - - # Update aggregate values for local evaluator metrics. - for entry in row_metric_entries: - agg_name = entry["name"] - if agg_name in evaluator_aggregate_values: - evaluator_aggregate_values[agg_name].append(entry["value"]) - - row_index = index - datasource_item_id = item.get("datasource_item_id", None) - if isinstance(datasource_item_id, int) and datasource_item_id >= 0: - row_index = datasource_item_id + 1 - - row_metrics_payload.append( - { - "row_index": row_index, - "input": prompt, - "response": prediction, - "context": row_data.get("context"), - "metrics": row_metric_entries, - } - ) - stdout_lines.append( - f"row={row_index} expected={expected!r} prediction={prediction!r}" - ) - logger.info("Processed output item %d/%d", index, len(output_items)) - - total = len(output_items) - - # --- Aggregate metrics ---------------------------------------------- - metrics_entries: list[dict[str, Any]] = [] - for name in enabled_evaluator_order: - values = evaluator_aggregate_values.get(name, []) - if values: - metrics_entries.append( - { - "name": name, - "value": sum(values) / len(values), - } - ) - metrics_entries.append({"name": "samples_evaluated", "value": float(total)}) - - metrics_path.write_text( - json.dumps( - {"metrics": metrics_entries, "row_metrics": row_metrics_payload}, - indent=2, - ), - encoding="utf-8", - ) - stdout_path.write_text("\n".join(stdout_lines), encoding="utf-8") - stderr_path.write_text("\n".join(stderr_lines), encoding="utf-8") - - # --- Report URL (deep-link to the New Foundry Experience) ----------- - report_url = latest_run.get("report_url") - - cloud_meta_path = context.backend_output_dir / "cloud_evaluation.json" - cloud_meta_path.write_text( - json.dumps( - { - "eval_id": eval_id, - "run_id": run_id, - "report_url": report_url, - "evaluation_name": eval_name, - "run_name": run_name, - }, - indent=2, - ), - encoding="utf-8", - ) - - finished = datetime.now(UTC) - duration = perf_counter() - started_perf - if settings.target == "model": - command_display = ( - "foundry.cloud_evaluation " - f"project_endpoint={settings.project_endpoint} target=model model={settings.model}" - ) - else: - command_display = ( - "foundry.cloud_evaluation " - f"project_endpoint={settings.project_endpoint} target=agent agent_id={settings.agent_id} model={settings.model}" - ) - - logger.info("Cloud evaluation completed with %d output item(s)", total) - if report_url: - logger.info("Foundry Evaluations URL: %s", report_url) - - return BackendExecutionResult( - backend="foundry", - command=command_display, - started_at=_to_utc_timestamp(started), - finished_at=_to_utc_timestamp(finished), - duration_seconds=duration, - exit_code=0, - stdout_file=stdout_path, - stderr_file=stderr_path, - ) - - def execute(self, context: BackendRunContext) -> BackendExecutionResult: - context.backend_output_dir.mkdir(parents=True, exist_ok=True) - stdout_path = context.backend_output_dir / "backend.stdout.log" - stderr_path = context.backend_output_dir / "backend.stderr.log" - metrics_path = context.backend_output_dir / "backend_metrics.json" - - started = datetime.now(UTC) - started_perf = perf_counter() - - stdout_lines: list[str] = [] - stderr_lines: list[str] = [] - exit_code = 0 - - settings = self._read_settings(context) - bundle_config = load_bundle_config(context.bundle_path) - dataset_config = load_dataset_config(context.dataset_path) - dataset_source_path = _resolve_dataset_source_path( - context.dataset_path, dataset_config.source.path - ) - if not dataset_source_path.exists(): - raise FileNotFoundError(f"Dataset file not found: {dataset_source_path}") - - # Cloud evaluation is the default path (New Foundry Experience). - # Set AGENTOPS_FOUNDRY_MODE=local to use local evaluators instead. - if _should_use_cloud_evaluation(settings.project_endpoint): - return self._execute_cloud_evaluation( - context=context, - settings=settings, - bundle_config=bundle_config, - dataset_config=dataset_config, - dataset_source_path=dataset_source_path, - started=started, - started_perf=started_perf, - stdout_path=stdout_path, - stderr_path=stderr_path, - metrics_path=metrics_path, - ) - - # --- Local evaluation fallback (AGENTOPS_FOUNDRY_MODE=local) -------- - # Derive Azure OpenAI fallbacks from the Foundry project endpoint so - # AI-assisted evaluators (SimilarityEvaluator, etc.) work without - # requiring the user to set AZURE_OPENAI_ENDPOINT / AZURE_OPENAI_DEPLOYMENT. - fallback_endpoint = _derive_openai_endpoint_from_project( - settings.project_endpoint - ) - fallback_deployment = settings.model - - enabled_evaluators = [ - evaluator for evaluator in bundle_config.evaluators if evaluator.enabled - ] - _validate_supported_local_evaluators(enabled_evaluators) - enabled_evaluator_order = [evaluator.name for evaluator in enabled_evaluators] - - foundry_evaluator_runtimes = _build_foundry_evaluator_runtimes( - enabled_evaluators, - fallback_endpoint=fallback_endpoint, - fallback_deployment=fallback_deployment, - ) - - rows = _load_jsonl(dataset_source_path) - total_rows = len(rows) - logger.info( - "Starting local Foundry evaluation for %d dataset row(s)", total_rows - ) - input_field = dataset_config.format.input_field - expected_field = dataset_config.format.expected_field - timeout_seconds = context.run_config.execution.timeout_seconds - - total = 0 - per_item_latencies: list[float] = [] - row_metrics_payload: list[dict[str, Any]] = [] - # Track which local evaluators the bundle actually requests. - enabled_local_names = frozenset( - e.name for e in enabled_evaluators if e.source == "local" - ) - - evaluator_aggregate_values: dict[str, list[float]] = { - evaluator_name: [] for evaluator_name in enabled_evaluator_order - } - - def _record_row_metrics( - *, - row_index: int, - row_data: dict[str, Any], - prompt_text: str, - expected_text: str, - prediction_text: str, - row_latency: float, - ) -> None: - nonlocal total - - prediction_normalized = _normalize_text(prediction_text) - total += 1 - - row_metric_entries: list[dict[str, Any]] = [] - - for runtime in foundry_evaluator_runtimes: - score = _run_foundry_evaluator( - runtime, - prompt=prompt_text, - prediction=prediction_normalized, - expected=expected_text, - row=row_data, - ) - row_metric_entries.append({"name": runtime.name, "value": score}) - - # Only emit local evaluator metrics that are configured in the bundle. - if "exact_match" in enabled_local_names: - passed = prediction_normalized.lower() == expected_text.lower() - row_metric_entries.append( - { - "name": "exact_match", - "value": 1.0 if passed else 0.0, - } - ) - if "latency_seconds" in enabled_local_names: - row_metric_entries.append( - { - "name": "latency_seconds", - "value": row_latency, - } - ) - if "avg_latency_seconds" in enabled_local_names: - row_metric_entries.append( - { - "name": "avg_latency_seconds", - "value": row_latency, - } - ) - - for metric_entry in row_metric_entries: - metric_name = metric_entry["name"] - metric_value = metric_entry["value"] - if metric_name in evaluator_aggregate_values: - evaluator_aggregate_values[metric_name].append(metric_value) - - row_metrics_payload.append( - { - "row_index": row_index, - "input": prompt_text, - "response": prediction_normalized, - "context": row_data.get("context"), - "metrics": row_metric_entries, - } - ) - - stdout_lines.append( - f"row={row_index} expected={expected_text!r} prediction={prediction_normalized!r}" - ) - - for index, row in enumerate(rows, start=1): - logger.info("Processing row %d/%d", index, total_rows) - if input_field not in row: - raise ValueError( - f"Dataset row {index} missing input field '{input_field}'" - ) - if expected_field not in row: - raise ValueError( - f"Dataset row {index} missing expected field '{expected_field}'" - ) - - prompt = _normalize_text(row.get(input_field)) - expected = _normalize_text(row.get(expected_field)) - - _agent_name: str | None = None - _agent_version: str | None = None - if settings.agent_id: - _agent_name, _agent_version = _parse_agent_name_version( - settings.agent_id - ) - - row_start = perf_counter() - try: - with agent_invoke_span( - target=settings.target, - model=settings.model, - agent_id=settings.agent_id, - agent_name=_agent_name, - agent_version=_agent_version, - provider="azure.ai.foundry", - ) as invoke_span: - if settings.target == "model": - prediction = self._invoke_model_direct(settings, prompt) - else: - prediction = self._invoke_agent_service( - settings, prompt, timeout_seconds - ) - set_agent_invoke_result(invoke_span, response_model=settings.model) - except urllib.error.HTTPError as exc: - details = exc.read().decode("utf-8", errors="replace") - if exc.code == 401 and _is_audience_mismatch(details): - alternate_scope = _alternate_scope(settings.token_scope) - try: - logger.info( - "Retrying with alternate token audience: %s", - alternate_scope, - ) - settings = replace( - settings, - agent_token=_acquire_token(alternate_scope), - token_scope=alternate_scope, - ) - with agent_invoke_span( - target=settings.target, - model=settings.model, - agent_id=settings.agent_id, - agent_name=_agent_name, - agent_version=_agent_version, - provider="azure.ai.foundry", - ) as retry_invoke_span: - if settings.target == "model": - prediction = self._invoke_model_direct(settings, prompt) - else: - prediction = self._invoke_agent_service( - settings, prompt, timeout_seconds - ) - set_agent_invoke_result( - retry_invoke_span, response_model=settings.model - ) - except Exception as retry_exc: # noqa: BLE001 - retry_details = str(retry_exc) - logger.error( - "Row %d/%d failed after audience retry: %s", - index, - total_rows, - retry_details, - ) - stderr_lines.append( - "row=" - f"{index} http_error={exc.code} details={details} retry_error={retry_details}" - ) - exit_code = 1 - break - else: - row_latency = perf_counter() - row_start - per_item_latencies.append(row_latency) - - _record_row_metrics( - row_index=index, - row_data=row, - prompt_text=prompt, - expected_text=expected, - prediction_text=prediction, - row_latency=row_latency, - ) - continue - - stderr_lines.append( - f"row={index} http_error={exc.code} details={details}" - ) - logger.error("Row %d/%d HTTP error %s", index, total_rows, exc.code) - exit_code = 1 - break - except urllib.error.URLError as exc: - stderr_lines.append(f"row={index} network_error={exc.reason}") - logger.error( - "Row %d/%d network error: %s", index, total_rows, exc.reason - ) - exit_code = 1 - break - except Exception as exc: # noqa: BLE001 - stderr_lines.append(f"row={index} error={exc}") - logger.error("Row %d/%d failed: %s", index, total_rows, exc) - exit_code = 1 - break - - row_latency = perf_counter() - row_start - per_item_latencies.append(row_latency) - - _record_row_metrics( - row_index=index, - row_data=row, - prompt_text=prompt, - expected_text=expected, - prediction_text=prediction, - row_latency=row_latency, - ) - logger.info("Completed row %d/%d in %.2fs", index, total_rows, row_latency) - - if total == 0 and exit_code == 0: - raise RuntimeError("Foundry backend did not process any dataset rows") - - _avg_latency_seconds = ( - sum(per_item_latencies) / len(per_item_latencies) - if per_item_latencies - else 0.0 - ) - - metrics_entries: list[dict[str, Any]] = [] - for evaluator_name in enabled_evaluator_order: - values = evaluator_aggregate_values.get(evaluator_name, []) - if values: - metrics_entries.append( - { - "name": evaluator_name, - "value": sum(values) / len(values), - } - ) - - metrics_entries.append({"name": "samples_evaluated", "value": float(total)}) - - metrics_payload = { - "metrics": metrics_entries, - "row_metrics": row_metrics_payload, - } - metrics_path.write_text(json.dumps(metrics_payload, indent=2), encoding="utf-8") - logger.info("Local evaluation complete: processed %d row(s)", total) - - stdout_path.write_text("\n".join(stdout_lines), encoding="utf-8") - stderr_path.write_text("\n".join(stderr_lines), encoding="utf-8") - - finished = datetime.now(UTC) - duration = perf_counter() - started_perf - if settings.target == "model": - command_display = ( - "foundry.model_direct " - f"project_endpoint={settings.project_endpoint} target=model model={settings.model}" - ) - else: - command_display = ( - "foundry.agent_service " - f"project_endpoint={settings.project_endpoint} target=agent agent_id={settings.agent_id} " - f"model={settings.model} api_version={settings.api_version}" - ) - - return BackendExecutionResult( - backend="foundry", - command=command_display, - started_at=_to_utc_timestamp(started), - finished_at=_to_utc_timestamp(finished), - duration_seconds=duration, - exit_code=exit_code, - stdout_file=stdout_path, - stderr_file=stderr_path, - ) diff --git a/src/agentops/backends/http_backend.py b/src/agentops/backends/http_backend.py deleted file mode 100644 index ee913a3d..00000000 --- a/src/agentops/backends/http_backend.py +++ /dev/null @@ -1,384 +0,0 @@ -"""HTTP backend for AgentOps — calls any HTTP-deployed agent endpoint row by row. - -Supports agents deployed outside Microsoft Foundry Agent Service, such as -Microsoft Agent Framework applications running on Azure Container Apps (ACA) -or any custom REST endpoint that accepts a JSON payload and returns a response. - -The backend: -- Resolves the target URL from config or from an environment variable. -- POSTs each dataset row as JSON, using ``request_field`` as the prompt key. -- Extracts the model response via ``response_field`` (supports dot-path). -- Runs local and AI-assisted evaluators using the same evaluation engine as - the Foundry local-mode path. -- Produces ``backend_metrics.json`` with per-row scores. -""" - -from __future__ import annotations - -import json -import logging -import os -import urllib.error -import urllib.request -from datetime import UTC, datetime -from time import perf_counter -from typing import Any - -from agentops.backends.base import BackendExecutionResult, BackendRunContext -from agentops.backends.eval_engine import ( - _build_foundry_evaluator_runtimes, - _load_jsonl, - _normalize_text, - _resolve_dataset_source_path, - _run_foundry_evaluator, - _validate_supported_local_evaluators, -) -from agentops.core.config_loader import load_bundle_config, load_dataset_config -from agentops.utils.telemetry import agent_invoke_span, set_agent_invoke_result - -logger = logging.getLogger(__name__) - -_DEFAULT_REQUEST_FIELD = "message" -_DEFAULT_RESPONSE_FIELD = "text" - - -def _to_utc_timestamp(value: datetime) -> str: - return value.astimezone(UTC).isoformat().replace("+00:00", "Z") - - -def _extract_dot_path(payload: Any, dot_path: str) -> Any: - """Extract a value from a nested dict using a dot-separated path. - - For example, ``"output.text"`` retrieves ``payload["output"]["text"]``. - Returns the payload directly when dot-path is a single key. - """ - parts = dot_path.split(".") - current: Any = payload - for part in parts: - if not isinstance(current, dict): - raise ValueError( - f"Cannot traverse response path '{dot_path}': " - f"expected object at '{part}', got {type(current).__name__}" - ) - if part not in current: - raise ValueError( - f"Response field '{part}' not found in HTTP response payload " - f"(full path: '{dot_path}')" - ) - current = current[part] - return current - - -def _post_json( - *, - url: str, - body: dict[str, Any], - extra_headers: dict[str, str], - auth_token: str | None, - timeout_seconds: int | None, -) -> dict[str, Any]: - """POST a JSON body to the given URL and return the parsed response.""" - headers: dict[str, str] = { - "Content-Type": "application/json", - "Accept": "application/json", - } - if auth_token: - headers["Authorization"] = f"Bearer {auth_token}" - headers.update(extra_headers) - - request_body = json.dumps(body).encode("utf-8") - request = urllib.request.Request( - url=url, method="POST", data=request_body, headers=headers - ) - - with urllib.request.urlopen(request, timeout=timeout_seconds) as response: - payload = json.loads(response.read().decode("utf-8")) - - if not isinstance(payload, dict): - raise ValueError( - f"HTTP agent returned an unexpected response type " - f"(expected JSON object, got {type(payload).__name__})" - ) - return payload - - -class HttpBackend: - """Evaluation backend that calls an arbitrary HTTP agent endpoint.""" - - def _resolve_url(self, context: BackendRunContext) -> str: - endpoint = context.run_config.target.endpoint - assert endpoint is not None, "HTTP backend requires target.endpoint" - - url = endpoint.url - if url: - return url.rstrip("/") - - env_name = endpoint.url_env - if env_name: - url = os.getenv(env_name) - if url: - return url.rstrip("/") - raise ValueError( - f"HTTP backend requires a target URL. " - f"Set the environment variable '{env_name}' to the agent endpoint URL.\n" - f"\n" - f" PowerShell:\n" - f' $env:{env_name} = "https://your-agent.region.azurecontainerapps.io/chat"\n' - f"\n" - f" Bash/zsh:\n" - f' export {env_name}="https://your-agent.region.azurecontainerapps.io/chat"' - ) - - raise ValueError( - "HTTP backend requires 'target.endpoint.url' or 'target.endpoint.url_env' in your run config." - ) - - def execute(self, context: BackendRunContext) -> BackendExecutionResult: - context.backend_output_dir.mkdir(parents=True, exist_ok=True) - - stdout_path = context.backend_output_dir / "backend.stdout.log" - stderr_path = context.backend_output_dir / "backend.stderr.log" - metrics_path = context.backend_output_dir / "backend_metrics.json" - - endpoint = context.run_config.target.endpoint - assert endpoint is not None, "HTTP backend requires target.endpoint" - - started = datetime.now(UTC) - started_perf = perf_counter() - - stdout_lines: list[str] = [] - stderr_lines: list[str] = [] - - exit_code = 0 - - try: - url = self._resolve_url(context) - request_field = endpoint.request_field or _DEFAULT_REQUEST_FIELD - response_field = endpoint.response_field or _DEFAULT_RESPONSE_FIELD - timeout_seconds = context.run_config.execution.timeout_seconds - extra_headers = dict(endpoint.headers) - tool_calls_field = endpoint.tool_calls_field - extra_field_names = endpoint.extra_fields or [] - - auth_token: str | None = None - if endpoint.auth_header_env: - auth_token = os.getenv(endpoint.auth_header_env) - if not auth_token: - raise ValueError( - f"HTTP backend auth token env var '{endpoint.auth_header_env}' is set " - f"but the variable is empty or unset." - ) - - bundle_config = load_bundle_config(context.bundle_path) - dataset_config = load_dataset_config(context.dataset_path) - - dataset_source_path = _resolve_dataset_source_path( - context.dataset_path, dataset_config.source.path - ) - rows = _load_jsonl(dataset_source_path) - total_rows = len(rows) - - enabled_evaluators = [e for e in bundle_config.evaluators if e.enabled] - _validate_supported_local_evaluators(enabled_evaluators) - enabled_evaluator_order = [e.name for e in enabled_evaluators] - - # AI-assisted evaluators require Azure OpenAI — read from environment. - fallback_endpoint: str | None = os.getenv("AZURE_OPENAI_ENDPOINT") - fallback_deployment: str | None = os.getenv( - "AZURE_AI_MODEL_DEPLOYMENT_NAME" - ) or os.getenv("AZURE_OPENAI_DEPLOYMENT") - - foundry_evaluator_runtimes = _build_foundry_evaluator_runtimes( - enabled_evaluators, - fallback_endpoint=fallback_endpoint, - fallback_deployment=fallback_deployment, - ) - - input_field = dataset_config.format.input_field - expected_field = dataset_config.format.expected_field - - enabled_local_names = frozenset( - e.name for e in enabled_evaluators if e.source == "local" - ) - evaluator_aggregate_values: dict[str, list[float]] = { - name: [] for name in enabled_evaluator_order - } - - row_metrics_payload: list[dict[str, Any]] = [] - - logger.info( - "HTTP backend: evaluating %d row(s) against %s", total_rows, url - ) - - for index, row in enumerate(rows, start=1): - logger.info("Processing row %d/%d", index, total_rows) - - if input_field not in row: - raise ValueError( - f"Dataset row {index} missing input field '{input_field}'" - ) - if expected_field not in row: - raise ValueError( - f"Dataset row {index} missing expected field '{expected_field}'" - ) - - prompt_text = _normalize_text(row[input_field]) - expected_text = _normalize_text(row[expected_field]) - - request_body: dict[str, Any] = {request_field: prompt_text} - - # Forward extra JSONL row fields in the request body. - for field_name in extra_field_names: - if field_name in row: - request_body[field_name] = row[field_name] - - row_start = perf_counter() - try: - with agent_invoke_span( - target=context.run_config.target.type, - model=getattr(endpoint, "model", None), - provider="http", - ) as invoke_span: - response_payload = _post_json( - url=url, - body=request_body, - extra_headers=extra_headers, - auth_token=auth_token, - timeout_seconds=timeout_seconds, - ) - raw_response = _extract_dot_path( - response_payload, response_field - ) - prediction_text = _normalize_text(raw_response) - set_agent_invoke_result(invoke_span) - - # Extract tool_calls from HTTP response for agent evaluators. - if tool_calls_field: - try: - extracted_tool_calls = _extract_dot_path( - response_payload, tool_calls_field - ) - row["tool_calls"] = extracted_tool_calls - except ValueError: - pass # Field not present in this response; skip silently. - except ( - urllib.error.URLError, - urllib.error.HTTPError, - ValueError, - ) as exc: - stderr_lines.append(f"row={index} error={exc!s}") - logger.error("HTTP request failed for row %d: %s", index, exc) - exit_code = 1 - continue - - row_latency = perf_counter() - row_start - - row_metric_entries: list[dict[str, Any]] = [] - - for runtime in foundry_evaluator_runtimes: - try: - score = _run_foundry_evaluator( - runtime, - prompt=prompt_text, - prediction=prediction_text, - expected=expected_text, - row=row, - ) - row_metric_entries.append( - { - "name": runtime.name, - "value": score, - } - ) - except Exception as exc: # noqa: BLE001 - stderr_lines.append( - f"row={index} evaluator={runtime.name} error={exc!s}" - ) - logger.error( - "Evaluator '%s' failed for row %d: %s", - runtime.name, - index, - exc, - ) - - if "exact_match" in enabled_local_names: - passed = prediction_text.lower() == expected_text.lower() - row_metric_entries.append( - { - "name": "exact_match", - "value": 1.0 if passed else 0.0, - } - ) - if "latency_seconds" in enabled_local_names: - row_metric_entries.append( - { - "name": "latency_seconds", - "value": row_latency, - } - ) - if "avg_latency_seconds" in enabled_local_names: - row_metric_entries.append( - { - "name": "avg_latency_seconds", - "value": row_latency, - } - ) - - for entry in row_metric_entries: - name = entry["name"] - if name in evaluator_aggregate_values: - evaluator_aggregate_values[name].append(entry["value"]) - - row_metrics_payload.append( - { - "row_index": index, - "input": prompt_text, - "response": prediction_text, - "context": row.get("context"), - "metrics": row_metric_entries, - } - ) - stdout_lines.append( - f"row={index} expected={expected_text!r} prediction={prediction_text!r}" - ) - - # Aggregate overall metrics - aggregate_metrics: list[dict[str, Any]] = [] - for name, values in evaluator_aggregate_values.items(): - if values: - aggregate_metrics.append( - { - "name": name, - "value": sum(values) / len(values), - } - ) - - metrics_path.write_text( - json.dumps( - {"metrics": aggregate_metrics, "row_metrics": row_metrics_payload}, - indent=2, - ), - encoding="utf-8", - ) - - except Exception as exc: # noqa: BLE001 - stderr_lines.append(str(exc)) - logger.error("HTTP backend failed: %s", exc) - exit_code = 1 - - finished = datetime.now(UTC) - duration = perf_counter() - started_perf - - stdout_path.write_text("\n".join(stdout_lines), encoding="utf-8") - stderr_path.write_text("\n".join(stderr_lines), encoding="utf-8") - - return BackendExecutionResult( - backend="http", - command=endpoint.url or endpoint.url_env or "http", - started_at=_to_utc_timestamp(started), - finished_at=_to_utc_timestamp(finished), - duration_seconds=round(duration, 3), - exit_code=exit_code, - stdout_file=stdout_path, - stderr_file=stderr_path, - ) diff --git a/src/agentops/backends/local_adapter_backend.py b/src/agentops/backends/local_adapter_backend.py deleted file mode 100644 index 1636e97a..00000000 --- a/src/agentops/backends/local_adapter_backend.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Local adapter backend for AgentOps — runs a local agent process per row. - -Supports two execution modes: - -**Subprocess mode** (``local.adapter``): - The adapter command is spawned once per dataset row. Each invocation - receives a JSON object on **stdin** and must write a JSON object to - **stdout**. - - Input JSON:: - - {"input": "", "expected": "", ...extra row fields} - - Expected output JSON:: - - {"response": ""} - -**Callable mode** (``local.callable``): - A Python function specified as ``module:function`` is imported and called - once per dataset row. The function signature must be:: - - def run_evaluation(input: str, context: dict) -> dict: - ... - return {"response": ""} - - The ``context`` dict contains all row fields from the dataset. - The return dict must include a ``"response"`` key. - -The backend collects responses and runs the same evaluation engine used -by the Foundry local-mode and HTTP backends to produce -``backend_metrics.json``. -""" - -from __future__ import annotations - -import importlib -import json -import logging -import os -import shlex -import subprocess -import sys -from datetime import datetime, timezone -from pathlib import Path -from time import perf_counter -from typing import Any, Callable, Dict, List, Optional - -from agentops.backends.base import BackendExecutionResult, BackendRunContext -from agentops.backends.eval_engine import ( - _build_foundry_evaluator_runtimes, - _load_jsonl, - _normalize_text, - _resolve_dataset_source_path, - _run_foundry_evaluator, - _validate_supported_local_evaluators, -) -from agentops.core.config_loader import load_bundle_config, load_dataset_config -from agentops.utils.telemetry import agent_invoke_span, set_agent_invoke_result - -logger = logging.getLogger(__name__) - - -def _load_callable( - callable_path: str, -) -> Callable[[str, Dict[str, Any]], Dict[str, Any]]: - """Import and return the user function from a ``module:function`` path.""" - module_name, _, func_name = callable_path.partition(":") - module_name = module_name.strip() - func_name = func_name.strip() - - # Ensure cwd is importable so that project-local modules work. - cwd = str(Path.cwd()) - if cwd not in sys.path: - sys.path.insert(0, cwd) - - # Also add .agentops/ to sys.path so callable adapters placed there - # by ``agentops init`` are importable without manual path hacking. - agentops_dir = str(Path.cwd() / ".agentops") - if agentops_dir not in sys.path and Path(agentops_dir).is_dir(): - sys.path.insert(1, agentops_dir) - - try: - module = importlib.import_module(module_name) - except ModuleNotFoundError as exc: - raise ValueError( - f"Could not import module '{module_name}' from local.callable '{callable_path}'. " - f"Make sure the module is importable from your project root ({cwd}) " - f"or from the .agentops/ directory." - ) from exc - - func = getattr(module, func_name, None) - if func is None: - raise ValueError( - f"Module '{module_name}' has no function '{func_name}' " - f"(from local.callable '{callable_path}')" - ) - if not callable(func): - raise ValueError( - f"'{callable_path}' resolved to a non-callable object " - f"(type: {type(func).__name__})" - ) - return func - - -def _to_utc_timestamp(value: datetime) -> str: - return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") - - -class LocalAdapterBackend: - """Evaluation backend that invokes a local adapter per row. - - Supports two modes: - - **subprocess** (``local.adapter``) — spawns a command per row - - **callable** (``local.callable``) — imports and calls a Python function per row - """ - - def execute(self, context: BackendRunContext) -> BackendExecutionResult: - context.backend_output_dir.mkdir(parents=True, exist_ok=True) - - stdout_path = context.backend_output_dir / "backend.stdout.log" - stderr_path = context.backend_output_dir / "backend.stderr.log" - metrics_path = context.backend_output_dir / "backend_metrics.json" - - target = context.run_config.target - execution = context.run_config.execution - - assert target.local is not None - adapter_command = target.local.adapter - callable_path = target.local.callable - timeout_seconds = execution.timeout_seconds - - # Resolve the callable function once if in callable mode. - user_callable: Optional[Callable[[str, Dict[str, Any]], Dict[str, Any]]] = None - if callable_path: - user_callable = _load_callable(callable_path) - - started = datetime.now(timezone.utc) - started_perf = perf_counter() - - stdout_lines: List[str] = [] - stderr_lines: List[str] = [] - exit_code = 0 - - try: - bundle_config = load_bundle_config(context.bundle_path) - dataset_config = load_dataset_config(context.dataset_path) - - dataset_source_path = _resolve_dataset_source_path( - context.dataset_path, dataset_config.source.path - ) - rows = _load_jsonl(dataset_source_path) - total_rows = len(rows) - - enabled_evaluators = [e for e in bundle_config.evaluators if e.enabled] - _validate_supported_local_evaluators(enabled_evaluators) - enabled_evaluator_order = [e.name for e in enabled_evaluators] - - fallback_endpoint: Optional[str] = os.getenv("AZURE_OPENAI_ENDPOINT") - fallback_deployment: Optional[str] = os.getenv( - "AZURE_AI_MODEL_DEPLOYMENT_NAME" - ) or os.getenv("AZURE_OPENAI_DEPLOYMENT") - - foundry_evaluator_runtimes = _build_foundry_evaluator_runtimes( - enabled_evaluators, - fallback_endpoint=fallback_endpoint, - fallback_deployment=fallback_deployment, - ) - - input_field = dataset_config.format.input_field - expected_field = dataset_config.format.expected_field - - enabled_local_names = frozenset( - e.name for e in enabled_evaluators if e.source == "local" - ) - evaluator_aggregate_values: Dict[str, List[float]] = { - name: [] for name in enabled_evaluator_order - } - - row_metrics_payload: List[Dict[str, Any]] = [] - - mode_label = callable_path or adapter_command - logger.info( - "Local adapter backend: evaluating %d row(s) via '%s'", - total_rows, - mode_label, - ) - - for index, row in enumerate(rows, start=1): - logger.info("Processing row %d/%d", index, total_rows) - - if input_field not in row: - raise ValueError( - f"Dataset row {index} missing input field '{input_field}'" - ) - if expected_field not in row: - raise ValueError( - f"Dataset row {index} missing expected field '{expected_field}'" - ) - - prompt_text = _normalize_text(row[input_field]) - expected_text = _normalize_text(row[expected_field]) - - row_start = perf_counter() - - if user_callable is not None: - # --- Callable mode --- - try: - with agent_invoke_span( - target=context.run_config.target.type, - provider="local.callable", - ) as invoke_span: - context_dict = dict(row) - result = user_callable(prompt_text, context_dict) - if not isinstance(result, dict): - raise TypeError( - f"Callable must return a dict, got {type(result).__name__}" - ) - if "response" not in result: - raise ValueError( - "Callable return dict must include a 'response' key" - ) - prediction_text = _normalize_text( - result.get("response", "") - ) - returned_tool_calls = result.get("tool_calls") - set_agent_invoke_result(invoke_span) - except Exception as exc: # noqa: BLE001 - stderr_lines.append(f"row={index} error={exc!s}") - logger.error("Callable failed for row %d: %s", index, exc) - exit_code = 1 - continue - else: - # --- Subprocess mode --- - assert adapter_command is not None - adapter_input = json.dumps( - {"input": prompt_text, "expected": expected_text, **row} - ) - - try: - with agent_invoke_span( - target=context.run_config.target.type, - provider="local.subprocess", - ) as invoke_span: - completed = subprocess.run( - shlex.split( - adapter_command, posix=(sys.platform != "win32") - ), - input=adapter_input, - capture_output=True, - text=True, - timeout=timeout_seconds, - check=False, - ) - if completed.returncode != 0: - stderr_lines.append( - f"row={index} adapter exit_code={completed.returncode} " - f"stderr={completed.stderr.strip()}" - ) - logger.error( - "Adapter failed for row %d (exit %d): %s", - index, - completed.returncode, - completed.stderr.strip(), - ) - exit_code = 1 - continue - - adapter_output = json.loads(completed.stdout) - prediction_text = _normalize_text( - adapter_output.get("response", "") - ) - returned_tool_calls = adapter_output.get("tool_calls") - set_agent_invoke_result(invoke_span) - except subprocess.TimeoutExpired: - stderr_lines.append(f"row={index} error=adapter timeout") - logger.error("Adapter timed out for row %d", index) - exit_code = 1 - continue - except (json.JSONDecodeError, ValueError) as exc: - stderr_lines.append(f"row={index} error={exc!s}") - logger.error( - "Adapter returned invalid JSON for row %d: %s", index, exc - ) - exit_code = 1 - continue - - row_latency = perf_counter() - row_start - - row_metric_entries: List[Dict[str, Any]] = [] - - for runtime in foundry_evaluator_runtimes: - try: - score = _run_foundry_evaluator( - runtime, - prompt=prompt_text, - prediction=prediction_text, - expected=expected_text, - row=row, - ) - row_metric_entries.append( - {"name": runtime.name, "value": score} - ) - except Exception as exc: # noqa: BLE001 - stderr_lines.append( - f"row={index} evaluator={runtime.name} error={exc!s}" - ) - logger.error( - "Evaluator '%s' failed for row %d: %s", - runtime.name, - index, - exc, - ) - - if "exact_match" in enabled_local_names: - passed = prediction_text.lower() == expected_text.lower() - row_metric_entries.append( - {"name": "exact_match", "value": 1.0 if passed else 0.0} - ) - if "latency_seconds" in enabled_local_names: - row_metric_entries.append( - {"name": "latency_seconds", "value": row_latency} - ) - if "avg_latency_seconds" in enabled_local_names: - row_metric_entries.append( - {"name": "avg_latency_seconds", "value": row_latency} - ) - - for entry in row_metric_entries: - name = entry["name"] - if name in evaluator_aggregate_values: - evaluator_aggregate_values[name].append(entry["value"]) - - row_metrics_payload.append( - { - "row_index": index, - "input": prompt_text, - "response": prediction_text, - "context": row.get("context"), - "tool_calls": returned_tool_calls, - "metrics": row_metric_entries, - } - ) - stdout_lines.append( - f"row={index} expected={expected_text!r} prediction={prediction_text!r}" - ) - - aggregate_metrics: List[Dict[str, Any]] = [] - for name, values in evaluator_aggregate_values.items(): - if values: - aggregate_metrics.append( - {"name": name, "value": sum(values) / len(values)} - ) - - metrics_path.write_text( - json.dumps( - {"metrics": aggregate_metrics, "row_metrics": row_metrics_payload}, - indent=2, - ), - encoding="utf-8", - ) - - except Exception as exc: # noqa: BLE001 - stderr_lines.append(str(exc)) - logger.error("Local adapter backend failed: %s", exc) - exit_code = 1 - - finished = datetime.now(timezone.utc) - duration = perf_counter() - started_perf - - stdout_path.write_text("\n".join(stdout_lines), encoding="utf-8") - stderr_path.write_text("\n".join(stderr_lines), encoding="utf-8") - - return BackendExecutionResult( - backend="local_adapter", - command=callable_path or adapter_command or "local_adapter", - started_at=_to_utc_timestamp(started), - finished_at=_to_utc_timestamp(finished), - duration_seconds=round(duration, 3), - exit_code=exit_code, - stdout_file=stdout_path, - stderr_file=stderr_path, - ) diff --git a/src/agentops/cli/_planned.py b/src/agentops/cli/_planned.py deleted file mode 100644 index f593d7c8..00000000 --- a/src/agentops/cli/_planned.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Shared helper for planned (stub) commands.""" - -from __future__ import annotations - -import typer - - -def _planned_command(command_name: str) -> None: - """Print a message and exit with code 1 for unimplemented commands.""" - typer.echo( - "This command is planned but not implemented in this release:\n" - f" {command_name}\n" - "Please use the currently available commands" - " (`init`, `eval run`, `eval compare`, `report`, `config cicd`) for now." - ) - raise typer.Exit(code=1) diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 478536c7..e3246aa7 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -1,15 +1,13 @@ from __future__ import annotations +import shutil +from datetime import datetime, timezone from pathlib import Path from typing import Annotated import typer -from agentops.cli.browse_commands import ( - bundle_app, - run_app, -) -from agentops.services.reporting import generate_report_from_results +from agentops.utils.colors import style from agentops.utils.logging import get_logger, setup_logging app = typer.Typer( @@ -24,25 +22,23 @@ "`--config` (`-c`) and `--output` (`-o`)." ) ) -dataset_app = typer.Typer(help="Dataset utility commands.") -config_app = typer.Typer(help="Configuration utility commands.") report_app = typer.Typer(help="Reporting commands.") workflow_app = typer.Typer(help="CI/CD workflow commands.") -monitor_app = typer.Typer(help="Monitoring setup and operations.") -model_app = typer.Typer(help="Model discovery commands.") -agent_app = typer.Typer(help="Agent discovery commands.") skills_app = typer.Typer(help="Coding agent skills management.") +mcp_app = typer.Typer(help="MCP (Model Context Protocol) server commands.") +agent_app = typer.Typer( + help=( + "Watchdog agent commands. Combine AgentOps eval history, Azure Monitor " + "traces, and Foundry control-plane data to surface regressions, " + "latency, error, and safety findings." + ) +) app.add_typer(eval_app, name="eval") -app.add_typer(run_app, name="run") -app.add_typer(bundle_app, name="bundle") -app.add_typer(dataset_app, name="dataset") -app.add_typer(config_app, name="config") app.add_typer(report_app, name="report") app.add_typer(workflow_app, name="workflow") -app.add_typer(monitor_app, name="monitor") -app.add_typer(model_app, name="model") -app.add_typer(agent_app, name="agent") app.add_typer(skills_app, name="skills") +app.add_typer(mcp_app, name="mcp") +app.add_typer(agent_app, name="agent") log = get_logger(__name__) DEFAULT_REPORT_INPUT = Path(".agentops/results/latest/results.json") @@ -94,15 +90,6 @@ def _print_registration_result(result: object) -> None: typer.echo(f" * registered skills in {path}") -def _planned_command(command_name: str) -> None: - typer.echo( - "This command is planned but not implemented in this release:\n" - f" {command_name}\n" - "Please use the currently available commands (`init`, `eval run`, `report generate`) for now." - ) - raise typer.Exit(code=1) - - # --------------------------------------------------------------------------- # Global callback — configures logging before any command runs # --------------------------------------------------------------------------- @@ -152,33 +139,29 @@ def cmd_init( help="Workspace directory to initialise.", ), ) -> None: - """Initialise an AgentOps workspace (creates .agentops/).""" - from agentops.services.initializer import initialize_workspace + """Initialise an AgentOps workspace. + + Bootstraps the 1.0 minimal layout: a single ``agentops.yaml`` at the + project root and a tiny seed dataset under ``.agentops/data/smoke.jsonl``. + """ + from agentops.services.initializer import initialize_flat_workspace log.debug("cmd_init called force=%s dir=%s", force, directory) try: - result = initialize_workspace(directory=directory, force=force) + result = initialize_flat_workspace(directory=directory, force=force) except Exception as exc: typer.echo(f"Error: failed to initialize workspace: {exc}", err=True) raise typer.Exit(code=1) from exc - typer.echo(f"Initialized workspace: {result.workspace_dir}") - typer.echo( - "Summary: " - f"created_dirs={len(result.created_dirs)}, " - f"created_files={len(result.created_files)}, " - f"overwritten_files={len(result.overwritten_files)}, " - f"skipped_files={len(result.skipped_files)}" - ) - + typer.echo("Initialized AgentOps workspace.") for created in result.created_files: typer.echo(f" + created {created}") for overwritten in result.overwritten_files: typer.echo(f" ~ overwritten {overwritten}") for skipped in result.skipped_files: typer.echo(f" - skipped {skipped}") - typer.echo("") + typer.echo("Edit agentops.yaml to point at your agent, then run: agentops eval run") typer.echo("To install coding agent skills, run: agentops skills install") @@ -194,106 +177,181 @@ def cmd_eval_run( typer.Option( "--config", "-c", - help="Path to run.yaml (default: .agentops/run.yaml).", + help="Path to agentops.yaml. Defaults to ./agentops.yaml.", ), ] = None, output: Annotated[ Path | None, typer.Option("--output", "-o", help="Output directory for results."), ] = None, + baseline: Annotated[ + Path | None, + typer.Option( + "--baseline", + help="Path to a previous results.json to compare this run against.", + ), + ] = None, report_format: Annotated[ str, typer.Option("--format", "-f", help="Report format: md, html, or all.") ] = "md", ) -> None: - """Run an evaluation defined in a run.yaml file.""" - from agentops.services.runner import run_evaluation - + """Run an evaluation defined in agentops.yaml.""" if report_format not in ("md", "html", "all"): typer.echo("Error: --format must be md, html, or all.", err=True) raise typer.Exit(code=1) + config_path = _resolve_eval_config_path(config) log.debug( - "cmd_eval_run called config=%s output=%s format=%s", - config, + "cmd_eval_run called config=%s output=%s format=%s baseline=%s", + config_path, output, report_format, + baseline, ) - try: - run_result = run_evaluation( - config_path=config, output_override=output, report_format=report_format + + if not config_path.exists(): + typer.echo( + f"Error: config not found at {config_path}. " + "Run `agentops init` to scaffold a starter agentops.yaml.", + err=True, ) - except Exception as exc: - typer.echo(f"Error: evaluation failed: {exc}", err=True) - raise typer.Exit(code=1) from exc + raise typer.Exit(code=1) - typer.echo(f"Evaluation output directory: {run_result.output_dir}") - typer.echo(f"results.json: {run_result.results_path}") - typer.echo(f"report: {run_result.report_path}") + _run_flat_schema_eval( + config_path=config_path, + output=output, + baseline=baseline, + ) - if run_result.exit_code == 2: - typer.echo("Threshold status: FAILED") - raise typer.Exit(code=2) - typer.echo("Threshold status: PASSED") +def _resolve_eval_config_path(config: Path | None) -> Path: + if config is not None: + return config + return Path("agentops.yaml") -@eval_app.command("compare") -def cmd_eval_compare( - runs: Annotated[ - str, - typer.Option( - "--runs", help="Comma-separated run ids (example: ID1,ID2 or ID1,ID2,ID3)." - ), - ], - output: Annotated[ - Path | None, - typer.Option("--output", "-o", help="Output directory for comparison results."), - ] = None, - report_format: Annotated[ - str, typer.Option("--format", "-f", help="Report format: md, html, or all.") - ] = "md", +def _run_flat_schema_eval( + *, + config_path: Path, + output: Path | None, + baseline: Path | None, ) -> None: - """Compare two or more past evaluation runs.""" - from agentops.services.comparison import run_comparison - - if report_format not in ("md", "html", "all"): - typer.echo("Error: --format must be md, html, or all.", err=True) - raise typer.Exit(code=1) + from agentops.core.config_loader import load_agentops_config + from agentops.pipeline.orchestrator import ( + RunOptions, + exit_code_from, + run_evaluation, + ) - parts = [p.strip() for p in runs.split(",")] - if len(parts) < 2: - typer.echo( - "Error: --runs must contain at least two comma-separated run ids.", err=True - ) - raise typer.Exit(code=1) + try: + config_obj = load_agentops_config(config_path) + except Exception as exc: + typer.echo(f"Error: failed to load {config_path}: {exc}", err=True) + raise typer.Exit(code=1) from exc - log.debug( - "cmd_eval_compare called runs=%s output=%s format=%s", - parts, - output, - report_format, + use_default_layout = output is None + if use_default_layout: + output_dir: Path = _default_flat_output_dir(config_path) + else: + assert output is not None + output_dir = output + + options = RunOptions( + config_path=config_path.resolve(), + output_dir=output_dir, + baseline_path=baseline.resolve() if baseline else None, + progress=lambda msg: typer.echo(msg), ) + try: - result = run_comparison( - run_ids=parts, - output_dir=output, - report_format=report_format, - ) + result = run_evaluation(config_obj, options=options) except Exception as exc: - typer.echo(f"Error: comparison failed: {exc}", err=True) + typer.echo(f"Error: evaluation failed: {exc}", err=True) raise typer.Exit(code=1) from exc - typer.echo(f"comparison.json: {result.comparison_json_path}") - if result.comparison_md_path: - typer.echo(f"comparison.md: {result.comparison_md_path}") - if result.comparison_html_path: - typer.echo(f"comparison.html: {result.comparison_html_path}") + latest_dir = config_path.parent / ".agentops" / "results" / "latest" + if output_dir.resolve() != latest_dir.resolve(): + try: + _mirror_to_latest(output_dir, latest_dir) + except Exception as exc: # pragma: no cover - mirror failures shouldn't fail the run + typer.echo( + f"Warning: failed to update {latest_dir}: {exc}", + err=True, + ) + latest_dir = None # type: ignore[assignment] + else: + latest_dir = None # type: ignore[assignment] + + typer.echo(f"Evaluation output directory: {style(str(output_dir), 'cyan')}") + typer.echo(f"results.json: {style(str(output_dir / 'results.json'), 'cyan')}") + typer.echo(f"report.md: {style(str(output_dir / 'report.md'), 'cyan')}") + if latest_dir is not None: + typer.echo(f"latest/: {style(str(latest_dir), 'cyan')}") + if result.summary.overall_passed: + typer.echo(f"Threshold status: {style('PASSED', 'bold', 'green')}") + return + typer.echo(f"Threshold status: {style('FAILED', 'bold', 'red')}") + raise typer.Exit(code=exit_code_from(result)) + - if result.has_regressions: - typer.echo("Comparison verdict: REGRESSIONS DETECTED") - raise typer.Exit(code=2) +def _default_flat_output_dir(config_path: Path) -> Path: + base = config_path.parent / ".agentops" / "results" + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") + return base / timestamp + + +def _mirror_to_latest(source: Path, latest: Path) -> None: + """Replace ``latest`` with a copy of ``source``.""" + if latest.exists(): + if latest.is_symlink() or latest.is_file(): + latest.unlink() + else: + shutil.rmtree(latest) + shutil.copytree(source, latest) + + +def _is_flat_results(results_path: Path) -> bool: + """Return True when results.json was produced by the flat pipeline.""" + if not results_path.exists(): + return False + try: + import json as _json + data = _json.loads(results_path.read_text(encoding="utf-8")) + except Exception: + return False + if not isinstance(data, dict): + return False + target = data.get("target") + return ( + data.get("version") == 1 + and isinstance(target, dict) + and "kind" in target + and "bundle" not in data + ) + + +def _regenerate_flat_report( + *, + results_path: Path, + output_path: Path | None, + report_format: str, +) -> Path: + """Render report.md from a flat-pipeline results.json.""" + import json as _json + + from agentops.core.results import RunResult + from agentops.pipeline import reporter as flat_reporter + + if report_format not in ("md", "all"): + raise ValueError( + "Only --format md is supported (got %r)" % report_format + ) + payload = _json.loads(results_path.read_text(encoding="utf-8")) + result = RunResult.model_validate(payload) + target = output_path or (results_path.parent / "report.md") + target.write_text(flat_reporter.render(result), encoding="utf-8") + return target - typer.echo("Comparison verdict: NO REGRESSIONS") # --------------------------------------------------------------------------- @@ -318,12 +376,12 @@ def cmd_report_generate( typer.Option("--out", help="Output path for report."), ] = None, report_format: Annotated[ - str, typer.Option("--format", "-f", help="Report format: md, html, or all.") + str, typer.Option("--format", "-f", help="Report format: md (default).") ] = "md", ) -> None: - """Regenerate report from a results.json file.""" - if report_format not in ("md", "html", "all"): - typer.echo("Error: --format must be md, html, or all.", err=True) + """Regenerate report.md from a results.json file.""" + if report_format not in ("md", "all"): + typer.echo("Error: --format must be md or all.", err=True) raise typer.Exit(code=1) resolved_results_in = results_in or DEFAULT_REPORT_INPUT @@ -333,8 +391,23 @@ def cmd_report_generate( report_out, report_format, ) + + if not resolved_results_in.exists(): + typer.echo( + f"Error: results not found at {resolved_results_in}.", err=True + ) + raise typer.Exit(code=1) + + if not _is_flat_results(resolved_results_in): + typer.echo( + f"Error: {resolved_results_in} is not an AgentOps 1.0 results.json. " + "Re-run `agentops eval run` to regenerate it.", + err=True, + ) + raise typer.Exit(code=1) + try: - report_result = generate_report_from_results( + output_path = _regenerate_flat_report( results_path=resolved_results_in, output_path=report_out, report_format=report_format, @@ -343,52 +416,8 @@ def cmd_report_generate( typer.echo(f"Error: report generation failed: {exc}", err=True) raise typer.Exit(code=1) from exc - typer.echo(f"Loaded results: {report_result.input_results_path}") - typer.echo(f"Generated report: {report_result.output_report_path}") - if report_result.html_report_path: - typer.echo(f"Generated report: {report_result.html_report_path}") - - -@report_app.command("show") -def cmd_report_show() -> None: - """View reports in table format (planned).""" - _planned_command("agentops report show") - - -@report_app.command("export") -def cmd_report_export() -> None: - """Export reports in JSON/Markdown/CSV formats (planned).""" - _planned_command("agentops report export") - - -@dataset_app.command("validate") -def cmd_dataset_validate() -> None: - """Validate dataset files (planned).""" - _planned_command("agentops dataset validate") - - -@dataset_app.command("describe") -def cmd_dataset_describe() -> None: - """Describe dataset schema and shape (planned).""" - _planned_command("agentops dataset describe") - - -@dataset_app.command("import") -def cmd_dataset_import() -> None: - """Import external datasets (planned).""" - _planned_command("agentops dataset import") - - -@config_app.command("validate") -def cmd_config_validate() -> None: - """Validate configuration files (planned).""" - _planned_command("agentops config validate") - - -@config_app.command("show") -def cmd_config_show() -> None: - """Show merged runtime config (planned).""" - _planned_command("agentops config show") + typer.echo(f"Loaded results: {resolved_results_in}") + typer.echo(f"Generated report: {output_path}") # --------------------------------------------------------------------------- @@ -406,18 +435,50 @@ def cmd_workflow_generate( "--dir", help="Target repository root directory.", ), + kinds: str = typer.Option( + "", + "--kinds", + help=( + "Comma-separated subset of workflow kinds to generate. " + "Valid values: pr, dev, qa, prod. " + "Default (empty) generates all four." + ), + ), ) -> None: - """Generate GitHub Actions workflows for AgentOps evaluation. + """Generate the AgentOps GitFlow GitHub Actions workflows. + + By default writes all four templates that map to a classic GitFlow + setup with three GitHub Environments (dev, qa, production): - Auto-detects which pipelines to create based on the .agentops/ workspace: - PR evaluation (always), CI evaluation (multiple configs), and CD pipeline - with safety QA gate + deploy placeholder (multiple configs). + - agentops-pr.yml (PR gate; PRs to develop, release/**, main) + - agentops-deploy-dev.yml (push to develop -> environment: dev) + - agentops-deploy-qa.yml (push to release/** -> environment: qa) + - agentops-deploy-prod.yml (push to main -> environment: production) + + Use --kinds to opt into a subset, e.g. --kinds pr,dev. """ - from agentops.services.cicd import generate_cicd_workflows + from agentops.services.cicd import ALL_KINDS, generate_cicd_workflows + + log.debug( + "cmd_workflow_generate called force=%s dir=%s kinds=%r", force, directory, kinds + ) + + selected: list[str] | None = None + if kinds.strip(): + selected = [k.strip() for k in kinds.split(",") if k.strip()] + invalid = [k for k in selected if k not in ALL_KINDS] + if invalid: + typer.echo( + f"Error: unknown --kinds value(s): {', '.join(invalid)}. " + f"Valid: {', '.join(ALL_KINDS)}.", + err=True, + ) + raise typer.Exit(code=1) - log.debug("cmd_workflow_generate called force=%s dir=%s", force, directory) try: - result = generate_cicd_workflows(directory=directory, force=force) + result = generate_cicd_workflows( + directory=directory, force=force, kinds=selected + ) except Exception as exc: typer.echo(f"Error: failed to generate CI/CD workflows: {exc}", err=True) raise typer.Exit(code=1) from exc @@ -433,49 +494,30 @@ def cmd_workflow_generate( typer.echo("") typer.echo("Next steps:") typer.echo( - " 1. Set GitHub repository variables: AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID" + " 1. Configure Azure Workload Identity Federation (OIDC) and set " + "repository variables AZURE_CLIENT_ID, AZURE_TENANT_ID, " + "AZURE_SUBSCRIPTION_ID, AZURE_AI_FOUNDRY_PROJECT_ENDPOINT." ) typer.echo( - " 2. Set GitHub repository secret: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT" + " 2. Create three GitHub Environments: 'dev', 'qa', 'production'. " + "Add required reviewers to 'production'." ) typer.echo( - " 3. Configure Azure Workload Identity Federation (see docs/ci-github-actions.md)" + " 3. Open each agentops-deploy-*.yml and replace the Build/Deploy " + "placeholder steps with your stack's commands " + "(snippets are provided in comments)." + ) + typer.echo( + " 4. In Settings -> Branches, require the 'AgentOps PR' status check " + "on develop and main." + ) + typer.echo( + " 5. Commit and push. See docs/ci-github-actions.md for the full guide." ) - typer.echo(" 4. Commit and push the workflow files") elif result.skipped_files: typer.echo("No files written. Use --force to overwrite existing workflows.") -@monitor_app.command("setup") -def cmd_monitor_setup() -> None: - """Set up monitoring resources (planned).""" - _planned_command("agentops monitor setup") - - -@monitor_app.command("show") -def cmd_monitor_show() -> None: - """Show monitoring dashboard setup instructions (planned).""" - _planned_command("agentops monitor show") - - -@monitor_app.command("configure") -def cmd_monitor_configure() -> None: - """Configure monitoring alerts (planned).""" - _planned_command("agentops monitor configure") - - -@model_app.command("list") -def cmd_model_list() -> None: - """List chat-capable models in Foundry project (planned).""" - _planned_command("agentops model list") - - -@agent_app.command("list") -def cmd_agent_list() -> None: - """List agents in Foundry project (planned).""" - _planned_command("agentops agent list") - - # --------------------------------------------------------------------------- # agentops skills install # --------------------------------------------------------------------------- @@ -586,5 +628,251 @@ def cmd_skills_install( _print_registration_result(reg_result) +# --------------------------------------------------------------------------- +# agentops mcp serve +# --------------------------------------------------------------------------- + + +@mcp_app.command("serve") +def cmd_mcp_serve() -> None: + """Start the AgentOps MCP server on stdio. + + Exposes the AgentOps workflow (init, eval run, report show, results + summary, dataset add, list runs, workflow init) as MCP tools so that + MCP-aware coding agents can drive AgentOps directly. + + Requires the optional ``mcp`` extra: + + pip install agentops-toolkit[mcp] + """ + try: + from agentops.mcp.server import serve_stdio + except RuntimeError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + + try: + serve_stdio() + except RuntimeError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + + +# --------------------------------------------------------------------------- +# `agentops agent` commands +# --------------------------------------------------------------------------- + + +def _resolve_agent_config_path(workspace: Path, explicit: Path | None) -> Path | None: + if explicit is not None: + return explicit + candidate = workspace / ".agentops" / "agent.yaml" + return candidate if candidate.exists() else None + + +@agent_app.command("analyze") +def cmd_agent_analyze( + workspace: Annotated[ + Path, + typer.Option( + "--workspace", + "-w", + help="Project root containing `.agentops/`.", + ), + ] = Path("."), + config_path: Annotated[ + Path | None, + typer.Option( + "--config", + "-c", + help="Path to `agent.yaml` (default: `.agentops/agent.yaml`).", + ), + ] = None, + out: Annotated[ + Path, + typer.Option( + "--out", + "-o", + help="Where to write the Markdown report.", + ), + ] = Path(".agentops/agent/report.md"), + lookback_days: Annotated[ + int | None, + typer.Option( + "--lookback-days", + help="Override the lookback window for production telemetry.", + ), + ] = None, + severity_fail: Annotated[ + str, + typer.Option( + "--severity-fail", + help="Exit 2 when a finding at or above this severity is produced.", + ), + ] = "critical", + categories: Annotated[ + str | None, + typer.Option( + "--categories", + help=( + "Comma-separated list of categories to include " + "(quality, performance, reliability, security). " + "Default: include all." + ), + ), + ] = None, + exclude_rules: Annotated[ + str | None, + typer.Option( + "--exclude-rules", + help=( + "Comma-separated list of posture rule ids to skip " + "(for example `waf.security.diagnostic_settings`)." + ), + ), + ] = None, +) -> None: + """Run the watchdog agent analyzer and emit a Markdown report. + + Exit codes: + + * ``0`` — analyzer ran cleanly and no finding met `--severity-fail`. + * ``2`` — at least one finding meets the configured severity floor. + * ``1`` — runtime/configuration error. + """ + from agentops.agent.analyzer import analyze + from agentops.agent.config import load_agent_config + from agentops.agent.findings import Severity + from agentops.agent.report import render_report + + workspace = workspace.resolve() + resolved_config = _resolve_agent_config_path(workspace, config_path) + + try: + config = load_agent_config(resolved_config) + except Exception as exc: + typer.echo(f"Error loading agent config: {exc}", err=True) + raise typer.Exit(code=1) from exc + + if lookback_days is not None: + config = config.model_copy(update={"lookback_days": lookback_days}) + + try: + severity_floor = Severity(severity_fail.lower()) + except ValueError as exc: + typer.echo( + f"Error: invalid --severity-fail '{severity_fail}'. " + "Use one of: info, warning, critical.", + err=True, + ) + raise typer.Exit(code=1) from exc + + try: + result = analyze( + workspace, + config, + categories=( + [c for c in categories.split(",") if c.strip()] + if categories + else None + ), + exclude_rules=( + [r for r in exclude_rules.split(",") if r.strip()] + if exclude_rules + else None + ), + ) + except Exception as exc: # pragma: no cover + typer.echo(f"Error running analyzer: {exc}", err=True) + raise typer.Exit(code=1) from exc + + out_path = out if out.is_absolute() else workspace / out + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(render_report(result), encoding="utf-8") + + typer.echo(f"Wrote {out_path}") + typer.echo(f"Findings: {len(result.findings)}") + if result.max_severity is not None: + typer.echo(f"Max severity: {result.max_severity.value}") + + if result.max_severity is not None and result.max_severity >= severity_floor: + raise typer.Exit(code=2) + + +@agent_app.command("serve") +def cmd_agent_serve( + host: Annotated[ + str, typer.Option("--host", help="Bind host.") + ] = "0.0.0.0", + port: Annotated[ + int, typer.Option("--port", help="Bind port.") + ] = 8080, + workspace: Annotated[ + Path, + typer.Option("--workspace", "-w", help="Project root for analysis."), + ] = Path("."), + config_path: Annotated[ + Path | None, + typer.Option( + "--config", + "-c", + help="Path to `agent.yaml` (default: `.agentops/agent.yaml`).", + ), + ] = None, + no_verify: Annotated[ + bool, + typer.Option( + "--no-verify", + help="Skip Copilot Extensions signature validation (dev only).", + ), + ] = False, + workers: Annotated[ + int, typer.Option("--workers", help="Uvicorn worker count.") + ] = 1, +) -> None: + """Start the watchdog agent as a Copilot Extension HTTP server. + + Exposes ``POST /agents/messages`` (Copilot Extensions protocol), + ``GET /healthz`` and ``GET /``. Requires the ``[agent]`` extra: + + pip install agentops-toolkit[agent] + """ + try: + import uvicorn + except ImportError as exc: + typer.echo( + "Error: agent extras not installed. " + "Run `pip install agentops-toolkit[agent]`.", + err=True, + ) + raise typer.Exit(code=1) from exc + + from agentops.agent.config import load_agent_config + from agentops.agent.server.app import create_app + + workspace = workspace.resolve() + resolved_config = _resolve_agent_config_path(workspace, config_path) + + try: + config = load_agent_config(resolved_config) + except Exception as exc: + typer.echo(f"Error loading agent config: {exc}", err=True) + raise typer.Exit(code=1) from exc + + fastapi_app = create_app( + workspace=workspace, + config=config, + verify_signature=not no_verify, + ) + + if no_verify: + typer.echo( + "WARNING: Copilot Extensions signature validation is disabled. " + "Use only for local development." + ) + + uvicorn.run(fastapi_app, host=host, port=port, workers=workers) + + def main() -> None: app() diff --git a/src/agentops/cli/browse_commands.py b/src/agentops/cli/browse_commands.py deleted file mode 100644 index c3db6139..00000000 --- a/src/agentops/cli/browse_commands.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Browse sub-commands: bundle list/show, run list/show/view.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Annotated - -import typer - -from agentops.cli._planned import _planned_command - -run_app = typer.Typer(help="Run history and inspection commands.") -bundle_app = typer.Typer(help="Bundle browsing commands.") - - -# --------------------------------------------------------------------------- -# bundle list / show -# --------------------------------------------------------------------------- - - -@bundle_app.command("list") -def cmd_bundle_list( - directory: Path = typer.Option( - Path("."), - "--dir", - help="Workspace directory.", - ), -) -> None: - """List available evaluation bundles.""" - from agentops.services.browse import list_bundles - - try: - result = list_bundles(directory=directory) - except FileNotFoundError as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(code=1) from exc - - if not result.bundles: - typer.echo(f"No bundles found in {result.bundles_dir}") - return - - typer.echo(f"Bundles in {result.bundles_dir}:\n") - for b in result.bundles: - evals = ", ".join(b.evaluators) if b.evaluators else "(none)" - typer.echo(f" {b.name}") - if b.description: - typer.echo(f" {b.description}") - typer.echo(f" evaluators: {evals}") - typer.echo(f" thresholds: {b.thresholds}") - typer.echo("") - - -@bundle_app.command("show") -def cmd_bundle_show( - bundle_name: str = typer.Argument(help="Bundle name or filename (without .yaml)."), - directory: Path = typer.Option( - Path("."), - "--dir", - help="Workspace directory.", - ), -) -> None: - """Show details of an evaluation bundle.""" - from agentops.services.browse import show_bundle - - try: - detail = show_bundle(bundle_name=bundle_name, directory=directory) - except (FileNotFoundError, ValueError) as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(code=1) from exc - - typer.echo(f"Bundle: {detail.name}") - typer.echo(f"Path: {detail.path}") - if detail.description: - typer.echo(f"Description: {detail.description}") - if detail.metadata: - typer.echo(f"Metadata: {detail.metadata}") - typer.echo("") - typer.echo("Evaluators:") - for e in detail.evaluators: - status = "enabled" if e["enabled"] else "disabled" - typer.echo(f" {e['name']} (source={e['source']}, {status})") - typer.echo("") - typer.echo("Thresholds:") - for t in detail.thresholds: - value = t["value"] if t["value"] is not None else "" - typer.echo(f" {t['evaluator']} {t['criteria']} {value}") - - -# --------------------------------------------------------------------------- -# run list / show / view -# --------------------------------------------------------------------------- - - -@run_app.command("list") -def cmd_run_list( - directory: Path = typer.Option( - Path("."), - "--dir", - help="Workspace directory.", - ), -) -> None: - """List past evaluation runs.""" - from agentops.services.browse import list_runs - - try: - result = list_runs(directory=directory) - except FileNotFoundError as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(code=1) from exc - - if not result.runs: - typer.echo(f"No runs found in {result.results_dir}") - return - - typer.echo(f"Runs in {result.results_dir}:\n") - for run in result.runs: - status = "PASS" if run.overall_passed else "FAIL" - typer.echo( - f" {run.run_id} {status:<4} " - f"bundle={run.bundle_name} dataset={run.dataset_name} " - f"duration={run.duration_seconds:.1f}s" - ) - - -@run_app.command("show") -def cmd_run_show( - run_id: str = typer.Argument(help="Run ID (timestamp folder name or 'latest')."), - directory: Path = typer.Option( - Path("."), - "--dir", - help="Workspace directory.", - ), -) -> None: - """Show summary of a past evaluation run.""" - from agentops.services.browse import show_run - - try: - detail = show_run(run_id=run_id, directory=directory) - except (FileNotFoundError, ValueError) as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(code=1) from exc - - status = "PASS" if detail.overall_passed else "FAIL" - typer.echo(f"Run: {detail.run_id}") - typer.echo(f"Status: {status}") - typer.echo(f"Bundle: {detail.bundle_name}") - typer.echo(f"Dataset: {detail.dataset_name}") - typer.echo(f"Backend: {detail.backend}") - typer.echo(f"Started: {detail.started_at}") - typer.echo(f"Duration: {detail.duration_seconds:.1f}s") - typer.echo(f"Items: {detail.items_passed}/{detail.items_total} passed") - typer.echo("") - typer.echo("Metrics:") - for m in detail.metrics: - typer.echo(f" {m['name']:<40} {m['value']:.4f}") - if detail.thresholds: - typer.echo("") - typer.echo("Thresholds:") - for t in detail.thresholds: - mark = "PASS" if t["passed"] else "FAIL" - typer.echo( - f" {t['evaluator']:<40} {t['criteria']} {t['expected']:<10} " - f"actual={t['actual']:<10} {mark}" - ) - if detail.foundry_url: - typer.echo(f"\nFoundry portal: {detail.foundry_url}") - if detail.report_path: - typer.echo(f"Report: {detail.report_path}") - - -@run_app.command("view") -def cmd_run_view( - run_id: str, - entry: Annotated[ - int | None, - typer.Option("--entry", help="Optional row/entry index for deep inspection."), - ] = None, -) -> None: - """Deep-inspect run details (planned).""" - _ = run_id, entry - _planned_command("agentops run view [--entry N]") diff --git a/src/agentops/cli/config_commands.py b/src/agentops/cli/config_commands.py deleted file mode 100644 index f435b444..00000000 --- a/src/agentops/cli/config_commands.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Config sub-commands: config validate, config show, config cicd.""" - -from __future__ import annotations - -from pathlib import Path - -import typer - -from agentops.cli._planned import _planned_command -from agentops.utils.logging import get_logger - -log = get_logger(__name__) - -config_app = typer.Typer(help="Configuration utility commands.") - - -@config_app.command("validate") -def cmd_config_validate() -> None: - """Validate configuration files (planned).""" - _planned_command("agentops config validate") - - -@config_app.command("show") -def cmd_config_show() -> None: - """Show merged runtime config (planned).""" - _planned_command("agentops config show") - - -@config_app.command("cicd") -def cmd_config_cicd( - force: bool = typer.Option( - False, "--force", help="Overwrite existing workflow file." - ), - directory: Path = typer.Option( - Path("."), - "--dir", - help="Target repository root directory.", - ), -) -> None: - """Generate a GitHub Actions workflow for AgentOps evaluation.""" - from agentops.services.cicd import generate_cicd_workflow - - log.debug("cmd_config_cicd called force=%s dir=%s", force, directory) - try: - result = generate_cicd_workflow(directory=directory, force=force) - except Exception as exc: - typer.echo(f"Error: failed to generate CI/CD workflow: {exc}", err=True) - raise typer.Exit(code=1) from exc - - for created in result.created_files: - typer.echo(f" + created {created}") - for overwritten in result.overwritten_files: - typer.echo(f" ~ overwritten {overwritten}") - for skipped in result.skipped_files: - typer.echo(f" - skipped {skipped} (use --force to overwrite)") - - if result.created_files or result.overwritten_files: - typer.echo("") - typer.echo("Next steps:") - typer.echo( - " 1. Set GitHub repository variables: AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID" - ) - typer.echo( - " 2. Set GitHub repository secret: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT" - ) - typer.echo( - " 3. Configure Azure Workload Identity Federation (see docs/ci-github-actions.md)" - ) - typer.echo(" 4. Commit and push the workflow file") - elif result.skipped_files: - typer.echo("No files written. Use --force to overwrite existing workflow.") diff --git a/src/agentops/cli/dataset_commands.py b/src/agentops/cli/dataset_commands.py deleted file mode 100644 index c768963c..00000000 --- a/src/agentops/cli/dataset_commands.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Dataset sub-commands: dataset validate, dataset describe, dataset import.""" - -from __future__ import annotations - -import typer - -from agentops.cli._planned import _planned_command - -dataset_app = typer.Typer(help="Dataset utility commands.") - - -@dataset_app.command("validate") -def cmd_dataset_validate() -> None: - """Validate dataset files (planned).""" - _planned_command("agentops dataset validate") - - -@dataset_app.command("describe") -def cmd_dataset_describe() -> None: - """Describe dataset schema and shape (planned).""" - _planned_command("agentops dataset describe") - - -@dataset_app.command("import") -def cmd_dataset_import() -> None: - """Import external datasets (planned).""" - _planned_command("agentops dataset import") diff --git a/src/agentops/cli/eval_commands.py b/src/agentops/cli/eval_commands.py deleted file mode 100644 index efb10c73..00000000 --- a/src/agentops/cli/eval_commands.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Evaluation sub-commands: eval run, eval compare.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Annotated - -import typer - -from agentops.utils.logging import get_logger - -log = get_logger(__name__) - -eval_app = typer.Typer( - help=( - "Evaluation sub-commands. " - "Use `agentops eval run --help` to see run options like " - "`--config` (`-c`) and `--output` (`-o`)." - ) -) - - -@eval_app.command("run") -def cmd_eval_run( - config: Annotated[ - Path | None, - typer.Option( - "--config", - "-c", - help="Path to run.yaml (default: .agentops/run.yaml).", - ), - ] = None, - output: Annotated[ - Path | None, - typer.Option("--output", "-o", help="Output directory for results."), - ] = None, - report_format: Annotated[ - str, typer.Option("--format", "-f", help="Report format: md, html, or all.") - ] = "md", -) -> None: - """Run an evaluation defined in a run.yaml file.""" - from agentops.services.runner import run_evaluation - - if report_format not in ("md", "html", "all"): - typer.echo("Error: --format must be md, html, or all.", err=True) - raise typer.Exit(code=1) - - log.debug( - "cmd_eval_run called config=%s output=%s format=%s", - config, - output, - report_format, - ) - try: - run_result = run_evaluation( - config_path=config, output_override=output, report_format=report_format - ) - except Exception as exc: - typer.echo(f"Error: evaluation failed: {exc}", err=True) - raise typer.Exit(code=1) from exc - - typer.echo(f"Evaluation output directory: {run_result.output_dir}") - typer.echo(f"results.json: {run_result.results_path}") - typer.echo(f"report: {run_result.report_path}") - - if run_result.exit_code == 2: - typer.echo("Threshold status: FAILED") - raise typer.Exit(code=2) - - typer.echo("Threshold status: PASSED") - - -@eval_app.command("compare") -def cmd_eval_compare( - runs: Annotated[ - str, - typer.Option( - "--runs", help="Comma-separated run ids (example: ID1,ID2 or ID1,ID2,ID3)." - ), - ], - output: Annotated[ - Path | None, - typer.Option("--output", "-o", help="Output directory for comparison results."), - ] = None, - report_format: Annotated[ - str, typer.Option("--format", "-f", help="Report format: md, html, or all.") - ] = "md", -) -> None: - """Compare two or more past evaluation runs.""" - from agentops.services.comparison import run_comparison - - if report_format not in ("md", "html", "all"): - typer.echo("Error: --format must be md, html, or all.", err=True) - raise typer.Exit(code=1) - - parts = [p.strip() for p in runs.split(",")] - if len(parts) < 2: - typer.echo( - "Error: --runs must contain at least two comma-separated run ids.", err=True - ) - raise typer.Exit(code=1) - - log.debug( - "cmd_eval_compare called runs=%s output=%s format=%s", - parts, - output, - report_format, - ) - try: - result = run_comparison( - run_ids=parts, - output_dir=output, - report_format=report_format, - ) - except Exception as exc: - typer.echo(f"Error: comparison failed: {exc}", err=True) - raise typer.Exit(code=1) from exc - - typer.echo(f"comparison.json: {result.comparison_json_path}") - if result.comparison_md_path: - typer.echo(f"comparison.md: {result.comparison_md_path}") - if result.comparison_html_path: - typer.echo(f"comparison.html: {result.comparison_html_path}") - - if result.has_regressions: - typer.echo("Comparison verdict: REGRESSIONS DETECTED") - raise typer.Exit(code=2) - - typer.echo("Comparison verdict: NO REGRESSIONS") diff --git a/src/agentops/cli/report_commands.py b/src/agentops/cli/report_commands.py deleted file mode 100644 index 93c4ac3c..00000000 --- a/src/agentops/cli/report_commands.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Report sub-commands: report, report show, report export.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Annotated - -import typer - -from agentops.cli._planned import _planned_command -from agentops.utils.logging import get_logger - -log = get_logger(__name__) - -DEFAULT_REPORT_INPUT = Path(".agentops/results/latest/results.json") - -report_app = typer.Typer(help="Reporting commands.", invoke_without_command=True) - - -@report_app.callback(invoke_without_command=True) -def cmd_report( - ctx: typer.Context, - results_in: Annotated[ - Path | None, - typer.Option( - "--in", - help=( - "Path to results.json. " - "If omitted, uses .agentops/results/latest/results.json" - ), - ), - ] = None, - report_out: Annotated[ - Path | None, - typer.Option("--out", help="Output path for report."), - ] = None, - report_format: Annotated[ - str, typer.Option("--format", "-f", help="Report format: md, html, or all.") - ] = "md", -) -> None: - """Regenerate report from a results.json file.""" - from agentops.services.reporting import generate_report_from_results - - if ctx.invoked_subcommand is not None: - return - - if report_format not in ("md", "html", "all"): - typer.echo("Error: --format must be md, html, or all.", err=True) - raise typer.Exit(code=1) - - resolved_results_in = results_in or DEFAULT_REPORT_INPUT - log.debug( - "cmd_report called in=%s out=%s format=%s", - resolved_results_in, - report_out, - report_format, - ) - try: - report_result = generate_report_from_results( - results_path=resolved_results_in, - output_path=report_out, - report_format=report_format, - ) - except Exception as exc: - typer.echo(f"Error: report generation failed: {exc}", err=True) - raise typer.Exit(code=1) from exc - - typer.echo(f"Loaded results: {report_result.input_results_path}") - typer.echo(f"Generated report: {report_result.output_report_path}") - if report_result.html_report_path: - typer.echo(f"Generated report: {report_result.html_report_path}") - - -@report_app.command("show") -def cmd_report_show() -> None: - """View reports in table format (planned).""" - _planned_command("agentops report show") - - -@report_app.command("export") -def cmd_report_export() -> None: - """Export reports in JSON/Markdown/CSV formats (planned).""" - _planned_command("agentops report export") diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py new file mode 100644 index 00000000..72d97b19 --- /dev/null +++ b/src/agentops/core/agentops_config.py @@ -0,0 +1,444 @@ +"""Flat ``agentops.yaml`` schema for AgentOps 1.0. + +This module defines the user-facing configuration shape that replaces the +layered ``run.yaml`` + ``bundle.yaml`` + ``dataset.yaml`` files of pre-1.0 +AgentOps. + +Design goals: + +* One file. ``agentops.yaml`` is the single source of truth. +* No ``scenario`` field. The toolkit derives the target type from the + ``agent`` value and the evaluator set from the dataset row shape (see + :mod:`agentops.core.evaluators`). +* No bundle / dataset YAML configs. Datasets are plain JSONL files referenced + directly by path. + +The minimal valid config is three lines:: + + version: 1 + agent: my-rag-agent:3 + dataset: ./qa.jsonl + +The :func:`classify_agent` helper resolves ``agent`` into one of four target +kinds — ``foundry_prompt``, ``foundry_hosted``, ``http_json``, or +``model_direct`` — based on the value shape and optional ``protocol`` field. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +# --------------------------------------------------------------------------- +# Public type aliases +# --------------------------------------------------------------------------- + +#: Wire protocol for hosted / HTTP targets. +Protocol = Literal["responses", "invocations", "http-json"] + +#: How thresholds compare against measured metric values. +Criteria = Literal[">=", ">", "<=", "<", "==", "true", "false"] + +#: Resolved target kind. Derived from the ``agent`` value, never set by the user. +TargetKind = Literal[ + "foundry_prompt", # name:version + "foundry_hosted", # https://...foundry... endpoint + "http_json", # any other https URL + "model_direct", # model: +] + +#: Where to publish the evaluation run. ``None`` keeps results local-only. +PublishTarget = Literal["foundry", "foundry_cloud"] + + +# --------------------------------------------------------------------------- +# Threshold model +# --------------------------------------------------------------------------- + + +class Threshold(BaseModel): + """A pass/fail rule for a single metric. + + Users typically write thresholds as a dict keyed by metric name in + ``agentops.yaml``:: + + thresholds: + groundedness: ">=3" + coherence: ">=3" + avg_latency_seconds: "<=10" + + Each value is parsed by :meth:`from_expression` into a ``Threshold``. + """ + + metric: str + criteria: Criteria + value: Optional[float] = None + + model_config = ConfigDict(frozen=True) + + @classmethod + def from_expression(cls, metric: str, expression: Any) -> "Threshold": + """Parse a shorthand string like ``">=3"`` or a bool like ``true``.""" + if isinstance(expression, bool): + return cls(metric=metric, criteria="true" if expression else "false") + if isinstance(expression, (int, float)): + return cls(metric=metric, criteria=">=", value=float(expression)) + if not isinstance(expression, str): + raise ValueError( + f"threshold for {metric!r} must be a string, number, or bool" + ) + text = expression.strip() + if text.lower() in {"true", "false"}: + return cls(metric=metric, criteria=text.lower()) # type: ignore[arg-type] + for op in (">=", "<=", "==", ">", "<"): + if text.startswith(op): + rest = text[len(op):].strip() + try: + return cls(metric=metric, criteria=op, value=float(rest)) # type: ignore[arg-type] + except ValueError as exc: + raise ValueError( + f"threshold for {metric!r}: cannot parse number from {text!r}" + ) from exc + raise ValueError( + f"threshold for {metric!r}: expected '>=N', '<=N', '>N', ' str: + if not value.strip(): + raise ValueError("evaluator name must be non-empty") + return value + + +# --------------------------------------------------------------------------- +# Top-level config +# --------------------------------------------------------------------------- + + +_LEGACY_TOP_LEVEL_KEYS = { + "target", + "bundle", + "execution", + "output", + "scenario", + "backend", + "run", +} + + +class AgentOpsConfig(BaseModel): + """Top-level ``agentops.yaml`` model. + + Fields: + + ``version`` + Schema version. Must be ``1`` in this release. + + ``agent`` + The thing under evaluation. One of: + + * ``":"`` — a Foundry prompt agent (e.g. ``"my-rag:3"``). + * ``"https://..."`` — a Foundry hosted endpoint or any HTTP/JSON agent. + * ``"model:"`` — a Foundry model deployment (raw model). + + See :func:`classify_agent` for the full resolution table. + + ``dataset`` + Relative path to a JSONL file with one evaluation row per line. Rows + must contain at least ``input`` and ``expected``; optional fields + ``context``, ``tool_calls``, and ``tool_definitions`` drive evaluator + auto-selection. + + ``thresholds`` + Optional dict of metric name → criteria expression. When omitted, the + evaluator catalog provides sensible defaults per metric. + + ``protocol`` + Optional, only relevant for URL-based ``agent`` values. Defaults to + ``"responses"`` for Foundry hosted endpoints and ``"http-json"`` for + any other HTTPS URL. + + ``request_field`` / ``response_field`` / ``tool_calls_field`` + ``http-json`` and ``invocations`` only. JSON keys / dot-paths used to + marshal each dataset row into the request body and to extract the + response. Defaults are sensible for OpenAI-compatible / ACA endpoints. + + ``headers`` / ``auth_header_env`` + Optional HTTP request configuration for ``http-json`` and + ``invocations`` targets. + + ``evaluators`` + Optional escape hatch: explicit list of evaluator names that overrides + the auto-selection rules. Most users should leave this unset. + """ + + version: int = Field(..., description="Schema version. Must be 1.") + agent: str = Field(..., description="Target identifier (name:version, URL, or model:deployment)") + dataset: Path = Field(..., description="Path to a JSONL dataset file") + + thresholds: Dict[str, Any] = Field( + default_factory=dict, + description="Metric name -> criteria expression (e.g. '>=3').", + ) + + protocol: Optional[Protocol] = None + request_field: Optional[str] = None + response_field: Optional[str] = None + tool_calls_field: Optional[str] = None + headers: Dict[str, str] = Field(default_factory=dict) + auth_header_env: Optional[str] = None + + evaluators: Optional[List[EvaluatorOverride]] = None + + publish: Optional[PublishTarget] = Field( + None, + description=( + "Optional opt-in publish target.\n" + "- 'foundry' (Classic): runs locally, uploads computed metrics " + "to the Classic Foundry Evaluations panel via OneDP.\n" + "- 'foundry_cloud' (preview): submits the run to the New Foundry " + "experience via the OpenAI Evals API. The agent and evaluators " + "execute server-side; agent must be a 'name:version' Foundry " + "agent." + ), + ) + project_endpoint: Optional[str] = Field( + None, + description=( + "Optional Foundry project endpoint URL used by 'publish: foundry'. " + "When omitted, AGENTOPS reads AZURE_AI_FOUNDRY_PROJECT_ENDPOINT." + ), + ) + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="before") + @classmethod + def _reject_legacy(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + legacy = _LEGACY_TOP_LEVEL_KEYS & set(data.keys()) + if legacy: + raise ValueError( + "agentops.yaml uses the new flat schema (see docs/concepts.md). " + f"Remove legacy keys: {sorted(legacy)}. The minimal config is " + "version + agent + dataset." + ) + return data + + @field_validator("version") + @classmethod + def _check_version(cls, value: int) -> int: + if value != 1: + raise ValueError( + f"agentops.yaml version must be 1 (got {value!r})" + ) + return value + + @field_validator("agent") + @classmethod + def _agent_non_empty(cls, value: str) -> str: + if not value.strip(): + raise ValueError("agent must be non-empty") + return value.strip() + + @model_validator(mode="after") + def _validate_protocol_compat(self) -> "AgentOpsConfig": + kind = classify_agent(self.agent, self.protocol).kind + if kind == "foundry_prompt" and self.protocol is not None: + raise ValueError( + "agent of the form 'name:version' is a Foundry prompt agent " + "and does not accept a 'protocol' field" + ) + if kind == "model_direct" and self.protocol is not None: + raise ValueError( + "agent of the form 'model:' does not accept a " + "'protocol' field" + ) + if kind != "http_json" and ( + self.request_field + or self.response_field + or self.tool_calls_field + or self.headers + or self.auth_header_env + ): + # Foundry hosted (responses/invocations) defines its own wire + # format. HTTP-only request/response shaping is invalid there. + if kind == "foundry_hosted" and self.protocol == "invocations": + # Invocations passes JSON through; users may need headers. + pass + else: + raise ValueError( + "request_field / response_field / tool_calls_field / " + "headers / auth_header_env are only valid for HTTP/JSON " + "or Foundry hosted (invocations) targets" + ) + return self + + def parsed_thresholds(self) -> List[Threshold]: + """Return the threshold dict parsed into structured rules.""" + return [ + Threshold.from_expression(metric, expression) + for metric, expression in self.thresholds.items() + ] + + def resolved_target(self) -> "TargetResolution": + """Return the resolved target classification.""" + return classify_agent(self.agent, self.protocol) + + +# --------------------------------------------------------------------------- +# Agent classifier +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class TargetResolution: + """Result of classifying the ``agent`` field.""" + + kind: TargetKind + protocol: Optional[Protocol] + raw: str + #: For ``foundry_prompt``: the agent name (left of the colon). + name: Optional[str] = None + #: For ``foundry_prompt``: the version (right of the colon). + version: Optional[str] = None + #: For ``foundry_hosted`` / ``http_json``: the target URL. + url: Optional[str] = None + #: For ``model_direct``: the deployment name. + deployment: Optional[str] = None + + +def _looks_like_foundry_url(url: str) -> bool: + """Return ``True`` when ``url`` matches a Foundry hosted endpoint pattern. + + Heuristic — Foundry URLs include the segment ``/agents/`` and the host + ends in a Foundry-recognized domain. We err on the side of accepting more + URLs as Foundry hosted (the user can force ``http-json`` via ``protocol``). + """ + lowered = url.lower() + foundry_domains = ( + ".azure.com", + ".azureml.ms", + ".cognitiveservices.azure.com", + ".services.ai.azure.com", + ".inference.ml.azure.com", + ".azurewebsites.net", # rare; users can override + ) + return any(domain in lowered for domain in foundry_domains) + + +def classify_agent( + agent: str, + protocol: Optional[Protocol] = None, +) -> TargetResolution: + """Classify the ``agent`` value into a target kind. + + Resolution table: + + +-------------------------+--------------------------+-----------------------+ + | ``agent`` value | ``protocol`` | ``TargetKind`` | + +=========================+==========================+=======================+ + | ``model:gpt-4o`` | n/a | ``model_direct`` | + +-------------------------+--------------------------+-----------------------+ + | ``my-rag:3`` | n/a | ``foundry_prompt`` | + +-------------------------+--------------------------+-----------------------+ + | ``https://...foundry`` | omitted or ``responses`` | ``foundry_hosted`` | + | (foundry-shaped URL) | | (responses) | + +-------------------------+--------------------------+-----------------------+ + | ``https://...foundry`` | ``invocations`` | ``foundry_hosted`` | + | | | (invocations) | + +-------------------------+--------------------------+-----------------------+ + | ``https://other-host`` | omitted or ``http-json`` | ``http_json`` | + +-------------------------+--------------------------+-----------------------+ + """ + raw = agent.strip() + + if raw.lower().startswith("model:"): + deployment = raw.split(":", 1)[1].strip() + if not deployment: + raise ValueError("model: prefix requires a deployment name") + return TargetResolution( + kind="model_direct", + protocol=None, + raw=raw, + deployment=deployment, + ) + + lowered = raw.lower() + if lowered.startswith(("http://", "https://")): + if _looks_like_foundry_url(raw): + resolved_protocol: Protocol = protocol or "responses" + if resolved_protocol not in {"responses", "invocations"}: + raise ValueError( + "Foundry hosted endpoints accept only protocol " + "'responses' or 'invocations'" + ) + return TargetResolution( + kind="foundry_hosted", + protocol=resolved_protocol, + raw=raw, + url=raw, + ) + + resolved_protocol = protocol or "http-json" + if resolved_protocol != "http-json": + raise ValueError( + "non-Foundry URLs must use protocol 'http-json' " + f"(got {resolved_protocol!r})" + ) + return TargetResolution( + kind="http_json", + protocol="http-json", + raw=raw, + url=raw, + ) + + if ":" in raw: + name, _, version = raw.partition(":") + name = name.strip() + version = version.strip() + if not name or not version: + raise ValueError( + "Foundry prompt agent must be 'name:version' " + f"(got {raw!r})" + ) + return TargetResolution( + kind="foundry_prompt", + protocol=None, + raw=raw, + name=name, + version=version, + ) + + raise ValueError( + f"unrecognized agent value {raw!r}: expected 'name:version', " + "'https://...', or 'model:'" + ) diff --git a/src/agentops/core/config_loader.py b/src/agentops/core/config_loader.py index 7a22abd3..1f5ea55d 100644 --- a/src/agentops/core/config_loader.py +++ b/src/agentops/core/config_loader.py @@ -4,98 +4,19 @@ import logging from pathlib import Path -from typing import Type, TypeVar -from pydantic import BaseModel, ValidationError +from pydantic import ValidationError -from agentops.core.models import ( - BundleConfig, - BundleRef, - DatasetConfig, - DatasetRef, - RunConfig, - WorkspaceConfig, -) +from agentops.core.agentops_config import AgentOpsConfig from agentops.utils.yaml import load_yaml logger = logging.getLogger(__name__) -TModel = TypeVar("TModel", bound=BaseModel) - -def _load_model(path: Path, model_cls: Type[TModel], label: str) -> TModel: - data = load_yaml(path) - try: - return model_cls.model_validate(data) - except ValidationError as exc: - raise ValueError(f"{label} validation error: {exc}") from exc - - -def load_workspace_config(path: Path) -> WorkspaceConfig: - return _load_model(path, WorkspaceConfig, "WorkspaceConfig") - - -def load_bundle_config(path: Path) -> BundleConfig: - return _load_model(path, BundleConfig, "BundleConfig") - - -def load_dataset_config(path: Path) -> DatasetConfig: - return _load_model(path, DatasetConfig, "DatasetConfig") - - -def load_run_config(path: Path) -> RunConfig: +def load_agentops_config(path: Path) -> AgentOpsConfig: + """Load the flat 1.0 ``agentops.yaml`` schema.""" data = load_yaml(path) - if isinstance(data, dict) and "backend" in data: - raise ValueError( - "Invalid run config: the top-level 'backend' key is not supported. " - "Did you mean 'target.hosting'? The backend is now determined by the " - "'target' section (type, hosting, execution_mode). Remove the 'backend' " - "key and configure 'target.hosting' and 'target.execution_mode' instead. " - "See docs/how-it-works.md for the current schema." - ) try: - return RunConfig.model_validate(data) + return AgentOpsConfig.model_validate(data) except ValidationError as exc: - raise ValueError(f"RunConfig validation error: {exc}") from exc - - -def resolve_bundle_ref(ref: BundleRef, base_dir: Path, workspace_dir: Path) -> Path: - """Resolve a bundle reference to an absolute path. - - If ``ref.path`` is set, resolve relative to *base_dir*. - If ``ref.name`` is set, resolve to ``/bundles/.yaml``. - """ - if ref.path is not None: - if ref.path.is_absolute(): - return ref.path - candidate = (base_dir / ref.path).resolve() - if candidate.exists(): - return candidate - fallback = (Path.cwd() / ref.path).resolve() - if fallback.exists(): - return fallback - return candidate - - assert ref.name is not None - return (workspace_dir / "bundles" / f"{ref.name}.yaml").resolve() - - -def resolve_dataset_ref(ref: DatasetRef, base_dir: Path, workspace_dir: Path) -> Path: - """Resolve a dataset reference to an absolute path. - - If ``ref.path`` is set, resolve relative to *base_dir*. - If ``ref.name`` is set, resolve to ``/datasets/.yaml``. - """ - if ref.path is not None: - if ref.path.is_absolute(): - return ref.path - candidate = (base_dir / ref.path).resolve() - if candidate.exists(): - return candidate - fallback = (Path.cwd() / ref.path).resolve() - if fallback.exists(): - return fallback - return candidate - - assert ref.name is not None - return (workspace_dir / "datasets" / f"{ref.name}.yaml").resolve() + raise ValueError(f"AgentOpsConfig validation error: {exc}") from exc diff --git a/src/agentops/core/evaluators.py b/src/agentops/core/evaluators.py new file mode 100644 index 00000000..935f6c8f --- /dev/null +++ b/src/agentops/core/evaluators.py @@ -0,0 +1,413 @@ +"""Evaluator catalog and auto-selection for AgentOps 1.0. + +This module replaces the layered ``bundle.yaml`` system. There is no +user-facing ``scenario`` concept. Evaluators are picked from two inputs: + +1. The resolved target kind (agent vs model). Model targets only get the + baseline quality evaluators — agent-specific evaluators are skipped even + if the dataset contains those fields. +2. The shape of the dataset rows: + + * Always: baseline quality evaluators (Coherence, Fluency, Similarity, + F1Score). + * If rows include ``context``: add RAG evaluators (Groundedness, + Retrieval, Relevance, ResponseCompleteness). + * If rows include ``tool_calls`` or ``tool_definitions``: add agent + evaluators (ToolCallAccuracy, IntentResolution, TaskAdherence). + +The :func:`select_evaluators` function returns a list of resolved +:class:`EvaluatorPreset` objects. Each preset carries its class name, the +input mapping it requires, the score key it produces, and a default +threshold. The runner uses these presets to instantiate +``azure-ai-evaluation`` evaluator classes against each dataset row. + +Power users can override the auto-selection by listing evaluator names in +``agentops.yaml`` under ``evaluators:``. When set, the override list is the +final word — no auto-detection runs. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, FrozenSet, List, Optional, Tuple + +from agentops.core.agentops_config import TargetKind, TargetResolution, Threshold + + +# --------------------------------------------------------------------------- +# Catalog +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EvaluatorPreset: + """Metadata for a single evaluator known to AgentOps. + + ``input_mapping`` keys are the parameter names the evaluator class + expects; values use the placeholder syntax ``$prompt``, ``$prediction``, + ``$context``, ``$expected``, ``$tool_calls``, ``$tool_definitions`` + which the runner resolves per row. + """ + + name: str + class_name: str + score_key: str + input_mapping: Dict[str, str] + default_threshold: Optional[Threshold] = None + #: Categories that this evaluator belongs to. Used by the inference rules. + categories: FrozenSet[str] = field(default_factory=frozenset) + #: Set when this evaluator is not safe to run for raw model deployments. + agent_only: bool = False + #: When True and the row carries ``tool_calls``, the runner upgrades the + #: ``query`` and ``response`` kwargs from plain strings to conversation + #: message lists that include the agent's tool_call + tool_result trace. + #: This is required for evaluators that judge agent reasoning (e.g. + #: TaskAdherence, IntentResolution) — without the trace they only see a + #: short final answer and consistently score it as 1/5. + needs_conversation: bool = False + + +def _t(metric: str, criteria: str, value: float) -> Threshold: + return Threshold(metric=metric, criteria=criteria, value=value) # type: ignore[arg-type] + + +_QUALITY_BASELINE: Tuple[EvaluatorPreset, ...] = ( + EvaluatorPreset( + name="CoherenceEvaluator", + class_name="CoherenceEvaluator", + score_key="coherence", + input_mapping={"query": "$prompt", "response": "$prediction"}, + default_threshold=_t("coherence", ">=", 3.0), + categories=frozenset({"quality"}), + ), + EvaluatorPreset( + name="FluencyEvaluator", + class_name="FluencyEvaluator", + score_key="fluency", + input_mapping={"response": "$prediction"}, + default_threshold=_t("fluency", ">=", 3.0), + categories=frozenset({"quality"}), + ), + EvaluatorPreset( + name="SimilarityEvaluator", + class_name="SimilarityEvaluator", + score_key="similarity", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "ground_truth": "$expected", + }, + default_threshold=_t("similarity", ">=", 3.0), + categories=frozenset({"quality"}), + ), + EvaluatorPreset( + name="F1ScoreEvaluator", + class_name="F1ScoreEvaluator", + score_key="f1_score", + input_mapping={ + "response": "$prediction", + "ground_truth": "$expected", + }, + default_threshold=_t("f1_score", ">=", 0.5), + categories=frozenset({"quality"}), + ), +) + + +_RAG_EVALUATORS: Tuple[EvaluatorPreset, ...] = ( + EvaluatorPreset( + name="GroundednessEvaluator", + class_name="GroundednessEvaluator", + score_key="groundedness", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "context": "$context", + }, + default_threshold=_t("groundedness", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), + EvaluatorPreset( + name="RelevanceEvaluator", + class_name="RelevanceEvaluator", + score_key="relevance", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "context": "$context", + }, + default_threshold=_t("relevance", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), + EvaluatorPreset( + name="RetrievalEvaluator", + class_name="RetrievalEvaluator", + score_key="retrieval", + input_mapping={"query": "$prompt", "context": "$context"}, + default_threshold=_t("retrieval", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), + EvaluatorPreset( + name="ResponseCompletenessEvaluator", + class_name="ResponseCompletenessEvaluator", + score_key="response_completeness", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "ground_truth": "$expected", + }, + default_threshold=_t("response_completeness", ">=", 3.0), + categories=frozenset({"rag"}), + agent_only=True, + ), +) + + +_TOOL_USE_EVALUATORS: Tuple[EvaluatorPreset, ...] = ( + EvaluatorPreset( + name="ToolCallAccuracyEvaluator", + class_name="ToolCallAccuracyEvaluator", + score_key="tool_call_accuracy", + input_mapping={ + "query": "$prompt", + "tool_calls": "$tool_calls", + "tool_definitions": "$tool_definitions", + }, + default_threshold=_t("tool_call_accuracy", ">=", 0.7), + categories=frozenset({"agent"}), + agent_only=True, + ), + EvaluatorPreset( + name="IntentResolutionEvaluator", + class_name="IntentResolutionEvaluator", + score_key="intent_resolution", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "tool_definitions": "$tool_definitions", + }, + default_threshold=_t("intent_resolution", ">=", 3.0), + categories=frozenset({"agent"}), + agent_only=True, + needs_conversation=True, + ), + EvaluatorPreset( + name="TaskAdherenceEvaluator", + class_name="TaskAdherenceEvaluator", + score_key="task_adherence", + input_mapping={ + "query": "$prompt", + "response": "$prediction", + "tool_definitions": "$tool_definitions", + }, + # azure-ai-evaluation's TaskAdherenceEvaluator returns a binary + # 0/1 score (0 = flagged, 1 = adheres) — *not* a 1–5 Likert scale + # like IntentResolutionEvaluator. We default to >=0.5 so a score + # of 1.0 passes and 0.0 fails. + default_threshold=_t("task_adherence", ">=", 0.5), + categories=frozenset({"agent"}), + agent_only=True, + needs_conversation=True, + ), +) + + +_LATENCY = EvaluatorPreset( + name="avg_latency_seconds", + class_name="_latency", + score_key="avg_latency_seconds", + input_mapping={}, + default_threshold=_t("avg_latency_seconds", "<=", 10.0), + categories=frozenset({"runtime"}), +) + + +CATALOG: Dict[str, EvaluatorPreset] = { + preset.name: preset + for preset in ( + *_QUALITY_BASELINE, + *_RAG_EVALUATORS, + *_TOOL_USE_EVALUATORS, + _LATENCY, + ) +} + + +# --------------------------------------------------------------------------- +# Dataset shape detection +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DatasetShape: + """Boolean flags summarising the columns present in a dataset.""" + + has_context: bool + has_tool_calls: bool + has_tool_definitions: bool + row_count: int + + @property + def looks_rag(self) -> bool: + return self.has_context + + @property + def looks_tool_use(self) -> bool: + return self.has_tool_calls or self.has_tool_definitions + + +def detect_dataset_shape(dataset_path: Path, *, sample: int = 50) -> DatasetShape: + """Inspect up to ``sample`` rows of ``dataset_path`` and report the shape. + + Truthy values are required — empty strings, empty lists, and ``None`` do + not count as the field being present. + """ + if not dataset_path.exists(): + raise FileNotFoundError(f"dataset file not found: {dataset_path}") + + has_context = False + has_tool_calls = False + has_tool_definitions = False + count = 0 + + with dataset_path.open("r", encoding="utf-8") as handle: + for line in handle: + stripped = line.strip() + if not stripped: + continue + count += 1 + try: + row = json.loads(stripped) + except json.JSONDecodeError as exc: + raise ValueError( + f"{dataset_path}: invalid JSON on line {count}: {exc}" + ) from exc + if not isinstance(row, dict): + raise ValueError( + f"{dataset_path}: line {count} is not a JSON object" + ) + + if not has_context and row.get("context"): + has_context = True + if not has_tool_calls and row.get("tool_calls"): + has_tool_calls = True + if not has_tool_definitions and row.get("tool_definitions"): + has_tool_definitions = True + + if count >= sample and ( + has_context and (has_tool_calls or has_tool_definitions) + ): + # Already saw both signals; no need to keep reading. + break + + if count == 0: + raise ValueError(f"{dataset_path}: dataset is empty") + + return DatasetShape( + has_context=has_context, + has_tool_calls=has_tool_calls, + has_tool_definitions=has_tool_definitions, + row_count=count, + ) + + +# --------------------------------------------------------------------------- +# Selection +# --------------------------------------------------------------------------- + + +def select_evaluators( + target: TargetResolution, + shape: DatasetShape, + *, + overrides: Optional[List[str]] = None, +) -> List[EvaluatorPreset]: + """Return the ordered list of evaluators to run. + + When ``overrides`` is provided it wins outright — the inference rules are + bypassed. Each name must exist in :data:`CATALOG` or a ``ValueError`` is + raised. + + Otherwise the rules are: + + * Always include the four baseline quality evaluators. + * If the target is a raw model, stop here. Agent-specific evaluators are + not meaningful (no tool calls, no retrieved context). + * If the dataset has ``context`` rows, add the RAG evaluators. + * If the dataset has ``tool_calls`` or ``tool_definitions``, add the agent + evaluators. + * Always append the runtime ``avg_latency_seconds`` evaluator. + """ + if overrides: + resolved: List[EvaluatorPreset] = [] + for name in overrides: + preset = CATALOG.get(name) + if preset is None: + known = ", ".join(sorted(CATALOG.keys())) + raise ValueError( + f"unknown evaluator override {name!r}. " + f"Known evaluators: {known}" + ) + resolved.append(preset) + return resolved + + selected: List[EvaluatorPreset] = list(_QUALITY_BASELINE) + + if _is_agent_target(target.kind): + if shape.looks_rag: + selected.extend(_RAG_EVALUATORS) + if shape.looks_tool_use: + selected.extend(_TOOL_USE_EVALUATORS) + # F1ScoreEvaluator and SimilarityEvaluator compare the + # assistant's natural-language reply against ``expected``. In + # tool-using datasets ``expected`` is conventionally a behavior + # description (e.g. "Calls lookup_order with order_id='ORD-12345'") + # rather than the literal reply, so token overlap and semantic + # similarity are meaningless and gate well-behaved agents on a + # metric that does not apply. Drop both from the selection. + _drop = {"F1ScoreEvaluator", "SimilarityEvaluator"} + selected = [p for p in selected if p.name not in _drop] + + selected.append(_LATENCY) + return selected + + +def _is_agent_target(kind: TargetKind) -> bool: + return kind in {"foundry_prompt", "foundry_hosted", "http_json"} + + +def merge_thresholds( + presets: List[EvaluatorPreset], + user_thresholds: List[Threshold], +) -> List[Threshold]: + """Combine evaluator default thresholds with user overrides. + + User entries override the preset default for the same metric. Metrics + listed by the user that don't correspond to any selected preset are kept + as-is — the threshold engine will report them as unmet rather than + silently drop them. + """ + by_metric: Dict[str, Threshold] = {} + for preset in presets: + if preset.default_threshold is not None: + by_metric[preset.default_threshold.metric] = preset.default_threshold + for override in user_thresholds: + by_metric[override.metric] = override + # Preserve preset order, then append user-only metrics in original order. + ordered: List[Threshold] = [] + seen: set[str] = set() + for preset in presets: + if preset.default_threshold is not None: + metric = preset.default_threshold.metric + ordered.append(by_metric[metric]) + seen.add(metric) + for override in user_thresholds: + if override.metric not in seen: + ordered.append(override) + seen.add(override.metric) + return ordered diff --git a/src/agentops/core/models.py b/src/agentops/core/models.py deleted file mode 100644 index 928aacc2..00000000 --- a/src/agentops/core/models.py +++ /dev/null @@ -1,582 +0,0 @@ -"""Pydantic models for AgentOps schemas.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, Dict, List, Literal, Optional - -from pydantic import BaseModel, Field, field_validator, model_validator - -ComparisonCriteria = Literal[">=", ">", "<=", "<", "=="] -Criteria = Literal[">=", ">", "<=", "<", "==", "true", "false"] -EvaluatorSource = Literal["local", "foundry"] - - -class WorkspacePaths(BaseModel): - bundles_dir: Path - datasets_dir: Path - data_dir: Path - results_dir: Path - - -class WorkspaceDefaults(BaseModel): - backend: str - timeout_seconds: int - - -class WorkspaceReport(BaseModel): - generate_markdown: bool = True - - -class WorkspaceConfig(BaseModel): - version: int - paths: WorkspacePaths - defaults: WorkspaceDefaults - report: WorkspaceReport - - -class EvaluatorConfig(BaseModel): - name: str - source: EvaluatorSource = "local" - enabled: bool = True - config: Dict[str, Any] = Field(default_factory=dict) - - @field_validator("name") - @classmethod - def _name_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("name must be non-empty") - return value - - -class ThresholdRule(BaseModel): - evaluator: str - criteria: Criteria - value: Optional[float] = Field(None, description="Numeric threshold target") - - @model_validator(mode="before") - @classmethod - def _normalize_legacy_fields(cls, data: Any) -> Any: - if not isinstance(data, dict): - return data - - normalized = dict(data) - - if "evaluator" not in normalized and "metric" in normalized: - normalized["evaluator"] = normalized["metric"] - - if "criteria" not in normalized and "operator" in normalized: - normalized["criteria"] = normalized["operator"] - - if isinstance(normalized.get("criteria"), bool): - normalized["criteria"] = "true" if normalized["criteria"] else "false" - - criteria_value = normalized.get("criteria") - if isinstance(criteria_value, str): - normalized["criteria"] = criteria_value.strip().lower() - - return normalized - - @field_validator("evaluator") - @classmethod - def _evaluator_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("evaluator must be non-empty") - return value - - @field_validator("value", mode="before") - @classmethod - def _value_is_number(cls, value: Any) -> Any: - if value is None: - return value - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError("value must be numeric") - return value - - @model_validator(mode="after") - def _validate_criteria(self) -> "ThresholdRule": - if self.criteria in {"true", "false"}: - if self.value is not None: - raise ValueError("value must be omitted for boolean criteria") - return self - - if self.value is None: - raise ValueError("value is required for comparison criteria") - return self - - -class BundleConfig(BaseModel): - version: int - name: str - description: Optional[str] = None - evaluators: List[EvaluatorConfig] = Field(default_factory=list) - thresholds: List[ThresholdRule] = Field(default_factory=list) - metadata: Dict[str, Any] = Field(default_factory=dict) - - @field_validator("name") - @classmethod - def _name_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("name must be non-empty") - return value - - -class DatasetSource(BaseModel): - type: str - path: Path - - @field_validator("path", mode="before") - @classmethod - def _path_non_empty(cls, value: Any) -> Any: - if isinstance(value, str) and not value.strip(): - raise ValueError("path must be non-empty") - return value - - -class DatasetFormat(BaseModel): - type: str - input_field: str - expected_field: str - context_field: Optional[str] = None - - -class DatasetConfig(BaseModel): - version: int - name: str - description: Optional[str] = None - source: DatasetSource - format: DatasetFormat - metadata: Dict[str, Any] = Field(default_factory=dict) - - @field_validator("name") - @classmethod - def _name_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("name must be non-empty") - return value - - -# --------------------------------------------------------------------------- -# Run configuration — orthogonal target / hosting / execution_mode model -# --------------------------------------------------------------------------- - -TargetType = Literal["agent", "model"] -Hosting = Literal["local", "foundry", "aks", "containerapps"] -ExecutionMode = Literal["local", "remote"] -AgentMode = Literal["prompt", "hosted"] -Framework = Literal["agent_framework", "langgraph", "custom"] -EndpointKind = Literal["foundry_agent", "http"] - - -class TargetEndpointConfig(BaseModel): - """Remote endpoint configuration for the evaluation target.""" - - kind: EndpointKind - - # Foundry agent fields - agent_id: Optional[str] = None - project_endpoint: Optional[str] = None - project_endpoint_env: Optional[str] = None - api_version: Optional[str] = None - poll_interval_seconds: Optional[float] = None - max_poll_attempts: Optional[int] = None - model: Optional[str] = None - - # HTTP fields - url: Optional[str] = None - url_env: Optional[str] = None - request_field: Optional[str] = None - response_field: Optional[str] = None - headers: Dict[str, str] = Field(default_factory=dict) - auth_header_env: Optional[str] = None - tool_calls_field: Optional[str] = None - extra_fields: Optional[List[str]] = None - - @field_validator("model") - @classmethod - def _reject_placeholder_model(cls, value: Optional[str]) -> Optional[str]: - if value is None: - return value - normalized = value.strip() - looks_like_placeholder = ( - normalized.startswith("<") and normalized.endswith(">") - ) or "replace-with" in normalized.lower() - if looks_like_placeholder: - raise ValueError( - "endpoint.model must be replaced with a real Foundry model deployment name" - ) - return normalized - - @model_validator(mode="after") - def _validate_endpoint_fields(self) -> "TargetEndpointConfig": - if self.kind == "foundry_agent": - if self.max_poll_attempts is not None and self.max_poll_attempts <= 0: - raise ValueError("endpoint.max_poll_attempts must be > 0") - if ( - self.poll_interval_seconds is not None - and self.poll_interval_seconds <= 0 - ): - raise ValueError("endpoint.poll_interval_seconds must be > 0") - elif self.kind == "http": - if not self.url and not self.url_env: - raise ValueError( - "HTTP endpoint requires 'endpoint.url' or 'endpoint.url_env'" - ) - return self - - -class LocalAdapterConfig(BaseModel): - """Configuration for local adapter execution. - - Exactly one of ``adapter`` (subprocess command) or ``callable`` - (``module:function`` path) must be provided. - """ - - adapter: Optional[str] = None - callable: Optional[str] = None - - @field_validator("adapter") - @classmethod - def _adapter_non_empty(cls, value: Optional[str]) -> Optional[str]: - if value is not None and not value.strip(): - raise ValueError("local.adapter must be non-empty") - return value - - @field_validator("callable") - @classmethod - def _callable_format(cls, value: Optional[str]) -> Optional[str]: - if value is None: - return value - if not value.strip(): - raise ValueError("local.callable must be non-empty") - if ":" not in value: - raise ValueError( - "local.callable must use 'module:function' format " - "(e.g. 'my_workflow:run_evaluation')" - ) - module_part, _, func_part = value.partition(":") - if not module_part.strip() or not func_part.strip(): - raise ValueError( - "local.callable must use 'module:function' format " - "(e.g. 'my_workflow:run_evaluation')" - ) - return value - - @model_validator(mode="after") - def _require_adapter_xor_callable(self) -> "LocalAdapterConfig": - has_adapter = self.adapter is not None - has_callable = self.callable is not None - if has_adapter and has_callable: - raise ValueError( - "local config must specify either 'adapter' or 'callable', not both" - ) - if not has_adapter and not has_callable: - raise ValueError( - "local config must specify either 'adapter' (subprocess command) " - "or 'callable' (module:function path)" - ) - return self - - -class TargetConfig(BaseModel): - """Defines what is being evaluated and how the toolkit interacts with it.""" - - type: TargetType - hosting: Hosting - execution_mode: ExecutionMode - agent_mode: Optional[AgentMode] = None - framework: Optional[Framework] = None - endpoint: Optional[TargetEndpointConfig] = None - local: Optional[LocalAdapterConfig] = None - - @model_validator(mode="after") - def _validate_target(self) -> "TargetConfig": - if self.agent_mode is not None and self.hosting != "foundry": - raise ValueError( - "target.agent_mode is only valid when hosting is 'foundry'" - ) - if self.framework is not None and self.type != "agent": - raise ValueError( - "target.framework is only valid when type is 'agent'" - ) - if self.execution_mode == "remote": - if self.endpoint is None: - raise ValueError( - "target.endpoint is required when execution_mode is 'remote'" - ) - if self.execution_mode == "local": - if self.local is None: - raise ValueError( - "target.local is required when execution_mode is 'local'" - ) - return self - - -class BundleRef(BaseModel): - name: Optional[str] = None - path: Optional[Path] = None - - @model_validator(mode="after") - def _require_name_or_path(self) -> "BundleRef": - if not self.name and not self.path: - raise ValueError("bundle requires 'name' or 'path'") - return self - - -class DatasetRef(BaseModel): - name: Optional[str] = None - path: Optional[Path] = None - - @model_validator(mode="after") - def _require_name_or_path(self) -> "DatasetRef": - if not self.name and not self.path: - raise ValueError("dataset requires 'name' or 'path'") - return self - - -class ExecutionConfig(BaseModel): - concurrency: int = 1 - timeout_seconds: int = 300 - - -class RunMetadata(BaseModel): - name: Optional[str] = None - description: Optional[str] = None - - -class OutputConfig(BaseModel): - path: Optional[Path] = None - write_report: bool = True - publish_foundry_evaluation: bool = True - fail_on_foundry_publish_error: bool = False - - -class RunConfig(BaseModel): - version: int - run: Optional[RunMetadata] = None - target: TargetConfig - bundle: BundleRef - dataset: DatasetRef - execution: ExecutionConfig = Field(default_factory=ExecutionConfig) - output: OutputConfig = Field(default_factory=OutputConfig) - - -class BundleInfo(BaseModel): - name: str - path: Path - - -class DatasetInfo(BaseModel): - name: str - path: Path - - -class ExecutionInfo(BaseModel): - backend: str - command: str - started_at: str - finished_at: str - duration_seconds: float - exit_code: int - - -class MetricResult(BaseModel): - name: str - value: float - - @field_validator("name") - @classmethod - def _name_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("name must be non-empty") - return value - - @field_validator("value", mode="before") - @classmethod - def _value_is_number(cls, value: Any) -> Any: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError("value must be numeric") - return value - - -class RowMetricsResult(BaseModel): - row_index: int - input: Optional[str] = None - response: Optional[str] = None - context: Optional[str] = None - metrics: List[MetricResult] = Field(default_factory=list) - - @field_validator("row_index") - @classmethod - def _row_index_positive(cls, value: int) -> int: - if value <= 0: - raise ValueError("row_index must be >= 1") - return value - - -class ThresholdEvaluationResult(BaseModel): - evaluator: str - criteria: Criteria - expected: str - actual: str - passed: bool - - @field_validator("evaluator") - @classmethod - def _evaluator_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("evaluator must be non-empty") - return value - - -class ItemThresholdEvaluationResult(BaseModel): - row_index: int - evaluator: str - criteria: Criteria - expected: str - actual: str - passed: bool - - @field_validator("row_index") - @classmethod - def _row_index_positive(cls, value: int) -> int: - if value <= 0: - raise ValueError("row_index must be >= 1") - return value - - @field_validator("evaluator") - @classmethod - def _evaluator_non_empty(cls, value: str) -> str: - if not value.strip(): - raise ValueError("evaluator must be non-empty") - return value - - -class ItemEvaluationResult(BaseModel): - row_index: int - passed_all: bool - thresholds: List[ItemThresholdEvaluationResult] = Field(default_factory=list) - - @field_validator("row_index") - @classmethod - def _row_index_positive(cls, value: int) -> int: - if value <= 0: - raise ValueError("row_index must be >= 1") - return value - - -class Summary(BaseModel): - metrics_count: int - thresholds_count: int - thresholds_passed: int - thresholds_failed: int - overall_passed: bool - - -class Artifacts(BaseModel): - backend_stdout: Optional[str] = None - backend_stderr: Optional[str] = None - foundry_eval_studio_url: Optional[str] = None - foundry_eval_name: Optional[str] = None - - -class RunResult(BaseModel): - version: int - status: str - bundle: BundleInfo - dataset: DatasetInfo - execution: ExecutionInfo - metrics: List[MetricResult] = Field(default_factory=list) - row_metrics: List[RowMetricsResult] = Field(default_factory=list) - item_evaluations: List[ItemEvaluationResult] = Field(default_factory=list) - run_metrics: List[MetricResult] = Field(default_factory=list) - thresholds: List[ThresholdEvaluationResult] = Field(default_factory=list) - summary: Summary - artifacts: Optional[Artifacts] = None - - -# --------------------------------------------------------------------------- -# Comparison models -# --------------------------------------------------------------------------- - -Direction = Literal["improved", "regressed", "unchanged"] - - -class RunReference(BaseModel): - run_id: str - bundle_name: str - dataset_name: str - started_at: str - backend: Optional[str] = None - target: Optional[str] = None - model: Optional[str] = None - agent_id: Optional[str] = None - project_endpoint: Optional[str] = None - overall_passed: Optional[bool] = None - - -class ComparisonMetricRow(BaseModel): - """One metric across all compared runs.""" - - name: str - values: List[float] = Field(default_factory=list) - deltas: List[Optional[float]] = Field(default_factory=list) - delta_percents: List[Optional[float]] = Field(default_factory=list) - directions: List[Direction] = Field(default_factory=list) - best_run_index: Optional[int] = None - - -class ComparisonThresholdRow(BaseModel): - """One threshold across all compared runs.""" - - evaluator: str - criteria: Criteria - target: Optional[str] = None - passed: List[bool] = Field(default_factory=list) - - -class ComparisonItemRow(BaseModel): - """One dataset item across all compared runs.""" - - row_index: int - passed_all: List[bool] = Field(default_factory=list) - scores: Dict[str, List[Optional[float]]] = Field(default_factory=dict) - - -ComparisonType = Literal[ - "agent", # Same dataset, different agent/agent version - "model", # Same dataset, different model - "dataset", # Same agent/model, different datasets - "general", # Multiple things differ -] - - -class ComparisonConditions(BaseModel): - """What's fixed vs varying across compared runs.""" - - comparison_type: ComparisonType - fixed: Dict[str, str] = Field(default_factory=dict) - varying: List[str] = Field(default_factory=list) - row_level_valid: bool = True - - -class ComparisonSummary(BaseModel): - run_count: int - any_regressions: bool - runs_with_regressions: List[int] = Field(default_factory=list) - - -class ComparisonResult(BaseModel): - """Unified comparison of 2 or more evaluation runs. - - The first entry in ``runs`` is always the baseline. - """ - - version: int = 1 - runs: List[RunReference] = Field(default_factory=list) - baseline_index: int = 0 - conditions: Optional[ComparisonConditions] = None - metric_rows: List[ComparisonMetricRow] = Field(default_factory=list) - threshold_rows: List[ComparisonThresholdRow] = Field(default_factory=list) - item_rows: List[ComparisonItemRow] = Field(default_factory=list) - summary: ComparisonSummary diff --git a/src/agentops/core/reporter.py b/src/agentops/core/reporter.py deleted file mode 100644 index 1206a63a..00000000 --- a/src/agentops/core/reporter.py +++ /dev/null @@ -1,973 +0,0 @@ -"""Report generation for AgentOps (Markdown and HTML).""" - -from __future__ import annotations - -import re - -from agentops.core.models import ComparisonResult, RunResult - -# --------------------------------------------------------------------------- -# Evaluator descriptions — one-line explanation of what each metric measures -# --------------------------------------------------------------------------- - -_EVALUATOR_DESCRIPTIONS: dict[str, str] = { - "groundedness": "Are claims supported by the retrieved context?", - "relevance": "Is the response relevant to the user's question?", - "coherence": "Is the response logically organized and easy to follow?", - "fluency": "Is the response grammatically correct and well-written?", - "similarity": "How closely does the response match the expected answer?", - "f1_score": "Token-level overlap between response and expected answer.", - "retrieval": "Did the retrieval step return relevant documents?", - "response_completeness": "Does the response fully address the question?", - "violence": "Does the response contain violent content? (lower is safer)", - "sexual": "Does the response contain sexual content? (lower is safer)", - "self_harm": "Does the response contain self-harm content? (lower is safer)", - "hate_unfairness": "Does the response contain hateful or unfair content? (lower is safer)", - "protected_material": "Does the response contain protected material? (lower is safer)", - "task_completion": "Did the agent complete the requested task?", - "tool_call_accuracy": "Did the agent invoke the correct tools with correct inputs?", - "intent_resolution": "Did the agent correctly resolve the user's intent?", - "task_adherence": "Did the agent follow the expected task workflow?", - "tool_selection": "Did the agent select the right tools for the task?", - "tool_input_accuracy": "Were the inputs passed to tools correct?", - "exact_match": "Does the response exactly match the expected answer?", - "latency_seconds": "Response time for this individual row.", - "avg_latency_seconds": "Average response time across all rows.", - "run_pass": "Did the overall run pass all thresholds? (1 = yes, 0 = no)", - "threshold_pass_rate": "Fraction of thresholds that passed.", - "items_total": "Total number of items evaluated.", - "items_passed_all": "Number of items that passed all thresholds.", - "items_pass_rate": "Fraction of items that passed all thresholds.", - "accuracy": "Overall accuracy score across all rows.", -} - -# Maximum characters of context to display in row details. -_MAX_CONTEXT_DISPLAY = 500 - - -def _format_metric_name(raw_name: str) -> str: - """Format a raw metric name into human-readable form. - - Examples: - groundedness → Groundedness - avg_latency_seconds → Avg Latency Seconds - SimilarityEvaluator → Similarity - GroundednessEvaluator_avg → Groundedness Avg - f1_score → F1 Score - """ - name = raw_name - # Strip "Evaluator" suffix (with optional _avg/_stddev) - name = re.sub(r"Evaluator(?:_(\w+))?$", r" \1", name).strip() - # CamelCase → spaced - name = re.sub(r"([a-z])([A-Z])", r"\1 \2", name) - # underscores → spaces - name = name.replace("_", " ") - # Collapse whitespace and title-case - name = " ".join(name.split()).title() - # Fix common abbreviations - name = name.replace("F1", "F1") - name = re.sub(r"\bAvg\b", "Avg.", name) - return name - - -def _get_evaluator_description(raw_name: str) -> str: - """Look up a description for a metric. Returns empty string if unknown.""" - # Try raw name first, then lowercase, then stripped of Evaluator suffix - key = raw_name.lower() - if key in _EVALUATOR_DESCRIPTIONS: - return _EVALUATOR_DESCRIPTIONS[key] - # Strip Evaluator suffix and _avg/_stddev - stripped = re.sub(r"evaluator(?:_\w+)?$", "", key).strip("_") - if stripped in _EVALUATOR_DESCRIPTIONS: - return _EVALUATOR_DESCRIPTIONS[stripped] - return "" - - -def _fmt_threshold_value(raw: str) -> str: - """Format a threshold value for display — drop unnecessary decimal zeros.""" - try: - val = float(raw) - return _fmt(val) - except (ValueError, TypeError): - return raw - - -def generate_report_markdown(result: RunResult) -> str: - overall_passed = result.summary.overall_passed - overall_icon = "✅" if overall_passed else "❌" - overall_label = "PASS" if overall_passed else "FAIL" - - lines: list[str] = [] - lines.append("# AgentOps Evaluation Report") - lines.append("") - lines.append("## Overview") - lines.append("") - lines.append(f"- Bundle: {result.bundle.name}") - lines.append(f"- Dataset: {result.dataset.name}") - lines.append(f"- Overall status: **{overall_icon} {overall_label}**") - lines.append("") - - # --- How pass/fail is determined --- - lines.append("## How Pass/Fail Is Determined") - lines.append("") - lines.append( - "Each evaluator scores every dataset row. Each score is compared against a threshold " - "(e.g., `>= 0.8`). A row passes if **all** its evaluator scores meet their thresholds. " - "The overall run passes only if **every** row passes **all** thresholds." - ) - lines.append("") - - # --- Execution Summary --- - lines.append("## Execution Summary") - lines.append("") - lines.append("| Field | Value |") - lines.append("|---|---|") - lines.append(f"| Backend | {result.execution.backend} |") - lines.append(f"| Duration (s) | {result.execution.duration_seconds:.3f} |") - lines.append(f"| Started at | {result.execution.started_at} |") - lines.append(f"| Finished at | {result.execution.finished_at} |") - lines.append(f"| Exit code | {result.execution.exit_code} |") - lines.append("") - - # --- Metrics (with descriptions) --- - lines.append("## Metrics") - lines.append("") - lines.append("Average scores across all dataset rows.") - - if result.metrics: - lines.append("") - lines.append("| Metric | Value | What It Measures |") - lines.append("|---|---:|---|") - for metric in result.metrics: - name = _format_metric_name(metric.name) - desc = _get_evaluator_description(metric.name) - lines.append(f"| {name} | {_fmt(metric.value)} | {desc} |") - else: - lines.append("") - lines.append("- No metrics found") - - # --- Run Metrics --- - lines.append("") - lines.append("## Run Metrics") - lines.append("") - lines.append("Derived summary statistics for the entire evaluation run.") - if result.run_metrics: - lines.append("") - lines.append("| Metric | Value |") - lines.append("|---|---:|") - for metric in result.run_metrics: - name = _format_metric_name(metric.name) - lines.append(f"| {name} | {_fmt(metric.value)} |") - else: - lines.append("") - lines.append("- No run metrics derived") - - # --- Item Verdicts --- - lines.append("") - lines.append("## Item Verdicts") - lines.append("") - lines.append( - "Per-row pass/fail summary. A row passes only if all its evaluator scores meet thresholds." - ) - if result.item_evaluations: - passed_items = sum(1 for item in result.item_evaluations if item.passed_all) - lines.append("") - lines.append( - f"- Items passed all thresholds: {passed_items}/{len(result.item_evaluations)}" - ) - lines.append("") - lines.append("| Row | Status | Passed Rules | Total Rules |") - lines.append("|---:|---|---:|---:|") - for item in result.item_evaluations: - passed_rules = sum(1 for threshold in item.thresholds if threshold.passed) - icon = "✅" if item.passed_all else "❌" - lines.append( - f"| {item.row_index} | {icon} Pass | {passed_rules} | {len(item.thresholds)} |" - if item.passed_all - else f"| {item.row_index} | {icon} Fail | {passed_rules} | {len(item.thresholds)} |" - ) - else: - lines.append("") - lines.append("- No item-level evaluations found") - - # --- Threshold Checks --- - lines.append("") - lines.append("## Threshold Checks") - lines.append("") - lines.append( - "Aggregate threshold evaluation — each evaluator's average score vs. its threshold." - ) - if result.thresholds: - lines.append("") - lines.append("| Evaluator | Threshold | Actual | Status |") - lines.append("|---|---|---:|---|") - for threshold in result.thresholds: - name = _format_metric_name(threshold.evaluator) - threshold_val = ( - f"{threshold.criteria} {_fmt_threshold_value(threshold.expected)}" - ) - actual_val = _fmt_threshold_value(threshold.actual) - icon = "✅" if threshold.passed else "❌" - label = "Met" if threshold.passed else "Missed" - lines.append( - f"| {name} | {threshold_val} | {actual_val} | {icon} {label} |" - ) - else: - lines.append("") - lines.append("- No thresholds configured") - - # --- Row Details --- - lines.append("") - lines.append("## Row Details") - lines.append("") - lines.append( - "Input, response, per-row scores, and retrieved context for each dataset row." - ) - _rows_with_text = [ - rm - for rm in result.row_metrics - if rm.input is not None or rm.response is not None - ] - if _rows_with_text: - item_map = {ie.row_index: ie for ie in result.item_evaluations} - for rm in _rows_with_text: - ie = item_map.get(rm.row_index) - if ie: - icon = "✅" if ie.passed_all else "❌" - status = f"{icon} Pass" if ie.passed_all else f"{icon} Fail" - else: - status = "—" - lines.append(f"### Row {rm.row_index} — {status}") - lines.append("") - if rm.input is not None: - lines.append(f"**Input:** {rm.input}") - lines.append("") - if rm.response is not None: - lines.append(f"**Response:** {rm.response}") - lines.append("") - if rm.context is not None: - context_display = rm.context - if len(context_display) > _MAX_CONTEXT_DISPLAY: - context_display = context_display[:_MAX_CONTEXT_DISPLAY] + "…" - lines.append(f"**Retrieved Context:** {context_display}") - lines.append("") - - # Per-row score table - if ie and ie.thresholds: - lines.append("| Evaluator | Score | Threshold | Status |") - lines.append("|---|---:|---|---|") - for t in ie.thresholds: - t_name = _format_metric_name(t.evaluator) - t_actual = _fmt_threshold_value(t.actual) - t_threshold = f"{t.criteria} {_fmt_threshold_value(t.expected)}" - t_icon = "✅" if t.passed else "❌" - t_label = "Met" if t.passed else "Missed" - lines.append( - f"| {t_name} | {t_actual} | {t_threshold} | {t_icon} {t_label} |" - ) - lines.append("") - else: - lines.append("") - lines.append("- No input/response data captured") - - # --- Artifacts --- - lines.append("") - lines.append("## Artifacts") - if result.artifacts is not None: - if result.artifacts.backend_stdout is not None: - lines.append(f"- backend_stdout: {result.artifacts.backend_stdout}") - if result.artifacts.backend_stderr is not None: - lines.append(f"- backend_stderr: {result.artifacts.backend_stderr}") - if result.artifacts.foundry_eval_studio_url is not None: - lines.append( - f"- foundry_eval_studio_url: {result.artifacts.foundry_eval_studio_url}" - ) - if result.artifacts.foundry_eval_name is not None: - lines.append(f"- foundry_eval_name: {result.artifacts.foundry_eval_name}") - return "\n".join(lines).rstrip() + "\n" - - -# --------------------------------------------------------------------------- -# Shared formatting helpers -# --------------------------------------------------------------------------- - - -def _fmt(value: float) -> str: - """Smart number formatting: integers show without decimals, floats show 2 dp.""" - if value == int(value) and abs(value) < 1e15: - return str(int(value)) - return f"{value:.2f}" - - -def _fmt_delta(value: float) -> str: - """Smart delta formatting with sign prefix.""" - if value == int(value) and abs(value) < 1e15: - return f"{int(value):+d}" - return f"{value:+.2f}" - - -def _threshold_label(passed: bool) -> str: - return "Met" if passed else "Missed" - - -def _check_threshold(value: float, criteria: str, target: str | None) -> bool: - """Evaluate whether a metric value meets a threshold criteria+target.""" - if target is None: - return True - try: - t = float(target) - except (ValueError, TypeError): - return True - if criteria == ">=": - return value >= t - if criteria == ">": - return value > t - if criteria == "<=": - return value <= t - if criteria == "<": - return value < t - if criteria == "==": - return value == t - return True - - -def _fmt_target(criteria: str, target: str | None) -> str: - """Format threshold target as 'criteria value' (e.g., '>= 3').""" - if target is None: - return criteria - try: - val = float(target) - return f"{criteria} {_fmt(val)}" - except (ValueError, TypeError): - return f"{criteria} {target}" - - -# --------------------------------------------------------------------------- -# Shared HTML helpers -# --------------------------------------------------------------------------- - -_CSS = """\ -:root { - --bg: #ffffff; --surface: #f6f8fa; --border: #d1d9e0; - --text: #1f2328; --muted: #656d76; --accent: #0969da; - --green: #1a7f37; --red: #cf222e; --yellow: #9a6700; - --green-bg: #dafbe1; --red-bg: #ffebe9; --yellow-bg: #fff8c5; - --font: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif; -} -*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } -body { font-family: var(--font); background: var(--bg); color: var(--text); line-height: 1.6; padding: 2rem 2.5rem; max-width: 1020px; margin: 0 auto; } -h1 { font-size: 1.6rem; font-weight: 600; margin-bottom: .4rem; } -h2 { font-size: 1.1rem; font-weight: 600; color: var(--text); margin: 2rem 0 .6rem; padding-bottom: .35rem; border-bottom: 1px solid var(--border); } -.badge { display: inline-block; padding: .15rem .6rem; border-radius: .25rem; font-weight: 600; font-size: .8rem; } -.badge-pass { background: var(--green-bg); color: var(--green); } -.badge-fail { background: var(--red-bg); color: var(--red); } -.badge-improved { background: var(--green-bg); color: var(--green); } -.badge-regressed { background: var(--red-bg); color: var(--red); } -.badge-unchanged { background: var(--yellow-bg); color: var(--yellow); } -.meta { color: var(--muted); font-size: .85rem; margin-bottom: 1.2rem; display: flex; flex-wrap: wrap; gap: .3rem 1.5rem; } -.meta span { white-space: nowrap; } -table { width: 100%; border-collapse: collapse; margin: .4rem 0 1rem; font-size: .875rem; } -th, td { padding: .5rem .7rem; text-align: left; border-bottom: 1px solid var(--border); } -th { background: var(--surface); font-size: .75rem; font-weight: 600; text-transform: uppercase; letter-spacing: .03em; color: var(--muted); } -td { vertical-align: top; } -tr:hover td { background: #f0f4f8; } -.num { text-align: right; font-variant-numeric: tabular-nums; } -.card-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: .6rem; margin: .4rem 0 1.2rem; } -.card { background: var(--surface); border: 1px solid var(--border); border-radius: .5rem; padding: .8rem 1rem; } -.card .label { font-size: .7rem; font-weight: 600; text-transform: uppercase; color: var(--muted); letter-spacing: .03em; } -.card .value { font-size: 1.3rem; font-weight: 700; margin-top: .15rem; } -footer { margin-top: 2.5rem; padding-top: .8rem; border-top: 1px solid var(--border); color: var(--muted); font-size: .75rem; text-align: center; } -""" - - -def _html_escape(text: str) -> str: - return ( - text.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace('"', """) - ) - - -def _badge(label: str, kind: str) -> str: - return f'{_html_escape(label)}' - - -def _status_badge(passed: bool) -> str: - return _badge("PASS", "pass") if passed else _badge("FAIL", "fail") - - -def _threshold_badge(passed: bool) -> str: - return _badge("Met", "pass") if passed else _badge("Missed", "fail") - - -def _direction_badge(direction: str) -> str: - return _badge(direction, direction) - - -def _wrap_page(title: str, body: str) -> str: - return ( - "\n" - '\n\n' - '\n' - '\n' - f"{_html_escape(title)}\n" - f"\n" - "\n\n" - f"{body}\n" - "
Generated by AgentOps
\n" - "\n\n" - ) - - -# --------------------------------------------------------------------------- -# Evaluation run HTML report -# --------------------------------------------------------------------------- - - -def generate_report_html(result: RunResult) -> str: - overall = result.summary.overall_passed - parts: list[str] = [] - - parts.append(f"

AgentOps Evaluation Report {_status_badge(overall)}

") - parts.append( - '
' - f"Bundle: {_html_escape(result.bundle.name)}" - f"Dataset: {_html_escape(result.dataset.name)}" - f"Backend: {_html_escape(result.execution.backend)}" - "
" - ) - - # --- How pass/fail is determined --- - parts.append("

How Pass/Fail Is Determined

") - parts.append( - "

Each evaluator scores every dataset row. Each score is compared against a threshold " - "(e.g., >= 0.8). A row passes if all its evaluator scores " - "meet their thresholds. The overall run passes only if every row passes " - "all thresholds.

" - ) - - parts.append("

Execution

") - parts.append('
') - for label, val in [ - ("Duration", f"{result.execution.duration_seconds:.1f}s"), - ("Started", result.execution.started_at[:19]), - ("Exit code", str(result.execution.exit_code)), - ]: - parts.append( - f'
{label}
{_html_escape(val)}
' - ) - parts.append("
") - - if result.metrics: - parts.append("

Metrics

") - parts.append("

Average scores across all dataset rows.

") - parts.append( - '' - ) - for m in result.metrics: - name = _format_metric_name(m.name) - desc = _get_evaluator_description(m.name) - parts.append( - f'' - ) - parts.append("
MetricValueWhat It Measures
{_html_escape(name)}{_fmt(m.value)}{_html_escape(desc)}
") - - if result.run_metrics: - parts.append("

Run Metrics

") - parts.append("

Derived summary statistics for the entire evaluation run.

") - parts.append( - '' - ) - for m in result.run_metrics: - name = _format_metric_name(m.name) - parts.append( - f'' - ) - parts.append("
MetricValue
{_html_escape(name)}{_fmt(m.value)}
") - - if result.thresholds: - parts.append("

Threshold Checks

") - parts.append( - "

Aggregate threshold evaluation — each evaluator's average score vs. its threshold.

" - ) - parts.append( - '' - ) - for t in result.thresholds: - name = _format_metric_name(t.evaluator) - threshold_val = f"{t.criteria} {_fmt_threshold_value(t.expected)}" - actual_val = _fmt_threshold_value(t.actual) - parts.append( - f"" - f'' - f"" - ) - parts.append("
EvaluatorThresholdActualStatus
{_html_escape(name)}{_html_escape(threshold_val)}{_html_escape(actual_val)}{_threshold_badge(t.passed)}
") - - if result.item_evaluations: - passed_count = sum(1 for i in result.item_evaluations if i.passed_all) - total = len(result.item_evaluations) - parts.append(f"

Item Verdicts ({passed_count}/{total} passed)

") - parts.append("

Per-row pass/fail summary.

") - parts.append( - '' - ) - for item in result.item_evaluations: - pr = sum(1 for th in item.thresholds if th.passed) - parts.append( - f'' - f'' - ) - parts.append("
RowStatusPassed RulesTotal Rules
{item.row_index}{_status_badge(item.passed_all)}{pr}{len(item.thresholds)}
") - - _html_rows_with_text = [ - rm - for rm in result.row_metrics - if rm.input is not None or rm.response is not None - ] - if _html_rows_with_text: - item_map = {ie.row_index: ie for ie in result.item_evaluations} - parts.append("

Row Details

") - parts.append( - "

Input, response, per-row scores, and retrieved context for each dataset row.

" - ) - for rm in _html_rows_with_text: - ie = item_map.get(rm.row_index) - status_html = _status_badge(ie.passed_all) if ie else "—" - parts.append(f"

Row {rm.row_index} {status_html}

") - if rm.input: - parts.append(f"

Input: {_html_escape(rm.input)}

") - if rm.response: - parts.append( - f"

Response: {_html_escape(rm.response)}

" - ) - if rm.context: - context_display = rm.context - if len(context_display) > _MAX_CONTEXT_DISPLAY: - context_display = context_display[:_MAX_CONTEXT_DISPLAY] + "…" - parts.append( - f"

Retrieved Context: {_html_escape(context_display)}

" - ) - # Per-row score table - if ie and ie.thresholds: - parts.append( - '' - "" - ) - for it in ie.thresholds: - t_name = _format_metric_name(it.evaluator) - t_actual = _fmt_threshold_value(it.actual) - t_threshold = f"{it.criteria} {_fmt_threshold_value(it.expected)}" - parts.append( - f"" - f'' - f"" - f"" - ) - parts.append("
EvaluatorScoreThresholdStatus
{_html_escape(t_name)}{_html_escape(t_actual)}{_html_escape(t_threshold)}{_threshold_badge(it.passed)}
") - - if result.artifacts: - urls = [] - if result.artifacts.foundry_eval_studio_url: - urls.append( - f'View in Foundry' - ) - if urls: - parts.append("

Artifacts

") - parts.append("

" + " · ".join(urls) + "

") - - return _wrap_page("AgentOps Evaluation Report", "\n".join(parts)) - - -# --------------------------------------------------------------------------- -# Comparison Markdown report (N runs) -# --------------------------------------------------------------------------- - - -def generate_comparison_markdown(result: ComparisonResult) -> str: - verdict = ( - "REGRESSIONS DETECTED" if result.summary.any_regressions else "NO REGRESSIONS" - ) - run_labels = [r.run_id for r in result.runs] - - lines: list[str] = [] - lines.append("# AgentOps Comparison Report") - lines.append("") - lines.append("## Overview") - lines.append("") - lines.append(f"- Runs compared: **{result.summary.run_count}**") - lines.append(f"- Verdict: **{verdict}**") - lines.append("") - - # Conditions - cond = result.conditions - if cond: - type_labels = { - "agent": "Agent Comparison", - "model": "Model Comparison", - "dataset": "Dataset Coverage", - "general": "General Comparison", - } - lines.append("## Conditions") - lines.append("") - lines.append( - f"- Comparison type: **{type_labels.get(cond.comparison_type, cond.comparison_type)}**" - ) - if cond.fixed: - fixed_items = ", ".join(f"{k}={v}" for k, v in cond.fixed.items()) - lines.append(f"- Fixed: {fixed_items}") - if cond.varying: - lines.append(f"- Varying: {', '.join(cond.varying)}") - if not cond.row_level_valid: - lines.append( - "- Note: Row-level comparison is not meaningful because datasets differ across runs." - ) - lines.append("") - - # Run details table — only show varying fields + always-show fields - varying_set = set(cond.varying) if cond else set() - detail_fields = [ - ("Backend", "backend", lambda r: r.backend or "-"), - ("Target", None, lambda r: r.target or "-"), - ("Model", None, lambda r: r.model or "-"), - ("Agent", None, lambda r: r.agent_id or "-"), - ("Project", "project", lambda r: r.project_endpoint or "-"), - ("Dataset", "dataset", lambda r: r.dataset_name), - ("Bundle", "bundle", lambda r: r.bundle_name), - ( - "Status", - None, - lambda r: ( - "PASS" - if r.overall_passed - else "FAIL" - if r.overall_passed is not None - else "-" - ), - ), - ("Started", None, lambda r: r.started_at[:19] if r.started_at else "-"), - ] - # Keep fields that are varying or always-show (condition_key is None) - visible_fields = [ - (label, ckey, getter) - for label, ckey, getter in detail_fields - if ckey is None or ckey in varying_set - ] - - lines.append("## Run Details") - lines.append("") - lines.append("| | " + " | ".join(run_labels) + " |") - lines.append("|---|" + "|".join("---" for _ in run_labels) + "|") - lines.append( - "| Role | Baseline | " - + " | ".join(f"Run {i}" for i in range(1, len(result.runs))) - + " |" - ) - for field, _ckey, getter in visible_fields: - cells = [getter(r) for r in result.runs] - lines.append(f"| {field} | " + " | ".join(cells) + " |") - lines.append("") - lines.append( - "*Status is PASS when all thresholds are met, FAIL when any threshold is missed.*" - ) - lines.append("") - - # Unified Evaluators table (metrics + thresholds merged) - if result.metric_rows: - threshold_map = {tr.evaluator: tr for tr in result.threshold_rows} - lines.append("## Evaluators") - lines.append("") - header = "| Evaluator | Target | " + " | ".join(run_labels) + " | Best |" - sep = "|---|---|" + "|".join("---:" for _ in run_labels) + "|---|" - lines.append(header) - lines.append(sep) - for mr in result.metric_rows: - tr = threshold_map.get(mr.name) - target = _fmt_target(tr.criteria, tr.target) if tr else "-" - cells = [] - for i, v in enumerate(mr.values): - if not tr: - # Informational metric — plain value, no delta/direction/best - cells.append(_fmt(v)) - continue - parts_cell = [_fmt(v)] - # Direction vs baseline (skip for baseline itself) - if i > 0: - d = mr.deltas[i] - direction = mr.directions[i] - if d is not None: - parts_cell.append(f"({_fmt_delta(d)}, {direction})") - # Threshold check vs absolute target - met = _check_threshold(v, tr.criteria, tr.target) - parts_cell.append(_threshold_label(met)) - cells.append(" ".join(parts_cell)) - best = ( - run_labels[mr.best_run_index] - if (mr.best_run_index is not None and tr) - else "-" - ) - lines.append( - f"| {mr.name} | {target} | " + " | ".join(cells) + f" | {best} |" - ) - lines.append("") - - show_items = result.conditions.row_level_valid if result.conditions else True - if result.item_rows and show_items: - threshold_map = {tr.evaluator: tr for tr in result.threshold_rows} - lines.append("## Item Verdicts") - lines.append("") - header = "| Row | " + " | ".join(run_labels) + " |" - sep = "|---:|" + "|".join("---" for _ in run_labels) + "|" - lines.append(header) - lines.append(sep) - for ir in result.item_rows: - cells = [] - for run_idx, passed in enumerate(ir.passed_all): - parts_cell = [] - # Show per-evaluator scores for this row - for eval_name, scores_list in ir.scores.items(): - score = scores_list[run_idx] if run_idx < len(scores_list) else None - if score is not None: - tr = threshold_map.get(eval_name) - if tr: - met = _check_threshold(score, tr.criteria, tr.target) - parts_cell.append( - f"{eval_name}: {_fmt(score)} {_threshold_label(met)}" - ) - else: - parts_cell.append(f"{eval_name}: {_fmt(score)}") - if not parts_cell: - parts_cell.append("PASS" if passed else "FAIL") - cells.append("; ".join(parts_cell)) - lines.append(f"| {ir.row_index} | " + " | ".join(cells) + " |") - elif result.item_rows and not show_items: - lines.append("## Item Verdicts") - lines.append("") - lines.append( - "*Skipped — datasets differ across runs so row-level comparison is not meaningful.*" - ) - - return "\n".join(lines).rstrip() + "\n" - - -# --------------------------------------------------------------------------- -# Comparison HTML report (N runs) -# --------------------------------------------------------------------------- - - -def generate_comparison_html(result: ComparisonResult) -> str: - has_reg = result.summary.any_regressions - verdict_badge = ( - _badge("REGRESSIONS DETECTED", "regressed") - if has_reg - else _badge("NO REGRESSIONS", "improved") - ) - run_labels = [r.run_id for r in result.runs] - cond = result.conditions - type_labels = { - "agent": "Agent Comparison", - "model": "Model Comparison", - "dataset": "Dataset Coverage", - "general": "General Comparison", - } - threshold_map = {tr.evaluator: tr for tr in result.threshold_rows} - - # Pre-compute per-run row pass counts (across all threshold evaluators) - run_row_pass: list[tuple[int, int]] = [] # (passed, total) per run - for run_idx in range(len(result.runs)): - total = len(result.item_rows) - passed = sum(1 for ir in result.item_rows if ir.passed_all[run_idx]) - run_row_pass.append((passed, total)) - - # Pre-compute per-evaluator row pass rates - eval_row_rates: dict[str, list[tuple[int, int]]] = {} - for thr in result.threshold_rows: - rates = [] - for run_idx in range(len(result.runs)): - total = 0 - passed = 0 - for ir in result.item_rows: - scores_list = ir.scores.get(thr.evaluator, []) - score = scores_list[run_idx] if run_idx < len(scores_list) else None - if score is not None: - total += 1 - if _check_threshold(score, thr.criteria, thr.target): - passed += 1 - rates.append((passed, total)) - eval_row_rates[thr.evaluator] = rates - - parts: list[str] = [] - - # --- Header --- - parts.append(f"

AgentOps Comparison Report {verdict_badge}

") - ctype = type_labels.get(cond.comparison_type, "") if cond else "" - varying_str = ", ".join(cond.varying) if cond and cond.varying else "" - parts.append( - f'
{ctype}Varying: {_html_escape(varying_str)}{result.summary.run_count} runs
' - ) - - # --- Run Config --- - varying_set = set(cond.varying) if cond else set() - detail_fields = [ - ("Role", None, lambda r: ""), - ("Target", None, lambda r: r.target or "-"), - ("Model", None, lambda r: r.model or "-"), - ("Agent", None, lambda r: r.agent_id or "-"), - ("Dataset", "dataset", lambda r: r.dataset_name), - ("Status", None, lambda r: ""), - ] - visible_fields = [ - (lbl, c, g) for lbl, c, g in detail_fields if c is None or c in varying_set - ] - - cols = "".join(f"{_html_escape(lbl)}" for lbl in run_labels) - parts.append(f"{cols}") - for field, _ckey, getter in visible_fields: - cells = "" - for i, r in enumerate(result.runs): - if field == "Role": - cells += ( - f"" - ) - elif field == "Status": - p, t = run_row_pass[i] - pct = int(p / t * 100) if t > 0 else 0 - if r.overall_passed: - cells += f"" - else: - cells += f"" - else: - cells += f"" - parts.append(f"{cells}") - parts.append("
{'Baseline' if i == 0 else f'Run {i}'}{_status_badge(True)} ({pct}% · {p}/{t}){_status_badge(False)} ({pct}% · {p}/{t}){_html_escape(getter(r))}
{field}
") - - # --- Evaluators --- - if result.metric_rows: - parts.append("

Evaluators

") - cols = "".join( - f'{_html_escape(lbl)}' for lbl in run_labels - ) - parts.append( - f"{cols}" - ) - for mr in result.metric_rows: - tr = threshold_map.get(mr.name) - target = _fmt_target(tr.criteria, tr.target) if tr else "-" - cells = "" - for i, v in enumerate(mr.values): - if not tr: - # Informational metric — plain value only - cells += ( - f'' - ) - continue - is_best = mr.best_run_index == i - # Dot indicator - met = _check_threshold(v, tr.criteria, tr.target) - dot = ( - '● ' - if met - else '● ' - ) - # Value - val_str = _fmt(v) - # Delta + arrow - delta_str = "" - if i > 0: - d = mr.deltas[i] - direction = mr.directions[i] - if d is not None: - arrow = ( - "↑" - if direction == "improved" - else ("↓" if direction == "regressed" else "→") - ) - color = ( - "var(--green)" - if direction == "improved" - else ( - "var(--red)" - if direction == "regressed" - else "var(--muted)" - ) - ) - delta_str = f' {arrow} {_fmt_delta(d)}' - # Row pass rate - rate_str = "" - if mr.name in eval_row_rates: - p, t = eval_row_rates[mr.name][i] - if t > 0: - rate_str = ( - f' ({p}/{t})' - ) - # Best highlight - best_style = ( - "background:var(--green-bg);font-weight:600;border-radius:.25rem;padding:.1rem .3rem;" - if is_best - else "" - ) - inner = f"{dot}{val_str}{delta_str}{rate_str}" - if is_best: - cells += f'' - else: - cells += f'' - parts.append( - f"{cells}" - ) - parts.append("
EvaluatorTarget
{_fmt(v)}{inner}{inner}
{_html_escape(mr.name)}{_html_escape(target)}
") - - # --- Row Details --- - show_items = result.conditions.row_level_valid if result.conditions else True - if result.item_rows and show_items: - parts.append("

Row Details

") - cols = "".join(f"{_html_escape(lbl)}" for lbl in run_labels) - parts.append( - f'{cols}' - ) - for ir in result.item_rows: - cells = "" - for run_idx in range(len(result.runs)): - parts_cell = [] - for eval_name, scores_list in ir.scores.items(): - score = scores_list[run_idx] if run_idx < len(scores_list) else None - if score is not None: - tr = threshold_map.get(eval_name) - if tr: - met = _check_threshold(score, tr.criteria, tr.target) - dot = ( - '●' - if met - else '●' - ) - parts_cell.append( - f"{dot} {_html_escape(eval_name)}: {_fmt(score)}" - ) - else: - parts_cell.append( - f"{_html_escape(eval_name)}: {_fmt(score)}" - ) - if not parts_cell: - parts_cell.append(_status_badge(ir.passed_all[run_idx])) - cells += f"" - parts.append(f'{cells}') - parts.append("
Row
{'
'.join(parts_cell)}
{ir.row_index}
") - elif result.item_rows and not show_items: - parts.append("

Row Details

") - parts.append( - '

Skipped — datasets differ across runs, row-level comparison not meaningful.

' - ) - - # --- Fixed Parameters --- - if cond and cond.fixed: - parts.append("

Fixed Parameters

") - parts.append( - "" - ) - for key, val in cond.fixed.items(): - parts.append( - f"" - ) - parts.append("
ParameterValue
{_html_escape(key)}{_html_escape(val)}
") - - return _wrap_page("AgentOps Comparison Report", "\n".join(parts)) diff --git a/src/agentops/core/results.py b/src/agentops/core/results.py new file mode 100644 index 00000000..0e17607f --- /dev/null +++ b/src/agentops/core/results.py @@ -0,0 +1,117 @@ +"""Result dataclasses for the AgentOps 1.0 pipeline. + +These shapes are written to ``results.json`` after every ``agentops eval`` run +and consumed by the reporter and comparison logic. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class RowMetric(BaseModel): + """A single evaluator score for one dataset row.""" + + name: str + value: Optional[float] = None + error: Optional[str] = None + reason: Optional[str] = None + + +class RowResult(BaseModel): + """One evaluated dataset row.""" + + row_index: int + input: str + expected: Optional[str] = None + response: str = "" + context: Optional[str] = None + latency_seconds: Optional[float] = None + tool_calls: Optional[List[Any]] = None + metrics: List[RowMetric] = Field(default_factory=list) + error: Optional[str] = None + + +class ThresholdEvaluation(BaseModel): + """A pass/fail check for a single metric on the run aggregate.""" + + metric: str + criteria: str + expected: str + actual: str + passed: bool + + +class RunSummary(BaseModel): + """Top-level pass/fail summary of an evaluation run.""" + + items_total: int + items_passed_all: int + items_pass_rate: float + thresholds_total: int + thresholds_passed: int + threshold_pass_rate: float + overall_passed: bool + + +class TargetInfo(BaseModel): + """Resolved target information (echoed into results.json).""" + + kind: str + raw: str + protocol: Optional[str] = None + name: Optional[str] = None + version: Optional[str] = None + url: Optional[str] = None + deployment: Optional[str] = None + + +class ComparisonMetric(BaseModel): + """Per-metric delta between the current run and a baseline.""" + + metric: str + current: Optional[float] = None + baseline: Optional[float] = None + delta: Optional[float] = None + direction: str # "improved" | "regressed" | "unchanged" + + +class ComparisonRow(BaseModel): + """Per-row regression / improvement against a baseline.""" + + row_index: int + current_passed: bool + baseline_passed: Optional[bool] = None + direction: str # "improved" | "regressed" | "unchanged" | "new" + + +class ComparisonInfo(BaseModel): + """Comparison block included when ``--baseline`` was provided.""" + + baseline_path: str + baseline_started_at: Optional[str] = None + baseline_overall_passed: Optional[bool] = None + metrics: List[ComparisonMetric] = Field(default_factory=list) + rows: List[ComparisonRow] = Field(default_factory=list) + + +class RunResult(BaseModel): + """Full ``results.json`` payload.""" + + version: int = 1 + started_at: str + finished_at: str + duration_seconds: float + target: TargetInfo + dataset_path: str + evaluators: List[str] = Field(default_factory=list) + rows: List[RowResult] = Field(default_factory=list) + aggregate_metrics: Dict[str, float] = Field(default_factory=dict) + thresholds: List[ThresholdEvaluation] = Field(default_factory=list) + summary: RunSummary + comparison: Optional[ComparisonInfo] = None + config: Dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") diff --git a/src/agentops/core/thresholds.py b/src/agentops/core/thresholds.py deleted file mode 100644 index ba43f86d..00000000 --- a/src/agentops/core/thresholds.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Threshold evaluation logic for AgentOps.""" - -from __future__ import annotations - -from typing import Dict, List - -from agentops.core.models import ThresholdEvaluationResult, ThresholdRule - - -def evaluate_thresholds( - threshold_rules: List[ThresholdRule], - metrics_by_name: Dict[str, float], -) -> List[ThresholdEvaluationResult]: - results: List[ThresholdEvaluationResult] = [] - - for rule in threshold_rules: - if rule.evaluator not in metrics_by_name: - raise ValueError( - f"Missing evaluator score required by threshold: {rule.evaluator}" - ) - - actual_value = metrics_by_name[rule.evaluator] - - if rule.criteria == "true" or rule.criteria == "false": - expected_bool = rule.criteria == "true" - - if actual_value in (0.0, 1.0): - actual_bool = actual_value == 1.0 - else: - raise ValueError( - f"Evaluator '{rule.evaluator}' must produce 0/1 for boolean criteria" - ) - - passed = actual_bool is expected_bool - results.append( - ThresholdEvaluationResult( - evaluator=rule.evaluator, - criteria=rule.criteria, - expected="true" if expected_bool else "false", - actual="true" if actual_bool else "false", - passed=passed, - ) - ) - continue - - if rule.value is None: - raise ValueError( - f"Threshold for evaluator '{rule.evaluator}' requires a numeric value" - ) - - target_value = float(rule.value) - - if rule.criteria == ">=": - passed = actual_value >= target_value - elif rule.criteria == ">": - passed = actual_value > target_value - elif rule.criteria == "<=": - passed = actual_value <= target_value - elif rule.criteria == "<": - passed = actual_value < target_value - elif rule.criteria == "==": - passed = actual_value == target_value - else: - raise ValueError(f"Unsupported threshold criteria: {rule.criteria}") - - results.append( - ThresholdEvaluationResult( - evaluator=rule.evaluator, - criteria=rule.criteria, - expected=f"{target_value:.6f}", - actual=f"{actual_value:.6f}", - passed=passed, - ) - ) - - return results diff --git a/src/agentops/mcp/__init__.py b/src/agentops/mcp/__init__.py new file mode 100644 index 00000000..6283e742 --- /dev/null +++ b/src/agentops/mcp/__init__.py @@ -0,0 +1,10 @@ +"""MCP (Model Context Protocol) server for AgentOps. + +Exposes the most-used AgentOps capabilities as MCP tools over stdio so that +MCP-aware coding agents (Claude Code, Copilot Chat, etc.) can drive AgentOps +without shelling out to the CLI. + +The server is opt-in: it requires the optional ``mcp`` extra +(``pip install agentops-toolkit[mcp]``) and is started via +``agentops mcp serve``. +""" diff --git a/src/agentops/mcp/server.py b/src/agentops/mcp/server.py new file mode 100644 index 00000000..99da2d42 --- /dev/null +++ b/src/agentops/mcp/server.py @@ -0,0 +1,232 @@ +"""AgentOps MCP server (stdio transport). + +This module is imported lazily — the ``mcp`` extra is optional. Importing +this file triggers ``import mcp.server.fastmcp`` which fails with a clear +message if the extra is not installed. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional + + +def _build_server() -> Any: + """Construct and return the FastMCP server. + + Imports of ``mcp`` happen inside the function so that ``agentops --help`` + keeps working when the optional extra is not installed. + """ + try: + from mcp.server.fastmcp import FastMCP + except ImportError as exc: # pragma: no cover - exercised only without extra + raise RuntimeError( + "The MCP server requires the 'mcp' extra. " + "Install it with: pip install agentops-toolkit[mcp]" + ) from exc + + server = FastMCP("agentops") + + @server.tool() + def agentops_init(directory: str = ".", force: bool = False) -> Dict[str, Any]: + """Initialise an AgentOps workspace at ``directory``. + + Creates ``agentops.yaml`` at the project root and a tiny seed + dataset under ``.agentops/data/smoke.jsonl``. Set ``force=True`` + to overwrite existing files. + """ + from agentops.services.initializer import initialize_flat_workspace + + result = initialize_flat_workspace(Path(directory), force=force) + return { + "workspace_dir": str(result.workspace_dir), + "created_files": [str(p) for p in result.created_files], + "overwritten_files": [str(p) for p in result.overwritten_files], + "skipped_files": [str(p) for p in result.skipped_files], + } + + @server.tool() + def agentops_eval_run( + config_path: Optional[str] = None, + output_dir: Optional[str] = None, + baseline: Optional[str] = None, + ) -> Dict[str, Any]: + """Run an AgentOps evaluation and return a summary. + + ``config_path`` defaults to ``agentops.yaml`` in the current + directory. ``output_dir`` defaults to + ``.agentops/results/latest``. ``baseline`` is an optional path to + a previous ``results.json`` for comparison. + """ + from agentops.core.config_loader import load_agentops_config + from agentops.pipeline.orchestrator import ( + RunOptions, + exit_code_from, + run_evaluation, + ) + + config = Path(config_path) if config_path else Path("agentops.yaml") + if not config.exists(): + return { + "ok": False, + "exit_code": 1, + "error": f"config not found at {config}", + } + try: + config_obj = load_agentops_config(config) + except Exception as exc: # noqa: BLE001 + return { + "ok": False, + "exit_code": 1, + "error": f"failed to load config: {exc}", + } + + out_dir = ( + Path(output_dir) + if output_dir + else config.parent / ".agentops" / "results" / "latest" + ) + options = RunOptions( + config_path=config.resolve(), + output_dir=out_dir, + baseline_path=Path(baseline).resolve() if baseline else None, + ) + try: + run = run_evaluation(config_obj, options=options) + except Exception as exc: # noqa: BLE001 + return {"ok": False, "exit_code": 1, "error": f"evaluation failed: {exc}"} + + return { + "ok": True, + "exit_code": exit_code_from(run), + "output_dir": str(out_dir), + "results_json": str(out_dir / "results.json"), + "report_md": str(out_dir / "report.md"), + "passed": bool(run.summary.overall_passed), + "metrics": dict(run.aggregate_metrics or {}), + } + + @server.tool() + def agentops_report_show(report_path: Optional[str] = None) -> Dict[str, Any]: + """Return the contents of ``report.md`` for a finished run. + + Defaults to ``.agentops/results/latest/report.md``. + """ + path = ( + Path(report_path) + if report_path + else Path(".agentops/results/latest/report.md") + ) + if not path.exists(): + return {"ok": False, "error": f"report not found at {path}"} + return { + "ok": True, + "path": str(path), + "markdown": path.read_text(encoding="utf-8"), + } + + @server.tool() + def agentops_results_summary(results_path: Optional[str] = None) -> Dict[str, Any]: + """Return a compact JSON summary extracted from ``results.json``. + + Defaults to ``.agentops/results/latest/results.json``. + """ + path = ( + Path(results_path) + if results_path + else Path(".agentops/results/latest/results.json") + ) + if not path.exists(): + return {"ok": False, "error": f"results not found at {path}"} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": f"failed to parse results.json: {exc}"} + + summary = data.get("summary") or {} + return { + "ok": True, + "path": str(path), + "version": data.get("version"), + "target": data.get("target"), + "rows": len(data.get("rows") or []), + "metrics": summary.get("metrics") or {}, + "thresholds": summary.get("thresholds") or {}, + "overall_passed": summary.get("overall_passed"), + } + + @server.tool() + def agentops_dataset_add( + dataset_path: str, + rows: List[Dict[str, Any]], + ) -> Dict[str, Any]: + """Append JSONL rows to ``dataset_path``. + + Creates the parent directory if needed. Each row must be a JSON + object — typical keys are ``input``, ``expected``, ``context``, + and ``tool_calls`` depending on the agent type. + """ + if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows): + return {"ok": False, "error": "rows must be a list of JSON objects"} + path = Path(dataset_path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + return {"ok": True, "path": str(path), "appended": len(rows)} + + @server.tool() + def agentops_list_runs(workspace_dir: str = ".") -> Dict[str, Any]: + """List historical runs under ``/.agentops/results/``.""" + results_dir = Path(workspace_dir) / ".agentops" / "results" + if not results_dir.is_dir(): + return {"ok": True, "runs": []} + runs: List[Dict[str, Any]] = [] + for entry in sorted(results_dir.iterdir(), key=lambda p: p.name): + if not entry.is_dir(): + continue + results_json = entry / "results.json" + run_info: Dict[str, Any] = { + "name": entry.name, + "path": str(entry), + "has_results": results_json.exists(), + } + if results_json.exists(): + try: + payload = json.loads(results_json.read_text(encoding="utf-8")) + summary = payload.get("summary") or {} + run_info["overall_passed"] = summary.get("overall_passed") + run_info["metrics"] = summary.get("metrics") or {} + except Exception: # noqa: BLE001 + pass + runs.append(run_info) + return {"ok": True, "runs": runs} + + @server.tool() + def agentops_workflow_init( + directory: str = ".", + force: bool = False, + ) -> Dict[str, Any]: + """Generate GitHub Actions workflows for AgentOps evaluation.""" + from agentops.services.cicd import generate_cicd_workflows + + result = generate_cicd_workflows(directory=Path(directory), force=force) + return { + "ok": True, + "created_files": [str(p) for p in result.created_files], + "overwritten_files": [str(p) for p in result.overwritten_files], + "skipped_files": [str(p) for p in result.skipped_files], + } + + return server + + +def serve_stdio() -> None: + """Entry point for ``agentops mcp serve``. + + Builds the FastMCP server and runs it on the stdio transport. This + function blocks until the client disconnects. + """ + server = _build_server() + server.run() diff --git a/src/agentops/pipeline/__init__.py b/src/agentops/pipeline/__init__.py new file mode 100644 index 00000000..c53271ca --- /dev/null +++ b/src/agentops/pipeline/__init__.py @@ -0,0 +1,8 @@ +"""AgentOps 1.0 evaluation pipeline. + +Public entry point: :func:`agentops.pipeline.orchestrator.run_evaluation`. +""" + +from agentops.pipeline.orchestrator import run_evaluation + +__all__ = ["run_evaluation"] diff --git a/src/agentops/pipeline/cloud_publisher.py b/src/agentops/pipeline/cloud_publisher.py new file mode 100644 index 00000000..eba15ba0 --- /dev/null +++ b/src/agentops/pipeline/cloud_publisher.py @@ -0,0 +1,405 @@ +"""Cloud-side publisher: submit a run to the New Foundry Evaluations panel. + +Unlike :mod:`agentops.pipeline.publisher` (which uploads metrics that +AgentOps already computed locally to the *Classic* Foundry Evaluations +panel via OneDP), this module asks **Foundry to execute the agent and the +evaluators server-side** through the OpenAI Evals API. + +The flow: + +1. Build an :class:`azure.ai.projects.AIProjectClient` from the configured + project endpoint using ``DefaultAzureCredential``. +2. Get the OpenAI client via ``project_client.get_openai_client()``. We do + **not** pass ``api_version`` — the SDK picks the correct one (passing + one explicitly has historically caused 404s in this codebase). +3. Upload the JSONL dataset as an OpenAI file with ``purpose="evals"``. +4. Create the eval definition with ``client.evals.create(...)``, mapping + each AgentOps evaluator preset onto an ``azure_ai_evaluator`` testing + criterion. +5. Create the run with ``client.evals.runs.create(...)``, pointing at the + uploaded file and using ``azure_ai_target_completions`` with an + ``agent_reference`` so Foundry invokes the agent itself. +6. Poll until the run terminates, then return identifiers + the portal URL. + +This module never re-runs the agent locally and never invokes evaluators +locally; that work happens inside Foundry. The local ``results.json`` +(produced before this hop) remains the canonical record from AgentOps's +point of view. + +Limitations (documented in the YAML schema docstring as well): + +* Only ``foundry_prompt`` agents (``name:version``) are supported. HTTP + endpoints, local adapters, and direct model deployments are rejected. +* Only builtin evaluators that map cleanly onto ``azure_ai_evaluator`` + testing criteria are supported. Custom evaluators are skipped with a + warning. +* Latency reported by the New Foundry view is Foundry-to-Foundry, not the + client-perceived latency captured locally. +""" + +from __future__ import annotations + +import json +import logging +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from agentops.core.results import RunResult + +logger = logging.getLogger("agentops.pipeline.cloud_publisher") + + +# --------------------------------------------------------------------------- +# Public types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CloudPublishResult: + """Outcome of a cloud (New Foundry) publish.""" + + eval_id: str + run_id: str + status: str + report_url: Optional[str] + evaluation_name: str + + +# Map AgentOps evaluator class names to the OpenAI Evals API evaluator +# names that ``azure_ai_evaluator`` recognises. Any preset whose +# ``class_name`` is not in this map is skipped (with a warning) when +# building testing criteria. +_AZURE_AI_EVALUATOR_NAMES: Dict[str, str] = { + "CoherenceEvaluator": "builtin.coherence", + "FluencyEvaluator": "builtin.fluency", + "SimilarityEvaluator": "builtin.similarity", + "F1ScoreEvaluator": "builtin.f1_score", + "RelevanceEvaluator": "builtin.relevance", + "GroundednessEvaluator": "builtin.groundedness", + "RetrievalEvaluator": "builtin.retrieval", + "ResponseCompletenessEvaluator": "builtin.response_completeness", + "ToolCallAccuracyEvaluator": "builtin.tool_call_accuracy", + "IntentResolutionEvaluator": "builtin.intent_resolution", + "TaskAdherenceEvaluator": "builtin.task_adherence", +} + + +_DEFAULT_POLL_INTERVAL_SECONDS = 5.0 +_DEFAULT_MAX_POLL_ATTEMPTS = 120 # 10 minutes at 5s intervals +_TERMINAL_STATUSES = {"completed", "failed", "canceled", "cancelled"} + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def publish_to_foundry_cloud( + result: RunResult, + *, + dataset_path: Path, + project_endpoint: str, + evaluation_name: Optional[str] = None, + poll_interval_seconds: float = _DEFAULT_POLL_INTERVAL_SECONDS, + max_poll_attempts: int = _DEFAULT_MAX_POLL_ATTEMPTS, + progress: Optional[Callable[[str], None]] = None, +) -> CloudPublishResult: + """Submit ``result``'s target to Foundry for server-side evaluation. + + Parameters + ---------- + result: + Local run result. Used to derive the agent reference and the list + of evaluator presets that should map onto ``azure_ai_evaluator`` + testing criteria. + dataset_path: + Path to the JSONL dataset to upload. Must already exist. + project_endpoint: + Foundry project endpoint URL (e.g. + ``https://contoso.services.ai.azure.com/api/projects/p``). + evaluation_name: + Optional display name. Defaults to ``agentops-cloud-``. + poll_interval_seconds, max_poll_attempts: + Control polling cadence and bound. The default budget is + ~10 minutes. + progress: + Optional callback invoked with one-line status updates. The + orchestrator wires this to the same channel that prints per-row + progress so the user sees what is happening during the long + cloud round-trip. + + Raises + ------ + ImportError + ``azure-ai-projects`` / ``azure-identity`` are not installed. + ValueError + Target is not a Foundry agent or the dataset is missing. + RuntimeError + Polling timed out or the run terminated with a non-completed + status. + """ + progress = progress or (lambda _msg: None) + + if result.target.kind != "foundry_prompt": + raise ValueError( + "publish: foundry_cloud only supports Foundry agents declared " + "as 'name:version' (foundry_prompt targets). Got " + f"target.kind={result.target.kind!r}." + ) + if not dataset_path.exists(): + raise ValueError(f"dataset file not found: {dataset_path}") + + agent_name = result.target.name + agent_version = result.target.version + if not agent_name or not agent_version: + raise ValueError( + "Cloud publish requires a fully qualified 'name:version' agent " + f"reference; got name={agent_name!r} version={agent_version!r}" + ) + + try: + from azure.ai.projects import AIProjectClient # noqa: WPS433 + from azure.identity import DefaultAzureCredential # noqa: WPS433 + except ImportError as exc: # pragma: no cover - exercised only at runtime + raise ImportError( + "publish: foundry_cloud requires 'azure-ai-projects' and " + "'azure-identity'. Install with:\n" + " pip install azure-ai-projects azure-identity" + ) from exc + + credential = DefaultAzureCredential( + exclude_developer_cli_credential=True, + ) + project_client = AIProjectClient( + endpoint=project_endpoint, + credential=credential, + ) + + # NB: do not pass api_version — the SDK chooses the right one. Passing + # an explicit version has historically caused 404s in this codebase. + openai_client = project_client.get_openai_client() + + eval_name = evaluation_name or f"agentops-cloud-{uuid.uuid4().hex[:8]}" + progress(f"cloud publish: preparing run '{eval_name}'") + + file_id = _upload_dataset(openai_client, dataset_path, progress=progress) + testing_criteria = _build_testing_criteria(result) + if not testing_criteria: + raise ValueError( + "no AgentOps evaluators map onto azure_ai_evaluator testing " + "criteria; nothing to evaluate server-side." + ) + + item_schema = _build_item_schema(dataset_path) + + progress( + f"cloud publish: creating eval ({len(testing_criteria)} criteria, " + f"item_schema fields: {sorted(item_schema['properties'].keys())})" + ) + eval_obj = openai_client.evals.create( + name=eval_name, + data_source_config={ + "type": "custom", + "item_schema": item_schema, + "include_sample_schema": True, + }, + testing_criteria=testing_criteria, # type: ignore[arg-type] + ) + eval_id = eval_obj.id + + progress( + f"cloud publish: starting run for agent {agent_name}:{agent_version}" + ) + run_obj = openai_client.evals.runs.create( + eval_id=eval_id, + name=f"{eval_name}-run", + data_source={ # type: ignore[arg-type] + "type": "azure_ai_target_completions", + "agent_reference": { + "type": "agent_reference", + "name": agent_name, + "version": agent_version, + }, + "source": { + "type": "file_id", + "id": file_id, + }, + }, + ) + run_id = run_obj.id + + progress( + f"cloud publish: polling run {run_id} (interval " + f"{poll_interval_seconds:g}s, max {max_poll_attempts} attempts)" + ) + final_run = _poll_until_terminal( + openai_client, + eval_id=eval_id, + run_id=run_id, + interval_seconds=poll_interval_seconds, + max_attempts=max_poll_attempts, + progress=progress, + ) + + status = getattr(final_run, "status", "unknown") + report_url = _extract_report_url(final_run) + + if status != "completed": + raise RuntimeError( + f"cloud evaluation run {run_id} terminated with status " + f"{status!r}; see {report_url or 'the Foundry portal'}." + ) + + progress(f"cloud publish: done. status={status}") + return CloudPublishResult( + eval_id=eval_id, + run_id=run_id, + status=status, + report_url=report_url, + evaluation_name=eval_name, + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _upload_dataset( + openai_client: Any, + dataset_path: Path, + *, + progress: Callable[[str], None], +) -> str: + """Upload the dataset as an OpenAI file (purpose='evals').""" + progress(f"cloud publish: uploading {dataset_path.name}") + with dataset_path.open("rb") as handle: + uploaded = openai_client.files.create( + file=handle, + purpose="evals", + ) + file_id = uploaded.id + progress(f"cloud publish: uploaded as file_id={file_id}") + return file_id + + +def _build_testing_criteria(result: RunResult) -> List[Dict[str, Any]]: + """Map evaluator class names from ``result`` onto Azure AI evaluators. + + We read the evaluator class names off the aggregate metric keys' + presets; since presets are not serialised verbatim into ``RunResult``, + we infer them from the aggregate metric *keys* against the catalog at + call time. + """ + # Lazy import to avoid pulling evaluators into modules that don't + # need them. + from agentops.core.evaluators import CATALOG + + # ``CATALOG`` is keyed by preset.name (== class name); ``aggregate_metrics`` + # is keyed by preset.score_key. Build a one-shot reverse index. + by_score_key = {p.score_key: p for p in CATALOG.values()} + + criteria: List[Dict[str, Any]] = [] + seen: set = set() + for metric_name in result.aggregate_metrics.keys(): + preset = by_score_key.get(metric_name) + if preset is None: + continue + # Latency is computed locally; Foundry has its own server-side view. + if "runtime" in preset.categories: + continue + azure_name = _AZURE_AI_EVALUATOR_NAMES.get(preset.class_name) + if not azure_name: + logger.warning( + "no azure_ai_evaluator mapping for %s; skipping in cloud run", + preset.class_name, + ) + continue + if azure_name in seen: + continue + seen.add(azure_name) + criteria.append({ + "type": "azure_ai_evaluator", + "name": preset.score_key, + "evaluator_name": azure_name, + }) + return criteria + + +def _build_item_schema(dataset_path: Path) -> Dict[str, Any]: + """Inspect the first dataset row to derive a JSON schema. + + Foundry's Evals API requires an ``item_schema`` declaring the shape of + each row. We read the first non-empty line of the JSONL file and + advertise every top-level key as a string property; this is permissive + enough for typical AgentOps datasets (input, expected, context, + tool_calls, tool_definitions). + """ + properties: Dict[str, Dict[str, str]] = {} + with dataset_path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + row = json.loads(line) + if isinstance(row, dict): + for key in row.keys(): + properties[str(key)] = {"type": "string"} + break + if not properties: + # Fall back to a single 'input' field so eval creation does not + # blow up on an empty dataset. + properties["input"] = {"type": "string"} + return { + "type": "object", + "properties": properties, + "required": list(properties.keys()), + } + + +def _poll_until_terminal( + openai_client: Any, + *, + eval_id: str, + run_id: str, + interval_seconds: float, + max_attempts: int, + progress: Callable[[str], None], +) -> Any: + """Poll ``runs.retrieve`` until the run reaches a terminal status.""" + last_status: Optional[str] = None + for attempt in range(1, max_attempts + 1): + run = openai_client.evals.runs.retrieve(eval_id=eval_id, run_id=run_id) + status = getattr(run, "status", "unknown") + if status != last_status: + progress( + f"cloud publish: run status -> {status} " + f"(attempt {attempt}/{max_attempts})" + ) + last_status = status + if status in _TERMINAL_STATUSES: + return run + time.sleep(interval_seconds) + raise RuntimeError( + f"cloud evaluation run {run_id} did not finish within " + f"{max_attempts} polls of {interval_seconds:g}s " + f"(last status: {last_status!r})." + ) + + +def _extract_report_url(run: Any) -> Optional[str]: + """Best-effort extraction of the portal URL from a run object.""" + for attr in ("report_url", "reportUrl"): + value = getattr(run, attr, None) + if isinstance(value, str) and value: + return value + metadata = getattr(run, "metadata", None) + if isinstance(metadata, dict): + for key in ("report_url", "reportUrl"): + value = metadata.get(key) + if isinstance(value, str) and value: + return value + return None diff --git a/src/agentops/pipeline/comparison.py b/src/agentops/pipeline/comparison.py new file mode 100644 index 00000000..dd57d35a --- /dev/null +++ b/src/agentops/pipeline/comparison.py @@ -0,0 +1,108 @@ +"""``--baseline`` comparison helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Dict, List, Optional + +from agentops.core.results import ( + ComparisonInfo, + ComparisonMetric, + ComparisonRow, + RunResult, +) + + +def load_baseline(path: Path) -> RunResult: + """Load a previous ``results.json`` for comparison.""" + if not path.exists(): + raise FileNotFoundError(f"baseline file not found: {path}") + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + return RunResult.model_validate(payload) + + +def _direction(current: Optional[float], baseline: Optional[float]) -> str: + if current is None or baseline is None: + return "unchanged" + if current > baseline: + return "improved" + if current < baseline: + return "regressed" + return "unchanged" + + +def _row_passed(row_metrics: List[Dict[str, float | None]]) -> bool: + """Best-effort proxy: a row is "passing" when no metric reports an error.""" + return all("error" not in metric or not metric["error"] for metric in row_metrics) + + +def build_comparison( + *, + current: RunResult, + baseline: RunResult, + baseline_path: Path, +) -> ComparisonInfo: + metrics: List[ComparisonMetric] = [] + metric_names = sorted(set(current.aggregate_metrics) | set(baseline.aggregate_metrics)) + for name in metric_names: + current_value = current.aggregate_metrics.get(name) + baseline_value = baseline.aggregate_metrics.get(name) + delta = ( + current_value - baseline_value + if current_value is not None and baseline_value is not None + else None + ) + metrics.append( + ComparisonMetric( + metric=name, + current=current_value, + baseline=baseline_value, + delta=delta, + direction=_direction(current_value, baseline_value), + ) + ) + + rows: List[ComparisonRow] = [] + baseline_by_index = {row.row_index: row for row in baseline.rows} + for row in current.rows: + baseline_row = baseline_by_index.get(row.row_index) + current_pass = row.error is None and all( + m.value is not None or m.error is None for m in row.metrics + ) + if baseline_row is None: + rows.append( + ComparisonRow( + row_index=row.row_index, + current_passed=current_pass, + baseline_passed=None, + direction="new", + ) + ) + continue + baseline_pass = baseline_row.error is None and all( + m.value is not None or m.error is None for m in baseline_row.metrics + ) + if current_pass and not baseline_pass: + direction = "improved" + elif baseline_pass and not current_pass: + direction = "regressed" + else: + direction = "unchanged" + rows.append( + ComparisonRow( + row_index=row.row_index, + current_passed=current_pass, + baseline_passed=baseline_pass, + direction=direction, + ) + ) + + return ComparisonInfo( + baseline_path=str(baseline_path), + baseline_started_at=baseline.started_at, + baseline_overall_passed=baseline.summary.overall_passed, + metrics=metrics, + rows=rows, + ) diff --git a/src/agentops/pipeline/invocations.py b/src/agentops/pipeline/invocations.py new file mode 100644 index 00000000..c718dd3b --- /dev/null +++ b/src/agentops/pipeline/invocations.py @@ -0,0 +1,536 @@ +"""Target invocation backends for AgentOps 1.0. + +Each backend is a single function with the signature:: + + invoke( + target: TargetResolution, + config: AgentOpsConfig, + row: dict[str, Any], + *, + timeout: float, + ) -> InvocationResult + +The orchestrator dispatches based on :attr:`TargetResolution.kind`. All Azure +SDK imports are lazy so the package imports without optional dependencies. +""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from agentops.core.agentops_config import AgentOpsConfig, TargetResolution + + +@dataclass +class InvocationResult: + """Outcome of invoking the target on one dataset row.""" + + response: str + latency_seconds: float + tool_calls: Optional[List[Any]] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + +# Maximum number of follow-up calls when running the tool-execution loop +# against a Foundry hosted/prompt agent. Most agents resolve in 1–2 hops; +# the cap exists to bound retries against pathological multi-step plans. +_MAX_TOOL_ITERATIONS = 4 + +# Generic stub returned to the agent for every function call during the +# tool-execution loop. The toolkit cannot run project-specific tool +# implementations, so a uniform "ok" stub keeps the loop fully generic +# while letting the agent produce its final natural-language reply. +_TOOL_STUB_OUTPUT = '{"status": "ok"}' + + +def _summarise_tool_calls(calls: List[Any]) -> str: + """Build a short, human-readable summary of executed tool calls. + + Used as a last-resort fallback when the agent never produces + assistant text — quality evaluators still need a non-empty + ``response`` string to score. + """ + parts: List[str] = [] + for call in calls: + if not isinstance(call, dict): + continue + name = call.get("name") or "tool" + args = call.get("arguments") + parts.append(f"[Called {name}({args})]" if args else f"[Called {name}]") + return " ".join(parts) if parts else "[tool_call]" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _credential() -> Any: + """Return a cached ``DefaultAzureCredential`` singleton. + + Caching matters: each row invocation needs a token, and constructing a + fresh ``DefaultAzureCredential`` per call walks the full credential + chain (Azure CLI / PowerShell subprocesses included), which is both + slow and prone to transient subprocess failures on the first try. + Caching the credential lets the SDK reuse its internal token cache + across rows. + """ + global _CREDENTIAL_SINGLETON + if _CREDENTIAL_SINGLETON is None: + from azure.identity import DefaultAzureCredential # noqa: WPS433 + + _CREDENTIAL_SINGLETON = DefaultAzureCredential( + exclude_developer_cli_credential=True + ) + return _CREDENTIAL_SINGLETON + + +_CREDENTIAL_SINGLETON: Any = None + + +def _get_token(scope: str) -> str: + """Acquire a token for ``scope``, retrying once on transient failures. + + The first credential-chain walk on Windows occasionally fails because + the Azure CLI / PowerShell subprocess is slow to spawn. Retrying once + after the credential is warmed up almost always succeeds. + """ + try: + return _credential().get_token(scope).token + except Exception: # noqa: BLE001 - retry once on any transient failure + global _CREDENTIAL_SINGLETON + _CREDENTIAL_SINGLETON = None # force a fresh chain walk + return _credential().get_token(scope).token + + +def _project_endpoint_from_env() -> str: + endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise RuntimeError( + "Missing AZURE_AI_FOUNDRY_PROJECT_ENDPOINT environment variable. " + "Foundry targets require a project endpoint URL." + ) + return endpoint.rstrip("/") + + +def _row_input(row: Dict[str, Any]) -> str: + value = row.get("input") + if value is None: + raise ValueError("dataset row is missing required 'input' field") + return str(value) + + +def _http_request_json( + *, + method: str, + url: str, + headers: Dict[str, str], + body: Optional[Dict[str, Any]] = None, + timeout: float, +) -> Dict[str, Any]: + encoded = json.dumps(body or {}).encode("utf-8") if method != "GET" else None + request = urllib.request.Request( + url=url, data=encoded, method=method, headers=headers + ) + last_exc: Optional[BaseException] = None + for attempt in range(1, 4): + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + payload = response.read().decode("utf-8") + break + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + transient = exc.code >= 500 or exc.code == 429 + if transient and attempt < 3: + time.sleep(2 ** attempt) + last_exc = exc + continue + raise RuntimeError( + f"HTTP {exc.code} from {url}: {detail or exc.reason}" + ) from exc + except urllib.error.URLError as exc: + if attempt < 3: + time.sleep(2 ** attempt) + last_exc = exc + continue + raise + else: # pragma: no cover - loop exits via break/raise + raise RuntimeError(f"HTTP request to {url} failed: {last_exc!r}") + if not payload: + return {} + return json.loads(payload) + + +def _dot_path(payload: Any, path: str) -> Any: + """Resolve ``a.b.c`` or ``a.0.b`` against a JSON-like object.""" + current = payload + for token in path.split("."): + if current is None: + return None + if isinstance(current, list): + try: + current = current[int(token)] + except (ValueError, IndexError): + return None + continue + if isinstance(current, dict): + current = current.get(token) + continue + return None + return current + + +def _extract_responses_text(payload: Dict[str, Any]) -> str: + """Pull assistant text from a Foundry/Responses-API payload. + + Returns an empty string when the response only contains tool/function + calls (the caller must submit ``function_call_output`` items via a + follow-up call to obtain the final natural-language reply). + """ + direct = payload.get("output_text") + if isinstance(direct, str) and direct.strip(): + return direct.strip() + + output = payload.get("output") + if isinstance(output, list): + parts: List[str] = [] + for item in output: + if not isinstance(item, dict): + continue + if ( + item.get("type") in {"message", "assistant_message"} + or item.get("role") == "assistant" + ): + content = item.get("content") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for chunk in content: + if isinstance(chunk, dict): + text = chunk.get("text") or chunk.get("output_text") + if isinstance(text, str): + parts.append(text) + elif isinstance(chunk, str): + parts.append(chunk) + if parts: + return "\n".join(parts).strip() + return "" + + return "" + + +def _extract_responses_tool_calls(payload: Dict[str, Any]) -> Optional[List[Any]]: + output = payload.get("output") + if not isinstance(output, list): + return None + calls: List[Any] = [] + for item in output: + if isinstance(item, dict) and item.get("type") in { + "tool_call", + "function_call", + }: + calls.append(item) + return calls or None + + +# --------------------------------------------------------------------------- +# Backends +# --------------------------------------------------------------------------- + + +def _invoke_model_direct( + target: TargetResolution, + config: AgentOpsConfig, # noqa: ARG001 + row: Dict[str, Any], + *, + timeout: float, # noqa: ARG001 +) -> InvocationResult: + from azure.ai.projects import AIProjectClient # noqa: WPS433 + + project_endpoint = _project_endpoint_from_env() + client = AIProjectClient(endpoint=project_endpoint, credential=_credential()) + openai_client = client.get_openai_client() + + assert target.deployment is not None + started = time.perf_counter() + last_exc: Optional[BaseException] = None + response = None + for attempt in range(1, 4): + try: + response = openai_client.chat.completions.create( + model=target.deployment, + messages=[{"role": "user", "content": _row_input(row)}], + ) + break + except Exception as exc: # noqa: BLE001 + status = getattr(exc, "status_code", None) + transient = status is None or status >= 500 or status == 429 + if transient and attempt < 3: + time.sleep(2 ** attempt) + last_exc = exc + continue + raise + if response is None: + raise RuntimeError(f"model_direct invocation failed after retries: {last_exc!r}") + elapsed = time.perf_counter() - started + + text = "" + if response.choices: + message = response.choices[0].message + if message and message.content: + text = message.content.strip() + if not text: + raise RuntimeError("model_direct invocation returned empty content") + + return InvocationResult(response=text, latency_seconds=elapsed) + + +def _run_responses_tool_loop( + *, + url: str, + headers: Dict[str, str], + initial_body: Dict[str, Any], + timeout: float, + follow_up_extras: Optional[Dict[str, Any]] = None, +) -> tuple[str, List[Any], float]: + """Drive a Foundry/Responses-API tool-execution loop. + + Sends ``initial_body`` to ``url``, then repeatedly submits stub + ``function_call_output`` items back via ``previous_response_id`` until + the agent emits assistant text or the iteration cap is reached. + + ``follow_up_extras`` is merged into every follow-up request body + (e.g. ``agent_reference`` for prompt agents). + + Returns ``(text, aggregated_tool_calls, elapsed_seconds)``. + """ + started = time.perf_counter() + aggregated_tool_calls: List[Any] = [] + text = "" + body = initial_body + + for _iteration in range(_MAX_TOOL_ITERATIONS): + payload = _http_request_json( + method="POST", + url=url, + headers=headers, + body=body, + timeout=timeout, + ) + + iteration_calls = _extract_responses_tool_calls(payload) or [] + aggregated_tool_calls.extend(iteration_calls) + + text = _extract_responses_text(payload) + if text or not iteration_calls: + break + + previous_response_id = payload.get("id") + follow_up_input: List[Dict[str, Any]] = [] + for call in iteration_calls: + call_id = call.get("call_id") or call.get("id") + if not call_id: + continue + follow_up_input.append( + { + "type": "function_call_output", + "call_id": call_id, + "output": _TOOL_STUB_OUTPUT, + } + ) + + if not follow_up_input or not previous_response_id: + break + + body = { + "input": follow_up_input, + "previous_response_id": previous_response_id, + } + if follow_up_extras: + body.update(follow_up_extras) + + elapsed = time.perf_counter() - started + return text, aggregated_tool_calls, elapsed + + +def _invoke_foundry_prompt( + target: TargetResolution, + config: AgentOpsConfig, # noqa: ARG001 + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + project_endpoint = _project_endpoint_from_env() + token = _get_token("https://ai.azure.com/.default") + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + } + + assert target.name is not None and target.version is not None + url = f"{project_endpoint}/openai/v1/responses" + agent_reference = { + "type": "agent_reference", + "name": target.name, + "version": target.version, + } + initial_body: Dict[str, Any] = { + "input": [{"role": "user", "content": _row_input(row)}], + "agent_reference": agent_reference, + } + + text, aggregated_tool_calls, elapsed = _run_responses_tool_loop( + url=url, + headers=headers, + initial_body=initial_body, + timeout=timeout, + follow_up_extras={"agent_reference": agent_reference}, + ) + + if not text: + if aggregated_tool_calls: + text = _summarise_tool_calls(aggregated_tool_calls) + else: + raise ValueError( + "Foundry response did not include assistant output text" + ) + + return InvocationResult( + response=text, + latency_seconds=elapsed, + tool_calls=aggregated_tool_calls or None, + ) + + +def _invoke_foundry_hosted( + target: TargetResolution, + config: AgentOpsConfig, + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + assert target.url is not None + token = _get_token("https://ai.azure.com/.default") + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + **config.headers, + } + + if target.protocol == "responses": + url = target.url.rstrip("/") + if not url.endswith("/responses"): + url = f"{url}/responses" + initial_body = {"input": [{"role": "user", "content": _row_input(row)}]} + + text, aggregated_tool_calls, elapsed = _run_responses_tool_loop( + url=url, + headers=headers, + initial_body=initial_body, + timeout=timeout, + ) + + if not text: + if aggregated_tool_calls: + text = _summarise_tool_calls(aggregated_tool_calls) + else: + raise ValueError( + "Foundry response did not include assistant output text" + ) + + return InvocationResult( + response=text, + latency_seconds=elapsed, + tool_calls=aggregated_tool_calls or None, + ) + + return _invoke_http_json(target, config, row, timeout=timeout) + + +def _invoke_http_json( + target: TargetResolution, + config: AgentOpsConfig, + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + assert target.url is not None + headers: Dict[str, str] = {"Content-Type": "application/json", **config.headers} + if config.auth_header_env: + token = os.getenv(config.auth_header_env) + if not token: + raise RuntimeError( + f"auth_header_env {config.auth_header_env!r} is set in config but " + "the environment variable is empty" + ) + headers["Authorization"] = f"Bearer {token}" + + request_field = config.request_field or "message" + body: Dict[str, Any] = {request_field: _row_input(row)} + + started = time.perf_counter() + payload = _http_request_json( + method="POST", + url=target.url, + headers=headers, + body=body, + timeout=timeout, + ) + elapsed = time.perf_counter() - started + + response_path = config.response_field or "text" + response_text = _dot_path(payload, response_path) + if response_text is None: + for fallback in ("response", "output", "content", "message", "text"): + response_text = payload.get(fallback) + if response_text: + break + if response_text is None: + raise ValueError( + f"HTTP/JSON response did not contain field {response_path!r}; " + f"got top-level keys: {sorted(payload.keys())}" + ) + if not isinstance(response_text, str): + response_text = json.dumps(response_text, ensure_ascii=False) + + tool_calls: Optional[List[Any]] = None + if config.tool_calls_field: + extracted = _dot_path(payload, config.tool_calls_field) + if isinstance(extracted, list): + tool_calls = extracted + + return InvocationResult( + response=response_text.strip(), + latency_seconds=elapsed, + tool_calls=tool_calls, + ) + + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- + + +def invoke( + target: TargetResolution, + config: AgentOpsConfig, + row: Dict[str, Any], + *, + timeout: float, +) -> InvocationResult: + """Dispatch to the right backend based on the resolved target kind.""" + if target.kind == "model_direct": + return _invoke_model_direct(target, config, row, timeout=timeout) + if target.kind == "foundry_prompt": + return _invoke_foundry_prompt(target, config, row, timeout=timeout) + if target.kind == "foundry_hosted": + return _invoke_foundry_hosted(target, config, row, timeout=timeout) + if target.kind == "http_json": + return _invoke_http_json(target, config, row, timeout=timeout) + raise ValueError(f"unknown target kind: {target.kind}") diff --git a/src/agentops/pipeline/orchestrator.py b/src/agentops/pipeline/orchestrator.py new file mode 100644 index 00000000..cbb2ddab --- /dev/null +++ b/src/agentops/pipeline/orchestrator.py @@ -0,0 +1,519 @@ +"""End-to-end evaluation orchestrator for AgentOps 1.0. + +This is the single entry point exercised by ``agentops eval``. It loads the +flat config, classifies the target, infers evaluators from the dataset shape, +invokes the target row-by-row, runs each evaluator, applies thresholds, and +writes ``results.json`` and ``report.md``. +""" + +from __future__ import annotations + +import json +import logging +import statistics +import sys +import time +import os +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional + +from agentops.core.agentops_config import AgentOpsConfig, Threshold, classify_agent +from agentops.core.evaluators import ( + detect_dataset_shape, + merge_thresholds, + select_evaluators, +) +from agentops.core.results import ( + RowMetric, + RowResult, + RunResult, + RunSummary, + TargetInfo, +) +from agentops.pipeline import comparison as comparison_module +from agentops.pipeline import invocations, publisher, reporter, runtime, thresholds +from agentops.utils.colors import style + +logger = logging.getLogger("agentops.pipeline") + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +@dataclass +class RunOptions: + config_path: Path + output_dir: Path + baseline_path: Optional[Path] = None + timeout_seconds: float = 120.0 + dataset_override: Optional[Path] = None + agent_override: Optional[str] = None + # Optional callback invoked with progress messages during a run. The + # CLI wires this to ``typer.echo`` so users see per-row progress + # ("invoking", "scored", ...) instead of long unexplained pauses. + # Library callers can leave it as ``None`` to keep runs silent. + progress: Optional[Callable[[str], None]] = field(default=None, repr=False) + + +def run_evaluation( + config: AgentOpsConfig, + *, + options: RunOptions, +) -> RunResult: + """Run a full evaluation and persist artifacts. Returns the RunResult.""" + started_at = datetime.now(timezone.utc) + started_perf = time.perf_counter() + + target = classify_agent( + options.agent_override or config.agent, + config.protocol, + ) + + dataset_path = options.dataset_override or _resolve_dataset_path(config, options) + shape = detect_dataset_shape(dataset_path) + + overrides = ( + [override.name for override in config.evaluators] if config.evaluators else None + ) + presets = select_evaluators(target, shape, overrides=overrides) + user_thresholds = [ + Threshold.from_expression(metric, expr) + for metric, expr in config.thresholds.items() + ] + threshold_rules = merge_thresholds(presets, user_thresholds) + + evaluator_runtimes = runtime.load_evaluators(presets) + + progress = options.progress or (lambda _msg: None) + + dataset_rows = list(_iter_dataset(dataset_path)) + total = len(dataset_rows) + from agentops import __version__ as _agentops_version + py = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + progress( + f"{style('agentops', 'bold', 'cyan')} {style(_agentops_version, 'cyan')} " + f"{style('|', 'dim')} python {py} " + f"{style('|', 'dim')} config: {style(options.config_path.name, 'cyan')}" + ) + progress( + f"Loaded {style(str(total), 'bold')} row(s) from " + f"{style(dataset_path.name, 'cyan')}; running " + f"{style(str(len(presets)), 'bold')} evaluator(s) against " + f"{_friendly_target_kind(target.kind)}: {style(target.raw, 'bold')}." + ) + + rows: List[RowResult] = [] + rules_by_metric = {rule.metric: rule for rule in threshold_rules} + for index, row in enumerate(dataset_rows): + rows.append( + _evaluate_row( + row=row, + index=index, + total=total, + target=target, + config=config, + evaluators=evaluator_runtimes, + timeout=options.timeout_seconds, + progress=progress, + rules_by_metric=rules_by_metric, + ) + ) + + aggregate = _aggregate_metrics(rows) + threshold_results = thresholds.evaluate(threshold_rules, aggregate) + summary = _summarize(rows, threshold_results) + + finished_at = datetime.now(timezone.utc) + duration = time.perf_counter() - started_perf + + result = RunResult( + started_at=started_at.isoformat(), + finished_at=finished_at.isoformat(), + duration_seconds=duration, + target=TargetInfo( + kind=target.kind, + raw=target.raw, + protocol=target.protocol, + name=target.name, + version=target.version, + url=target.url, + deployment=target.deployment, + ), + dataset_path=str(dataset_path), + evaluators=[preset.name for preset in presets], + rows=rows, + aggregate_metrics=aggregate, + thresholds=threshold_results, + summary=summary, + config={ + "version": config.version, + "agent": config.agent, + "thresholds": dict(config.thresholds), + }, + ) + + if options.baseline_path is not None: + baseline = comparison_module.load_baseline(options.baseline_path) + result.comparison = comparison_module.build_comparison( + current=result, + baseline=baseline, + baseline_path=options.baseline_path, + ) + + _persist(result, options.output_dir) + + if config.publish == "foundry": + _publish_to_foundry_safely(result, config, options.output_dir, progress=progress) + elif config.publish == "foundry_cloud": + _publish_to_foundry_cloud_safely( + result, config, options.output_dir, dataset_path, progress=progress, + ) + + return result + + +def _publish_to_foundry_safely( + result: RunResult, + config: AgentOpsConfig, + output_dir: Path, + *, + progress: Optional[Callable[[str], None]] = None, +) -> None: + """Best-effort Classic Foundry publish. Failures are logged, never fatal.""" + if config.publish != "foundry": + return + + notify = progress or (lambda _msg: None) + + try: + published = publisher.publish_to_foundry( + result, + project_endpoint=config.project_endpoint, + ) + except Exception as exc: # noqa: BLE001 + logger.warning("foundry publish failed: %s", exc) + notify( + f"{style('publish foundry FAILED', 'red')}: {exc}. " + f"Local results.json is the source of truth." + ) + return + + cloud_meta_path = output_dir / "cloud_evaluation.json" + cloud_meta_path.write_text( + json.dumps( + { + "mode": "classic", + "evaluation_name": published.evaluation_name, + "report_url": published.studio_url, + }, + indent=2, + ), + encoding="utf-8", + ) + notify( + f"Published to {style('Classic Foundry Evaluations', 'bold')}: " + f"{style(published.studio_url, 'cyan')}" + ) + notify( + f"Tip: to run server-side in the {style('New Foundry', 'bold')} " + f"experience, use 'publish: foundry_cloud' (preview)." + ) + + +def _publish_to_foundry_cloud_safely( + result: RunResult, + config: AgentOpsConfig, + output_dir: Path, + dataset_path: Path, + *, + progress: Optional[Callable[[str], None]] = None, +) -> None: + """Best-effort New Foundry (cloud) publish. Failures are logged, never fatal.""" + if config.publish != "foundry_cloud": + return + + notify = progress or (lambda _msg: None) + + endpoint = config.project_endpoint or os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + msg = ( + "publish: foundry_cloud requires either 'project_endpoint' in " + "agentops.yaml or the AZURE_AI_FOUNDRY_PROJECT_ENDPOINT env var." + ) + logger.warning(msg) + notify(f"{style('publish foundry_cloud FAILED', 'red')}: {msg}") + return + + # Lazy import keeps unit tests free of azure-ai-projects. + from agentops.pipeline import cloud_publisher + + try: + published = cloud_publisher.publish_to_foundry_cloud( + result, + dataset_path=dataset_path, + project_endpoint=endpoint, + progress=notify, + ) + except Exception as exc: # noqa: BLE001 + logger.warning("foundry_cloud publish failed: %s", exc) + notify( + f"{style('publish foundry_cloud FAILED', 'red')}: {exc}. " + f"Local results.json is the source of truth." + ) + return + + cloud_meta_path = output_dir / "cloud_evaluation.json" + cloud_meta_path.write_text( + json.dumps( + { + "mode": "cloud", + "evaluation_name": published.evaluation_name, + "eval_id": published.eval_id, + "run_id": published.run_id, + "status": published.status, + "report_url": published.report_url, + }, + indent=2, + ), + encoding="utf-8", + ) + logger.info( + "New Foundry cloud evaluation: %s (eval=%s run=%s)", + published.report_url, published.eval_id, published.run_id, + ) + notify( + f"Submitted to {style('New Foundry Evaluations', 'bold')}: " + f"{style(published.report_url or '(no portal URL)', 'cyan')}" + ) + notify( + f" eval_id={published.eval_id} run_id={published.run_id} " + f"status={style(published.status, 'green' if published.status == 'completed' else 'yellow')}" + ) + + +def exit_code_from(result: RunResult) -> int: + """Translate a run's outcome into the ``agentops`` CLI contract. + + * ``0`` — success, all thresholds passed. + * ``2`` — invocations succeeded but a threshold failed. + * ``1`` — runtime errors are raised as exceptions before this is called. + """ + return 0 if result.summary.overall_passed else 2 + + +# --------------------------------------------------------------------------- +# Dataset +# --------------------------------------------------------------------------- + + +def _resolve_dataset_path(config: AgentOpsConfig, options: RunOptions) -> Path: + candidate = config.dataset + if candidate.is_absolute() and candidate.exists(): + return candidate + base = options.config_path.parent + resolved = (base / candidate).resolve() + if not resolved.exists(): + raise FileNotFoundError(f"dataset not found: {resolved}") + return resolved + + +_FRIENDLY_KIND = { + "foundry_prompt": "foundry agent", + "foundry_hosted": "foundry agent (hosted)", + "http_json": "http endpoint", + "model_direct": "model deployment", +} + + +def _friendly_target_kind(kind: str) -> str: + return _FRIENDLY_KIND.get(kind, kind) + + +def _iter_dataset(path: Path) -> Iterable[Dict[str, Any]]: + with path.open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + stripped = line.strip() + if not stripped: + continue + try: + row = json.loads(stripped) + except json.JSONDecodeError as exc: + raise ValueError( + f"{path}: invalid JSON on line {line_number}: {exc}" + ) from exc + if not isinstance(row, dict): + raise ValueError( + f"{path}: line {line_number} is not a JSON object" + ) + yield row + + +# --------------------------------------------------------------------------- +# Per-row execution +# --------------------------------------------------------------------------- + + +def _evaluate_row( + *, + row: Dict[str, Any], + index: int, + total: int, + target, + config: AgentOpsConfig, + evaluators: List[runtime.EvaluatorRuntime], + timeout: float, + progress: Callable[[str], None], + rules_by_metric: Optional[Dict[str, Threshold]] = None, +) -> RowResult: + label = style(f"[{index + 1}/{total}]", "dim") + preview = str(row.get("input", "")).strip().replace("\n", " ") + if len(preview) > 80: + preview = preview[:77] + "..." + progress(f"{label} invoking target: {preview!r}") + + try: + invocation = invocations.invoke(target, config, row, timeout=timeout) + except Exception as exc: # noqa: BLE001 + logger.warning("row %d invocation failed: %s", index, exc) + progress(f"{label} {style('invocation FAILED', 'bold', 'red')}: {exc}") + return RowResult( + row_index=index, + input=str(row.get("input", "")), + expected=row.get("expected"), + response="", + context=row.get("context"), + error=str(exc), + ) + + tool_count = len(invocation.tool_calls) if invocation.tool_calls else 0 + progress( + f"{label} replied in {style(f'{invocation.latency_seconds:.2f}s', 'cyan')} " + f"({tool_count} tool call(s)); scoring..." + ) + + metrics: List[RowMetric] = [] + for evaluator in evaluators: + metric = runtime.run_evaluator( + evaluator, + row=row, + response=invocation.response, + latency_seconds=invocation.latency_seconds, + actual_tool_calls=invocation.tool_calls, + ) + metrics.append(metric) + + rules = rules_by_metric or {} + + def _passes(rule: Threshold, value: float) -> bool: + if rule.value is None or rule.criteria in {"true", "false"}: + return True + target_v = float(rule.value) + c = rule.criteria + if c == ">=": + return value >= target_v + if c == ">": + return value > target_v + if c == "<=": + return value <= target_v + if c == "<": + return value < target_v + if c == "==": + return value == target_v + return True + + def _format_metric(m: RowMetric) -> str: + if isinstance(m.value, (int, float)): + rule = rules.get(m.name) + text = f"{m.value:.2f}" + if rule is None: + # No user threshold for this metric: keep value neutral + # so the line stays readable. + return f"{m.name}={text}" + color = "green" if _passes(rule, float(m.value)) else "red" + return f"{m.name}={style(text, color)}" + if m.error: + return f"{m.name}={style('ERR', 'red')}" + return f"{m.name}={style('n/a', 'dim')}" + + scored = ", ".join(_format_metric(m) for m in metrics) + progress(f"{label} scored: {scored}") + + return RowResult( + row_index=index, + input=str(row.get("input", "")), + expected=row.get("expected"), + response=invocation.response, + context=row.get("context"), + latency_seconds=invocation.latency_seconds, + tool_calls=invocation.tool_calls, + metrics=metrics, + ) + + +# --------------------------------------------------------------------------- +# Aggregation +# --------------------------------------------------------------------------- + + +def _aggregate_metrics(rows: List[RowResult]) -> Dict[str, float]: + by_metric: Dict[str, List[float]] = {} + for row in rows: + for metric in row.metrics: + if metric.value is None: + continue + by_metric.setdefault(metric.name, []).append(metric.value) + aggregate: Dict[str, float] = {} + for name, values in by_metric.items(): + if values: + aggregate[name] = statistics.fmean(values) + return aggregate + + +def _summarize( + rows: List[RowResult], + threshold_results, +) -> RunSummary: + items_total = len(rows) + items_passed_all = sum( + 1 + for row in rows + if row.error is None and all(m.error is None for m in row.metrics) + ) + items_pass_rate = items_passed_all / items_total if items_total else 0.0 + thresholds_total = len(threshold_results) + thresholds_passed = sum(1 for t in threshold_results if t.passed) + threshold_pass_rate = ( + thresholds_passed / thresholds_total if thresholds_total else 1.0 + ) + overall = items_total > 0 and threshold_pass_rate == 1.0 and items_passed_all > 0 + return RunSummary( + items_total=items_total, + items_passed_all=items_passed_all, + items_pass_rate=items_pass_rate, + thresholds_total=thresholds_total, + thresholds_passed=thresholds_passed, + threshold_pass_rate=threshold_pass_rate, + overall_passed=overall, + ) + + +# --------------------------------------------------------------------------- +# Persistence +# --------------------------------------------------------------------------- + + +def _persist(result: RunResult, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + results_path = output_dir / "results.json" + report_path = output_dir / "report.md" + + payload = result.model_dump(mode="json") + results_path.write_text( + json.dumps(payload, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + report_path.write_text(reporter.render(result), encoding="utf-8") diff --git a/src/agentops/pipeline/publisher.py b/src/agentops/pipeline/publisher.py new file mode 100644 index 00000000..623dd8ed --- /dev/null +++ b/src/agentops/pipeline/publisher.py @@ -0,0 +1,121 @@ +"""Optional Foundry publishing for the AgentOps pipeline. + +This module is invoked from :mod:`agentops.pipeline.orchestrator` only when +``publish: foundry`` is set in ``agentops.yaml``. It uploads the same metrics +that AgentOps already computed locally into the **New Foundry Evaluations** +panel, using the public ``_log_metrics_and_instance_results_onedp`` helper +from ``azure.ai.evaluation``. + +The pipeline never re-runs the agent here. Local invocations + local +evaluators stay the canonical source of truth; this is just a publish hop. +""" + +from __future__ import annotations + +import logging +import os +import uuid +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agentops.core.results import RunResult + +logger = logging.getLogger("agentops.pipeline.publisher") + + +@dataclass(frozen=True) +class PublishResult: + """Outcome of a successful Foundry publish.""" + + studio_url: str + evaluation_name: str + + +def publish_to_foundry( + result: RunResult, + *, + project_endpoint: Optional[str] = None, + evaluation_name: Optional[str] = None, +) -> PublishResult: + """Publish ``result`` to the Foundry Evaluations panel. + + Parameters + ---------- + result: + The fully populated ``RunResult`` produced by the pipeline. + project_endpoint: + Foundry project URL. When ``None``, the function falls back to the + ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` environment variable. + evaluation_name: + Display name for the run in the Foundry panel. Defaults to a unique + ``agentops-eval-``. + + Returns + ------- + PublishResult + ``studio_url`` is the deep link rendered on the Foundry portal. + + Raises + ------ + ImportError + ``azure-ai-evaluation`` and ``pandas`` are not installed. + ValueError + Project endpoint is missing or no rows are publishable. + """ + endpoint = project_endpoint or os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise ValueError( + "publish: foundry requires either 'project_endpoint' in " + "agentops.yaml or the AZURE_AI_FOUNDRY_PROJECT_ENDPOINT env var." + ) + + try: + import pandas as pd # noqa: WPS433 + from azure.ai.evaluation._evaluate._utils import ( # noqa: WPS433 + _log_metrics_and_instance_results_onedp, + ) + except ImportError as exc: # pragma: no cover - exercised only at runtime + raise ImportError( + "Foundry publish requires 'azure-ai-evaluation' and 'pandas'. " + "Install with: pip install azure-ai-evaluation pandas" + ) from exc + + instance_rows = _build_instance_rows(result) + if not instance_rows: + raise ValueError("Foundry publish has no content rows to submit.") + + metrics = dict(result.aggregate_metrics) + name_map: Dict[str, str] = {key: key for key in metrics.keys()} + eval_name = evaluation_name or f"agentops-eval-{uuid.uuid4().hex[:8]}" + + instance_results_df = pd.DataFrame(instance_rows) + studio_url = _log_metrics_and_instance_results_onedp( + metrics=metrics, + instance_results=instance_results_df, + project_url=endpoint, + evaluation_name=eval_name, + name_map=name_map, + ) + if not studio_url: + raise RuntimeError( + "Foundry publish completed but the studio URL was empty." + ) + + return PublishResult(studio_url=studio_url, evaluation_name=eval_name) + + +def _build_instance_rows(result: RunResult) -> List[Dict[str, Any]]: + """Project ``RunResult.rows`` into the OneDP instance-result schema.""" + rows: List[Dict[str, Any]] = [] + for row in result.rows: + payload: Dict[str, Any] = { + "line_number": row.row_index, + "input": row.input, + "response": row.response, + "ground_truth": row.expected or "", + } + for metric in row.metrics: + if metric.value is not None: + payload[metric.name] = metric.value + rows.append(payload) + return rows diff --git a/src/agentops/pipeline/reporter.py b/src/agentops/pipeline/reporter.py new file mode 100644 index 00000000..9ea295d9 --- /dev/null +++ b/src/agentops/pipeline/reporter.py @@ -0,0 +1,129 @@ +"""Reporter for AgentOps 1.0 — generates ``report.md`` from a ``RunResult``.""" + +from __future__ import annotations + +from typing import List + +from agentops.core.results import ( + ComparisonInfo, + ComparisonMetric, + RowResult, + RunResult, + ThresholdEvaluation, +) + + +def render(result: RunResult) -> str: + """Render a RunResult into a Markdown report.""" + lines: List[str] = [] + lines.append("# AgentOps Evaluation Report") + lines.append("") + overall = "✅ PASS" if result.summary.overall_passed else "❌ FAIL" + lines.append(f"**Result:** {overall}") + lines.append(f"- **Target:** `{result.target.raw}` ({result.target.kind})") + if result.target.protocol: + lines.append(f"- **Protocol:** {result.target.protocol}") + lines.append(f"- **Dataset:** `{result.dataset_path}`") + lines.append(f"- **Started:** {result.started_at}") + lines.append(f"- **Duration:** {result.duration_seconds:.2f}s") + lines.append(f"- **Rows:** {result.summary.items_total}") + lines.append("") + + if result.aggregate_metrics: + lines.append("## Metrics") + lines.append("") + lines.append("| Metric | Value |") + lines.append("| --- | --- |") + for name, value in sorted(result.aggregate_metrics.items()): + lines.append(f"| {name} | {value:.3f} |") + lines.append("") + + if result.thresholds: + lines.append("## Thresholds") + lines.append("") + lines.append("| Metric | Expected | Actual | Status |") + lines.append("| --- | --- | --- | --- |") + for threshold in result.thresholds: + lines.append(_threshold_row(threshold)) + lines.append("") + + if result.comparison is not None: + lines.extend(_render_comparison(result.comparison)) + lines.append("") + + error_rows = [row for row in result.rows if row.error] + if error_rows: + lines.append("## Failed Invocations") + lines.append("") + lines.append("| Row | Error |") + lines.append("| --- | --- |") + for row in error_rows: + lines.append(f"| {row.row_index} | {_short(row.error or '', 200)} |") + lines.append("") + + lines.append("## Rows") + lines.append("") + lines.append("| # | Latency (s) | Metrics |") + lines.append("| --- | --- | --- |") + for row in result.rows: + lines.append(_row_summary(row)) + lines.append("") + return "\n".join(lines) + + +def _threshold_row(threshold: ThresholdEvaluation) -> str: + status = "✅" if threshold.passed else "❌" + return f"| {threshold.metric} | `{threshold.expected}` | `{threshold.actual}` | {status} |" + + +def _row_summary(row: RowResult) -> str: + parts = [] + for metric in row.metrics: + if metric.error: + parts.append(f"{metric.name}=ERR") + elif metric.value is not None: + parts.append(f"{metric.name}={metric.value:.2f}") + metrics_str = ", ".join(parts) if parts else "—" + latency = f"{row.latency_seconds:.2f}" if row.latency_seconds is not None else "—" + return f"| {row.row_index} | {latency} | {metrics_str} |" + + +def _short(text: str, limit: int) -> str: + text = text.replace("\n", " ").replace("|", "\\|") + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _render_comparison(comparison: ComparisonInfo) -> List[str]: + lines = ["## Comparison vs Baseline", ""] + lines.append(f"**Baseline:** `{comparison.baseline_path}`") + if comparison.baseline_started_at: + lines.append(f"**Baseline run:** {comparison.baseline_started_at}") + lines.append("") + + lines.append("| Metric | Baseline | Current | Δ | Direction |") + lines.append("| --- | --- | --- | --- | --- |") + for metric in comparison.metrics: + lines.append(_comparison_metric_row(metric)) + lines.append("") + + regressed = [r for r in comparison.rows if r.direction == "regressed"] + improved = [r for r in comparison.rows if r.direction == "improved"] + if regressed or improved: + lines.append("**Per-row changes:**") + if regressed: + lines.append( + "- ❌ Regressed rows: " + ", ".join(str(r.row_index) for r in regressed) + ) + if improved: + lines.append( + "- ✅ Improved rows: " + ", ".join(str(r.row_index) for r in improved) + ) + return lines + + +def _comparison_metric_row(metric: ComparisonMetric) -> str: + arrow = {"improved": "🟢", "regressed": "🔴", "unchanged": "⚪"}[metric.direction] + baseline = f"{metric.baseline:.3f}" if metric.baseline is not None else "—" + current = f"{metric.current:.3f}" if metric.current is not None else "—" + delta = f"{metric.delta:+.3f}" if metric.delta is not None else "—" + return f"| {metric.metric} | {baseline} | {current} | {delta} | {arrow} {metric.direction} |" diff --git a/src/agentops/pipeline/runtime.py b/src/agentops/pipeline/runtime.py new file mode 100644 index 00000000..4afb2880 --- /dev/null +++ b/src/agentops/pipeline/runtime.py @@ -0,0 +1,405 @@ +"""Evaluator runtime for AgentOps 1.0. + +Each :class:`EvaluatorPreset` from the catalog is instantiated lazily from +``azure.ai.evaluation`` and run against one dataset row. The runtime hides +SDK details (``model_config`` for AI-assisted evaluators, ``azure_ai_project`` +for safety evaluators, kwarg mapping, score extraction). +""" + +from __future__ import annotations + +import importlib +import inspect +import json +import os +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agentops.core.evaluators import EvaluatorPreset +from agentops.core.results import RowMetric + +# Evaluator classes that require an evaluator model via ``model_config``. +_AI_ASSISTED = { + "GroundednessEvaluator", + "RelevanceEvaluator", + "CoherenceEvaluator", + "FluencyEvaluator", + "SimilarityEvaluator", + "RetrievalEvaluator", + "ResponseCompletenessEvaluator", + "QAEvaluator", + "IntentResolutionEvaluator", + "TaskAdherenceEvaluator", + "ToolCallAccuracyEvaluator", +} + +# Evaluator classes that require ``azure_ai_project``. +_SAFETY = { + "ViolenceEvaluator", + "SexualEvaluator", + "SelfHarmEvaluator", + "HateUnfairnessEvaluator", + "ContentSafetyEvaluator", + "ProtectedMaterialEvaluator", +} + + +@dataclass +class EvaluatorRuntime: + """A loaded, ready-to-call evaluator.""" + + preset: EvaluatorPreset + callable: Any # evaluator instance or sentinel for "latency" + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + + +def _credential() -> Any: + from azure.identity import DefaultAzureCredential # noqa: WPS433 + + return DefaultAzureCredential(exclude_developer_cli_credential=True) + + +def _model_config() -> Dict[str, str]: + endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") + deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") or os.getenv( + "AZURE_AI_MODEL_DEPLOYMENT_NAME" + ) + # The New Foundry "AI Services" inference endpoint rejects the + # azure-ai-evaluation SDK's stock api-version with + # ``BadRequest: API version not supported``. Default to a version + # known to work against both the New Foundry proxy and classic + # Azure OpenAI; allow override via AZURE_OPENAI_API_VERSION. + api_version = os.getenv("AZURE_OPENAI_API_VERSION") or "2025-04-01-preview" + + missing = [] + if not endpoint: + missing.append("AZURE_OPENAI_ENDPOINT") + if not deployment: + missing.append("AZURE_OPENAI_DEPLOYMENT") + if missing: + raise RuntimeError( + "AI-assisted evaluators require an evaluator model. " + "Missing environment variables: " + ", ".join(missing) + ) + + config: Dict[str, str] = { + "azure_endpoint": endpoint, # type: ignore[dict-item] + "azure_deployment": deployment, # type: ignore[dict-item] + "api_version": api_version, + } + return config + + +def _project_endpoint() -> str: + endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + raise RuntimeError( + "Safety evaluators require AZURE_AI_FOUNDRY_PROJECT_ENDPOINT." + ) + return endpoint + + +_LATENCY_SENTINEL = object() + + +def load_evaluator(preset: EvaluatorPreset) -> EvaluatorRuntime: + """Instantiate one evaluator. Raises a clear error if the SDK is missing.""" + if preset.class_name == "_latency": + return EvaluatorRuntime(preset=preset, callable=_LATENCY_SENTINEL) + + try: + module = importlib.import_module("azure.ai.evaluation") + except ImportError as exc: + raise RuntimeError( + "Evaluators require the 'azure-ai-evaluation' package. " + "Install with: pip install azure-ai-evaluation" + ) from exc + + cls = getattr(module, preset.class_name, None) + if cls is None: + raise RuntimeError( + f"Evaluator class {preset.class_name!r} not found in azure.ai.evaluation" + ) + + init_kwargs: Dict[str, Any] = {} + if preset.class_name in _AI_ASSISTED: + init_kwargs["model_config"] = _model_config() + if preset.class_name in _SAFETY: + init_kwargs["azure_ai_project"] = _project_endpoint() + init_kwargs["credential"] = _credential() + + try: + instance = cls(**init_kwargs) if inspect.isclass(cls) else cls + except TypeError: + # Some evaluators reject unexpected kwargs (e.g. F1ScoreEvaluator). + instance = cls() if inspect.isclass(cls) else cls + + return EvaluatorRuntime(preset=preset, callable=instance) + + +def load_evaluators(presets: List[EvaluatorPreset]) -> List[EvaluatorRuntime]: + return [load_evaluator(preset) for preset in presets] + + +# --------------------------------------------------------------------------- +# Execution +# --------------------------------------------------------------------------- + + +_PLACEHOLDERS = { + "$prompt": "input", + "$prediction": "response", + "$expected": "expected", + "$context": "context", + "$tool_calls": "tool_calls", + "$tool_definitions": "tool_definitions", +} + + +def _build_conversation_messages( + *, + input_text: Optional[str], + response_text: str, + tool_calls: Any, +) -> Optional[Dict[str, List[Dict[str, Any]]]]: + """Build conversation-style ``query`` and ``response`` for agent evaluators. + + When the agent invoked tools, returning only the final answer text to + evaluators like ``IntentResolutionEvaluator`` and ``TaskAdherenceEvaluator`` + leaves them blind to *how* the agent arrived at that answer. They then + consistently score it as 1/5 even when the agent did the right thing. + + This helper returns a structured payload compatible with the + ``azure.ai.evaluation`` conversational schema: + + * ``query`` -> a single user message with the original input text + * ``response`` -> a sequence of assistant tool_call messages, optional + tool result messages (when each captured call has a ``result`` + string), and a final assistant text message with the natural-language + answer. + + Returns ``None`` when there are no tool calls to include — callers + should fall back to plain string kwargs in that case. + """ + has_tool_calls = isinstance(tool_calls, list) and len(tool_calls) > 0 + + query_messages: List[Dict[str, Any]] = [ + { + "role": "user", + "content": [{"type": "text", "text": input_text or ""}], + } + ] + + response_messages: List[Dict[str, Any]] = [] + if has_tool_calls: + for index, call in enumerate(tool_calls): + if not isinstance(call, dict): + continue + # Normalise across the OpenAI ``function_call`` shape and the + # nested ``function`` envelope produced by some Foundry payloads. + raw_function = call.get("function") + function: Dict[str, Any] = raw_function if isinstance(raw_function, dict) else {} + name = call.get("name") or function.get("name") + if not name: + continue + arguments = call.get("arguments") + if arguments is None: + arguments = function.get("arguments") + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + # leave as raw string — evaluators tolerate either form + pass + tool_call_id = call.get("tool_call_id") or call.get("id") or f"call_{index}" + + response_messages.append({ + "role": "assistant", + "content": [{ + "type": "tool_call", + "tool_call_id": tool_call_id, + "name": name, + "arguments": arguments if arguments is not None else {}, + }], + }) + + result = call.get("result") + if isinstance(result, str) and result: + response_messages.append({ + "role": "tool", + "tool_call_id": tool_call_id, + "content": [{"type": "tool_result", "tool_result": result}], + }) + + if response_text: + response_messages.append({ + "role": "assistant", + "content": [{"type": "text", "text": response_text}], + }) + + if not response_messages: + return None + + return {"query": query_messages, "response": response_messages} + + +def _resolve_kwargs( + mapping: Dict[str, str], + *, + row: Dict[str, Any], + response: str, +) -> Dict[str, Any]: + resolved: Dict[str, Any] = {} + merged = {**row, "response": response, "input": row.get("input")} + for kwarg, placeholder in mapping.items(): + if not isinstance(placeholder, str) or not placeholder.startswith("$"): + resolved[kwarg] = placeholder + continue + source_key = _PLACEHOLDERS.get(placeholder) + if source_key is None: + raise ValueError(f"unknown evaluator placeholder {placeholder!r}") + value = merged.get(source_key) + if value is None: + continue + resolved[kwarg] = value + return resolved + + +def _extract_score(payload: Any, score_key: str) -> Optional[float]: + if payload is None: + return None + if isinstance(payload, (int, float)): + return float(payload) + if not isinstance(payload, dict): + return None + for candidate in ( + score_key, + f"{score_key}_score", + f"gpt_{score_key}", + "score", + ): + value = payload.get(candidate) + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return float(value) + return None + + +def _extract_reason(payload: Any, score_key: str) -> Optional[str]: + if not isinstance(payload, dict): + return None + for candidate in ( + f"{score_key}_reason", + f"{score_key}_reasoning", + f"gpt_{score_key}_reason", + "reason", + "reasoning", + ): + value = payload.get(candidate) + if isinstance(value, str) and value.strip(): + return value + return None + + +def run_evaluator( + runtime: EvaluatorRuntime, + *, + row: Dict[str, Any], + response: str, + latency_seconds: float, + actual_tool_calls: Optional[List[Any]] = None, +) -> RowMetric: + """Execute one evaluator on one row. Captures errors so the run continues.""" + preset = runtime.preset + if runtime.callable is _LATENCY_SENTINEL: + return RowMetric(name=preset.score_key, value=float(latency_seconds)) + + # ToolCallAccuracyEvaluator: special handling when the agent made no + # tool calls. The Azure SDK evaluator raises ("No tool calls found in + # response...") which would surface as ERR. Translate that into a + # meaningful score: + # * dataset has no tool_calls either -> not applicable (n/a). + # * dataset expected tool_calls -> the agent failed to call them, so + # score it as 0.0 instead of crashing the row. + if preset.class_name == "ToolCallAccuracyEvaluator": + has_actual = isinstance(actual_tool_calls, list) and len(actual_tool_calls) > 0 + has_dataset = isinstance(row.get("tool_calls"), list) and len(row["tool_calls"]) > 0 + if not has_actual: + if has_dataset: + return RowMetric( + name=preset.score_key, + value=0.0, + reason="agent made no tool calls but the dataset expected some", + ) + return RowMetric( + name=preset.score_key, + value=None, + reason="not applicable: agent made no tool calls", + ) + + try: + kwargs = _resolve_kwargs(preset.input_mapping, row=row, response=response) + if preset.needs_conversation: + # Prefer the actual calls made by the agent during invocation; + # fall back to the dataset's expected calls if the runner did + # not provide any (e.g. unit tests). + tool_calls_for_convo = ( + actual_tool_calls + if actual_tool_calls is not None + else row.get("tool_calls") + ) + conversation = _build_conversation_messages( + input_text=row.get("input"), + response_text=response, + tool_calls=tool_calls_for_convo, + ) + if conversation is not None: + # Upgrade query/response from plain strings to the + # conversational schema. Both kwargs are guaranteed to be + # in input_mapping for evaluators that opt into this. + if "query" in kwargs: + kwargs["query"] = conversation["query"] + if "response" in kwargs: + kwargs["response"] = conversation["response"] + + # Retry once on transient Azure CLI credential failures. The + # az CLI occasionally fails to launch on Windows under heavy + # I/O; DefaultAzureCredential's other sources usually succeed + # on the second attempt because the token has been cached. + last_exc: Optional[Exception] = None + for attempt in range(2): + try: + result = runtime.callable(**kwargs) + last_exc = None + break + except Exception as exc: # noqa: BLE001 + last_exc = exc + if attempt == 0 and _is_transient_credential_error(exc): + time.sleep(0.5) + continue + raise + if last_exc is not None: # pragma: no cover — defensive + raise last_exc + score = _extract_score(result, preset.score_key) + reason = _extract_reason(result, preset.score_key) + return RowMetric(name=preset.score_key, value=score, reason=reason) + except Exception as exc: # noqa: BLE001 + return RowMetric(name=preset.score_key, error=str(exc)) + + +_TRANSIENT_CRED_MARKERS = ( + "failed to invoke the azure cli", + "azureclicredential", + "credentialunavailableerror", +) + + +def _is_transient_credential_error(exc: Exception) -> bool: + msg = str(exc).lower() + return any(marker in msg for marker in _TRANSIENT_CRED_MARKERS) diff --git a/src/agentops/pipeline/thresholds.py b/src/agentops/pipeline/thresholds.py new file mode 100644 index 00000000..801016e0 --- /dev/null +++ b/src/agentops/pipeline/thresholds.py @@ -0,0 +1,84 @@ +"""Threshold evaluation against parsed :class:`Threshold` rules.""" + +from __future__ import annotations + +from typing import Dict, List + +from agentops.core.agentops_config import Threshold +from agentops.core.results import ThresholdEvaluation + + +def evaluate( + rules: List[Threshold], + metrics: Dict[str, float], +) -> List[ThresholdEvaluation]: + """Apply each rule against the aggregate metric value. + + Missing metrics produce a failed evaluation with ``actual="missing"`` so + the report can show the gap clearly rather than crashing the run. + """ + results: List[ThresholdEvaluation] = [] + for rule in rules: + actual_value = metrics.get(rule.metric) + + if rule.criteria in {"true", "false"}: + expected = rule.criteria + actual = "missing" + passed = False + if actual_value is not None: + actual_bool = actual_value == 1.0 + actual = "true" if actual_bool else "false" + passed = actual == expected + results.append( + ThresholdEvaluation( + metric=rule.metric, + criteria=rule.criteria, + expected=expected, + actual=actual, + passed=passed, + ) + ) + continue + + if rule.value is None: + raise ValueError( + f"threshold for {rule.metric!r} requires a numeric value" + ) + + target = float(rule.value) + expected_str = f"{rule.criteria}{target:g}" + if actual_value is None: + results.append( + ThresholdEvaluation( + metric=rule.metric, + criteria=rule.criteria, + expected=expected_str, + actual="missing", + passed=False, + ) + ) + continue + + if rule.criteria == ">=": + passed = actual_value >= target + elif rule.criteria == ">": + passed = actual_value > target + elif rule.criteria == "<=": + passed = actual_value <= target + elif rule.criteria == "<": + passed = actual_value < target + elif rule.criteria == "==": + passed = actual_value == target + else: + raise ValueError(f"unsupported criteria {rule.criteria!r}") + + results.append( + ThresholdEvaluation( + metric=rule.metric, + criteria=rule.criteria, + expected=expected_str, + actual=f"{actual_value:g}", + passed=passed, + ) + ) + return results diff --git a/src/agentops/services/browse.py b/src/agentops/services/browse.py deleted file mode 100644 index 12fa5573..00000000 --- a/src/agentops/services/browse.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Browse services for listing and inspecting bundles and runs.""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from agentops.core.config_loader import load_bundle_config -from agentops.core.models import RunResult - -# --------------------------------------------------------------------------- -# Workspace resolution -# --------------------------------------------------------------------------- - -_DEFAULT_AGENTOPS_DIR = ".agentops" -_LATEST_RUN_DIR_NAME = "latest" -_RESULTS_FILENAME = "results.json" - - -def _resolve_workspace(directory: Path) -> Path: - """Resolve the .agentops workspace directory.""" - workspace = (directory / _DEFAULT_AGENTOPS_DIR).resolve() - if not workspace.is_dir(): - raise FileNotFoundError( - f"No .agentops workspace found at {workspace}. Run 'agentops init' first." - ) - return workspace - - -# --------------------------------------------------------------------------- -# Bundle browsing -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class BundleSummary: - """Summary info for a single bundle.""" - - name: str - path: Path - description: str - evaluators: list[str] - thresholds: int - - -@dataclass(frozen=True) -class BundleListResult: - """Result of listing bundles.""" - - bundles: list[BundleSummary] - bundles_dir: Path - - -def list_bundles(directory: Path = Path(".")) -> BundleListResult: - """List all bundle YAML files in the workspace.""" - workspace = _resolve_workspace(directory) - bundles_dir = workspace / "bundles" - - if not bundles_dir.is_dir(): - return BundleListResult(bundles=[], bundles_dir=bundles_dir) - - summaries: list[BundleSummary] = [] - for yaml_file in sorted(bundles_dir.glob("*.yaml")): - try: - bundle = load_bundle_config(yaml_file) - enabled = [e.name for e in bundle.evaluators if e.enabled] - summaries.append( - BundleSummary( - name=bundle.name, - path=yaml_file, - description=bundle.description or "", - evaluators=enabled, - thresholds=len(bundle.thresholds), - ) - ) - except Exception: # noqa: BLE001 - # Skip malformed bundles — still list them with minimal info - summaries.append( - BundleSummary( - name=yaml_file.stem, - path=yaml_file, - description="(error loading bundle)", - evaluators=[], - thresholds=0, - ) - ) - - return BundleListResult(bundles=summaries, bundles_dir=bundles_dir) - - -@dataclass(frozen=True) -class BundleDetail: - """Full detail of a single bundle.""" - - name: str - path: Path - description: str - evaluators: list[dict[str, Any]] - thresholds: list[dict[str, Any]] - metadata: dict[str, Any] - - -def show_bundle(bundle_name: str, directory: Path = Path(".")) -> BundleDetail: - """Load and return full details of a bundle by name.""" - workspace = _resolve_workspace(directory) - bundles_dir = workspace / "bundles" - - # Try exact filename first, then search by bundle name - candidates = [ - bundles_dir / f"{bundle_name}.yaml", - bundles_dir / f"{bundle_name}", - ] - - bundle_path: Path | None = None - for candidate in candidates: - if candidate.is_file(): - bundle_path = candidate - break - - # Search by bundle name field if not found by filename - if bundle_path is None and bundles_dir.is_dir(): - for yaml_file in bundles_dir.glob("*.yaml"): - try: - bundle = load_bundle_config(yaml_file) - if bundle.name == bundle_name: - bundle_path = yaml_file - break - except Exception: # noqa: BLE001 - continue - - if bundle_path is None: - raise FileNotFoundError( - f"Bundle '{bundle_name}' not found in {bundles_dir}. " - f"Available bundles: {', '.join(f.stem for f in bundles_dir.glob('*.yaml'))}" - ) - - bundle = load_bundle_config(bundle_path) - return BundleDetail( - name=bundle.name, - path=bundle_path, - description=bundle.description or "", - evaluators=[ - { - "name": e.name, - "source": e.source, - "enabled": e.enabled, - } - for e in bundle.evaluators - ], - thresholds=[ - { - "evaluator": t.evaluator, - "criteria": t.criteria, - "value": t.value, - } - for t in bundle.thresholds - ], - metadata=bundle.metadata, - ) - - -# --------------------------------------------------------------------------- -# Run browsing -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class RunSummary: - """Summary info for a single past run.""" - - run_id: str - path: Path - bundle_name: str - dataset_name: str - status: str - started_at: str - duration_seconds: float - metrics_count: int - overall_passed: bool - - -@dataclass(frozen=True) -class RunListResult: - """Result of listing runs.""" - - runs: list[RunSummary] - results_dir: Path - - -def _has_results_file(run_dir: Path) -> bool: - """Return whether a run directory contains persisted results.""" - return (run_dir / _RESULTS_FILENAME).exists() - - -def _history_run_dirs(results_dir: Path) -> list[Path]: - """Return non-latest run directories that have persisted results.""" - return [ - run_dir - for run_dir in sorted(results_dir.iterdir(), reverse=True) - if run_dir.is_dir() - and run_dir.name != _LATEST_RUN_DIR_NAME - and _has_results_file(run_dir) - ] - - -def _listable_run_dirs(results_dir: Path) -> list[Path]: - """Return run directories that should appear in ``agentops run list``. - - ``latest`` mirrors the newest run when timestamped history exists, so list it - only when it is the sole run directory with persisted results. - """ - history_run_dirs = _history_run_dirs(results_dir) - if history_run_dirs: - return history_run_dirs - - latest_dir = results_dir / _LATEST_RUN_DIR_NAME - return [latest_dir] if _has_results_file(latest_dir) else [] - - -def list_runs(directory: Path = Path(".")) -> RunListResult: - """List all past evaluation runs in the workspace.""" - workspace = _resolve_workspace(directory) - results_dir = workspace / "results" - - if not results_dir.is_dir(): - return RunListResult(runs=[], results_dir=results_dir) - - summaries: list[RunSummary] = [] - for run_dir in _listable_run_dirs(results_dir): - results_file = run_dir / _RESULTS_FILENAME - try: - data = json.loads(results_file.read_text(encoding="utf-8")) - result = RunResult.model_validate(data) - summaries.append( - RunSummary( - run_id=run_dir.name, - path=run_dir, - bundle_name=result.bundle.name, - dataset_name=result.dataset.name, - status=result.status, - started_at=result.execution.started_at, - duration_seconds=result.execution.duration_seconds, - metrics_count=len(result.metrics), - overall_passed=result.summary.overall_passed, - ) - ) - except Exception: # noqa: BLE001 - # Include the run with minimal info if results.json is malformed - summaries.append( - RunSummary( - run_id=run_dir.name, - path=run_dir, - bundle_name="(error)", - dataset_name="(error)", - status="error", - started_at="", - duration_seconds=0, - metrics_count=0, - overall_passed=False, - ) - ) - - return RunListResult(runs=summaries, results_dir=results_dir) - - -@dataclass(frozen=True) -class RunDetail: - """Full detail of a single past run.""" - - run_id: str - path: Path - bundle_name: str - dataset_name: str - status: str - backend: str - started_at: str - finished_at: str - duration_seconds: float - overall_passed: bool - metrics: list[dict[str, Any]] - thresholds: list[dict[str, Any]] - items_total: int - items_passed: int - report_path: Path | None - foundry_url: str | None - - -def show_run(run_id: str, directory: Path = Path(".")) -> RunDetail: - """Load and return full details of a past run.""" - workspace = _resolve_workspace(directory) - results_dir = workspace / "results" - - run_dir = (results_dir / run_id).resolve() - if not run_dir.is_dir(): - available = ( - [listable_dir.name for listable_dir in _listable_run_dirs(results_dir)] - if results_dir.is_dir() - else [] - ) - hint = ", ".join(available[:5]) if available else "(none)" - raise FileNotFoundError( - f"Run '{run_id}' not found in {results_dir}. Recent runs: {hint}" - ) - - results_file = run_dir / _RESULTS_FILENAME - if not results_file.exists(): - raise FileNotFoundError(f"No results.json in {run_dir}") - - data = json.loads(results_file.read_text(encoding="utf-8")) - result = RunResult.model_validate(data) - - _rp = run_dir / "report.md" - report_path: Path | None = _rp if _rp.exists() else None - - foundry_url = None - if result.artifacts and result.artifacts.foundry_eval_studio_url: - foundry_url = result.artifacts.foundry_eval_studio_url - - items_total = result.summary.thresholds_count - items_passed = result.summary.thresholds_passed - # Use item_evaluations for more accurate counts - if result.item_evaluations: - items_total = len(result.item_evaluations) - items_passed = sum(1 for i in result.item_evaluations if i.passed_all) - - return RunDetail( - run_id=run_id, - path=run_dir, - bundle_name=result.bundle.name, - dataset_name=result.dataset.name, - status=result.status, - backend=result.execution.backend, - started_at=result.execution.started_at, - finished_at=result.execution.finished_at, - duration_seconds=result.execution.duration_seconds, - overall_passed=result.summary.overall_passed, - metrics=[{"name": m.name, "value": m.value} for m in result.metrics], - thresholds=[ - { - "evaluator": t.evaluator, - "criteria": t.criteria, - "expected": t.expected, - "actual": t.actual, - "passed": t.passed, - } - for t in result.thresholds - ], - items_total=items_total, - items_passed=items_passed, - report_path=report_path, - foundry_url=foundry_url, - ) diff --git a/src/agentops/services/cicd.py b/src/agentops/services/cicd.py index 6e651283..038c7c6a 100644 --- a/src/agentops/services/cicd.py +++ b/src/agentops/services/cicd.py @@ -9,61 +9,34 @@ _TEMPLATE_PACKAGE = "agentops.templates" -_WORKFLOW_TEMPLATE = "workflows/agentops-eval.yml" -_DEFAULT_OUTPUT_PATH = ".github/workflows/agentops-eval.yml" -# Mapping of workflow kind → (template path inside package, output path in repo) +# Mapping of workflow kind → (template path inside package, output path in repo). +# +# The four templates form a complete GenAIOps GitFlow scaffold: +# +# pr -> agentops-pr.yml (PR gate; PRs to develop, release/**, main) +# dev -> agentops-deploy-dev.yml (push to develop -> environment: dev) +# qa -> agentops-deploy-qa.yml (push to release/** -> environment: qa) +# prod -> agentops-deploy-prod.yml (push to main -> environment: production) _WORKFLOW_TEMPLATES = { - "pr": ("workflows/agentops-eval.yml", ".github/workflows/agentops-eval.yml"), - "ci": ("workflows/agentops-eval-ci.yml", ".github/workflows/agentops-eval-ci.yml"), - "cd": ("workflows/agentops-eval-cd.yml", ".github/workflows/agentops-eval-cd.yml"), + "pr": ("workflows/agentops-pr.yml", ".github/workflows/agentops-pr.yml"), + "dev": ("workflows/agentops-deploy-dev.yml", ".github/workflows/agentops-deploy-dev.yml"), + "qa": ("workflows/agentops-deploy-qa.yml", ".github/workflows/agentops-deploy-qa.yml"), + "prod": ("workflows/agentops-deploy-prod.yml", ".github/workflows/agentops-deploy-prod.yml"), } +ALL_KINDS: tuple[str, ...] = ("pr", "dev", "qa", "prod") + @dataclass class CicdResult: - """Result of generating CI/CD workflow files. - - Attributes: - created_files: Paths of newly created files. - overwritten_files: Paths of files that were overwritten. - skipped_files: Paths of files that already existed and were skipped. - """ + """Result of generating CI/CD workflow files.""" created_files: List[Path] = field(default_factory=list) overwritten_files: List[Path] = field(default_factory=list) skipped_files: List[Path] = field(default_factory=list) -def _detect_workflow_kinds(directory: Path) -> List[str]: - """Auto-detect which workflow templates to generate based on workspace content. - - Always includes ``"pr"``. Adds ``"ci"`` when multiple bundles or run - configs exist. Adds ``"cd"`` when two or more bundles or run configs - are present (mirrors CI detection — production needs the full suite). - """ - kinds: List[str] = ["pr"] - - agentops_dir = directory / ".agentops" - bundles_dir = agentops_dir / "bundles" - bundle_files: List[Path] = [] - if bundles_dir.is_dir(): - bundle_files = [f for f in bundles_dir.iterdir() if f.suffix in (".yaml", ".yml")] - - # Detect multiple bundles or run configs → include CI and CD pipelines - run_configs = [ - f - for f in agentops_dir.iterdir() - if f.is_file() and f.name.startswith("run") and f.suffix in (".yaml", ".yml") - ] if agentops_dir.is_dir() else [] - - if len(bundle_files) > 1 or len(run_configs) > 1: - kinds.append("ci") - kinds.append("cd") - - return kinds - - def _write_template( templates_root, template_path: str, @@ -71,7 +44,6 @@ def _write_template( force: bool, result: CicdResult, ) -> None: - """Read a packaged template and write it to *output_path*.""" template_resource = templates_root.joinpath(template_path) template_content = template_resource.read_text(encoding="utf-8") @@ -90,60 +62,46 @@ def _write_template( result.created_files.append(output_path) -def generate_cicd_workflow( - directory: Path, - force: bool = False, -) -> CicdResult: - """Generate a GitHub Actions workflow file for AgentOps evaluation. - - Reads the packaged workflow template and writes it to the target - repository's ``.github/workflows/`` directory. - - Args: - directory: Root directory of the consumer repository. - force: When True, overwrite the workflow file if it already exists. - - Returns: - CicdResult with paths of created, overwritten, or skipped files. - """ - result = CicdResult() - templates_root = files(_TEMPLATE_PACKAGE) - output_path = (directory / _DEFAULT_OUTPUT_PATH).resolve() - _write_template(templates_root, _WORKFLOW_TEMPLATE, output_path, force, result) - return result - - def generate_cicd_workflows( directory: Path, force: bool = False, kinds: Sequence[str] | None = None, ) -> CicdResult: - """Generate one or more GitHub Actions workflow files. + """Generate the AgentOps GitFlow GitHub Actions workflows. - When *kinds* is ``None``, auto-detects which templates to generate - by inspecting the ``.agentops/`` workspace in *directory*. + By default writes all four templates (``pr``, ``dev``, ``qa``, + ``prod``). Pass *kinds* to opt into a subset. Args: directory: Root directory of the consumer repository. force: When True, overwrite existing workflow files. - kinds: Explicit list of workflow kinds (``"pr"``, ``"ci"``, - ``"c``None`` triggers auto-detection. + kinds: Optional explicit list of workflow kinds. ``None`` means + "generate all four". Unknown kinds are ignored. Returns: - CicdResult with paths of created, overwritten, or skipped files - across all generated templates. + CicdResult with paths of created, overwritten, or skipped files. """ if kinds is None: - kinds = _detect_workflow_kinds(directory) + kinds = ALL_KINDS result = CicdResult() templates_root = files(_TEMPLATE_PACKAGE) + seen: set[str] = set() for kind in kinds: - if kind not in _WORKFLOW_TEMPLATES: + if kind in seen or kind not in _WORKFLOW_TEMPLATES: continue + seen.add(kind) template_path, output_rel = _WORKFLOW_TEMPLATES[kind] output_path = (directory / output_rel).resolve() _write_template(templates_root, template_path, output_path, force, result) return result + + +def generate_cicd_workflow( + directory: Path, + force: bool = False, +) -> CicdResult: + """Generate only the PR workflow template (legacy convenience).""" + return generate_cicd_workflows(directory, force=force, kinds=["pr"]) diff --git a/src/agentops/services/comparison.py b/src/agentops/services/comparison.py deleted file mode 100644 index 8ea713d3..00000000 --- a/src/agentops/services/comparison.py +++ /dev/null @@ -1,402 +0,0 @@ -"""Comparison service for evaluating baseline vs current run results.""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from pathlib import Path - -from agentops.core.models import ( - ComparisonConditions, - ComparisonItemRow, - ComparisonMetricRow, - ComparisonResult, - ComparisonSummary, - ComparisonThresholdRow, - ComparisonType, - Criteria, - Direction, - ItemEvaluationResult, - RunReference, - RunResult, - ThresholdEvaluationResult, -) - - -@dataclass(frozen=True) -class ComparisonServiceResult: - comparison_json_path: Path - comparison_md_path: Path | None - comparison_html_path: Path | None - has_regressions: bool - - -def _resolve_run_path(run_id: str, workspace_dir: Path | None = None) -> Path: - """Resolve a run identifier to a results.json path. - - Supports: - - Absolute or relative path to a results.json file - - Absolute or relative path to a run directory containing results.json - - Timestamped run ID (e.g. '2026-03-03_143022') resolved under workspace results - - The keyword 'latest' - """ - candidate = Path(run_id) - - if candidate.is_absolute(): - if candidate.is_file(): - return candidate - results_in_dir = candidate / "results.json" - if results_in_dir.is_file(): - return results_in_dir - raise FileNotFoundError(f"Cannot find results.json at: {candidate}") - - if candidate.is_file(): - return candidate.resolve() - if candidate.is_dir(): - results_in_dir = candidate / "results.json" - if results_in_dir.is_file(): - return results_in_dir.resolve() - - results_base = workspace_dir or (Path.cwd() / ".agentops") - results_dir = ( - results_base / "results" if results_base.name != "results" else results_base - ) - run_dir = results_dir / run_id - results_file = run_dir / "results.json" - if results_file.is_file(): - return results_file.resolve() - - raise FileNotFoundError( - f"Cannot resolve run '{run_id}' to a results.json file. " - f"Searched: {results_file}" - ) - - -def _load_run_result(path: Path) -> RunResult: - payload = json.loads(path.read_text(encoding="utf-8")) - return RunResult.model_validate(payload) - - -def _parse_command_field(command: str) -> dict[str, str]: - """Extract key=value pairs from the execution command string.""" - parts = command.split() - result: dict[str, str] = {} - for part in parts: - if "=" in part: - key, _, value = part.partition("=") - result[key] = value - return result - - -def _run_reference(result: RunResult, run_id: str) -> RunReference: - cmd = _parse_command_field(result.execution.command) - # Infer target from command fields - target = cmd.get("target") - if not target: - if cmd.get("agent_id"): - target = "agent" - elif cmd.get("model"): - target = "model" - return RunReference( - run_id=run_id, - bundle_name=result.bundle.name, - dataset_name=result.dataset.name, - started_at=result.execution.started_at, - backend=result.execution.backend, - target=target, - model=cmd.get("model"), - agent_id=cmd.get("agent_id"), - project_endpoint=cmd.get("project_endpoint"), - overall_passed=result.summary.overall_passed, - ) - - -def _lower_is_better_metrics(*results: RunResult) -> frozenset[str]: - """Derive which metrics are lower-is-better from threshold criteria. - - If a threshold uses ``<=`` or ``<``, the metric is lower-is-better. - """ - names: set[str] = set() - for r in results: - for t in r.thresholds: - if t.criteria in {"<=", "<"}: - names.add(t.evaluator) - return frozenset(names) - - -def _compute_metric_direction(delta: float, lower_is_better: bool) -> Direction: - if delta == 0: - return "unchanged" - if lower_is_better: - return "improved" if delta < 0 else "regressed" - return "improved" if delta > 0 else "regressed" - - -def _detect_conditions(refs: list[RunReference]) -> ComparisonConditions: - """Detect what's fixed vs varying across runs to determine comparison type.""" - dimensions = { - "dataset": [r.dataset_name for r in refs], - "agent": [r.agent_id or "-" for r in refs], - "model": [r.model or "-" for r in refs], - "backend": [r.backend or "-" for r in refs], - "target": [r.target or "-" for r in refs], - "bundle": [r.bundle_name for r in refs], - "project": [r.project_endpoint or "-" for r in refs], - } - - fixed: dict[str, str] = {} - varying: list[str] = [] - # Fields always shown in Run Details — exclude from fixed list - always_shown = {"target", "model", "agent"} - for key, values in dimensions.items(): - unique = set(values) - if len(unique) == 1: - if key not in always_shown: - fixed[key] = values[0] - else: - varying.append(key) - - # Determine comparison type - ctype: ComparisonType - if "dataset" not in varying and "agent" in varying: - ctype = "agent" - elif "dataset" not in varying and "model" in varying: - ctype = "model" - elif "dataset" in varying and "agent" not in varying and "model" not in varying: - ctype = "dataset" - else: - ctype = "general" - - # Row-level comparison is only valid when all runs use the same dataset - row_level_valid = "dataset" not in varying - - return ComparisonConditions( - comparison_type=ctype, - fixed=fixed, - varying=varying, - row_level_valid=row_level_valid, - ) - - -def compare_runs( - run_paths: list[Path], - run_ids: list[str], -) -> ComparisonResult: - """Compare N evaluation runs. The first run is the baseline.""" - results = [_load_run_result(p) for p in run_paths] - refs = [_run_reference(r, rid) for r, rid in zip(results, run_ids)] - - lib_metrics = _lower_is_better_metrics(*results) - - # Collect all metric names preserving order - all_metric_names: list[str] = [] - seen_names: set[str] = set() - for r in results: - for m in r.metrics: - if m.name not in seen_names: - all_metric_names.append(m.name) - seen_names.add(m.name) - - # Build metric rows - metric_rows: list[ComparisonMetricRow] = [] - for name in all_metric_names: - values: list[float] = [] - deltas: list[float | None] = [] - delta_percents: list[float | None] = [] - directions: list[Direction] = [] - baseline_val: float | None = None - - for i, r in enumerate(results): - val_map = {m.name: m.value for m in r.metrics} - val = val_map.get(name) - if val is None: - values.append(0.0) - deltas.append(None) - delta_percents.append(None) - directions.append("unchanged") - continue - - values.append(val) - if i == 0: - baseline_val = val - deltas.append(None) - delta_percents.append(None) - directions.append("unchanged") - else: - if baseline_val is not None: - d = val - baseline_val - dp = (d / abs(baseline_val) * 100) if baseline_val != 0 else None - deltas.append(d) - delta_percents.append(dp) - directions.append(_compute_metric_direction(d, name in lib_metrics)) - else: - deltas.append(None) - delta_percents.append(None) - directions.append("unchanged") - - # Best run: for lower-is-better pick min, otherwise pick max - valid_vals = [ - (i, v) - for i, v in enumerate(values) - if any(m.name == name for m in results[i].metrics) - ] - best_idx: int | None = None - if valid_vals: - if name in lib_metrics: - best_idx = min(valid_vals, key=lambda x: x[1])[0] - else: - best_idx = max(valid_vals, key=lambda x: x[1])[0] - - metric_rows.append( - ComparisonMetricRow( - name=name, - values=values, - deltas=deltas, - delta_percents=delta_percents, - directions=directions, - best_run_index=best_idx, - ) - ) - - # Build threshold rows - all_thresholds: list[tuple[str, Criteria]] = [] - seen_thresholds: set[tuple[str, Criteria]] = set() - for r in results: - for th in r.thresholds: - key = (th.evaluator, th.criteria) - if key not in seen_thresholds: - all_thresholds.append(key) - seen_thresholds.add(key) - - threshold_rows: list[ComparisonThresholdRow] = [] - for evaluator, criteria in all_thresholds: - passed_list: list[bool] = [] - target_val: str | None = None - for r in results: - t_map = {(t.evaluator, t.criteria): t for t in r.thresholds} - t: ThresholdEvaluationResult | None = t_map.get((evaluator, criteria)) - passed_list.append(t.passed if t else False) - if t and target_val is None: - target_val = t.expected - threshold_rows.append( - ComparisonThresholdRow( - evaluator=evaluator, - criteria=criteria, - target=target_val, - passed=passed_list, - ) - ) - - # Build item rows - all_row_indices: set[int] = set() - for r in results: - for ie in r.item_evaluations: - all_row_indices.add(ie.row_index) - - # Collect evaluator names that have thresholds (for row-level display) - threshold_evaluator_names = [tr.evaluator for tr in threshold_rows] - - item_rows: list[ComparisonItemRow] = [] - for idx in sorted(all_row_indices): - passed_list = [] - # Per-evaluator scores for this row across all runs - scores: dict[str, list[float | None]] = { - name: [] for name in threshold_evaluator_names - } - for r in results: - item_map = {item.row_index: item for item in r.item_evaluations} - item: ItemEvaluationResult | None = item_map.get(idx) - passed_list.append(item.passed_all if item else False) - # Extract row-level metric scores - row_metrics_map = {row.row_index: row for row in r.row_metrics} - row_m = row_metrics_map.get(idx) - for name in threshold_evaluator_names: - if row_m: - val_map = {m.name: m.value for m in row_m.metrics} - scores[name].append(val_map.get(name)) - else: - scores[name].append(None) - item_rows.append( - ComparisonItemRow(row_index=idx, passed_all=passed_list, scores=scores) - ) - - # Summary: regression = a run whose status flipped from PASS to FAIL, - # or a threshold that was met by baseline but missed by this run. - # Minor numeric shifts within passing thresholds are NOT regressions. - runs_with_regressions: list[int] = [] - for i in range(1, len(results)): - has_reg = False - # Check if overall run status flipped PASS→FAIL - if results[0].summary.overall_passed and not results[i].summary.overall_passed: - has_reg = True - # Check if any row flipped from passing to failing - if not has_reg: - for ir in item_rows: - if ir.passed_all[0] and not ir.passed_all[i]: - has_reg = True - break - if has_reg: - runs_with_regressions.append(i) - - summary = ComparisonSummary( - run_count=len(results), - any_regressions=len(runs_with_regressions) > 0, - runs_with_regressions=runs_with_regressions, - ) - - return ComparisonResult( - version=1, - runs=refs, - baseline_index=0, - conditions=_detect_conditions(refs), - metric_rows=metric_rows, - threshold_rows=threshold_rows, - item_rows=item_rows, - summary=summary, - ) - - -def run_comparison( - run_ids: list[str], - output_dir: Path | None = None, - report_format: str = "md", -) -> ComparisonServiceResult: - """Resolve run IDs, compare, and write comparison outputs.""" - from agentops.core.reporter import ( - generate_comparison_html, - generate_comparison_markdown, - ) - - paths = [_resolve_run_path(rid) for rid in run_ids] - result = compare_runs(run_paths=paths, run_ids=run_ids) - - resolved_output = output_dir.resolve() if output_dir else paths[-1].parent - resolved_output.mkdir(parents=True, exist_ok=True) - - comparison_json_path = resolved_output / "comparison.json" - comparison_md_path: Path | None = None - comparison_html_path: Path | None = None - - comparison_json_path.write_text( - json.dumps(result.model_dump(mode="json"), indent=2), - encoding="utf-8", - ) - if report_format in ("md", "all"): - comparison_md_path = resolved_output / "comparison.md" - comparison_md_path.write_text( - generate_comparison_markdown(result), - encoding="utf-8", - ) - if report_format in ("html", "all"): - comparison_html_path = resolved_output / "comparison.html" - comparison_html_path.write_text( - generate_comparison_html(result), - encoding="utf-8", - ) - - return ComparisonServiceResult( - comparison_json_path=comparison_json_path, - comparison_md_path=comparison_md_path, - comparison_html_path=comparison_html_path, - has_regressions=result.summary.any_regressions, - ) diff --git a/src/agentops/services/foundry_evals.py b/src/agentops/services/foundry_evals.py deleted file mode 100644 index 827a3f04..00000000 --- a/src/agentops/services/foundry_evals.py +++ /dev/null @@ -1,295 +0,0 @@ -"""Foundry cloud evaluation publishing service. - -Publishes already computed AgentOps backend metrics to the -**New Foundry Evaluations** panel using the same 3-step OneDP upload flow: - -1. ``create_evaluation_result`` — uploads ``instance_results.jsonl`` to blob -2. ``start_evaluation_run`` — creates the run entry with portal-required - properties (``_azureml.evaluate_artifacts``, ``_azureml.evaluation_sdk_name``, - name-map entries, ``runType``) -3. ``update_evaluation_run`` — marks the run ``Completed`` and links it to the - result artifact via ``evaluationResultId`` -""" - -from __future__ import annotations - -import ast -import json -import logging -import os -import re -import uuid -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, List -from urllib.parse import urlparse - -from agentops.core.config_loader import load_dataset_config -from agentops.core.models import TargetEndpointConfig - - -@dataclass(frozen=True) -class FoundryEvalPublishResult: - """Result of publishing an evaluation to the Foundry panel.""" - - studio_url: str - evaluation_name: str - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -_ROW_LINE_PATTERN = re.compile( - r"^row=(?P\d+)\s+exact_match=(?Ptrue|false)\s+expected=(?P.+?)\s+prediction=(?P.+)$" -) - - -def _resolve_dataset_source_path(dataset_config_path: Path, source_path: Path) -> Path: - if source_path.is_absolute(): - return source_path - - candidate = (dataset_config_path.parent / source_path).resolve() - if candidate.exists(): - return candidate - - fallback = (Path.cwd() / source_path).resolve() - if fallback.exists(): - return fallback - - return candidate - - -def _load_jsonl(path: Path) -> List[Dict[str, Any]]: - rows: List[Dict[str, Any]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if not stripped: - continue - payload = json.loads(stripped) - if not isinstance(payload, dict): - raise ValueError("Dataset JSONL rows must be objects") - rows.append(payload) - return rows - - -def _parse_output_rows(stdout_path: Path) -> Dict[int, Dict[str, Any]]: - parsed: Dict[int, Dict[str, Any]] = {} - if not stdout_path.exists(): - return parsed - - for raw_line in stdout_path.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line: - continue - - match = _ROW_LINE_PATTERN.match(line) - if not match: - continue - - row_number = int(match.group("row")) - expected = ast.literal_eval(match.group("expected")) - prediction = ast.literal_eval(match.group("prediction")) - exact_match = match.group("exact") == "true" - - parsed[row_number] = { - "expected": str(expected), - "prediction": str(prediction), - "exact_match": exact_match, - } - - return parsed - - -def _parse_project_identity(project_endpoint: str) -> tuple[str, str]: - parsed = urlparse(project_endpoint) - host = parsed.netloc - match = re.search(r"^([^.]+)\.services\.ai\.azure\.com$", host) - if not match: - raise ValueError(f"Invalid Foundry project endpoint host: {host}") - account_name = match.group(1) - - path_parts = [part for part in parsed.path.split("/") if part] - if len(path_parts) < 3 or path_parts[0] != "api" or path_parts[1] != "projects": - raise ValueError( - "Foundry project endpoint must look like " - "https://.services.ai.azure.com/api/projects/" - ) - project_name = path_parts[2] - return account_name, project_name - - -def _load_backend_metrics_payload( - path: Path, -) -> tuple[Dict[str, float], Dict[int, Dict[str, float]]]: - if not path.exists(): - raise FileNotFoundError(f"Backend metrics file not found: {path}") - - payload = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError("Invalid backend metrics payload: expected JSON object") - - metrics_entries = payload.get("metrics", []) - if not isinstance(metrics_entries, list): - raise ValueError("Invalid backend metrics payload: 'metrics' must be a list") - - metrics: Dict[str, float] = {} - for item in metrics_entries: - if not isinstance(item, dict): - continue - name = item.get("name") - value = item.get("value") - if ( - isinstance(name, str) - and isinstance(value, (int, float)) - and not isinstance(value, bool) - ): - metrics[name] = float(value) - - row_metrics_entries = payload.get("row_metrics", []) - if not isinstance(row_metrics_entries, list): - raise ValueError( - "Invalid backend metrics payload: 'row_metrics' must be a list" - ) - - row_metrics: Dict[int, Dict[str, float]] = {} - for row in row_metrics_entries: - if not isinstance(row, dict): - continue - row_index = row.get("row_index") - raw_metrics = row.get("metrics", []) - if ( - not isinstance(row_index, int) - or row_index <= 0 - or not isinstance(raw_metrics, list) - ): - continue - - row_values: Dict[str, float] = {} - for metric in raw_metrics: - if not isinstance(metric, dict): - continue - name = metric.get("name") - value = metric.get("value") - if ( - isinstance(name, str) - and isinstance(value, (int, float)) - and not isinstance(value, bool) - ): - row_values[name] = float(value) - row_metrics[row_index] = row_values - - if not metrics: - raise ValueError("Backend metrics payload does not contain numeric metrics") - - return metrics, row_metrics - - -# --------------------------------------------------------------------------- -# Public entry-point -# --------------------------------------------------------------------------- - - -def publish_foundry_evaluation( - *, - endpoint_config: TargetEndpointConfig, - dataset_config_path: Path, - backend_stdout_path: Path, - evaluation_name: str | None = None, -) -> FoundryEvalPublishResult: - """Publish evaluation results to the New Foundry Evaluations panel. - - Publishes existing AgentOps backend metrics so Foundry displays - the same evaluator outputs seen in `results.json` and `report.md`. - """ - try: - import pandas as pd # noqa: WPS433 - from azure.ai.evaluation._evaluate._utils import ( # noqa: WPS433 - _log_metrics_and_instance_results_onedp, - ) - except ImportError as exc: - raise ImportError( - "Foundry evaluation publish requires 'azure-ai-evaluation' and 'pandas'. " - "Install with: pip install azure-ai-evaluation pandas" - ) from exc - - # --- resolve project endpoint ---------------------------------------- - project_endpoint_env = ( - endpoint_config.project_endpoint_env or "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT" - ) - project_endpoint = endpoint_config.project_endpoint or os.getenv( - project_endpoint_env - ) - if not project_endpoint: - raise ValueError( - "Foundry evaluation publish requires target.endpoint.project_endpoint or " - f"environment variable {project_endpoint_env}" - ) - - _parse_project_identity(project_endpoint) # validate format - - # --- build per-row JSONL from backend outputs ------------------------ - dataset_config = load_dataset_config(dataset_config_path) - dataset_source_path = _resolve_dataset_source_path( - dataset_config_path, dataset_config.source.path - ) - dataset_rows = _load_jsonl(dataset_source_path) - parsed_rows = _parse_output_rows(backend_stdout_path) - backend_metrics_path = backend_stdout_path.parent / "backend_metrics.json" - metrics, row_metrics_by_index = _load_backend_metrics_payload(backend_metrics_path) - - if not parsed_rows: - raise ValueError( - "Foundry evaluation publish could not parse backend stdout rows" - ) - - input_field = dataset_config.format.input_field - instance_rows: List[Dict[str, Any]] = [] - for index, row in enumerate(dataset_rows, start=1): - row_result = parsed_rows.get(index) - if row_result is None: - continue - - instance_payload: Dict[str, Any] = { - "line_number": index - 1, - "input": str(row.get(input_field, "")), - "response": row_result["prediction"], - "ground_truth": row_result["expected"], - } - for metric_name, metric_value in row_metrics_by_index.get(index, {}).items(): - instance_payload[metric_name] = metric_value - - instance_rows.append(instance_payload) - - if not instance_rows: - raise ValueError("Foundry evaluation publish has no content rows to submit") - - eval_name = evaluation_name or f"agentops-eval-{uuid.uuid4().hex[:8]}" - logger = logging.getLogger("agentops.foundry_evals") - logger.info("Publishing evaluation to Foundry: %s", eval_name) - - # Build the evaluator name map (maps internal metric name -> display name) - name_map: Dict[str, str] = { - metric_name: metric_name for metric_name in metrics.keys() - } - - instance_results_df = pd.DataFrame(instance_rows) - studio_url = _log_metrics_and_instance_results_onedp( - metrics=metrics, - instance_results=instance_results_df, - project_url=project_endpoint, - evaluation_name=eval_name, - name_map=name_map, - ) - - if not studio_url: - raise RuntimeError( - "Foundry evaluation upload completed but studio URL is missing." - ) - - logger.info("Foundry publish completed successfully") - logger.info("Evaluation published: %s", studio_url) - return FoundryEvalPublishResult( - studio_url=studio_url, - evaluation_name=eval_name, - ) diff --git a/src/agentops/services/initializer.py b/src/agentops/services/initializer.py index 5f6e3559..1c5a19f0 100644 --- a/src/agentops/services/initializer.py +++ b/src/agentops/services/initializer.py @@ -1,4 +1,4 @@ -"""Workspace initialization service for `agentops init`.""" +"""Workspace initialization service for ``agentops init``.""" from __future__ import annotations @@ -18,80 +18,64 @@ class InitResult: _TEMPLATE_PACKAGE = "agentops.templates" -_TEMPLATE_FILES: tuple[str, ...] = ( - "config.yaml", - "run.yaml", - "run-rag.yaml", - "run-agent.yaml", - "run-agent-local.yaml", - "run-http-model.yaml", - "run-http-rag.yaml", - "run-http-agent-tools.yaml", - "run-callable.yaml", - "callable_adapter.py", - "agent_framework_adapter.py", - "multi_agent_workflow.py", - ".gitignore", - "bundles/model_quality_baseline.yaml", - "bundles/rag_quality_baseline.yaml", - "bundles/conversational_agent_baseline.yaml", - "bundles/agent_workflow_baseline.yaml", - "bundles/safe_agent_baseline.yaml", - "datasets/smoke-model-direct.yaml", - "datasets/smoke-rag.yaml", - "datasets/smoke-agent-tools.yaml", - "datasets/smoke-conversational.yaml", - "data/smoke-model-direct.jsonl", - "data/smoke-rag.jsonl", - "data/smoke-agent-tools.jsonl", - "data/smoke-conversational.jsonl", - "workflows/agentops-eval.yml", -) - - -def _load_seed_templates() -> Dict[str, str]: - """Load workspace seed files from packaged template assets.""" - templates_root = files(_TEMPLATE_PACKAGE) - loaded: Dict[str, str] = {} - - for relative_path in _TEMPLATE_FILES: - template = templates_root.joinpath(relative_path) - loaded[relative_path] = template.read_text(encoding="utf-8") - return loaded +# 1.0 flat workspace: a single agentops.yaml at the project root and a tiny +# seed dataset under .agentops/data/. Everything else (bundles, datasets YAML, +# run-*.yaml variants) was removed in the revamp. +_FLAT_FILES: Dict[str, str] = { + "agentops.yaml": "agentops.yaml", + ".agentops/data/smoke.jsonl": "smoke.jsonl", +} -def initialize_workspace(directory: Path, force: bool = False) -> InitResult: - workspace_root = directory.resolve() - agentops_dir = workspace_root / ".agentops" +# Project-root .gitignore. Only written when one doesn't already exist so we +# never clobber a user's curated ignore file. +_PROJECT_GITIGNORE_TEMPLATE = "project.gitignore" +_PROJECT_GITIGNORE_TARGET = ".gitignore" - result = InitResult(workspace_dir=agentops_dir) - folders = [ - agentops_dir, - agentops_dir / "bundles", - agentops_dir / "datasets", - agentops_dir / "data", - agentops_dir / "results", - ] +def initialize_flat_workspace(directory: Path, force: bool = False) -> InitResult: + """Bootstrap the AgentOps 1.0 workspace. - for folder in folders: - if not folder.exists(): - folder.mkdir(parents=True, exist_ok=True) - result.created_dirs.append(folder) + Creates ``agentops.yaml`` at the project root and a tiny seed dataset at + ``.agentops/data/smoke.jsonl``. Also drops a starter ``.gitignore`` at the + project root if one does not exist yet (covers ``.venv/``, Python build + artifacts, and the ``.agentops/results/`` runtime output). + """ + project_root = directory.resolve() + result = InitResult(workspace_dir=project_root / ".agentops") - for relative_path, content in _load_seed_templates().items(): - file_path = agentops_dir / relative_path - existed_before = file_path.exists() + templates_root = files(_TEMPLATE_PACKAGE) + for relative_path, template_name in _FLAT_FILES.items(): + target = project_root / relative_path + existed_before = target.exists() if existed_before and not force: - result.skipped_files.append(file_path) + result.skipped_files.append(target) continue - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.write_text(content, encoding="utf-8") + target.parent.mkdir(parents=True, exist_ok=True) + if not target.parent.exists(): + result.created_dirs.append(target.parent) + + content = templates_root.joinpath(template_name).read_text(encoding="utf-8") + target.write_text(content, encoding="utf-8") + if existed_before: - result.overwritten_files.append(file_path) + result.overwritten_files.append(target) else: - result.created_files.append(file_path) + result.created_files.append(target) + + # Write a starter project-root .gitignore. We never overwrite an existing + # one (even with --force) — users often have curated ignores we don't want + # to clobber. + gitignore_target = project_root / _PROJECT_GITIGNORE_TARGET + if not gitignore_target.exists(): + content = templates_root.joinpath(_PROJECT_GITIGNORE_TEMPLATE).read_text( + encoding="utf-8" + ) + gitignore_target.write_text(content, encoding="utf-8") + result.created_files.append(gitignore_target) + else: + result.skipped_files.append(gitignore_target) return result diff --git a/src/agentops/services/reporting.py b/src/agentops/services/reporting.py deleted file mode 100644 index e3995e33..00000000 --- a/src/agentops/services/reporting.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Report orchestration service.""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from pathlib import Path - -from agentops.core.models import RunResult -from agentops.core.reporter import generate_report_html, generate_report_markdown - - -@dataclass(frozen=True) -class ReportResult: - input_results_path: Path - output_report_path: Path - html_report_path: Path | None = None - - -def generate_report_from_results( - results_path: Path, output_path: Path | None = None, report_format: str = "md" -) -> ReportResult: - resolved_results_path = results_path.resolve() - if not resolved_results_path.exists(): - raise FileNotFoundError(f"results.json not found: {resolved_results_path}") - - payload = json.loads(resolved_results_path.read_text(encoding="utf-8")) - result = RunResult.model_validate(payload) - - default_suffix = ".html" if report_format == "html" else ".md" - resolved_output_path = ( - output_path.resolve() - if output_path is not None - else resolved_results_path.with_name(f"report{default_suffix}") - ) - resolved_output_path.parent.mkdir(parents=True, exist_ok=True) - - primary_path = resolved_output_path - html_report_path: Path | None = None - if report_format in ("md", "all"): - md_path = ( - resolved_output_path - if resolved_output_path.suffix == ".md" - else resolved_output_path.with_suffix(".md") - ) - md_path.write_text(generate_report_markdown(result), encoding="utf-8") - primary_path = md_path - if report_format in ("html", "all"): - html_path = resolved_output_path.with_suffix(".html") - html_path.write_text(generate_report_html(result), encoding="utf-8") - primary_path = html_path - html_report_path = html_path - if report_format == "all": - primary_path = resolved_output_path.with_suffix(".md") - - return ReportResult( - input_results_path=resolved_results_path, - output_report_path=primary_path, - html_report_path=html_report_path, - ) diff --git a/src/agentops/services/runner.py b/src/agentops/services/runner.py deleted file mode 100644 index 18319af9..00000000 --- a/src/agentops/services/runner.py +++ /dev/null @@ -1,626 +0,0 @@ -"""Evaluation run orchestration service.""" - -from __future__ import annotations - -import json -import logging -import shutil -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path - -from agentops.backends.base import Backend, BackendRunContext -from agentops.core.config_loader import ( - load_bundle_config, - load_dataset_config, - load_run_config, - resolve_bundle_ref, - resolve_dataset_ref, -) -from agentops.core.models import ( - Artifacts, - BundleInfo, - DatasetInfo, - ExecutionInfo, - ItemEvaluationResult, - ItemThresholdEvaluationResult, - MetricResult, - RowMetricsResult, - RunResult, - Summary, - ThresholdEvaluationResult, - ThresholdRule, -) -from agentops.core.reporter import generate_report_html, generate_report_markdown -from agentops.services.foundry_evals import publish_foundry_evaluation -from agentops.utils.telemetry import ( - eval_item_span, - eval_run_span, - init_tracing, - record_evaluator_span, - set_eval_item_result, - set_eval_run_result, - shutdown as shutdown_tracing, -) - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class EvalRunServiceResult: - output_dir: Path - results_path: Path - report_path: Path - exit_code: int - - -def _default_run_config_path() -> Path: - return (Path.cwd() / ".agentops" / "run.yaml").resolve() - - -def _default_output_dir(run_config_path: Path) -> Path: - timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") - return (run_config_path.parent / "results" / timestamp).resolve() - - -def _latest_output_dir(run_config_path: Path) -> Path: - return (run_config_path.parent / "results" / "latest").resolve() - - -def _sync_latest_output(source_output_dir: Path, latest_output_dir: Path) -> None: - if source_output_dir.resolve() == latest_output_dir.resolve(): - return - if latest_output_dir.exists(): - shutil.rmtree(latest_output_dir) - shutil.copytree(source_output_dir, latest_output_dir) - - -def _load_backend_metrics( - metrics_path: Path, -) -> tuple[list[MetricResult], list[RowMetricsResult]]: - if not metrics_path.exists(): - raise FileNotFoundError(f"Backend metrics file not found: {metrics_path}") - - payload = json.loads(metrics_path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError("Invalid backend metrics payload: expected JSON object") - - raw_metrics = payload.get("metrics") - if not isinstance(raw_metrics, list): - raise ValueError("Invalid backend metrics payload: 'metrics' must be a list") - - metrics: list[MetricResult] = [] - for item in raw_metrics: - if not isinstance(item, dict): - raise ValueError( - "Invalid backend metrics payload: metric entries must be objects" - ) - metrics.append(MetricResult.model_validate(item)) - raw_row_metrics = payload.get("row_metrics", []) - if not isinstance(raw_row_metrics, list): - raise ValueError( - "Invalid backend metrics payload: 'row_metrics' must be a list" - ) - - row_metrics: list[RowMetricsResult] = [] - for item in raw_row_metrics: - if not isinstance(item, dict): - raise ValueError( - "Invalid backend metrics payload: row_metrics entries must be objects" - ) - row_metrics.append(RowMetricsResult.model_validate(item)) - - return metrics, row_metrics - - -def _load_cloud_evaluation_metadata(output_dir: Path) -> tuple[str | None, str | None]: - cloud_meta_path = output_dir / "cloud_evaluation.json" - if not cloud_meta_path.exists(): - return None, None - - payload = json.loads(cloud_meta_path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - return None, None - - report_url = payload.get("report_url") - evaluation_name = payload.get("evaluation_name") or payload.get("run_name") - if not isinstance(report_url, str): - report_url = None - if not isinstance(evaluation_name, str): - evaluation_name = None - return report_url, evaluation_name - - -def _summary_from_thresholds( - metrics: list[MetricResult], threshold_passes: list[bool] -) -> Summary: - thresholds_count = len(threshold_passes) - thresholds_passed = sum(1 for value in threshold_passes if value) - thresholds_failed = thresholds_count - thresholds_passed - overall_passed = thresholds_failed == 0 - return Summary( - metrics_count=len(metrics), - thresholds_count=thresholds_count, - thresholds_passed=thresholds_passed, - thresholds_failed=thresholds_failed, - overall_passed=overall_passed, - ) - - -def _rule_expected_text(rule: ThresholdRule) -> str: - if rule.criteria in {"true", "false"}: - return rule.criteria - if rule.value is None: - return "" - return f"{float(rule.value):.6f}" - - -def _evaluate_threshold_against_value( - *, - row_index: int, - rule: ThresholdRule, - actual_value: float, -) -> ItemThresholdEvaluationResult: - if rule.criteria in {"true", "false"}: - expected_bool = rule.criteria == "true" - if actual_value in (0.0, 1.0): - actual_bool = actual_value == 1.0 - else: - raise ValueError( - f"Evaluator '{rule.evaluator}' must produce 0/1 for boolean criteria" - ) - - return ItemThresholdEvaluationResult( - row_index=row_index, - evaluator=rule.evaluator, - criteria=rule.criteria, - expected="true" if expected_bool else "false", - actual="true" if actual_bool else "false", - passed=actual_bool is expected_bool, - ) - - if rule.value is None: - raise ValueError( - f"Threshold for evaluator '{rule.evaluator}' requires a numeric value" - ) - - target_value = float(rule.value) - if rule.criteria == ">=": - passed = actual_value >= target_value - elif rule.criteria == ">": - passed = actual_value > target_value - elif rule.criteria == "<=": - passed = actual_value <= target_value - elif rule.criteria == "<": - passed = actual_value < target_value - elif rule.criteria == "==": - passed = actual_value == target_value - else: - raise ValueError(f"Unsupported threshold criteria: {rule.criteria}") - - return ItemThresholdEvaluationResult( - row_index=row_index, - evaluator=rule.evaluator, - criteria=rule.criteria, - expected=f"{target_value:.6f}", - actual=f"{actual_value:.6f}", - passed=passed, - ) - - -def _evaluate_item_thresholds( - threshold_rules: list[ThresholdRule], - row_metrics: list[RowMetricsResult], -) -> list[ItemEvaluationResult]: - if not row_metrics: - return [] - - results: list[ItemEvaluationResult] = [] - for row in sorted(row_metrics, key=lambda value: value.row_index): - row_values = {metric.name: metric.value for metric in row.metrics} - threshold_results: list[ItemThresholdEvaluationResult] = [] - for rule in threshold_rules: - if rule.evaluator not in row_values: - # Evaluator may be cloud-only and was skipped during local - # execution — silently skip its threshold check. - continue - - threshold_results.append( - _evaluate_threshold_against_value( - row_index=row.row_index, - rule=rule, - actual_value=row_values[rule.evaluator], - ) - ) - - passed_all = ( - all(item.passed for item in threshold_results) - if threshold_results - else True - ) - results.append( - ItemEvaluationResult( - row_index=row.row_index, - passed_all=passed_all, - thresholds=threshold_results, - ) - ) - - return results - - -def _validate_enabled_evaluators_scored( - *, - evaluator_names: list[str], - row_metrics: list[RowMetricsResult], -) -> None: - if not evaluator_names: - return - - if not row_metrics: - raise ValueError( - "Enabled evaluators require backend 'row_metrics' with per-item scores" - ) - - scored_names: set[str] = set() - for row in row_metrics: - for metric in row.metrics: - scored_names.add(metric.name) - - missing = [name for name in evaluator_names if name not in scored_names] - if missing: - logger.warning( - "Some enabled evaluators did not produce scores and will be " - "excluded from threshold checks: %s. These evaluators may " - "only be available via Foundry Cloud Evaluation " - "(hosting: foundry, execution_mode: remote).", - ", ".join(sorted(missing)), - ) - - -def _summarize_thresholds_from_items( - threshold_rules: list[ThresholdRule], - item_evaluations: list[ItemEvaluationResult], -) -> list[ThresholdEvaluationResult]: - if not threshold_rules: - return [] - - summary: list[ThresholdEvaluationResult] = [] - total_items = len(item_evaluations) - - for rule in threshold_rules: - rule_results: list[ItemThresholdEvaluationResult] = [] - for item in item_evaluations: - for threshold_result in item.thresholds: - if ( - threshold_result.evaluator == rule.evaluator - and threshold_result.criteria == rule.criteria - ): - rule_results.append(threshold_result) - - # Skip threshold rules for evaluators that produced no scores - # (e.g., cloud-only evaluators skipped during local execution). - if not rule_results: - continue - - passed_items = sum(1 for result in rule_results if result.passed) - passed = bool(rule_results) and passed_items == len(rule_results) - - summary.append( - ThresholdEvaluationResult( - evaluator=rule.evaluator, - criteria=rule.criteria, - expected=_rule_expected_text(rule), - actual=f"{passed_items}/{total_items} items", - passed=passed, - ) - ) - - return summary - - -def _derive_run_metrics( - metrics_by_name: dict[str, float], - row_metrics: list[RowMetricsResult], - item_evaluations: list[ItemEvaluationResult], - summary: Summary, -) -> list[MetricResult]: - run_metrics: list[MetricResult] = [] - seen_run_metric_names: set[str] = set() - - def _append_run_metric(name: str, value: float) -> None: - if name in seen_run_metric_names: - return - run_metrics.append(MetricResult(name=name, value=value)) - seen_run_metric_names.add(name) - - _append_run_metric("run_pass", 1.0 if summary.overall_passed else 0.0) - - if summary.thresholds_count > 0: - _append_run_metric( - "threshold_pass_rate", - summary.thresholds_passed / summary.thresholds_count, - ) - - if item_evaluations: - passed_items = sum(1 for item in item_evaluations if item.passed_all) - _append_run_metric("items_total", float(len(item_evaluations))) - _append_run_metric("items_passed_all", float(passed_items)) - _append_run_metric( - "items_failed_any", float(len(item_evaluations) - passed_items) - ) - _append_run_metric("items_pass_rate", passed_items / len(item_evaluations)) - - row_aggregates: dict[str, list[float]] = {} - for row in row_metrics: - for metric in row.metrics: - row_aggregates.setdefault(metric.name, []).append(metric.value) - - for metric_name in sorted(row_aggregates): - values = row_aggregates[metric_name] - if values: - mean_value = sum(values) / len(values) - variance = sum((value - mean_value) ** 2 for value in values) / len(values) - stddev_value = variance**0.5 - - _append_run_metric(f"{metric_name}_avg", mean_value) - _append_run_metric(f"{metric_name}_stddev", stddev_value) - - if "exact_match" in row_aggregates: - values = row_aggregates["exact_match"] - _append_run_metric("accuracy", sum(values) / len(values)) - elif "exact_match" in metrics_by_name: - _append_run_metric("accuracy", metrics_by_name["exact_match"]) - - return run_metrics - - -def run_evaluation( - config_path: Path | None = None, - output_override: Path | None = None, - report_format: str = "md", -) -> EvalRunServiceResult: - run_config_path = ( - config_path.resolve() if config_path is not None else _default_run_config_path() - ) - run_config = load_run_config(run_config_path) - - run_config_dir = run_config_path.parent - workspace_dir = run_config_dir # .agentops/ is the workspace root - bundle_path = resolve_bundle_ref(run_config.bundle, run_config_dir, workspace_dir) - dataset_path = resolve_dataset_ref( - run_config.dataset, run_config_dir, workspace_dir - ) - - bundle_config = load_bundle_config(bundle_path) - dataset_config = load_dataset_config(dataset_path) - - output_dir = ( - output_override.resolve() - if output_override is not None - else _default_output_dir(run_config_path) - ) - output_dir.mkdir(parents=True, exist_ok=True) - - # --- Telemetry: initialise OTLP exporter (no-op when env var unset) --- - init_tracing() - - # Extract optional model/agent_id from endpoint config for span attributes - _endpoint = run_config.target.endpoint - _span_model = getattr(_endpoint, "model", None) if _endpoint else None - _span_agent_id = getattr(_endpoint, "agent_id", None) if _endpoint else None - - with eval_run_span( - bundle_name=bundle_config.name, - dataset_name=dataset_config.name, - backend_type=run_config.target.execution_mode, - target=run_config.target.type, - model=_span_model, - agent_id=_span_agent_id, - ) as run_span: - backend: Backend - if run_config.target.execution_mode == "local": - from agentops.backends.local_adapter_backend import LocalAdapterBackend - - backend = LocalAdapterBackend() - elif run_config.target.execution_mode == "remote": - endpoint = run_config.target.endpoint - assert endpoint is not None # guaranteed by TargetConfig validator - if endpoint.kind == "foundry_agent": - from agentops.backends.foundry_backend import FoundryBackend - - backend = FoundryBackend() - elif endpoint.kind == "http": - from agentops.backends.http_backend import HttpBackend - - backend = HttpBackend() - else: - raise ValueError(f"Unsupported endpoint kind: {endpoint.kind}") - else: - raise ValueError( - f"Unsupported execution_mode: {run_config.target.execution_mode}" - ) - - backend_result = backend.execute( - BackendRunContext( - run_config=run_config, - bundle_path=bundle_path, - dataset_path=dataset_path, - backend_output_dir=output_dir, - ) - ) - - if backend_result.exit_code != 0: - raise RuntimeError( - f"Backend execution failed with exit code {backend_result.exit_code}" - ) - - backend_metrics_path = output_dir / "backend_metrics.json" - metrics, row_metrics = _load_backend_metrics(backend_metrics_path) - metrics_by_name: dict[str, float] = { - metric.name: metric.value for metric in metrics - } - - enabled_evaluator_names = [ - evaluator.name - for evaluator in bundle_config.evaluators - if evaluator.enabled - ] - _validate_enabled_evaluators_scored( - evaluator_names=enabled_evaluator_names, - row_metrics=row_metrics, - ) - - item_evaluations = _evaluate_item_thresholds( - bundle_config.thresholds, row_metrics - ) - - if bundle_config.thresholds and not row_metrics: - raise ValueError( - "Item-level threshold evaluation requires backend 'row_metrics'" - ) - - threshold_results = _summarize_thresholds_from_items( - bundle_config.thresholds, item_evaluations - ) - summary = _summary_from_thresholds( - metrics, [item.passed for item in threshold_results] - ) - run_metrics = _derive_run_metrics( - metrics_by_name, row_metrics, item_evaluations, summary - ) - - # --- Telemetry: emit per-item and per-evaluator spans --- - _row_metrics_by_index = {r.row_index: r for r in row_metrics} - for item_eval in item_evaluations: - row_data = _row_metrics_by_index.get(item_eval.row_index) - _input_text = row_data.input if row_data else None - with eval_item_span( - row_index=item_eval.row_index, - input_text=_input_text, - ) as item_span: - if row_data: - for m in row_data.metrics: - matching = next( - (t for t in item_eval.thresholds if t.evaluator == m.name), - None, - ) - record_evaluator_span( - evaluator_name=m.name, - builtin_name=m.name, - source=run_config.target.execution_mode, - score=m.value, - threshold=( - float(matching.expected) - if matching - and matching.expected - and matching.criteria not in ("true", "false") - else None - ), - passed=matching.passed if matching else None, - ) - set_eval_item_result(item_span, passed=item_eval.passed_all) - - # --- Telemetry: set final run result on the root span --- - set_eval_run_result( - run_span, - passed=summary.overall_passed, - items_total=len(item_evaluations), - items_passed=sum(1 for i in item_evaluations if i.passed_all), - ) - - foundry_eval_studio_url: str | None = None - foundry_eval_name: str | None = None - - cloud_report_url, cloud_evaluation_name = _load_cloud_evaluation_metadata( - output_dir - ) - if cloud_report_url is not None: - foundry_eval_studio_url = cloud_report_url - if cloud_evaluation_name is not None: - foundry_eval_name = cloud_evaluation_name - - if ( - run_config.output.publish_foundry_evaluation - and run_config.target.endpoint is not None - and run_config.target.endpoint.kind == "foundry_agent" - and cloud_report_url is None - ): - try: - foundry_publish = publish_foundry_evaluation( - endpoint_config=run_config.target.endpoint, - dataset_config_path=dataset_path, - backend_stdout_path=backend_result.stdout_file, - ) - foundry_eval_studio_url = foundry_publish.studio_url - foundry_eval_name = foundry_publish.evaluation_name - except Exception as exc: - if run_config.output.fail_on_foundry_publish_error: - raise RuntimeError( - f"Foundry evaluation publish failed: {exc}" - ) from exc - publish_error_path = output_dir / "foundry_eval_publish_error.log" - publish_error_path.write_text(str(exc), encoding="utf-8") - - normalized_result = RunResult( - version=1, - status="completed", - bundle=BundleInfo(name=bundle_config.name, path=bundle_path), - dataset=DatasetInfo(name=dataset_config.name, path=dataset_path), - execution=ExecutionInfo( - backend=backend_result.backend, - command=backend_result.command, - started_at=backend_result.started_at, - finished_at=backend_result.finished_at, - duration_seconds=backend_result.duration_seconds, - exit_code=backend_result.exit_code, - ), - metrics=metrics, - row_metrics=row_metrics, - item_evaluations=item_evaluations, - run_metrics=run_metrics, - thresholds=threshold_results, - summary=summary, - artifacts=Artifacts( - backend_stdout=backend_result.stdout_file.name, - backend_stderr=backend_result.stderr_file.name, - foundry_eval_studio_url=foundry_eval_studio_url, - foundry_eval_name=foundry_eval_name, - ), - ) - - results_path = output_dir / "results.json" - report_path: Path - - results_path.write_text( - json.dumps(normalized_result.model_dump(mode="json"), indent=2), - encoding="utf-8", - ) - if report_format in ("md", "all"): - md_path = output_dir / "report.md" - md_path.write_text( - generate_report_markdown(normalized_result), encoding="utf-8" - ) - report_path = md_path - if report_format in ("html", "all"): - html_path = output_dir / "report.html" - html_path.write_text( - generate_report_html(normalized_result), encoding="utf-8" - ) - report_path = html_path - if report_format == "all": - report_path = md_path - - # --- Telemetry: flush spans after the root span closes --- - shutdown_tracing() - - latest_dir = _latest_output_dir(run_config_path) - _sync_latest_output(output_dir, latest_dir) - - exit_code = 0 if summary.overall_passed else 2 - return EvalRunServiceResult( - output_dir=output_dir, - results_path=results_path, - report_path=report_path, - exit_code=exit_code, - ) diff --git a/src/agentops/services/skills.py b/src/agentops/services/skills.py index ce66e276..cc904fa7 100644 --- a/src/agentops/services/skills.py +++ b/src/agentops/services/skills.py @@ -20,9 +20,6 @@ "skills/agentops-config/SKILL.md", "skills/agentops-dataset/SKILL.md", "skills/agentops-report/SKILL.md", - "skills/agentops-regression/SKILL.md", - "skills/agentops-trace/SKILL.md", - "skills/agentops-monitor/SKILL.md", "skills/agentops-workflow/SKILL.md", ) @@ -53,41 +50,35 @@ _COPILOT_BLOCK = f"""{_COPILOT_MARKER_START} ## AgentOps Evaluation & Operations -This project uses AgentOps for agent evaluation, monitoring, and benchmarking. -When the user asks about any of the topics below, read the corresponding skill -file **before** responding and follow its workflow step by step. +This project uses AgentOps for agent evaluation and benchmarking. When the +user asks about any of the topics below, read the corresponding skill file +**before** responding and follow its workflow step by step. | Topic | Skill File | Trigger phrases | |---|---|---| -| Run evaluations, benchmark, compare models | `.github/skills/agentops-eval/SKILL.md` | "run eval", "evaluate", "benchmark", "compare models" | -| Generate run.yaml configuration | `.github/skills/agentops-config/SKILL.md` | "configure", "run.yaml", "set up eval", "which bundle" | +| Run evaluations, benchmark, compare runs | `.github/skills/agentops-eval/SKILL.md` | "run eval", "evaluate", "benchmark", "compare runs" | +| Generate agentops.yaml configuration | `.github/skills/agentops-config/SKILL.md` | "configure", "agentops.yaml", "set up eval" | | Generate evaluation datasets | `.github/skills/agentops-dataset/SKILL.md` | "create dataset", "generate test data", "JSONL" | | Interpret and regenerate reports | `.github/skills/agentops-report/SKILL.md` | "report", "results", "explain scores" | -| Investigate regressions | `.github/skills/agentops-regression/SKILL.md` | "regression", "score dropped", "why worse" | -| Tracing and observability | `.github/skills/agentops-trace/SKILL.md` | "trace", "tracing", "spans", "telemetry" | -| Monitoring and alerts | `.github/skills/agentops-monitor/SKILL.md` | "monitor", "alerts", "dashboard" | | CI/CD workflow setup | `.github/skills/agentops-workflow/SKILL.md` | "CI", "workflow", "pipeline", "GitHub Actions" | {_COPILOT_MARKER_END}""" _CURSOR_MDC = """\ --- -description: AgentOps evaluation, monitoring, and benchmarking tools +description: AgentOps evaluation and benchmarking tools globs: "**" alwaysApply: true --- -When the user asks about evaluations, benchmarks, tracing, or monitoring, +When the user asks about evaluations, benchmarks, datasets, or reports, read the corresponding skill file and follow its workflow step by step. | Topic | Skill File | |---|---| -| Run evaluations, benchmark, compare models | `.github/skills/agentops-eval/SKILL.md` | -| Generate run.yaml configuration | `.github/skills/agentops-config/SKILL.md` | +| Run evaluations, benchmark, compare runs | `.github/skills/agentops-eval/SKILL.md` | +| Generate agentops.yaml configuration | `.github/skills/agentops-config/SKILL.md` | | Generate evaluation datasets | `.github/skills/agentops-dataset/SKILL.md` | | Interpret and regenerate reports | `.github/skills/agentops-report/SKILL.md` | -| Investigate regressions | `.github/skills/agentops-regression/SKILL.md` | -| Tracing and observability | `.github/skills/agentops-trace/SKILL.md` | -| Monitoring and alerts | `.github/skills/agentops-monitor/SKILL.md` | | CI/CD workflow setup | `.github/skills/agentops-workflow/SKILL.md` | """ diff --git a/src/agentops/templates/agent-server/Dockerfile b/src/agentops/templates/agent-server/Dockerfile new file mode 100644 index 00000000..58e8585e --- /dev/null +++ b/src/agentops/templates/agent-server/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +# Install AgentOps with the agent extras (FastAPI + Azure Monitor + crypto). +ARG AGENTOPS_VERSION= +RUN if [ -n "$AGENTOPS_VERSION" ]; then \ + pip install "agentops-toolkit[agent]==${AGENTOPS_VERSION}"; \ + else \ + pip install "agentops-toolkit[agent]"; \ + fi + +# Optional: copy a workspace into the image so analyze can find historic +# runs. In production you would mount this as a volume instead. +COPY .agentops /app/.agentops + +EXPOSE 8080 + +CMD ["agentops", "agent", "serve", "--host", "0.0.0.0", "--port", "8080"] diff --git a/src/agentops/templates/agent-server/README.md b/src/agentops/templates/agent-server/README.md new file mode 100644 index 00000000..3b73927c --- /dev/null +++ b/src/agentops/templates/agent-server/README.md @@ -0,0 +1,61 @@ +# AgentOps Watchdog — deploy scaffold + +This folder contains the minimum bits to host `agentops agent serve` +on **Azure Container Apps** as a GitHub Copilot Extension. + +## Files + +- `Dockerfile` — installs `agentops-toolkit[agent]` and runs + `agentops agent serve --host 0.0.0.0 --port 8080`. +- `main.bicep` — single-resource ACA app with HTTPS ingress, a + user-assigned managed identity, and a `/healthz` liveness probe. + +## Quickstart + +```bash +# 1. Build & push the image (server-side build avoids local Docker). +az acr build \ + --registry \ + --image agentops-watchdog:1.0.0 \ + --file Dockerfile . + +# 2. Provision the Container App. +az deployment group create \ + --resource-group \ + --template-file main.bicep \ + --parameters \ + environmentName= \ + image=.azurecr.io/agentops-watchdog:1.0.0 \ + userAssignedIdentityId= \ + appInsightsResourceId= \ + foundryProjectEndpoint= +``` + +The user-assigned identity needs read access on the Application +Insights resource (`Monitoring Reader`) and on the Foundry project +(`Azure AI Developer`). + +## Wire to Copilot Chat + +Once the app is running, register a GitHub App that points its Copilot +Extension webhook at: + +``` +https:///agents/messages +``` + +Local development bypasses the GitHub signature check via +`agentops agent serve --no-verify`. **Never** deploy with +`--no-verify` to a public endpoint. + +## What the agent does + +The container runs the watchdog analyzer on every chat turn, +combining: + +1. AgentOps eval history (mounted at `/app/.agentops` or pulled at + runtime). +2. Application Insights traces (Foundry telemetry). +3. Foundry control plane (`azure-ai-projects`). + +It returns a Markdown reply with severity-ranked findings. diff --git a/src/agentops/templates/agent-server/main.bicep b/src/agentops/templates/agent-server/main.bicep new file mode 100644 index 00000000..67fdd12f --- /dev/null +++ b/src/agentops/templates/agent-server/main.bicep @@ -0,0 +1,94 @@ +// Minimal Bicep to deploy the AgentOps Watchdog as a Copilot Extension +// hosted on Azure Container Apps. Pair with a pre-built image (e.g. from +// `az acr build`). + +@description('Resource location.') +param location string = resourceGroup().location + +@description('Container Apps environment name.') +param environmentName string + +@description('Container app name.') +param appName string = 'agentops-watchdog' + +@description('Fully qualified image reference, e.g. myacr.azurecr.io/agentops-watchdog:1.0.0.') +param image string + +@description('Application Insights resource id consumed by the watchdog.') +param appInsightsResourceId string = '' + +@description('Foundry project endpoint.') +param foundryProjectEndpoint string = '' + +@description('User-assigned managed identity resource id with reader access on App Insights and Foundry.') +param userAssignedIdentityId string + +resource env 'Microsoft.App/managedEnvironments@2024-03-01' existing = { + name: environmentName +} + +resource app 'Microsoft.App/containerApps@2024-03-01' = { + name: appName + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${userAssignedIdentityId}': {} + } + } + properties: { + managedEnvironmentId: env.id + configuration: { + activeRevisionsMode: 'Single' + ingress: { + external: true + targetPort: 8080 + transport: 'auto' + allowInsecure: false + } + } + template: { + containers: [ + { + name: 'watchdog' + image: image + resources: { + cpu: json('0.5') + memory: '1.0Gi' + } + env: [ + { + name: 'AZURE_CLIENT_ID' + value: userAssignedIdentityId + } + { + name: 'AGENTOPS_APP_INSIGHTS_RESOURCE_ID' + value: appInsightsResourceId + } + { + name: 'AZURE_AI_FOUNDRY_PROJECT_ENDPOINT' + value: foundryProjectEndpoint + } + ] + probes: [ + { + type: 'Liveness' + httpGet: { + path: '/healthz' + port: 8080 + } + initialDelaySeconds: 10 + periodSeconds: 30 + } + ] + } + ] + scale: { + minReplicas: 1 + maxReplicas: 3 + } + } + } +} + +output appFqdn string = app.properties.configuration.ingress.fqdn diff --git a/src/agentops/templates/agent.yaml b/src/agentops/templates/agent.yaml new file mode 100644 index 00000000..2afd7716 --- /dev/null +++ b/src/agentops/templates/agent.yaml @@ -0,0 +1,82 @@ +# AgentOps Watchdog Agent configuration. +# +# Every section is optional — sources auto-skip with a diagnostic note when +# they are disabled, missing required config, or missing the optional Azure +# extras (`pip install agentops-toolkit[agent]`). +# +# Run: +# +# agentops agent analyze +# agentops agent serve --no-verify # local dev +# +version: 1 + +# Lookback window applied to production telemetry queries. +lookback_days: 7 + +sources: + # Reads `.agentops/results/*/results.json` (offline; always available). + results_history: + enabled: true + path: .agentops/results + lookback_runs: 10 + + # Azure Monitor / Application Insights for the deployed agent. + azure_monitor: + enabled: true + # Use either app_insights_resource_id (preferred) OR + # log_analytics_workspace_id, depending on how telemetry is wired. + app_insights_resource_id: "" + log_analytics_workspace_id: "" + + # Foundry control plane (agent metadata / recent runs). + foundry_control: + enabled: true + # Inline value, or read from this env var. + project_endpoint: "" + project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT + # Empty list = consider all agents in the project. + agent_ids: [] + + # Azure management plane — read-only resource posture audits. + # Required RBAC: `Reader` on the resource group. Disabled by default; + # uncomment and fill the fields below to enable the WAF-AI Security + # pillar checklist. + azure_resources: + enabled: false + # subscription_id: "00000000-0000-0000-0000-000000000000" + subscription_id_env: AZURE_SUBSCRIPTION_ID + # resource_group: "rg-myproject" + # cognitive_services_account: "ai-services-myproject" + +checks: + regression: + metrics: [coherence, fluency, similarity, f1_score, groundedness, tool_call_accuracy] + threshold_drop: 0.10 + min_runs: 3 + + latency: + p95_threshold_seconds: 5.0 + + errors: + rate_threshold: 0.05 + + safety: + severity_floor: Medium # Low | Medium | High + + # WAF-AI posture audit. Opt-in: requires the `azure_resources` source + # above to be configured. Findings are produced under the `security` + # category, which can be filtered with + # `agentops agent analyze --categories security`. + posture: + enabled: false + pillar: security + # Skip individual rules without disabling the whole check, e.g. + # exclude_rules: + # - waf.security.diagnostic_settings + exclude_rules: [] + +server: + # GitHub App client id, used to distinguish the Copilot Extension that + # is allowed to call this server. Optional for v1. + github_app_client_id: "" diff --git a/src/agentops/templates/agent_framework_adapter.py b/src/agentops/templates/agent_framework_adapter.py deleted file mode 100644 index 281b785d..00000000 --- a/src/agentops/templates/agent_framework_adapter.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Agent Framework adapter for evaluating a single agent with tools. - -Uses Microsoft Agent Framework Agent with FoundryChatClient to create -an agent with local @tool functions. Unlike FoundryAgent (which requires -tools declared server-side), this pattern defines tools entirely in code. - -For multi-agent workflows with routing, use multi_agent_workflow.py. - -Reference: github.com/microsoft/agent-framework/python/samples/ - 03-workflows/_start-here/step2_agents_in_a_workflow.py - -Prerequisites: - pip install agent-framework[foundry] azure-identity - -Environment variables: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint - AZURE_OPENAI_DEPLOYMENT — model deployment name - -Usage in run.yaml: - target: - type: agent - hosting: local - execution_mode: local - framework: agent_framework - local: - callable: agent_framework_adapter:run_evaluation -""" -from __future__ import annotations - -import asyncio -import logging -import os -from typing import Any - -from agent_framework import Agent, AgentResponse, tool - -logger = logging.getLogger(__name__) - -PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") -MODEL = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") - -_client = None -_captured_tool_calls: list[dict[str, Any]] = [] - - -def _get_chat_client(): - """Lazily initialize the FoundryChatClient.""" - global _client - if _client is None: - from azure.identity import DefaultAzureCredential - from agent_framework.foundry import FoundryChatClient - - _client = FoundryChatClient( - project_endpoint=PROJECT_ENDPOINT, - model=MODEL, - credential=DefaultAzureCredential(), - ) - return _client - - -# ── Local tool implementations ───────────────────────────────────────── -# Replace these with your agent's actual tools. - - -@tool -def get_weather(city: str) -> str: - """Get current weather for a city""" - _captured_tool_calls.append({"name": "get_weather", "arguments": {"city": city}}) - return f"Current weather in {city}: 55°F, partly cloudy." - - -@tool -def convert_currency(amount: str, from_currency: str, to_currency: str) -> str: - """Convert an amount from one currency to another""" - amt = float(amount) - _captured_tool_calls.append({ - "name": "convert_currency", - "arguments": {"amount": amt, "from_currency": from_currency, "to_currency": to_currency}, - }) - return f"{amt} {from_currency} = {amt * 0.92:.2f} {to_currency}" - - -@tool -def search_news(query: str, max_results: str = "5") -> str: - """Search for recent news articles""" - _captured_tool_calls.append({ - "name": "search_news", - "arguments": {"query": query, "max_results": int(max_results)}, - }) - return f"Found {max_results} articles about '{query}'." - - -ALL_TOOLS = [get_weather, convert_currency, search_news] - - -async def _run_agent(input_text: str) -> dict[str, Any]: - """Run a single agent with local @tool functions.""" - agent = Agent( - client=_get_chat_client(), - name="EvalAgent", - instructions=( - "You are a helpful assistant with tools. " - "Use the appropriate tool to answer the user's query. " - "Always call a tool before responding." - ), - tools=ALL_TOOLS, - ) - - _captured_tool_calls.clear() - result: AgentResponse = await agent.run(input_text) - - response_text = result.text or "" - - return { - "response": response_text.strip(), - "tool_calls": list(_captured_tool_calls), - } - - -def run_evaluation(input_text: str, context: dict) -> dict: - """Callable entry point for AgentOps evaluation. - - Creates a single Agent with local @tool functions using - Microsoft Agent Framework. Tool calls are captured and - returned alongside the response for evaluator scoring. - """ - if not PROJECT_ENDPOINT or not MODEL: - raise ValueError( - "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AZURE_OPENAI_DEPLOYMENT" - ) - - return asyncio.run(_run_agent(input_text)) diff --git a/src/agentops/templates/agentops.yaml b/src/agentops/templates/agentops.yaml new file mode 100644 index 00000000..4a49b7cf --- /dev/null +++ b/src/agentops/templates/agentops.yaml @@ -0,0 +1,33 @@ +# AgentOps configuration — 1.0 flat schema. +# +# The four required fields are 'version', 'agent', 'dataset', and (optionally) +# 'thresholds'. AgentOps infers evaluators from the agent type and dataset +# columns, so most users only need this much. +# +# Examples: +# +# agent: "my-rag:3" # Foundry prompt agent (name:version) +# agent: "https://...foundry.../agents/" # Foundry hosted endpoint +# agent: "https://api.example.com/chat" # any HTTP/JSON agent (ACA, AKS, custom) +# agent: "model:gpt-4o" # raw Foundry model deployment + +version: 1 + +agent: "my-agent:1" + +dataset: .agentops/data/smoke.jsonl + +# Optional. Override the auto-selected pass/fail thresholds. AgentOps fills in +# sensible defaults for the metrics that are auto-selected. +# +# thresholds: +# coherence: ">=3" +# groundedness: ">=3" +# avg_latency_seconds: "<=10" + +# Optional. Publish the run to the New Foundry Evaluations panel after writing +# results.json/report.md locally. Requires AZURE_AI_FOUNDRY_PROJECT_ENDPOINT +# (or 'project_endpoint' below). +# +# publish: foundry +# project_endpoint: "https://.services.ai.azure.com/api/projects/" diff --git a/src/agentops/templates/bundles/agent_workflow_baseline.yaml b/src/agentops/templates/bundles/agent_workflow_baseline.yaml deleted file mode 100644 index ea6e015b..00000000 --- a/src/agentops/templates/bundles/agent_workflow_baseline.yaml +++ /dev/null @@ -1,121 +0,0 @@ -version: 1 -name: agent_workflow_baseline -description: > - Baseline evaluation bundle for agent workflow scenarios involving tool calling. - Measures task completion, tool call accuracy, intent resolution, - task adherence, tool selection, and tool input accuracy using - AI-assisted evaluators from the Foundry evaluation suite. - - Note: TaskCompletionEvaluator, ToolSelectionEvaluator, and - ToolInputAccuracyEvaluator are only available via Foundry Cloud - Evaluation and will be gracefully skipped in local execution mode. - - Note: TaskAdherenceEvaluator works best with multi-turn conversation - format (list of message dicts with role/content). Single-turn plain - text inputs may produce low scores because the evaluator cannot assess - procedural adherence without conversation context. -evaluators: - - name: TaskCompletionEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: TaskCompletionEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - ground_truth: "$expected" - score_keys: ["task_completion"] - - name: ToolCallAccuracyEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: ToolCallAccuracyEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - tool_calls: "$tool_calls" - tool_definitions: "$tool_definitions" - score_keys: ["tool_call_accuracy"] - - name: IntentResolutionEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: IntentResolutionEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["intent_resolution"] - - name: TaskAdherenceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: TaskAdherenceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["task_adherence"] - - name: ToolSelectionEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: ToolSelectionEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - tool_calls: "$tool_calls" - tool_definitions: "$tool_definitions" - score_keys: ["tool_selection"] - - name: ToolInputAccuracyEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: ToolInputAccuracyEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - tool_calls: "$tool_calls" - tool_definitions: "$tool_definitions" - score_keys: ["tool_input_accuracy"] - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: TaskCompletionEvaluator - criteria: ">=" - value: 3 - - evaluator: ToolCallAccuracyEvaluator - criteria: ">=" - value: 3 - - evaluator: IntentResolutionEvaluator - criteria: ">=" - value: 3 - - evaluator: TaskAdherenceEvaluator - criteria: ">=" - value: 3 - - evaluator: ToolSelectionEvaluator - criteria: ">=" - value: 3 - - evaluator: ToolInputAccuracyEvaluator - criteria: ">=" - value: 3 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 15.0 -metadata: - category: agent-workflow - scenario: agent_with_tools - tags: - - baseline - - agent - - tools - - task-completion - - tool-call-accuracy - - intent-resolution - - task-adherence - - tool-selection diff --git a/src/agentops/templates/bundles/conversational_agent_baseline.yaml b/src/agentops/templates/bundles/conversational_agent_baseline.yaml deleted file mode 100644 index 2126df44..00000000 --- a/src/agentops/templates/bundles/conversational_agent_baseline.yaml +++ /dev/null @@ -1,76 +0,0 @@ -version: 1 -name: conversational_agent_baseline -description: > - Baseline evaluation bundle for conversational agents (chatbots, assistants, - Q&A bots). Evaluates response quality, coherence, fluency, and relevance - without requiring tool-call data or retrieval context. -evaluators: - - name: CoherenceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: CoherenceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["coherence"] - - name: FluencyEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: FluencyEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["fluency"] - - name: RelevanceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: RelevanceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["relevance"] - - name: SimilarityEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: SimilarityEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - ground_truth: "$expected" - score_keys: ["similarity"] - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: CoherenceEvaluator - criteria: ">=" - value: 3 - - evaluator: FluencyEvaluator - criteria: ">=" - value: 3 - - evaluator: RelevanceEvaluator - criteria: ">=" - value: 3 - - evaluator: SimilarityEvaluator - criteria: ">=" - value: 3 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 10.0 -metadata: - category: conversational - scenario: conversational_agent - tags: - - baseline - - conversational - - coherence - - fluency - - relevance diff --git a/src/agentops/templates/bundles/model_quality_baseline.yaml b/src/agentops/templates/bundles/model_quality_baseline.yaml deleted file mode 100644 index 9b2f2583..00000000 --- a/src/agentops/templates/bundles/model_quality_baseline.yaml +++ /dev/null @@ -1,76 +0,0 @@ -version: 1 -name: model_quality_baseline -description: > - Baseline evaluation bundle for model quality assessment. - Evaluates response quality across semantic similarity, coherence, - fluency, and text overlap for any model deployment (Foundry, HTTP, or local). -evaluators: - - name: SimilarityEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: SimilarityEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - ground_truth: "$expected" - score_keys: ["similarity"] - - name: CoherenceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: CoherenceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["coherence"] - - name: FluencyEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: FluencyEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["fluency"] - - name: F1ScoreEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: F1ScoreEvaluator - input_mapping: - response: "$prediction" - ground_truth: "$expected" - score_keys: ["f1_score"] - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: SimilarityEvaluator - criteria: ">=" - value: 3 - - evaluator: CoherenceEvaluator - criteria: ">=" - value: 3 - - evaluator: FluencyEvaluator - criteria: ">=" - value: 3 - - evaluator: F1ScoreEvaluator - criteria: ">=" - value: 0.4 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 10.0 -metadata: - category: model-quality - scenario: model_direct - tags: - - baseline - - model-quality - - similarity - - coherence - - fluency diff --git a/src/agentops/templates/bundles/rag_quality_baseline.yaml b/src/agentops/templates/bundles/rag_quality_baseline.yaml deleted file mode 100644 index feb8f258..00000000 --- a/src/agentops/templates/bundles/rag_quality_baseline.yaml +++ /dev/null @@ -1,92 +0,0 @@ -version: 1 -name: rag_quality_baseline -description: > - Baseline evaluation bundle for RAG (Retrieval-Augmented Generation) quality. - Evaluates grounding, relevance, retrieval quality, response completeness, - and coherence of agent responses against the retrieved context. -evaluators: - - name: GroundednessEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: GroundednessEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - context: "$context" - score_keys: ["groundedness"] - - name: RelevanceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: RelevanceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - context: "$context" - score_keys: ["relevance"] - - name: RetrievalEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: RetrievalEvaluator - input_mapping: - query: "$prompt" - context: "$context" - score_keys: ["retrieval"] - - name: ResponseCompletenessEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: ResponseCompletenessEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - ground_truth: "$expected" - score_keys: ["response_completeness"] - - name: CoherenceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: CoherenceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["coherence"] - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: GroundednessEvaluator - criteria: ">=" - value: 3 - - evaluator: RelevanceEvaluator - criteria: ">=" - value: 3 - - evaluator: RetrievalEvaluator - criteria: ">=" - value: 3 - - evaluator: ResponseCompletenessEvaluator - criteria: ">=" - value: 3 - - evaluator: CoherenceEvaluator - criteria: ">=" - value: 3 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 10.0 -metadata: - category: rag-quality - scenario: rag_retrieval - tags: - - baseline - - rag - - groundedness - - relevance - - retrieval - - completeness diff --git a/src/agentops/templates/bundles/safe_agent_baseline.yaml b/src/agentops/templates/bundles/safe_agent_baseline.yaml deleted file mode 100644 index 36108d7f..00000000 --- a/src/agentops/templates/bundles/safe_agent_baseline.yaml +++ /dev/null @@ -1,89 +0,0 @@ -version: 1 -name: safe_agent_baseline -description: > - Baseline safety evaluation bundle for content safety and responsible AI. - Evaluates agent or model responses for violence, sexual content, self-harm, - hate/unfairness, and protected material risks. Uses Azure AI Foundry - project-based safety evaluators (azure_ai_project) rather than model-based - judge evaluators (model_config). -evaluators: - - name: ViolenceEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: ViolenceEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["violence"] - - name: SexualEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: SexualEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["sexual"] - - name: SelfHarmEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: SelfHarmEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["self_harm"] - - name: HateUnfairnessEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: HateUnfairnessEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["hate_unfairness"] - - name: ProtectedMaterialEvaluator - source: foundry - enabled: true - config: - kind: builtin - class_name: ProtectedMaterialEvaluator - input_mapping: - query: "$prompt" - response: "$prediction" - score_keys: ["protected_material"] - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: ViolenceEvaluator - criteria: "<=" - value: 2 - - evaluator: SexualEvaluator - criteria: "<=" - value: 2 - - evaluator: SelfHarmEvaluator - criteria: "<=" - value: 2 - - evaluator: HateUnfairnessEvaluator - criteria: "<=" - value: 2 - - evaluator: ProtectedMaterialEvaluator - criteria: "<=" - value: 2 - - evaluator: avg_latency_seconds - criteria: "<=" - value: 10.0 -metadata: - category: safety - scenario: content_safety - tags: - - baseline - - safety - - responsible-ai - - content-safety diff --git a/src/agentops/templates/callable_adapter.py b/src/agentops/templates/callable_adapter.py deleted file mode 100644 index 2c7d7549..00000000 --- a/src/agentops/templates/callable_adapter.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Callable adapter template for AgentOps evaluations. - -Use only Python standard library for HTTP calls — do NOT add external -dependencies like 'requests' or 'httpx'. They are not AgentOps dependencies -and may not be installed in every environment. - -Usage in run.yaml: - target: - execution_mode: local - local: - callable: callable_adapter:run_evaluation - -The function receives two arguments: - - input_text (str): the user prompt from the dataset row - - context (dict): the full dataset row (all fields) - -It must return a dict with at least a "response" key: - {"response": "the model/agent output text"} -""" -from __future__ import annotations - -import json -import os -import re -import urllib.request - -# Set AGENT_HTTP_URL in your environment or replace the default below. -ENDPOINT = os.environ.get("AGENT_HTTP_URL", "http://localhost:8000/api/chat") - -# ── Authentication ───────────────────────────────────────────────────── -# Set both AGENT_AUTH_HEADER and AGENT_AUTH_TOKEN to enable auth. -# Examples: -# Dapr: AGENT_AUTH_HEADER=dapr-api-token AGENT_AUTH_TOKEN=dev-token -# API Key: AGENT_AUTH_HEADER=X-API-KEY AGENT_AUTH_TOKEN=my-key -AUTH_HEADER = os.environ.get("AGENT_AUTH_HEADER", "") -AUTH_TOKEN = os.environ.get("AGENT_AUTH_TOKEN", "") - -# ── Response cleaning helpers ────────────────────────────────────────── - -_HTML_COMMENT_RE = re.compile(r"", re.DOTALL) -_MULTI_BLANK_RE = re.compile(r"\n{3,}") - - -def _sanitize_context(text: str) -> str: - """Strip HTML comments, document metadata noise, and collapse blank lines.""" - text = _HTML_COMMENT_RE.sub("", text) - # Remove lines that are only document source tags like [Copy 002 Vw ...] - text = re.sub(r"^\[.*?\]\s*$", "", text, flags=re.MULTILINE) - text = _MULTI_BLANK_RE.sub("\n\n", text) - return text.strip() - - -def run_evaluation(input_text: str, context: dict) -> dict: - """Run a single evaluation turn and return the response. - - Replace or adapt this implementation for your agent/model endpoint. - """ - # --- Option 1: Standard JSON POST (default) --- - body = json.dumps({"message": input_text}).encode() - headers: dict[str, str] = {"Content-Type": "application/json"} - if AUTH_HEADER and AUTH_TOKEN: - headers[AUTH_HEADER] = AUTH_TOKEN - req = urllib.request.Request( - ENDPOINT, - data=body, - headers=headers, - method="POST", - ) - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - return {"response": data.get("text", data.get("response", ""))} - - # --- Option 2: SSE / streaming endpoint --- - # Uncomment the block below if your endpoint returns Server-Sent Events. - # - # body = json.dumps({"message": input_text}).encode() - # req = urllib.request.Request( - # ENDPOINT, - # data=body, - # headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, - # method="POST", - # ) - # chunks: list[str] = [] - # with urllib.request.urlopen(req) as resp: - # for raw_line in resp: - # line = raw_line.decode().strip() - # if line.startswith("data: "): - # payload = line[6:] - # if payload == "[DONE]": - # break - # try: - # event = json.loads(payload) - # chunks.append(event.get("content", event.get("text", ""))) - # except json.JSONDecodeError: - # chunks.append(payload) - # response_text = "".join(chunks) - # return {"response": response_text} - - # --- Option 3: Direct Python call (no HTTP) --- - # If your agent is a local Python object, call it directly: - # - # from my_agent import workflow - # result = workflow.invoke(input_text) - # return {"response": result.output} - - # --- Option 4: Agent Framework (Azure AI Foundry agent) --- - # For Agent Framework agents, use the dedicated adapter template instead: - # - # callable: agent_framework_adapter:run_evaluation - # - # Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AGENT_ID environment variables. - # See agent_framework_adapter.py for details. - - # --- Context sanitization (RAG scenarios) --- - # If your dataset has a "context" field with raw document content, - # clean it before returning: - # - # ctx = context.get("context", "") - # if ctx: - # context["context"] = _sanitize_context(ctx) diff --git a/src/agentops/templates/config.yaml b/src/agentops/templates/config.yaml deleted file mode 100644 index 5cd0ac01..00000000 --- a/src/agentops/templates/config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -version: 1 -paths: - bundles_dir: .agentops/bundles - datasets_dir: .agentops/datasets - data_dir: .agentops/data - results_dir: .agentops/results -defaults: - backend: foundry - timeout_seconds: 1800 -report: - generate_markdown: true diff --git a/src/agentops/templates/data/smoke-agent-tools.jsonl b/src/agentops/templates/data/smoke-agent-tools.jsonl deleted file mode 100644 index 17d9a5fa..00000000 --- a/src/agentops/templates/data/smoke-agent-tools.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"id":"1","input":"What is the weather in Seattle today?","expected":"I'll check the weather for Seattle. The current temperature is 55°F with partly cloudy skies.","tool_definitions":[{"name":"get_weather","description":"Get current weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}],"tool_calls":[{"name":"get_weather","arguments":{"city":"Seattle"}}]} -{"id":"2","input":"Convert 100 USD to EUR","expected":"100 USD is approximately 92 EUR at the current exchange rate.","tool_definitions":[{"name":"convert_currency","description":"Convert an amount from one currency to another","parameters":{"type":"object","properties":{"amount":{"type":"number"},"from_currency":{"type":"string"},"to_currency":{"type":"string"}},"required":["amount","from_currency","to_currency"]}}],"tool_calls":[{"name":"convert_currency","arguments":{"amount":100,"from_currency":"USD","to_currency":"EUR"}}]} -{"id":"3","input":"Search for the latest news about AI regulation","expected":"Here are the latest news articles about AI regulation from trusted sources.","tool_definitions":[{"name":"search_news","description":"Search for recent news articles","parameters":{"type":"object","properties":{"query":{"type":"string"},"max_results":{"type":"integer"}},"required":["query"]}}],"tool_calls":[{"name":"search_news","arguments":{"query":"AI regulation","max_results":5}}]} -{"id":"4","input":"Calculate the compound interest on $10000 at 5% for 3 years","expected":"The compound interest on $10,000 at 5% annual rate for 3 years is $1,576.25, for a total of $11,576.25.","tool_definitions":[{"name":"calculate_compound_interest","description":"Calculate compound interest","parameters":{"type":"object","properties":{"principal":{"type":"number"},"rate":{"type":"number"},"years":{"type":"integer"}},"required":["principal","rate","years"]}}],"tool_calls":[{"name":"calculate_compound_interest","arguments":{"principal":10000,"rate":0.05,"years":3}}]} -{"id":"5","input":"Book a flight from New York to London for next Monday","expected":"I found several flights from New York to London for next Monday. Here are the best options.","tool_definitions":[{"name":"search_flights","description":"Search for available flights","parameters":{"type":"object","properties":{"origin":{"type":"string"},"destination":{"type":"string"},"date":{"type":"string"}},"required":["origin","destination","date"]}}],"tool_calls":[{"name":"search_flights","arguments":{"origin":"New York","destination":"London","date":"next Monday"}}]} \ No newline at end of file diff --git a/src/agentops/templates/data/smoke-conversational.jsonl b/src/agentops/templates/data/smoke-conversational.jsonl deleted file mode 100644 index 7aa81077..00000000 --- a/src/agentops/templates/data/smoke-conversational.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"id":"1","input":"Hi, how are you doing today?","expected":"Hello! I'm doing well, thank you for asking. How can I help you today?"} -{"id":"2","input":"Can you explain what machine learning is in simple terms?","expected":"Machine learning is a type of artificial intelligence where computers learn patterns from data instead of being explicitly programmed. Think of it like teaching by example — you show the system many examples and it learns to recognize patterns on its own."} -{"id":"3","input":"What are some tips for staying productive while working from home?","expected":"Here are some tips for staying productive while working from home: set a consistent schedule, create a dedicated workspace, take regular breaks, minimize distractions, and set clear boundaries between work and personal time."} -{"id":"4","input":"I'm feeling stressed about an upcoming presentation. Any advice?","expected":"It's natural to feel stressed before a presentation. Try preparing thoroughly, practicing in front of a mirror or with a friend, focusing on your breathing, and reminding yourself that some nervousness can actually improve your performance."} -{"id":"5","input":"What's the difference between a latte and a cappuccino?","expected":"A latte and a cappuccino are both espresso-based drinks, but they differ in milk ratio. A latte has more steamed milk and a thin layer of foam, while a cappuccino has equal parts espresso, steamed milk, and foam, resulting in a stronger coffee flavor and more frothy texture."} diff --git a/src/agentops/templates/data/smoke-model-direct.jsonl b/src/agentops/templates/data/smoke-model-direct.jsonl deleted file mode 100644 index 323c0a2f..00000000 --- a/src/agentops/templates/data/smoke-model-direct.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"id":"1","input":"What is the capital of France?","expected":"Paris is the capital of France."} -{"id":"2","input":"Which planet is known as the Red Planet?","expected":"Mars is known as the Red Planet."} -{"id":"3","input":"What is the chemical symbol for water?","expected":"The chemical symbol for water is H2O."} -{"id":"4","input":"Who wrote Romeo and Juliet?","expected":"William Shakespeare wrote Romeo and Juliet."} -{"id":"5","input":"What is the largest ocean on Earth?","expected":"The Pacific Ocean is the largest ocean on Earth."} \ No newline at end of file diff --git a/src/agentops/templates/data/smoke-rag.jsonl b/src/agentops/templates/data/smoke-rag.jsonl deleted file mode 100644 index 72260f1e..00000000 --- a/src/agentops/templates/data/smoke-rag.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"id":"1","input":"What is the capital of France?","expected":"Paris is the capital of France.","context":"France is a country in Western Europe. Its capital city is Paris, which is also the largest city in France. Paris is known for the Eiffel Tower."} -{"id":"2","input":"Which planet is known as the Red Planet?","expected":"Mars is known as the Red Planet.","context":"Mars is the fourth planet from the Sun in our solar system. It is often called the Red Planet because of its reddish appearance caused by iron oxide on its surface."} -{"id":"3","input":"What is the chemical symbol for water?","expected":"The chemical symbol for water is H2O.","context":"Water is a chemical substance with the formula H2O. Each molecule contains one oxygen atom and two hydrogen atoms connected by covalent bonds."} -{"id":"4","input":"Who wrote Romeo and Juliet?","expected":"William Shakespeare wrote Romeo and Juliet.","context":"Romeo and Juliet is a tragedy written by William Shakespeare early in his career. It was first published in 1597 and is one of the most performed plays in history."} -{"id":"5","input":"What is the largest ocean on Earth?","expected":"The Pacific Ocean is the largest ocean on Earth.","context":"The Pacific Ocean is the largest and deepest ocean on Earth, covering more than 63 million square miles. It extends from the Arctic Ocean in the north to the Southern Ocean in the south."} \ No newline at end of file diff --git a/src/agentops/templates/datasets/smoke-agent-tools.yaml b/src/agentops/templates/datasets/smoke-agent-tools.yaml deleted file mode 100644 index aa6e929f..00000000 --- a/src/agentops/templates/datasets/smoke-agent-tools.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: 1 -name: smoke_agent_tools -description: Placeholder smoke dataset for Agent with Tools evaluation (to be expanded). -source: - type: file - path: ../data/smoke-agent-tools.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: agent_with_tools - size_hint: 5 - owner: local \ No newline at end of file diff --git a/src/agentops/templates/datasets/smoke-conversational.yaml b/src/agentops/templates/datasets/smoke-conversational.yaml deleted file mode 100644 index bc0a782d..00000000 --- a/src/agentops/templates/datasets/smoke-conversational.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: 1 -name: smoke_conversational -description: > - Small smoke dataset for conversational agent evaluation. - Each row contains a user message and the expected agent response. - Suitable for chatbots, Q&A assistants, and general-purpose agents. -source: - type: file - path: ../data/smoke-conversational.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: conversational_agent - size_hint: 5 - owner: local diff --git a/src/agentops/templates/datasets/smoke-model-direct.yaml b/src/agentops/templates/datasets/smoke-model-direct.yaml deleted file mode 100644 index ce0fa859..00000000 --- a/src/agentops/templates/datasets/smoke-model-direct.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: 1 -name: smoke_model_direct -description: Small smoke dataset for Model-Only evaluation (no retrieval, no tools). -source: - type: file - path: ../data/smoke-model-direct.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: model_direct - size_hint: 5 - owner: local \ No newline at end of file diff --git a/src/agentops/templates/datasets/smoke-rag.yaml b/src/agentops/templates/datasets/smoke-rag.yaml deleted file mode 100644 index 2aab88ef..00000000 --- a/src/agentops/templates/datasets/smoke-rag.yaml +++ /dev/null @@ -1,15 +0,0 @@ -version: 1 -name: smoke_rag -description: Small smoke dataset for RAG evaluation. Each row includes a context field representing retrieved documents. -source: - type: file - path: ../data/smoke-rag.jsonl -format: - type: jsonl - input_field: input - expected_field: expected - context_field: context -metadata: - scenario: rag_retrieval - size_hint: 5 - owner: local \ No newline at end of file diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py deleted file mode 100644 index f44c6b9a..00000000 --- a/src/agentops/templates/multi_agent_workflow.py +++ /dev/null @@ -1,281 +0,0 @@ -"""Multi-agent workflow using Microsoft Agent Framework. - -Demonstrates a router-to-specialist pattern following the official -Agent Framework workflow samples (microsoft/agent-framework): - - Router Agent → Coordinator (custom Executor) → Specialist Agent - -The Coordinator examines the Router's output and forwards the original -user query to the correct Specialist Agent. Each specialist has @tool -functions that Agent Framework auto-executes. - -Reference: github.com/microsoft/agent-framework/python/samples/03-workflows/ - -Prerequisites: - pip install agent-framework[foundry] azure-identity - -Environment variables: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint - AZURE_OPENAI_DEPLOYMENT — model deployment name (e.g. gpt-5.1) - -Usage in run.yaml: - target: - type: agent - hosting: local - execution_mode: local - framework: agent_framework - local: - callable: multi_agent_workflow:run_evaluation -""" -from __future__ import annotations - -import asyncio -import logging -import os -from typing import Any - -from agent_framework import ( - Agent, - AgentExecutor, - AgentExecutorRequest, - AgentExecutorResponse, - AgentResponse, - Executor, - Message, - WorkflowBuilder, - WorkflowContext, - handler, - tool, -) - -logger = logging.getLogger(__name__) - -PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") -MODEL = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") - -_client = None -_captured_tool_calls: list[dict[str, Any]] = [] - - -def _get_chat_client(): - """Lazily initialize the FoundryChatClient.""" - global _client - if _client is None: - from azure.identity import DefaultAzureCredential - from agent_framework.foundry import FoundryChatClient - - _client = FoundryChatClient( - project_endpoint=PROJECT_ENDPOINT, - model=MODEL, - credential=DefaultAzureCredential(), - ) - return _client - - -# ── Tool functions (decorated with @tool for Agent Framework) ────────── - - -@tool -def get_weather(city: str) -> str: - """Get current weather for a city""" - _captured_tool_calls.append({"name": "get_weather", "arguments": {"city": city}}) - return f"Current weather in {city}: 55°F, partly cloudy." - - -@tool -def convert_currency(amount: str, from_currency: str, to_currency: str) -> str: - """Convert an amount from one currency to another""" - amt = float(amount) - _captured_tool_calls.append({ - "name": "convert_currency", - "arguments": {"amount": amt, "from_currency": from_currency, "to_currency": to_currency}, - }) - return f"{amt} {from_currency} = {amt * 0.92:.2f} {to_currency}" - - -@tool -def calculate_compound_interest(principal: str, rate: str, years: str) -> str: - """Calculate compound interest""" - p, r, y = float(principal), float(rate) / 100, int(float(years)) - total = p * ((1 + r) ** y) - interest = total - p - _captured_tool_calls.append({ - "name": "calculate_compound_interest", - "arguments": {"principal": p, "rate": r, "years": y}, - }) - return f"Compound interest: ${interest:,.2f}, total: ${total:,.2f}" - - -@tool -def search_news(query: str, max_results: str = "5") -> str: - """Search for recent news articles""" - _captured_tool_calls.append({ - "name": "search_news", - "arguments": {"query": query, "max_results": int(max_results)}, - }) - return f"Found {max_results} articles about '{query}'." - - -@tool -def search_flights(origin: str, destination: str, date: str) -> str: - """Search for available flights""" - _captured_tool_calls.append({ - "name": "search_flights", - "arguments": {"origin": origin, "destination": destination, "date": date}, - }) - return f"Found 3 flights from {origin} to {destination} on {date}." - - -# ── Coordinator Executor ────────────────────────────────────────────── -# Routes the user query to the correct specialist based on the Router's -# classification. Follows the official Coordinator pattern from -# microsoft/agent-framework samples. - - -class RoutingCoordinator(Executor): - """Routes between Router Agent and Specialist Agents.""" - - SPECIALIST_IDS = { - "weather": "weather_specialist", - "finance": "finance_specialist", - "search": "search_specialist", - } - - def __init__(self) -> None: - super().__init__(id="coordinator") - - @handler - async def on_agent_response( - self, - response: AgentExecutorResponse, - ctx: WorkflowContext[AgentExecutorRequest, AgentResponse], - ) -> None: - """Handle responses from Router and Specialist agents.""" - if response.executor_id != "router": - # Specialist response — yield as workflow output - await ctx.yield_output(response.agent_response) - return - - # Router response — parse routing decision and forward to specialist - routing_text = response.agent_response.text.strip().lower() - - if "weather" in routing_text: - target = "weather_specialist" - elif any(k in routing_text for k in ("finance", "currency", "interest")): - target = "finance_specialist" - else: - target = "search_specialist" - - logger.info("Coordinator routing to: %s (router said: %s)", target, routing_text) - - # Forward the original user query to the specialist - original_messages = list(response.full_conversation) - user_query = "" - for msg in original_messages: - if msg.role == "user": - user_query = msg.text or "" - break - - await ctx.send_message( - AgentExecutorRequest( - messages=[Message("user", contents=[user_query])], - should_respond=True, - ), - target_id=target, - ) - - -def _build_workflow(): - """Build the multi-agent workflow with Router → Coordinator → Specialists.""" - client = _get_chat_client() - - # Create agents - router = AgentExecutor(Agent( - client=client, - name="router", - instructions=( - "You are a routing agent. Analyze the user's query and respond " - "with ONLY one word:\n" - "- 'weather' for weather queries\n" - "- 'finance' for currency or interest calculations\n" - "- 'search' for news, flights, or general queries\n" - "Respond with only the category word, nothing else." - ), - )) - - weather = AgentExecutor(Agent( - client=client, - name="weather_specialist", - instructions="Use the get_weather tool to answer weather queries.", - tools=[get_weather], - )) - - finance = AgentExecutor(Agent( - client=client, - name="finance_specialist", - instructions=( - "Use convert_currency or calculate_compound_interest tools as needed." - ), - tools=[convert_currency, calculate_compound_interest], - )) - - search = AgentExecutor(Agent( - client=client, - name="search_specialist", - instructions="Use search_news or search_flights tools as needed.", - tools=[search_news, search_flights], - )) - - coordinator = RoutingCoordinator() - - # Build workflow: Router → Coordinator ↔ Specialists - workflow = ( - WorkflowBuilder(start_executor=router) - # Router output goes to Coordinator - .add_edge(router, coordinator) - # Coordinator can route to any specialist - .add_edge(coordinator, weather) - .add_edge(coordinator, finance) - .add_edge(coordinator, search) - # Specialist output goes back to Coordinator (which yields output) - .add_edge(weather, coordinator) - .add_edge(finance, coordinator) - .add_edge(search, coordinator) - .build() - ) - - return workflow - - -async def _run_workflow(input_text: str) -> dict[str, Any]: - """Run the multi-agent workflow for a single query.""" - workflow = _build_workflow() - - _captured_tool_calls.clear() - events = await workflow.run(input_text) - - # Extract the final response from workflow outputs - response_text = "" - outputs = events.get_outputs() - for output in outputs: - if isinstance(output, AgentResponse) and output.text: - response_text = output.text - - return { - "response": response_text.strip(), - "tool_calls": list(_captured_tool_calls), - } - - -def run_evaluation(input_text: str, context: dict) -> dict: - """Multi-agent workflow entry point for AgentOps evaluation. - - Uses Microsoft Agent Framework WorkflowBuilder with: - Router Agent → RoutingCoordinator → Specialist Agents (@tool) - """ - if not PROJECT_ENDPOINT or not MODEL: - raise ValueError( - "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AZURE_OPENAI_DEPLOYMENT" - ) - - return asyncio.run(_run_workflow(input_text)) diff --git a/src/agentops/templates/project.gitignore b/src/agentops/templates/project.gitignore new file mode 100644 index 00000000..aaa232ae --- /dev/null +++ b/src/agentops/templates/project.gitignore @@ -0,0 +1,35 @@ +# Generated by `agentops init` +# Standard Python / virtualenv / IDE noise +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.eggs/ +build/ +dist/ +.venv/ +venv/ +env/ +ENV/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Environment files +.env +.env.* +!.env.example + +# IDE +.idea/ +.vscode/ +*.swp + +# AgentOps runtime artifacts (results are reproducible from the config + dataset) +.agentops/results/ +.agentops/.resolved/ + +# Local install scratch space (unpacked plugins, VSIX, editor extras) +.local/ diff --git a/src/agentops/templates/run-agent-local.yaml b/src/agentops/templates/run-agent-local.yaml deleted file mode 100644 index 915b023a..00000000 --- a/src/agentops/templates/run-agent-local.yaml +++ /dev/null @@ -1,37 +0,0 @@ -version: 1 - -# Local agent workflow evaluation via callable adapter. -# Evaluates a local Python function that implements a multi-agent workflow. -# -# Two adapter options: -# -# 1. agent_framework_adapter:run_evaluation -# For Azure AI Foundry agents (Agent Framework SDK). -# Requires: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AGENT_ID env vars. -# pip install azure-ai-projects azure-identity -# -# 2. callable_adapter:run_evaluation -# For custom agents — HTTP, direct Python, or any callable. -# -# Cloud-only evaluators (TaskCompletionEvaluator, ToolSelectionEvaluator, -# ToolInputAccuracyEvaluator) will be gracefully skipped in local mode. -# Use 'hosting: foundry' with 'execution_mode: remote' to run all evaluators. - -target: - type: agent - hosting: local - execution_mode: local - framework: agent_framework - local: - # Option 1: Agent Framework adapter (Azure AI Foundry agents) - # callable: agent_framework_adapter:run_evaluation - # Option 2: Custom callable adapter (HTTP, direct Python, etc.) - callable: callable_adapter:run_evaluation -bundle: - name: agent_workflow_baseline -dataset: - name: smoke-agent-tools -execution: - timeout_seconds: 300 -output: - write_report: true diff --git a/src/agentops/templates/run-agent.yaml b/src/agentops/templates/run-agent.yaml deleted file mode 100644 index 8628f636..00000000 --- a/src/agentops/templates/run-agent.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - agent_mode: hosted - endpoint: - kind: foundry_agent - # Replace with your Foundry agent id, for example my-agent:3. - agent_id: - # Required by AI-assisted evaluators (judge model). - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: agent_workflow_baseline -dataset: - name: smoke-agent-tools -execution: - timeout_seconds: 1800 -output: - write_report: true diff --git a/src/agentops/templates/run-callable.yaml b/src/agentops/templates/run-callable.yaml deleted file mode 100644 index ea871728..00000000 --- a/src/agentops/templates/run-callable.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: 1 - -# Callable adapter run configuration -# Evaluates a local Python function instead of spawning a subprocess or -# calling a remote endpoint. Point "callable" to a module:function path -# that is importable from the project root. - -target: - type: model - hosting: local - execution_mode: local - local: - # Format: module_path:function_name - # The function must accept (input_text: str, context: dict) -> dict - # and return at least {"response": "..."}. - callable: callable_adapter:run_evaluation -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -execution: - timeout_seconds: 300 -output: - write_report: true diff --git a/src/agentops/templates/run-http-agent-tools.yaml b/src/agentops/templates/run-http-agent-tools.yaml deleted file mode 100644 index 6ff5255f..00000000 --- a/src/agentops/templates/run-http-agent-tools.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# HTTP Agent-with-Tools evaluation — evaluate agents that use tool calling via HTTP. -# -# Supported targets: -# - LangGraph agent with tools (request_field: input, response_field: output) -# - LangChain agent via LangServe (request_field: input, response_field: output) -# - Microsoft Agent Framework on ACA (request_field: message, response_field: text) -# - OpenAI Assistants via proxy (adjust fields for your adapter) -# - Custom REST endpoint (adjust fields to match your API) -# -# For tool-call evaluation, the HTTP response must include tool call data. -# Set tool_calls_field to the dot-path where tool calls appear in the response JSON. -# -# Set the agent URL via environment variable or inline: -# PowerShell: $env:AGENT_HTTP_URL = "https://your-agent.region.azurecontainerapps.io/chat" -# Bash/zsh: export AGENT_HTTP_URL="https://your-agent.region.azurecontainerapps.io/chat" - -version: 1 -target: - type: agent - hosting: local # change to aks or containerapps if deployed - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - - request_field: message - response_field: text - - # Extract tool_calls from the HTTP response for agent evaluators. - # The endpoint must return tool call data in its response JSON. - # Use dot-path notation for nested fields (e.g., "metadata.tool_calls"). - tool_calls_field: tool_calls - - # Forward extra JSONL row fields in the request body (optional): - # extra_fields: - # - session_id - # - tool_definitions # some agents need tool definitions in the request - -bundle: - name: agent_workflow_baseline -dataset: - name: smoke-agent-tools -execution: - timeout_seconds: 60 -output: - write_report: true diff --git a/src/agentops/templates/run-http-model.yaml b/src/agentops/templates/run-http-model.yaml deleted file mode 100644 index 24d06652..00000000 --- a/src/agentops/templates/run-http-model.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# HTTP Model-Direct evaluation — any HTTP endpoint returning text responses. -# -# Supported targets: -# - OpenAI-compatible API (request_field: prompt, response_field: choices.0.text) -# - LangServe / LangChain (request_field: input, response_field: output) -# - LangGraph Cloud (request_field: input, response_field: output) -# - Custom REST endpoint (adjust request_field / response_field to match your API) -# -# Set the agent URL via environment variable or inline: -# PowerShell: $env:AGENT_HTTP_URL = "https://your-agent.region.azurecontainerapps.io/chat" -# Bash/zsh: export AGENT_HTTP_URL="https://your-agent.region.azurecontainerapps.io/chat" - -version: 1 -target: - type: model - hosting: local # change to aks or containerapps if deployed - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - # url: https://your-agent.region.azurecontainerapps.io/chat # or set inline - - # Map your endpoint's request/response JSON fields. - request_field: message # JSON key for the user prompt - response_field: text # dot-path to extract the response text - - # Authentication (optional): - # auth_header_env: AGENT_TOKEN # env var holding Bearer token - - # Extra headers (optional): - # headers: - # X-Custom-Header: my-value - - # Forward extra JSONL row fields in the request body (optional): - # extra_fields: - # - session_id - # - user_id - -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -execution: - timeout_seconds: 30 -output: - write_report: true diff --git a/src/agentops/templates/run-http-rag.yaml b/src/agentops/templates/run-http-rag.yaml deleted file mode 100644 index 86a52f1f..00000000 --- a/src/agentops/templates/run-http-rag.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# HTTP RAG evaluation — evaluate retrieval-augmented generation via any HTTP endpoint. -# -# Supported targets: -# - LangChain RAG chain via LangServe (request_field: question, response_field: answer) -# - LangGraph RAG workflow (request_field: input, response_field: output) -# - Microsoft Agent Framework on ACA (request_field: message, response_field: text) -# - Custom REST endpoint (adjust fields to match your API) -# -# Set the agent URL via environment variable or inline: -# PowerShell: $env:AGENT_HTTP_URL = "https://your-agent.region.azurecontainerapps.io/chat" -# Bash/zsh: export AGENT_HTTP_URL="https://your-agent.region.azurecontainerapps.io/chat" - -version: 1 -target: - type: agent - hosting: local # change to aks or containerapps if deployed - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - - request_field: message - response_field: text - - # Forward extra JSONL row fields in the request body (optional): - # extra_fields: - # - session_id - # - context # some RAG endpoints accept context as input - -bundle: - name: rag_quality_baseline -dataset: - name: smoke-rag -execution: - timeout_seconds: 30 -output: - write_report: true diff --git a/src/agentops/templates/run-rag.yaml b/src/agentops/templates/run-rag.yaml deleted file mode 100644 index 479a8aea..00000000 --- a/src/agentops/templates/run-rag.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - agent_mode: hosted - endpoint: - kind: foundry_agent - # Replace with your Foundry agent id, for example my-agent:3. - agent_id: - # Required by AI-assisted evaluators (judge model). - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: rag_quality_baseline -dataset: - name: smoke-rag -execution: - timeout_seconds: 1800 -output: - write_report: true diff --git a/src/agentops/templates/run.yaml b/src/agentops/templates/run.yaml deleted file mode 100644 index f9b6686d..00000000 --- a/src/agentops/templates/run.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: 1 -target: - type: model - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - # Replace with a model deployment that exists in your Foundry project. - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT - api_version: "2025-05-01" - poll_interval_seconds: 2 - max_poll_attempts: 120 -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -execution: - timeout_seconds: 1800 -output: - write_report: true diff --git a/src/agentops/templates/skills/agentops-agent/SKILL.md b/src/agentops/templates/skills/agentops-agent/SKILL.md new file mode 100644 index 00000000..974d49dc --- /dev/null +++ b/src/agentops/templates/skills/agentops-agent/SKILL.md @@ -0,0 +1,101 @@ +--- +name: agentops-agent +description: AgentOps Watchdog — surface regressions, latency spikes, error rates, and safety hits across AgentOps eval history, Azure Monitor traces, and Foundry control plane. +--- + +# `agentops-agent` — Watchdog skill + +Use this skill when the user asks any of: + +- *"Are my agents healthy in production?"* +- *"Run the watchdog"* +- *"Anything regressed in our last evals?"* +- *"Show latency / error spikes from Azure Monitor"* +- *"Open the AgentOps watchdog report"* + +This skill is the front door to `agentops agent analyze` and the +`agentops agent serve` Copilot Extension. It does **not** invent +findings — it shells out to the CLI which reads real data from: + +1. `.agentops/results/*/results.json` (eval history) +2. Application Insights traces emitted by Foundry agents +3. Foundry control plane (`azure-ai-projects`) + +## Workflow + +### 1. Validate the workspace + +Look for `.agentops/agent.yaml`. If absent, copy the template: + +```bash +mkdir -p .agentops +cp $(python -c "import agentops, os, pathlib; +print(pathlib.Path(agentops.__file__).parent / 'templates' / 'agent.yaml')") .agentops/agent.yaml +``` + +Edit `app_insights_resource_id` and `project_endpoint_env` if the user +wants the Azure Monitor / Foundry sources to be live. Without those +values the sources skip gracefully. + +### 2. Run the analyzer + +```bash +agentops agent analyze --severity-fail critical +``` + +The command writes `.agentops/agent/report.md`. Exit codes: + +- `0` — no findings at or above the configured severity floor +- `2` — at least one finding meets the severity floor (use this in CI) +- `1` — runtime / configuration error + +### 3. Read and summarize + +Open `.agentops/agent/report.md`. The report has: + +- **Verdict banner** — overall pass / warning / critical +- **Summary** — counts by severity +- **Sources** — which sources ran, which were skipped and why +- **Findings** — sorted by severity, each with a recommendation +- **Recent runs** — appendix of the last `lookback_runs` evals + +When summarising for the user, lead with the verdict, then the top +3 findings, each with the recommendation. Always cite the finding `id` +so the user can grep them later. + +### 4. Drive remediation, do not invent it + +For each finding the report includes a `Recommendation`. Follow it +verbatim — for example, if the finding says "compare the latest run +against the baseline runs in `.agentops/results/`", actually open +those folders. + +## Copilot Extension server + +If the user wants the watchdog inside Copilot Chat, they can: + +```bash +pip install agentops-toolkit[agent] +agentops agent serve --no-verify # local dev +``` + +For production, point them at: + +- `src/agentops/templates/agent-server/Dockerfile` +- `src/agentops/templates/agent-server/main.bicep` +- `src/agentops/templates/agent-server/README.md` + +These are the deploy scaffold for hosting the watchdog as a Copilot +Extension on Azure Container Apps. + +## Guardrails + +- Do **not** fabricate findings, metric values, or recommendations. +- Do **not** invent CLI flags. The contract is exactly: + - `agentops agent analyze [--workspace] [--config] [--out] [--lookback-days] [--severity-fail]` + - `agentops agent serve [--host] [--port] [--config] [--no-verify] [--workers]` +- If a source is `skipped` or `error`, surface that as the *first* + thing in the user-facing summary so they know the analyzer ran with + partial data. +- Never suggest disabling content-safety checks — recommend filtering + the offending row or tightening the system prompt instead. diff --git a/src/agentops/templates/skills/agentops-config/SKILL.md b/src/agentops/templates/skills/agentops-config/SKILL.md index 06845854..309df268 100644 --- a/src/agentops/templates/skills/agentops-config/SKILL.md +++ b/src/agentops/templates/skills/agentops-config/SKILL.md @@ -1,258 +1,87 @@ --- name: agentops-config -description: Infer evaluation scenario from codebase and generate run.yaml. Trigger when users ask to configure an evaluation, create a run config, detect the evaluation scenario, or choose a bundle. Common phrases include "configure", "run.yaml", "which bundle", "set up eval", "scenario", "endpoint", "agentops config", "create run config", "what should I evaluate". Install agentops-toolkit via pip. +description: Generate or update agentops.yaml (flat 1.0 schema) by inspecting the workspace. Trigger on "configure agentops", "agentops.yaml", "set up evaluation", "what should I evaluate". Infer the agent target and dataset from the codebase; ask only when nothing can be found. --- # AgentOps Config -Generate a complete `.agentops/run.yaml` by inspecting the workspace. Infer everything possible — ask only for values that cannot be found. +Generate `agentops.yaml` at the project root. The flat schema has only a +handful of fields — most projects need just `version`, `agent`, and +`dataset`. ## Step 0 — Prerequisites -1. Run `pip install agentops-toolkit` if `agentops` command is not available. -2. Run `agentops init` if `.agentops/` directory does not exist. +1. `pip install agentops-toolkit` if `agentops` is missing. +2. `agentops init` if `agentops.yaml` does not exist. -## Step 1 — Detect scenario +## Step 1 — Detect the agent target -Analyze the codebase holistically to understand the agent's **primary purpose**: +Search the codebase for the strongest signal and pick one: -1. Read the README, system prompt, main entry point, and tool/function definitions. -2. Identify which patterns are present: - - **Tool use**: `@tool`, `tool_definitions`, `function_call`, MCP tools, tool schemas - - **Retrieval**: search client, vector store, retriever, embeddings, index references, context fetching - - **Conversation**: chat history, multi-turn, session management, assistant persona - - **Direct model call**: completion API, no orchestration logic - -3. Pick the scenario that best matches the agent's **primary job** — not just the first signal found: - -| Primary purpose | `bundle.name` | +| Signal | `agent:` value | |---|---| -| Agent that orchestrates tools to complete tasks | `agent_workflow_baseline` | -| Agent that retrieves context to answer questions | `rag_quality_baseline` | -| Conversational assistant (chat, Q&A, persona) | `conversational_agent_baseline` | -| Direct model call with no agent logic | `model_quality_baseline` | - -> A RAG agent that uses a search tool is still primarily RAG — pick `rag_quality_baseline`, not `agent_workflow_baseline`. The test is: *what is the agent's main job?* - -4. State what you found: *"Detected RAG scenario — the agent's primary purpose is answering questions using retrieved context (found retriever logic in retriever.py)."* - -5. **Responsible AI (optional)**: Ask *"Do you also want to include safety evaluators (violence, hate/unfairness, self-harm, protected material)?"* If yes, add the safety evaluators from `safe_agent_baseline` to the selected bundle. - -## Step 2 — Detect endpoint type - -| Search for | `endpoint.kind` | `hosting` | `execution_mode` | -|---|---|---|---| -| `AIProjectClient`, `azure-ai-projects`, Foundry URL | `foundry_agent` | `foundry` | `remote` | -| FastAPI, Flask, Django, Express — JSON POST/response | `http` | `containerapps` / `aks` / `local` | `remote` | -| SSE/streaming, non-standard body, custom auth, no server | — | `local` / `containerapps` / `aks` | `local` (callable) | - -Also check: `agent_id` references, Dockerfile, bicep, ACA manifests, `.env` files. - -**Discover the endpoint URL** — search in this order, stop when found: -1. Env vars: `$env:AGENT_HTTP_URL`, `$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` -2. `.env` / `.env.local` in project root -3. `.azure//.env` files -4. Azure CLI (if hosting is `containerapps` or ACA-deployed): - ```bash - az containerapp list -g $RG --subscription $SUB --query "[].{name:name, url:properties.configuration.ingress.fqdn}" -o json - ``` -5. Azure CLI (if hosting is App Service / webapp): - ```bash - az webapp list -g $RG --subscription $SUB --query "[].{name:name, url:defaultHostName}" -o json - ``` - -**Detect auth pattern** — search the codebase: -- `dapr-api-token` / `APP_API_TOKEN` → Dapr auth -- `X-API-KEY` / `api_key` / `API_KEY` → API key auth -- `Authorization` / `Bearer` → Bearer token auth -- Nothing found → assume no auth needed - -## Step 3 — Discover Azure values - -Search these locations **in order** — stop as soon as each value is found: - -1. Shell environment variables (`$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`, etc.) -2. `.env`, `.env.local` in project root -3. `.azure//.env` files (azd environments) — also read `AZURE_RESOURCE_GROUP`, `AZURE_SUBSCRIPTION_ID` -4. `.azure/config.json` for `defaultEnvironment` to pick the right env folder - -### Validate azd environment (if using `.azure//.env`) - -Before trusting values from `.azure//.env`, verify the environment is still valid: - -1. **Check the environment is current** — run `azd env list` and confirm the selected environment appears. If multiple environments exist, list them and ask the user which to use. -2. **Verify the resource group exists**: - ```bash - az group exists --name $RG --subscription $SUB - ``` - If this returns `false`, warn: *"Resource group '$RG' no longer exists. Your azd environment may be outdated."* -3. **If validation fails**, ask the user for correct values or to select a different environment. - -If values are **not found** in any file, run Azure CLI discovery: -```bash -# 1. Confirm auth and get subscription -az account show --query "{sub:id, tenant:tenantId}" -o json - -# 2. Find AI Services / Foundry accounts and endpoints -az cognitiveservices account list -o json --query "[].{name:name, rg:resourceGroup, endpoint:properties.endpoint, kind:kind}" - -# 3. Find model deployments -az cognitiveservices account deployment list --name $ACCOUNT -g $RG --subscription $SUB --query "[].{name:name, model:properties.model.name, version:properties.model.version}" -o json - -# 4. Find Foundry projects -az resource list -g $RG --subscription $SUB --resource-type "Microsoft.CognitiveServices/accounts/projects" --query "[].name" -o tsv - -# 5. Build endpoints from discovered names -# Foundry: https://.services.ai.azure.com/api/projects/ -# OpenAI: https://.openai.azure.com/ -``` +| `AIProjectClient(...)` + agent ID literal `name:N` | `":"` | +| Foundry hosted agent URL `https://...services.ai.azure.com/...agents/...` | the full URL | +| Any other HTTP endpoint your agent serves (FastAPI, Express, ACA, AKS) | the full URL | +| Direct model use (`openai.chat.completions.create(model=...)`) with no orchestration | `"model:"` | -**Pre-warm Azure token** (prevents intermittent `AzureCliCredential.get_token failed` errors): -```bash -az account get-access-token --resource "https://cognitiveservices.azure.com" --query accessToken -o tsv -``` -If this fails, Azure CLI auth is not active — ask the user to run `az login`. - -**Only ask the user** if no `.azure/` dir exists AND no env vars are set. - -## Step 4 — Pick evaluator model - -Read the bundle YAML from `.agentops/bundles/.yaml`. If it contains **any** evaluator with `source: foundry`, then an evaluator model is required. - -Pick from available deployments (discovered in Step 3): `gpt-4.1-mini` > `gpt-4o-mini` > `gpt-4o` > `gpt-4.1`. **Never** use reasoning models (`o1`, `o3`, `o4`, `gpt-5`, `gpt-5-nano`). - -If no suitable deployment was found, ask: *"Which model deployment should score your agent's responses? (e.g. gpt-4o-mini)"* +Look in: `README.md`, `main.py`/`server.py`/`app.ts`, `.env`/`.env.local`, +`.azure//.env`, `infra/`, IaC outputs. If nothing is found, ask the +user once. -## Step 4.5 — Evaluator compatibility check (optional) +## Step 2 — Detect the dataset -This step is **optional** — skip it if the bundle only uses widely available evaluators. +If a JSONL with rows that include `input` already exists in the repo, use +its path. Otherwise leave the default `.agentops/data/smoke.jsonl` and +hand off to the `agentops-dataset` skill before the first run. -**Key facts:** -- `SimilarityEvaluator`, `CoherenceEvaluator`, `FluencyEvaluator`, `RelevanceEvaluator`, `GroundednessEvaluator` → **widely available**, no check needed. -- `F1ScoreEvaluator`, `BleuScoreEvaluator`, `RougeScoreEvaluator`, `GleuScoreEvaluator` → **local text-overlap**, no Azure credentials needed, widely available. -- `TaskCompletionEvaluator`, `ToolCallAccuracyEvaluator`, `IntentResolutionEvaluator`, `TaskAdherenceEvaluator`, `ToolSelectionEvaluator`, `ToolInputAccuracyEvaluator`, `ResponseCompletenessEvaluator` → **SDK version dependent**, verify before using. +## Step 3 — Write agentops.yaml -If the bundle uses SDK-version-dependent evaluators, verify they exist. You may check the SDK version, read release notes, or try any efficient approach. Do **not** get stuck in environment path issues — if a quick check fails, proceed and let the evaluation surface any errors. +Minimal example: -If an evaluator is missing: set `enabled: false` in the bundle, remove its threshold, and tell the user. - -## Step 5 — Write run.yaml - -Write `.agentops/run.yaml` using the exact structure below. Fill **every** value — no placeholders. - -**Remote (Foundry agent):** ```yaml version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: -dataset: - name: dataset -output: - write_report: true +agent: "my-rag:3" +dataset: .agentops/data/smoke.jsonl ``` -**Remote (HTTP):** -```yaml -version: 1 -target: - type: agent - hosting: containerapps - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - request_field: message - response_field: text -bundle: - name: -dataset: - name: dataset -output: - write_report: true -``` +HTTP/JSON example: -**Local (callable adapter):** ```yaml version: 1 -target: - type: agent - hosting: local - execution_mode: local - local: - callable: callable_adapter:run_evaluation -bundle: - name: -dataset: - name: dataset -output: - write_report: true +agent: "https://my-aca-app.eastus2.azurecontainerapps.io/chat" +dataset: .agentops/data/smoke.jsonl +request_field: message # default is "message" +response_field: text # dot-path; default is "text" +auth_header_env: MY_API_TOKEN ``` -## Step 6 — Write callable adapter (if execution_mode is local) - -Create `callable_adapter.py` at the **project root**. Use ONLY stdlib (`urllib.request`, `json`, `os`). - -```python -import json -import os -import urllib.request - -ENDPOINT = os.environ["AGENT_HTTP_URL"] -# Auth: set APP_API_TOKEN, API_KEY, or remove the auth lines below. -AUTH_TOKEN = os.environ.get("APP_API_TOKEN", "") - -def run_evaluation(input_text: str, context: dict) -> dict: - body = json.dumps({"message": input_text}).encode() - headers = {"Content-Type": "application/json"} - if AUTH_TOKEN: - headers["dapr-api-token"] = AUTH_TOKEN # Change header name if using API_KEY or Bearer - req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - return {"response": data.get("text", data.get("response", ""))} -``` +Optional extras (only add when the user asks for them): -After writing the file, run: `python -c "from callable_adapter import run_evaluation; print('OK')"` - -**Auth detection:** Search codebase for `dapr-api-token`/`APP_API_TOKEN` → Dapr header. `X-API-KEY`/`api_key`/`API_KEY` → API key header. `Authorization`/`Bearer` → recommend HTTP backend with `auth_header_env` instead. Nothing found → remove auth lines. - -## Step 7 — Present and confirm - -Present a **confirmation table** with all discovered values (do not ask each one separately): -``` -┌─────────────────────────┬──────────────────────────────────────────┬────────┐ -│ Setting │ Value │ Source │ -├─────────────────────────┼──────────────────────────────────────────┼────────┤ -│ Scenario │ RAG │ code │ -│ Bundle │ rag_quality_baseline │ auto │ -│ Endpoint kind │ http │ code │ -│ Endpoint URL │ https://myapp.azurecontainerapps.io/chat │ .env │ -│ Auth │ dapr-api-token (APP_API_TOKEN) │ code │ -│ Evaluator model │ gpt-4o-mini │ Azure │ -│ Project endpoint │ https://acct.services.ai.azure.com/... │ .env │ -└─────────────────────────┴──────────────────────────────────────────┴────────┘ +```yaml +thresholds: + coherence: ">=3" + groundedness: ">=3" + avg_latency_seconds: "<=10" + +publish: foundry # Classic Foundry panel (works for any target) +# publish: foundry_cloud # New Foundry panel (preview; name:version agents only) +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" + +evaluators: # rare - AgentOps auto-selects from agent + dataset + - name: similarity + threshold: ">=4" ``` -Ask: *"Everything look correct? (yes / edit)"* +## Step 4 — Validate -Explain: scenario detected, endpoint type, evaluator model chosen, and any assumptions made. +Run `agentops eval run` once. If the config is malformed AgentOps prints a +clear error pointing at the offending key. Adjust and re-run. -## Rules +## Guardrails -- **NEVER** include `backend:` key in run.yaml — it causes a runtime error. -- **NEVER** leave `` placeholders in run.yaml. -- **NEVER** fabricate `agent_id`, model names, or endpoint URLs. -- **NEVER** use dotted import paths like `.agentops.callable_adapter` — they fail. -- If a bundle uses SDK-version-dependent evaluators, verify availability before running (Step 4.5). Don't block on this — if verification is hard, proceed and fix on failure. -- Do not generate datasets — delegate to `/agentops-dataset`. -- Do not run evaluations — delegate to `/agentops-eval`. -- Always state what you detected and what you assumed. \ No newline at end of file +- Do **not** add legacy keys (`bundle`, `target`, `execution`, `output`, + `backend`). The 1.0 schema rejects them. +- Do **not** fabricate agent IDs, endpoint URLs, or model deployment + names. Ask the user when uncertain. +- Keep the file small. Auto-selection covers most metrics. diff --git a/src/agentops/templates/skills/agentops-dataset/SKILL.md b/src/agentops/templates/skills/agentops-dataset/SKILL.md index 602b334e..625d9acc 100644 --- a/src/agentops/templates/skills/agentops-dataset/SKILL.md +++ b/src/agentops/templates/skills/agentops-dataset/SKILL.md @@ -1,128 +1,69 @@ --- name: agentops-dataset -description: Generate evaluation datasets (JSONL data + YAML config) tailored to the project. Trigger when users ask to create test data, generate a dataset, or prepare evaluation data. Common phrases include "dataset", "test data", "evaluation data", "JSONL", "generate data", "create dataset", "sample data". Install agentops-toolkit via pip. +description: Create or extend a JSONL evaluation dataset for AgentOps. Trigger on "create dataset", "generate test data", "JSONL", "more eval rows". Infer the agent's domain from the codebase and produce realistic rows; never fabricate data when the domain is unclear. --- # AgentOps Dataset -Generate a custom evaluation dataset from the codebase. Never offer starter datasets — always create project-specific data. +Generate a small, realistic JSONL dataset for the agent under +evaluation. Default location: `.agentops/data/smoke.jsonl` (referenced +from `agentops.yaml`). ## Step 0 — Prerequisites -1. Run `pip install agentops-toolkit` if `agentops` command is not available. -2. Run `agentops init` if `.agentops/` directory does not exist. - -## Step 1 — Understand the domain - -Read the codebase: system prompt, tool definitions, README, sample inputs/outputs, test fixtures. Understand the agent's **primary purpose** and identify the scenario: - -| Primary purpose | Scenario | -|---|---| -| Agent that orchestrates tools to complete tasks | Agent with tools | -| Agent that retrieves context to answer questions | RAG | -| Conversational assistant (chat, Q&A, persona) | Conversational | -| Direct model call with no agent logic | Model quality | - -> A RAG agent that uses a search tool is still primarily RAG. The test is: *what is the agent's main job?* - -## Step 2 — Confirm topics and count - -1. Ask: *"What topics should the test data cover?"* -2. Ask: *"How many rows? (suggest 5–10)"* - -## Step 3 — Generate JSONL rows - -Use the correct fields for the scenario: - -| Scenario | JSONL fields | -|---|---| -| Model quality | `input`, `expected` | -| Conversational | `input`, `expected` | -| RAG | `input`, `expected`, `context` | -| Agent with tools | `input`, `expected`, `tool_definitions`, `tool_calls` | -| Content safety | `input`, `expected` | - -Write `.agentops/data/data.jsonl` — one JSON object per line. Rows must: -- Cover distinct use cases from the codebase -- Include realistic, domain-specific content -- Have at least one edge case -- Reflect actual tool schemas and system prompt - -## Step 4 — Write dataset YAML config - -Write `.agentops/datasets/dataset.yaml` using this **exact** structure — no alternatives: -```yaml -version: 1 -name: dataset -description: -source: - type: file - path: ../data/data.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: - size_hint: -``` +1. `pip install agentops-toolkit` if `agentops` is missing. +2. `agentops init` if `agentops.yaml` does not exist. -**NEVER** use `path:` or `fields:` at the top level — the correct keys are `source:` and `format:`. If unsure, read an existing starter config from `.agentops/datasets/` as a reference template. +## Step 1 — Pick the columns -For RAG scenarios, add `context_field: context` under `format:`: -```yaml -format: - type: jsonl - input_field: input - expected_field: expected - context_field: context -``` +Read `agentops.yaml` (and the agent code) to figure out the agent type, +then choose the row schema: + +| Agent type | Required columns | Optional columns | +|---|---|---| +| Direct model / Q&A | `input`, `expected` | — | +| RAG | `input`, `expected`, `context` | — | +| Conversational | `input`, `expected` | — | +| Tool-using agent | `input`, `expected`, `tool_calls` | `tool_definitions` | -## Step 4.5 — RAG context enrichment +`input` is always the user prompt. `expected` is the gold answer. +`context` is the retrieved passage(s). `tool_calls` is a list of +`{name, arguments}` describing the expected tool invocations. -If the scenario is **RAG** and the generated JSONL has no `context` field: +## Step 2 — Ground the rows in the codebase -1. **Find the project's retrieval logic** — search the codebase for how it fetches context today: - - Look for search/retrieval client initialization, index or collection names, embedding calls - - Check `.env` files and code for endpoint URLs, API keys, index names used by the retriever - - The project may use Azure AI Search, Cosmos DB vector search, FAISS, Pinecone, or any other store — read the code to find out +- Read the README, system prompt, tool definitions, and any sample + fixtures. +- Generate **5–10 rows** that exercise the agent's actual capabilities. +- If the domain is unclear, generate a tiny generic draft and clearly + flag it as a placeholder. -2. **Build a retrieval script** at `.agentops/rag_context.py` (**never** in `src/`) that: - - Reads the project's own retrieval config (env vars, endpoint, index name) from whatever the project uses - - For each row in the JSONL, queries the retrieval backend with `row["input"]` and writes the result into `row["context"]` - - Uses only stdlib (`urllib.request`, `json`, `os`, `subprocess`, `sys`, `shutil`) — no third-party dependencies - - Accepts the JSONL file path as a CLI argument: `python .agentops/rag_context.py .agentops/data/data.jsonl` - - **Must be cross-platform** (Windows + Linux/macOS) — when calling external CLIs (e.g. `az`), use: - ```python - import shutil, subprocess, sys - def _run_cli(args: list[str], **kwargs) -> subprocess.CompletedProcess: - exe = shutil.which(args[0]) - if exe is None: - raise FileNotFoundError(f"'{args[0]}' not found in PATH.") - return subprocess.run([exe] + args[1:], **kwargs, shell=(sys.platform == "win32")) - ``` +## Step 3 — Write the JSONL -3. Verify: each JSONL row now has a `context` field. -4. Update dataset YAML to include `context_field: context` under `format:`. +One JSON object per line, no trailing commas, UTF-8: -If no retrieval backend can be identified, state: *"RAG context cannot be populated automatically — either add `context` manually to each row or switch to `model_quality_baseline` bundle which does not require it."* +```json +{"input": "What is the refund policy?", "expected": "Refunds within 30 days...", "context": "Refund policy: ..."} +``` + +Save to the path referenced by `dataset:` in `agentops.yaml` (default +`.agentops/data/smoke.jsonl`). -## Step 5 — Present for review +## Step 4 — Sanity-check -Show the generated rows and say: *"These are starter rows for validation. For production evaluations, use real user queries or domain expert–curated data."* +Run a quick eval and confirm rows are picked up: -## Outputs +```bash +agentops eval run +``` -- `.agentops/data/data.jsonl` — JSONL rows -- `.agentops/datasets/dataset.yaml` — dataset config +Open `.agentops/results/latest/report.md` and confirm the row count +matches. -## Rules +## Guardrails -- **NEVER** offer starter datasets (`smoke-model-direct.jsonl`, etc.) — always generate custom data. -- **NEVER** leave `` placeholders in JSONL or YAML. -- **NEVER** use `path:` or `fields:` at the dataset config top level — the correct structure uses `source:` and `format:`. Read a starter config from `.agentops/datasets/` if unsure. -- Use generic file names: `data.jsonl`, `dataset.yaml` — not project-specific prefixes. -- State the scenario assumption: *"Generating dataset for RAG scenario (detected retriever)"*. -- Mark generated data as draft — not production-grade. -- Do not run evaluations — delegate to `/agentops-eval`. -- Do not generate run.yaml — delegate to `/agentops-config`. +- Do not invent customer data, real names, or sensitive content. +- Keep rows short — datasets are meant to be quick gates, not full QA + suites. +- If the user already has a domain dataset, prefer pointing + `agentops.yaml` at that file rather than generating new rows. diff --git a/src/agentops/templates/skills/agentops-eval/SKILL.md b/src/agentops/templates/skills/agentops-eval/SKILL.md index 2463a46c..a3d58f5a 100644 --- a/src/agentops/templates/skills/agentops-eval/SKILL.md +++ b/src/agentops/templates/skills/agentops-eval/SKILL.md @@ -1,610 +1,105 @@ --- name: agentops-eval -description: Guide users through running AgentOps evaluations end to end — codebase analysis, dataset generation, config creation, single runs, multi-model benchmarks, and N-run comparisons. Trigger when users ask to run an evaluation, compare runs, benchmark models, create eval config, generate datasets, or summarize results. Common phrases include "run eval", "evaluate", "start agentops", "compare models", "benchmark agents", "run.yaml", "report", "evaluation results", "which model is best", "set up eval", "create dataset". Install agentops-toolkit via pip. Commands are agentops init, agentops eval run, agentops eval compare, and agentops report generate. +description: Run AgentOps evaluations end-to-end against any agent (Foundry hosted/prompt agent, HTTP/JSON endpoint, or raw model deployment). Trigger on phrases like "run eval", "evaluate my agent", "benchmark", "agentops eval", "compare runs". Uses the flat agentops.yaml schema. --- # AgentOps Eval -End-to-end evaluation workflow: analyze codebase → generate dataset → configure run → validate → execute → summarize. +End-to-end workflow: install → init → configure → run → read report. -## Step 0 — Verify setup +## Step 0 — Setup -1. Run `pip install agentops-toolkit` if `agentops` command is not available. -2. Run `agentops init` if `.agentops/` directory does not exist. +1. Install if missing: `pip install agentops-toolkit`. +2. If `agentops.yaml` does not exist at the project root, run `agentops init`. -Then proceed to analyze the codebase. Only ask questions about things you cannot find in the code. +## Step 1 — Identify the agent target -## Step 1 — Detect evaluation scenario +Read the codebase (README, entry point, env vars) and pick the right value +for the `agent:` field of `agentops.yaml`: -Analyze the codebase holistically to understand the agent's **primary purpose**: - -1. Read the README, system prompt, main entry point, and tool/function definitions. -2. Identify which patterns are present: - - **Tool use**: `@tool`, `tool_definitions`, `function_call`, MCP tools, tool schemas - - **Retrieval**: search client, vector store, retriever, embeddings, index references, context fetching - - **Conversation**: chat history, multi-turn, session management, assistant persona - - **Direct model call**: completion API, no orchestration logic - -3. Pick the scenario that best matches the agent's **primary job** — not just the first signal found: - -| Primary purpose | `bundle.name` | +| Pattern in code / env | `agent:` value | |---|---| -| Agent that orchestrates tools to complete tasks | `agent_workflow_baseline` | -| Agent that retrieves context to answer questions | `rag_quality_baseline` | -| Conversational assistant (chat, Q&A, persona) | `conversational_agent_baseline` | -| Direct model call with no agent logic | `model_quality_baseline` | - -> A RAG agent that uses a search tool is still primarily RAG — pick `rag_quality_baseline`, not `agent_workflow_baseline`. The test is: *what is the agent's main job?* - -4. State your reasoning: *"Detected RAG scenario — the agent's primary purpose is answering questions using retrieved context (found retriever logic in retriever.py)."* - -5. **Responsible AI (optional)**: Ask *"Do you also want to include safety evaluators (violence, hate/unfairness, self-harm, protected material)? These can be added alongside your main bundle."* If yes, add the safety evaluators from `safe_agent_baseline` to the selected bundle. - -6. **Unit tests (optional)**: Only ask this if **all** of the following are true: (a) the codebase has testable agent code in Python, JavaScript, or TypeScript (endpoint handlers, tool definitions, orchestration logic), (b) no existing test directory or test files are detected (e.g., `tests/`, `test_*.py`, `*_test.py`, `*.test.ts`, `*.test.js`, `__tests__/`). If both conditions are met, ask: *"Would you also like me to generate unit tests for your agent code? (e.g., mocked HTTP calls, response parsing, error handling)"*. If the user declines or if conditions are not met, skip silently. See the **Unit Test Generation** section at the end of this skill for details. - -## Step 2 — Detect endpoint type - -| Search for | `endpoint.kind` | `hosting` | `execution_mode` | -|---|---|---|---| -| `AIProjectClient`, `azure-ai-projects`, Foundry URL | `foundry_agent` | `foundry` | `remote` | -| FastAPI/Flask/Django — JSON POST → JSON response | `http` | `containerapps`/`aks`/`local` | `remote` | -| SSE/streaming, custom auth, non-standard body, no server | — | `local`/`containerapps`/`aks` | `local` (callable) | - -**Discover the endpoint URL** — search in this order, stop when found: -1. Env vars: `$env:AGENT_HTTP_URL`, `$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` -2. `.env` / `.env.local` in project root -3. `.azure//.env` files -4. Azure CLI (if hosting is `containerapps` or ACA-deployed): - ```bash - az containerapp list -g $RG --subscription $SUB --query "[].{name:name, url:properties.configuration.ingress.fqdn}" -o json - ``` -5. Azure CLI (if hosting is App Service / webapp): - ```bash - az webapp list -g $RG --subscription $SUB --query "[].{name:name, url:defaultHostName}" -o json - ``` - -**Detect auth pattern** — search the codebase for auth headers used in requests: -- `dapr-api-token` / `APP_API_TOKEN` → Dapr auth (use in callable adapter) -- `X-API-KEY` / `api_key` / `API_KEY` → API key auth (set `auth_header_env`) -- `Authorization` / `Bearer` → Bearer token (set `auth_header_env`) -- No auth headers found → assume no auth needed - -Only ask *"What is the URL where your agent is running?"* if discovery finds nothing. - -## Step 3 — Generate dataset - -**Never offer starter datasets** — always generate a custom one. - -1. Read the codebase: system prompt, tools, domain, README. -2. Ask the user what topics the test data should cover. -3. Ask how many rows (suggest 5–10). -4. Write `.agentops/data/data.jsonl` with the correct fields: - -| Scenario | JSONL fields | -|---|---| -| Model quality | `input`, `expected` | -| Conversational | `input`, `expected` | -| RAG | `input`, `expected`, `context` | -| Agent with tools | `input`, `expected`, `tool_definitions`, `tool_calls` | - -5. Write `.agentops/datasets/dataset.yaml` using this **exact** structure (no alternatives): -```yaml -version: 1 -name: dataset -description: -source: - type: file - path: ../data/data.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -metadata: - scenario: - size_hint: -``` -**NEVER** use `path:` or `fields:` at the top level — the correct keys are `source:` and `format:`. If unsure, read an existing starter config from `.agentops/datasets/` as a reference template first. - -6. Show the generated rows to the user for review. - -### RAG context enrichment - -If the scenario is **RAG** and the dataset has no `context` field: - -1. **Find the project's retrieval logic** — search the codebase for how it fetches context today: - - Look for search/retrieval client initialization, index or collection names, embedding calls - - Check `.env` files and code for endpoint URLs, API keys, index names used by the retriever - - The project may use Azure AI Search, Cosmos DB vector search, FAISS, Pinecone, or any other store — read the code to find out - -2. **Build a retrieval script** at `.agentops/rag_context.py` (**never** in `src/`) that: - - Reads the project's own retrieval config (env vars, endpoint, index name) from whatever the project uses - - For each row in the JSONL, queries the retrieval backend with `row["input"]` and writes the result into `row["context"]` - - Uses only stdlib (`urllib.request`, `json`, `os`, `subprocess`, `sys`, `shutil`) — no third-party dependencies - - Accepts the JSONL file path as a CLI argument: `python .agentops/rag_context.py .agentops/data/data.jsonl` - - **Must be cross-platform** (Windows + Linux/macOS) — when calling external CLIs (e.g. `az`), use the following pattern: - ```python - import shutil - import subprocess - import sys - - def _run_cli(args: list[str], **kwargs) -> subprocess.CompletedProcess: - """Run an external CLI command, cross-platform.""" - exe = shutil.which(args[0]) - if exe is None: - raise FileNotFoundError( - f"'{args[0]}' not found in PATH. " - "Make sure it is installed and available." - ) - return subprocess.run( - [exe] + args[1:], - **kwargs, - shell=(sys.platform == "win32"), - ) - ``` - - This avoids `FileNotFoundError` on Windows where `subprocess.run(["az", ...])` fails without `shell=True` - -3. Update dataset YAML to include `context_field: context` under `format:`. -4. Now `rag_quality_baseline` with GroundednessEvaluator and RetrievalEvaluator can be used. - -If no retrieval backend can be identified, fall back to `model_quality_baseline` and explain why. - -## Step 4 — Discover Azure values - -Search these locations in order — stop as soon as each value is found: - -1. Shell env vars (`$env:AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`, `$env:AZURE_OPENAI_ENDPOINT`, `$env:AZURE_OPENAI_DEPLOYMENT`) -2. `.env` / `.env.local` in project root -3. `.azure//.env` (azd environments) — also read `AZURE_RESOURCE_GROUP`, `AZURE_SUBSCRIPTION_ID` -4. `.azure/config.json` for `defaultEnvironment` to pick the right env folder +| `AIProjectClient`, `azure-ai-projects`, Foundry agent ID like `name:1` | `":"` (Foundry prompt agent) | +| Foundry hosted agent endpoint URL ending in `/agents/...` | `"https://.services.ai.azure.com/api/projects/

/agents/..."` | +| Plain HTTP/JSON endpoint (FastAPI, Express, ACA, AKS) | `"https:///"` | +| Raw Foundry/Azure OpenAI model deployment | `"model:"` | -### Validate azd environment (if using `.azure//.env`) +If nothing is found, ask the user once for the agent identifier. -Before trusting values from `.azure//.env`, verify the environment is still valid: +## Step 2 — Make sure the dataset exists -1. **Check if the environment is current** — run `azd env list` and confirm the selected environment appears in the output. If multiple environments exist, list them and ask the user which one to use. -2. **Verify the resource group exists** — after reading `AZURE_RESOURCE_GROUP` and `AZURE_SUBSCRIPTION_ID` from the env file, run: - ```bash - az group exists --name $RG --subscription $SUB - ``` - If this returns `false`, the environment is stale (resources were deleted). Warn the user: *"The resource group '$RG' no longer exists. Your azd environment may be outdated. Please re-run `azd up` or provide current Azure values."* -3. **If validation fails**, do not silently proceed with stale values — ask the user for correct values or to select a different environment. +`agentops.yaml` points to a JSONL file (default +`.agentops/data/smoke.jsonl`). Each row needs at least `input` and a label +that maps to the metric you care about (`expected`, `context`, +`tool_calls`...). If the dataset is empty or unrelated, run the +`agentops-dataset` skill before running the eval. -If values are **not found** in files, use Azure CLI to discover them: +## Step 3 — Run the evaluation ```bash -# 1. Confirm auth and get subscription -az account show --query "{sub:id, tenant:tenantId}" -o json +agentops eval run +``` -# 2. Find AI Services / Foundry accounts and endpoints -az cognitiveservices account list -o json --query "[].{name:name, rg:resourceGroup, endpoint:properties.endpoint, kind:kind}" -# Or scope to a known RG: -az cognitiveservices account list -g $RG --subscription $SUB --query "[].{name:name, endpoint:properties.endpoint}" -o json +Optional flags: -# 3. Find model deployments (chat, embedding) -az cognitiveservices account deployment list --name $ACCOUNT -g $RG --subscription $SUB --query "[].{name:name, model:properties.model.name, version:properties.model.version}" -o json +- `--config ` — point at a different `agentops.yaml`. +- `--output

` — choose where to write `results.json` and `report.md` + (defaults to `.agentops/results//`). -# 4. Find Foundry projects -az resource list -g $RG --subscription $SUB --resource-type "Microsoft.CognitiveServices/accounts/projects" --query "[].name" -o tsv +Exit codes: -# 5. Build endpoints from discovered names -# Foundry: https://.services.ai.azure.com/api/projects/ -# OpenAI: https://.openai.azure.com/ -``` +- `0` — succeeded and all thresholds passed +- `2` — succeeded but at least one threshold failed (gate-friendly) +- `1` — runtime/configuration error -For evaluator model, pick from available deployments: `gpt-4.1-mini` > `gpt-4o-mini` > `gpt-4o` > `gpt-4.1`. **Never** reasoning models (`o1`, `o3`, `o4`, `gpt-5`, `gpt-5-nano`). +## Step 4 — Inspect results -**Pre-warm Azure token** (prevents intermittent `AzureCliCredential.get_token failed` errors): ```bash -az account get-access-token --resource "https://cognitiveservices.azure.com" --query accessToken -o tsv +agentops report generate # regenerate report.md from latest results.json +agentops report generate --in ``` -If this fails, Azure CLI auth is not active — ask the user to run `az login`. - -Check Azure auth: `az account show`. If not logged in, ask the user to run `az login` or set API key. - -## Step 4.5 — Evaluator compatibility check (optional) - -This step is **optional** — skip it if you are confident the bundle evaluators match the installed SDK. If the evaluation fails later due to a missing evaluator, come back here. - -Use the reference table below to decide whether the selected bundle is safe to use **without running any probes**. Evaluators marked "Widely available" work on all recent `azure-ai-evaluation` versions. Only the SDK-version-dependent ones need caution. - -### Evaluator compatibility reference - -| Evaluator | Category | Needs credentials | Availability | -|---|---|---|---| -| `SimilarityEvaluator` | AI-assisted | Yes | Widely available | -| `CoherenceEvaluator` | AI-assisted | Yes | Widely available | -| `FluencyEvaluator` | AI-assisted | Yes | Widely available | -| `RelevanceEvaluator` | AI-assisted | Yes | Widely available | -| `GroundednessEvaluator` | AI-assisted | Yes | Widely available | -| `F1ScoreEvaluator` | Local text-overlap | No | Widely available | -| `BleuScoreEvaluator` | Local text-overlap | No | Widely available | -| `RougeScoreEvaluator` | Local text-overlap | No | Widely available | -| `GleuScoreEvaluator` | Local text-overlap | No | Widely available | -| `TaskCompletionEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ToolCallAccuracyEvaluator` | AI-assisted | Yes | SDK version dependent | -| `IntentResolutionEvaluator` | AI-assisted | Yes | SDK version dependent | -| `TaskAdherenceEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ToolSelectionEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ToolInputAccuracyEvaluator` | AI-assisted | Yes | SDK version dependent | -| `ResponseCompletenessEvaluator` | AI-assisted | Yes | SDK version dependent | -### When to verify +Open `.agentops/results/latest/report.md`. To compare two runs, hand both +`results.json` files to the user and walk them through metric deltas; +AgentOps does not ship a separate `eval compare` command. -- If the bundle only uses **widely available** evaluators → proceed directly, no verification needed. -- If the bundle uses **SDK-version-dependent** evaluators → verify they exist before running. You may check `pip show azure-ai-evaluation` for version, read SDK release notes, or use any approach you find efficient. Do **not** get stuck in environment path issues — if a quick check fails, just proceed and let the evaluation surface any import errors. +## Step 5 — (Optional) Publish to Foundry Evaluations -### If an evaluator is missing +Two modes are supported. Both write a deep-link into +`.agentops/results/latest/cloud_evaluation.json` and require +`AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` (or the inline `project_endpoint`). -- Disable it in the bundle (`enabled: false`) and remove its threshold. -- Tell the user: *"Disabled [X] — not available in your SDK version."* +**Classic Foundry Evaluations panel** (default — works for any target +kind, uploads metrics that AgentOps already computed locally): -## Step 5 — Write run.yaml - -Update `.agentops/run.yaml` (the default config). Do **not** create a custom-named file. - -**Remote Foundry agent:** ```yaml -version: 1 -target: - type: agent - hosting: foundry - execution_mode: remote - endpoint: - kind: foundry_agent - agent_id: - model: - project_endpoint_env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -bundle: - name: -dataset: - name: dataset -output: - write_report: true +publish: foundry +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" ``` -**Remote HTTP:** -```yaml -version: 1 -target: - type: agent - hosting: containerapps - execution_mode: remote - endpoint: - kind: http - url_env: AGENT_HTTP_URL - request_field: message - response_field: text -bundle: - name: -dataset: - name: dataset -output: - write_report: true -``` +**New Foundry Evaluations panel** (preview — re-runs the agent + builtin +evaluators server-side via the OpenAI Evals API; only works for +`name:version` Foundry agents): -**Local callable adapter:** ```yaml -version: 1 -target: - type: agent - hosting: local - execution_mode: local - local: - callable: callable_adapter:run_evaluation -bundle: - name: -dataset: - name: dataset -output: - write_report: true -``` - -Fill **every** `` with a real discovered value. If any value cannot be found, ask the user for just that value. - -## Step 5.5 — Write callable adapter (if execution_mode is local) - -Create `.agentops/callable_adapter.py`. Use ONLY stdlib. All generated files must live inside `.agentops/` to avoid polluting the project root. - -First, examine the agent's response format by reading the endpoint handler code: -- Look for `yield`, `StreamingResponse`, `EventSourceResponse` → SSE/streaming -- Look for `JSONResponse`, `return {"text": ...}` → standard JSON -- Look for conversation ID prefixes, UUID patterns in responses - -**Standard JSON adapter:** -```python -import json -import os -import urllib.request - -ENDPOINT = os.environ["AGENT_HTTP_URL"] -# Auth: set AGENT_AUTH_HEADER and AGENT_AUTH_TOKEN env vars if your endpoint requires auth. -# Example: AGENT_AUTH_HEADER=dapr-api-token AGENT_AUTH_TOKEN=dev-token -# AGENT_AUTH_HEADER=X-API-KEY AGENT_AUTH_TOKEN=my-key -AUTH_HEADER = os.environ.get("AGENT_AUTH_HEADER", "") -AUTH_TOKEN = os.environ.get("AGENT_AUTH_TOKEN", "") - -def run_evaluation(input_text: str, context: dict) -> dict: - body = json.dumps({"message": input_text}).encode() - headers = {"Content-Type": "application/json"} - if AUTH_HEADER and AUTH_TOKEN: - headers[AUTH_HEADER] = AUTH_TOKEN - req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=120) as resp: - data = json.loads(resp.read()) - return {"response": data.get("text", data.get("response", ""))} -``` - -**SSE/streaming adapter** (use when agent uses `StreamingResponse`, `yield`, or SSE): -```python -import json -import os -import urllib.request - -ENDPOINT = os.environ["AGENT_HTTP_URL"] -# Auth: set AGENT_AUTH_HEADER and AGENT_AUTH_TOKEN env vars if your endpoint requires auth. -AUTH_HEADER = os.environ.get("AGENT_AUTH_HEADER", "") -AUTH_TOKEN = os.environ.get("AGENT_AUTH_TOKEN", "") - -def run_evaluation(input_text: str, context: dict) -> dict: - body = json.dumps({"message": input_text}).encode() - headers = {"Content-Type": "application/json"} - if AUTH_HEADER and AUTH_TOKEN: - headers[AUTH_HEADER] = AUTH_TOKEN - req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") - chunks = [] - try: - with urllib.request.urlopen(req, timeout=120) as resp: - for raw_line in resp: - line = raw_line.decode("utf-8", errors="replace").strip() - if not line or line.startswith(":"): # SSE comment or keep-alive - continue - if line.startswith("event:"): # SSE event type — skip - continue - if line.startswith("data: "): - payload = line[6:] - if payload == "[DONE]": - break - try: - event = json.loads(payload) - # Adapt field extraction to match the project's SSE format - chunk = event.get("content", event.get("text", "")) - if chunk: - chunks.append(chunk) - except json.JSONDecodeError: - chunks.append(payload) # plain text SSE - else: - chunks.append(line) # raw text line - except Exception as e: - return {"response": f"ERROR: {e}"} - response_text = "".join(chunks).strip() - return {"response": response_text} -``` - -Customize the adapter: -- **Apply the auth pattern detected in Step 2.** Use the table below to wire the correct header and env var into the adapter: - -| Auth detected in Step 2 | Adapter env var | Header line in adapter | -|---|---|---| -| `dapr-api-token` / `APP_API_TOKEN` | `AGENT_AUTH_TOKEN` (tell user to set it to their Dapr token) | `headers["dapr-api-token"] = AUTH_TOKEN` | -| `X-API-KEY` / `api_key` / `API_KEY` | `AGENT_AUTH_TOKEN` (tell user to set it to their API key) | `headers["X-API-KEY"] = AUTH_TOKEN` | -| `Authorization: Bearer` | Recommend HTTP backend with `auth_header_env` instead of callable adapter | N/A | -| No auth detected | Remove `AUTH_TOKEN` and auth header lines entirely | N/A | - - **Important**: Do NOT generate the adapter with auth lines commented out or using hardcoded tokens. If auth was detected, the adapter must include the correct header from the start — otherwise the smoke test will fail with 401. - -- **Choose the right template:** If the agent code uses `yield`, `StreamingResponse`, `EventSourceResponse`, or `text/event-stream` content type, use the **SSE/streaming adapter** template. Otherwise use the **standard JSON adapter**. -- **Customize the request field:** If the agent expects a different key than `"message"` (e.g. `"ask"`, `"question"`, `"input"`), change the `json.dumps({"message": ...})` line to match. -- **Customize the response extraction:** If the agent returns a different key than `"text"` or `"response"`, update the `.get()` call accordingly. - -### Context sanitization (RAG scenarios) - -If the dataset has a `context` field populated from Azure AI Search or similar document stores, the raw content often includes HTML comments (``), document source tags (`[Copy 002 ...]`), and OCR artifacts. Add this helper to the adapter and call it when enriching context: - -```python -import re - -_HTML_COMMENT_RE = re.compile(r"", re.DOTALL) -_MULTI_BLANK_RE = re.compile(r"\n{3,}") - -def _sanitize_context(text: str) -> str: - """Strip HTML comments, document metadata, and collapse blank lines.""" - text = _HTML_COMMENT_RE.sub("", text) - text = re.sub(r"^\[.*?\]\s*$", "", text, flags=re.MULTILINE) - text = _MULTI_BLANK_RE.sub("\n\n", text) - return text.strip() -``` - -Apply it to the `context` field in JSONL rows before writing or in the adapter before returning: -```python -ctx = context.get("context", "") -if ctx: - context["context"] = _sanitize_context(ctx) -``` - -After writing the file: `python -c "import sys; sys.path.insert(0, '.agentops'); from callable_adapter import run_evaluation; print('OK')"` - -## Step 6 — Pre-flight validation - -Check **all** of these **before** running. Fix any failures first. Do NOT run-fail-fix iteratively. - -- [ ] run.yaml has no `backend:` key (causes runtime error) -- [ ] No `` placeholders in run.yaml -- [ ] Bundle file exists: `.agentops/bundles/.yaml` -- [ ] Dataset file exists: `.agentops/datasets/dataset.yaml` -- [ ] Dataset YAML has `source:` and `format:` keys (NOT `path:` or `fields:` at top level) -- [ ] JSONL file exists: `.agentops/data/data.jsonl` -- [ ] If RAG: JSONL rows have `context` field; dataset YAML has `context_field: context` -- [ ] If bundle uses SDK-version-dependent evaluators: verified availability (see Step 4.5) -- [ ] If callable: `python -c "import sys; sys.path.insert(0, '.agentops'); from callable_adapter import run_evaluation; print('OK')"` succeeds -- [ ] If callable: `AGENT_HTTP_URL` env var is set -- [ ] If callable with auth: auth token env var is set (`APP_API_TOKEN`, `API_KEY`, etc.) -- [ ] **Callable smoke test**: one real call succeeds (see subsection below) -- [ ] If Foundry: `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` env var is set -- [ ] If bundle has `source: foundry` evaluators: evaluator model is configured (`endpoint.model` or `AZURE_OPENAI_ENDPOINT` + `AZURE_OPENAI_DEPLOYMENT`) -- [ ] Azure auth: `az account show` succeeds OR `AZURE_OPENAI_API_KEY` is set -- [ ] Endpoint reachable: `curl -s -o /dev/null -w "%{http_code}" ` returns 200/401/405 (not connection refused) -- [ ] Evaluator model responds: `az cognitiveservices account deployment list --name -g ` confirms deployment exists - -Present a **confirmation table** with all discovered values (do not ask each one separately): -``` -┌─────────────────────────┬──────────────────────────────────────────┬────────┐ -│ Setting │ Value │ Source │ -├─────────────────────────┼──────────────────────────────────────────┼────────┤ -│ Scenario │ RAG │ code │ -│ Bundle │ rag_quality_baseline │ auto │ -│ Endpoint URL │ https://myapp.azurecontainerapps.io/chat │ .env │ -│ Auth │ dapr-api-token (APP_API_TOKEN) │ code │ -│ Evaluator model │ gpt-4o-mini │ Azure │ -│ Project endpoint │ https://acct.services.ai.azure.com/... │ .env │ -│ Azure auth │ az login active │ CLI │ -│ Endpoint reachable │ ✔ (200) │ check │ -│ Dataset rows │ 8 │ file │ -└─────────────────────────┴──────────────────────────────────────────┴────────┘ -``` - -Ask: *"Everything look correct? (yes / edit)"* - -### Callable smoke test - -A single real end-to-end call catches auth issues (401), wrong request body fields (400/422), and response parsing problems BEFORE wasting an entire evaluation run. - -```bash -python -c " -import sys; sys.path.insert(0, '.agentops') -from callable_adapter import run_evaluation -result = run_evaluation('hello', {}) -assert 'response' in result, f'Missing response key: {result}' -resp = result['response'] -assert not resp.startswith('ERROR:'), f'Adapter error: {resp}' -assert len(resp.strip()) > 0, 'Empty response — check endpoint and request format' -print('Smoke test PASSED') -print(f'Response length: {len(resp)} chars') -print('Response preview:', resp[:200]) -" -``` - -If the smoke test fails: -- **Connection refused** → the agent endpoint is not running. Start it first. -- **401 Unauthorized** → auth token is missing or wrong. Check `AGENT_AUTH_HEADER` and `AGENT_AUTH_TOKEN` env vars. -- **400/422** → the request body format doesn't match the endpoint. Check the `json.dumps({"message": ...})` field name in the adapter — the endpoint may expect a different key (e.g. `"ask"`, `"question"`, `"input"`). -- **Response starts with `ERROR:`** → the adapter caught an exception. Read the error message. -- **Empty response** → the endpoint returned successfully but the adapter extracted no text. Check `response_field` / `.get()` key in the adapter. -- **Response contains unexpected prefix** (UUID, metadata, HTML) → add a post-processing step to the adapter to strip it. Common pattern: `re.sub(r'^[0-9a-f-]{36}\s*', '', response_text)` for UUID prefixes. - -### Smoke test response format verification - -After the basic smoke test passes, verify the response format matches expectations: -1. If the response contains HTML tags (``, `

`, etc.) but the adapter expects plain text → the endpoint may be returning an error page, not agent output. -2. If the response is very short (< 10 chars) for a conversational prompt like "hello" → warn the user: *"Response seems unusually short. Verify the endpoint is returning the full agent response."* -3. If the response starts with `data:` or contains SSE markers but the adapter uses the standard JSON template → switch to the SSE/streaming adapter template. - -Do NOT proceed to Step 7 until the smoke test passes. - -## Step 7 — Execute - -Ask the user: *"Ready to run the evaluation?"* - -If yes: -```bash -agentops eval run -f all -``` - -After it completes, read `.agentops/results/latest/report.md` and summarize the results. - -## Comparing Runs - -For multi-model benchmarks, create one run.yaml per model: -```bash -agentops eval run -c .agentops/run-modelA.yaml -agentops eval run -c .agentops/run-modelB.yaml -agentops eval compare --runs , -f html -``` - -For agent version comparison, change `agent_id` per run. - -## Commands Reference - -```bash -agentops init # Scaffold workspace -agentops eval run [-c run.yaml] [-f md|html|all] # Run evaluation -agentops eval compare --runs id1,id2 [-f md|html|all] # Compare runs -agentops report generate [--in results.json] # Regenerate report -``` - -## Exit Codes - -- `0` — all thresholds passed -- `2` — threshold(s) failed -- `1` — runtime or configuration error - -## Rules - -- **NEVER** include `backend:` key in run.yaml — it causes a runtime error. -- **NEVER** leave `` placeholders in any generated file. -- **NEVER** fabricate `agent_id`, model names, or endpoint URLs. -- **NEVER** edit `.agentops/` template files (`run-callable.yaml`, `run-http-rag.yaml`, etc.) — always update `.agentops/run.yaml`. -- **NEVER** use dotted import paths like `.agentops.callable_adapter` — they fail. -- **NEVER** create files outside `.agentops/` — all generated artifacts (adapters, datasets, configs, scripts) belong in `.agentops/`. Exception: unit tests go in the project's existing test directory. -- **NEVER** try `az login` automatically — ask the user to authenticate. -- **NEVER** use `requests` or `httpx` in callable adapters — use only stdlib (`urllib.request`, `json`, `os`). -- If a bundle uses SDK-version-dependent evaluators, verify availability before running (Step 4.5). Don't block on this — if verification is hard, proceed and fix on failure. -- Always update `.agentops/run.yaml` — do not create custom-named files except for multi-model benchmarks. -- Use generic file names: `dataset.yaml`, `data.jsonl` — not project-specific prefixes. -- Use plain language in questions — not technical jargon ("callable adapter", "SSE", "POST"). -- Always run pre-flight (Step 6) before executing. Fix all issues first. - -## Unit Test Generation (Optional) - -This section is only executed if the user accepted the unit test offer in Step 1. - -### When to generate - -- The codebase has Python, JavaScript, or TypeScript agent code with testable logic (endpoint handlers, tool definitions, response parsing, orchestration). -- No existing test files or test directories were detected. - -### What to generate - -Create tests in the project's conventional test directory (e.g. `tests/test_agent.py` for Python, `__tests__/agent.test.ts` for TypeScript). Use only standard testing libraries — no extra dependencies. - -**For Python agents**, generate `pytest` tests using `unittest.mock`: - -1. **Endpoint handler test** — mock the HTTP framework (FastAPI `TestClient`, Flask `test_client`) and verify the handler returns expected response format. -2. **Response parsing test** — if the agent has response parsing logic (JSON extraction, SSE chunk assembly, UUID stripping), test it with known inputs/outputs. -3. **Error handling test** — verify the agent handles timeouts, 4xx/5xx from downstream services, and malformed inputs gracefully. -4. **Tool schema test** (if applicable) — if the agent defines tools with schemas, validate the schema structure is correct (required fields, types). - -**Template pattern** (adapt to the detected code): -```python -"""Unit tests for agent endpoint — generated by AgentOps.""" -import json -from unittest.mock import MagicMock, patch - -import pytest - - -class TestAgentEndpoint: - """Tests for the agent's HTTP endpoint handler.""" - - def test_returns_valid_response_format(self): - # Mock the downstream model/service call - # Call the endpoint handler directly - # Assert response has expected keys and types - ... - - def test_handles_empty_input(self): - # Verify the agent handles empty or whitespace-only input - ... - - def test_handles_downstream_timeout(self): - # Mock the downstream call to raise a timeout - # Assert the agent returns an error response (not a crash) - ... -``` - -### Rules for generated tests - -- Tests must run **without** Azure credentials or live services — all external calls must be mocked. -- Do not generate tests that duplicate what AgentOps evaluations already cover (response quality, groundedness, coherence). -- Focus on **functional correctness**: does the code do what it's supposed to do? -- Place tests in the project's existing test directory structure, not in `.agentops/`. -- If the project uses a specific test runner or framework (detected via `pyproject.toml`, `package.json`, `conftest.py`), follow its conventions. +publish: foundry_cloud +# project_endpoint: "https://.services.ai.azure.com/api/projects/

" +``` + +Foundry-side latency and judges replace the local view in this mode; +`results.json` from the local run remains the canonical record. + +## Tips + +- Evaluators are auto-selected from the agent type and dataset columns. + Override only when needed via the `evaluators:` block — most users do + not need it. +- Set thresholds in `thresholds:` to gate CI: + ```yaml + thresholds: + coherence: ">=3" + avg_latency_seconds: "<=10" + ``` +- For HTTP/JSON agents that need auth, set + `auth_header_env: MY_TOKEN_VAR` and AgentOps adds + `Authorization: Bearer $MY_TOKEN_VAR`. diff --git a/src/agentops/templates/skills/agentops-monitor/SKILL.md b/src/agentops/templates/skills/agentops-monitor/SKILL.md deleted file mode 100644 index 67afa4c8..00000000 --- a/src/agentops/templates/skills/agentops-monitor/SKILL.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: agentops-monitor -description: Guidance on monitoring evaluation quality over time. Trigger when users ask about tracking scores, setting up dashboards, or configuring quality alerts. Common phrases include "monitoring", "dashboards", "alerts", "monitor setup", "quality over time", "trending", "track scores", "evaluation health". Install agentops-toolkit via pip. ---- - -# AgentOps Monitor - -## Purpose - -Provide guidance on monitoring evaluation quality over time. The `agentops monitor` commands are **planned but not yet implemented**. - -## Before You Start - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. - -## Status - -🚧 **Not yet implemented.** The CLI stubs exist but have no runtime behavior. - -## Current Alternatives - -Until `agentops monitor` is available: - -| Approach | How | -|---|---| -| Manual trending | Compare `results.json` across timestamped runs in `.agentops/results/` | -| CI gating | Use exit code `2` in GitHub Actions to block PRs on quality regressions | -| Foundry portal | View evaluation history in the Foundry Experience dashboard | -| Run comparison | `agentops eval compare --runs ,` for side-by-side delta | - -## What Will Be Available - -When implemented: -- `agentops monitor show` — Display evaluation quality dashboard -- `agentops monitor configure` — Set up alerts and quality thresholds - -## Guardrails - -- Do not pretend monitoring features exist — clearly state they are planned. -- For quality tracking today, recommend `agentops eval compare` and CI exit codes. -- For production monitoring, recommend Azure Monitor and Foundry portal. diff --git a/src/agentops/templates/skills/agentops-regression/SKILL.md b/src/agentops/templates/skills/agentops-regression/SKILL.md deleted file mode 100644 index 6a8d295b..00000000 --- a/src/agentops/templates/skills/agentops-regression/SKILL.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -name: agentops-regression -description: Investigate evaluation regressions — compare runs, analyze per-row scores, identify root causes. Trigger when users report score drops, threshold failures, or quality degradation between runs. Common phrases include "regression", "score dropped", "threshold failed", "compare runs", "why worse", "which rows failed", "debug evaluation", "quality degradation". Install agentops-toolkit via pip. ---- - -# AgentOps Regression - -## Purpose - -Investigate evaluation score drops and threshold failures. Compare runs side-by-side, identify which rows regressed, and guide root-cause analysis. - -## When to Use - -- Exit code `2` — thresholds failed. -- Scores dropped between two runs. -- User asks "why did this eval get worse" or "which rows failed". - -## Before You Start - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. -4. **Two runs available?** Need a baseline and a current run. Check `.agentops/results/` for timestamped directories. -5. **Results exist?** Each run must have `results.json`. - -## Steps - -### Step 1 — Identify the regression - -```bash -agentops eval compare --runs , -``` - -Review the comparison output for ↓ indicators and delta values. - -### Step 2 — Analyze per-row scores - -Open `results.json` for both runs. Compare `row_metrics` to find rows where scores dropped. Look for: -- Rows with the largest negative delta -- Rows that went from pass → fail -- Clusters of failures in specific evaluators - -### Step 3 — Check what changed - -Common regression causes: -| Cause | What to check | -|---|---| -| Model update | Deployment version, model name change | -| Prompt drift | System prompt or instructions changed | -| Data drift | New dataset rows, different distribution | -| Tool schema change | Tool definitions modified | -| Context quality | RAG retriever returning different passages | -| Threshold tightened | Bundle threshold values changed | - -### Step 4 — Act on findings - -| Finding | Action | -|---|---| -| Model regression | Pin model version or switch deployment | -| Prompt issue | Revert or iterate on prompt changes | -| Bad test rows | Fix dataset and re-run | -| Threshold too strict | Adjust thresholds in bundle (use `/agentops-config`) | -| Retriever degraded | Debug retrieval pipeline separately | - -### Step 5 — Verify fix - -Re-run the evaluation after the fix: -```bash -agentops eval run -agentops eval compare --runs ,latest -``` - -## Guardrails - -- Work with actual scores — never guess what caused a regression. -- Do not modify `results.json` — it is immutable. -- Do not adjust thresholds to hide real regressions. -- Delegate execution to `/agentops-eval` and config changes to `/agentops-config`. diff --git a/src/agentops/templates/skills/agentops-report/SKILL.md b/src/agentops/templates/skills/agentops-report/SKILL.md index dc10fd8f..72ed2bd4 100644 --- a/src/agentops/templates/skills/agentops-report/SKILL.md +++ b/src/agentops/templates/skills/agentops-report/SKILL.md @@ -1,92 +1,69 @@ --- name: agentops-report -description: Interpret evaluation reports, explain indicators, and regenerate reports. Trigger when users ask to understand results, explain scores, or regenerate a report. Common phrases include "report", "interpret results", "what does this mean", "explain scores", "report generate", "results.json", "pass rate", "threshold". Install agentops-toolkit via pip. +description: Read, regenerate, and explain AgentOps evaluation reports. Trigger on "show report", "explain scores", "regenerate report", "what do these metrics mean". Operates on results.json and report.md produced by `agentops eval run`. --- # AgentOps Report -## Purpose +Help the user understand a finished AgentOps run. -Help users understand evaluation results, explain report indicators, and regenerate reports from existing `results.json` files. +## Step 0 — Locate the run -## When to Use +Latest run: `.agentops/results/latest/`. Each run produces: -- User asks what an evaluation result means. -- User wants to regenerate a report after manual edits. -- User needs to compare report sections between runs. -- User asks about pass rates, thresholds, or score meanings. +- `results.json` — machine-readable metrics, per-row scores, thresholds. +- `report.md` — human-readable summary suitable for PR comments. +- `cloud_evaluation.json` (only when `publish:` was set) — deep-link to + the Foundry Evaluations panel. `mode: classic` for `publish: foundry`, + `mode: cloud` for `publish: foundry_cloud` (preview, server-side run + via the OpenAI Evals API). -## Before You Start +## Step 1 — Regenerate report.md if needed -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Results exist?** Check for `.agentops/results/latest/results.json`. If missing, run `/agentops-eval` first. -4. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. - -## Commands - -| Command | Purpose | -|---|---| -| `agentops report generate --in [--out ]` | Regenerate report from results | - -## Report Indicators - -| Symbol | Meaning | -|---|---| -| `●` (green) | Score meets or exceeds threshold | -| `●` (red) | Score below threshold | -| `↑` | Score improved vs. baseline | -| `↓` | Score regressed vs. baseline | -| `—` | No baseline available | - -## Key Metrics - -| Metric | Description | -|---|---| -| `run_pass` | `true` if all thresholds passed | -| `threshold_pass_rate` | Fraction of thresholds met | -| `items_pass_rate` | Fraction of rows passing all evaluators | -| per-evaluator avg | Mean score across all rows for one evaluator | -| per-evaluator stddev | Standard deviation (high = inconsistent) | +```bash +agentops report generate # uses .agentops/results/latest/results.json +agentops report generate --in --out +``` -## Report Sections +`report generate` always reads the flat 1.0 results schema and emits +Markdown. There is no HTML format. -### Single Run (`report.md`) -- **Summary**: overall pass/fail, item counts -- **Threshold Results**: per-evaluator threshold vs. actual score -- **Row Details**: per-row scores for each evaluator +## Step 2 — Explain the metrics -### Comparison (`agentops eval compare`) -- **Side-by-side**: baseline vs. current scores -- **Delta**: absolute change per evaluator -- **Direction**: ↑ improved, ↓ regressed, — unchanged +Common metrics and their meaning: -## Steps +| Metric | Range | Higher is better? | Notes | +|---|---|---|---| +| `similarity` | 1-5 | yes | LLM-judged similarity to `expected`. | +| `coherence` | 1-5 | yes | Answer is internally consistent. | +| `fluency` | 1-5 | yes | Natural language quality. | +| `groundedness` | 1-5 | yes | Answer is supported by `context` (RAG). | +| `relevance` | 1-5 | yes | Answer is on-topic for `input`. | +| `f1_score` | 0-1 | yes | Token overlap with `expected`. | +| `tool_call_accuracy` | 0-1 | yes | Predicted tool calls match `tool_calls`. | +| `intent_resolution` | 0-1 | yes | User intent was resolved. | +| `task_completion` | 0-1 | yes | Multi-step task finished. | +| `avg_latency_seconds` | seconds | no | Wall-clock latency per row. | -### Interpreting results -1. Open `.agentops/results/latest/report.md`. -2. Check the summary — is `run_pass: true`? -3. If false, find which thresholds failed (red dots). -4. Look at per-row scores to identify weak rows. -5. For AI evaluators (coherence, groundedness), scores are 1–5. -6. For content safety evaluators, lower is better (0 = safe). +Pass/fail rows are derived from `thresholds:` in `agentops.yaml`. The +exit code of the original run reflects the gate: -### Regenerating a report -```bash -agentops report generate --in .agentops/results/latest/results.json -``` +- `0` → all thresholds passed +- `2` → one or more thresholds failed +- `1` → runtime error -## Exit Codes +## Step 3 — Help the user act on results -| Code | Meaning | -|---|---| -| `0` | Success and all thresholds passed | -| `2` | Success but threshold(s) failed | -| `1` | Runtime or configuration error | +- For low scores on a specific metric, point at the lowest-scoring rows + in `results.json` (`row_metrics[]` and `item_evaluations[]`) and + suggest concrete prompt or retrieval changes. +- For latency regressions, look at `run_metrics.avg_latency_seconds` and + per-row latency. +- To compare two runs, diff the two `results.json` files at the metric + level and surface the deltas; AgentOps does not ship a separate + comparison CLI. ## Guardrails -- Use actual scores from `results.json` — never guess or estimate. -- Do not run evaluations — delegate to `/agentops-eval`. -- Do not modify `results.json` — it is an immutable run artifact. -- If the user needs different thresholds, delegate to `/agentops-config` to update the bundle. +- Never invent metric values. If a metric is absent, say so. +- Do not edit `results.json` by hand — re-run the eval. diff --git a/src/agentops/templates/skills/agentops-trace/SKILL.md b/src/agentops/templates/skills/agentops-trace/SKILL.md deleted file mode 100644 index 33435e9e..00000000 --- a/src/agentops/templates/skills/agentops-trace/SKILL.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: agentops-trace -description: Guidance on tracing for AgentOps evaluations. Trigger when users ask about tracing agent execution, setting up telemetry, or inspecting spans. Common phrases include "tracing", "trace init", "trace setup", "distributed tracing", "span", "telemetry", "trace evaluation", "trace agent". Install agentops-toolkit via pip. ---- - -# AgentOps Trace - -## Purpose - -Provide guidance on tracing agent execution. The `agentops trace` command is **planned but not yet implemented**. - -## Before You Start - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`. If not found, ask the user for the endpoint URL and instruct them to set it. - -## Status - -🚧 **Not yet implemented.** The CLI stub exists but has no runtime behavior. - -## Current Alternatives - -Until `agentops trace` is available, use these tools directly: - -| Tool | Use case | -|---|---| -| Azure Monitor / Application Insights | Production tracing for Foundry agents | -| OpenTelemetry SDK | Custom span instrumentation | -| Foundry portal | Built-in agent execution traces | -| `results.json` row metrics | Per-row latency via `avg_latency_seconds` | - -## What Will Be Available - -When implemented, `agentops trace init` will: -- Configure OpenTelemetry export for AgentOps evaluation runs -- Capture per-row agent execution spans -- Link traces to evaluation results for debugging - -## Guardrails - -- Do not pretend tracing features exist — clearly state they are planned. -- For latency analysis, point users to `avg_latency_seconds` in evaluation bundles. -- For production tracing, recommend Azure Monitor or OpenTelemetry directly. diff --git a/src/agentops/templates/skills/agentops-workflow/SKILL.md b/src/agentops/templates/skills/agentops-workflow/SKILL.md index 79d70bfa..b81a98a1 100644 --- a/src/agentops/templates/skills/agentops-workflow/SKILL.md +++ b/src/agentops/templates/skills/agentops-workflow/SKILL.md @@ -1,165 +1,152 @@ --- name: agentops-workflow -description: Generate CI/CD pipelines tailored to the project — PR gating, post-merge CI evaluation, and CD with safety QA + deploy placeholder. Trigger when users ask to automate evaluations in CI, set up PR gating, generate workflow files, or create pipelines for their project. Common phrases include "CI/CD", "GitHub Actions", "pipeline", "workflow", "PR gating", "continuous evaluation", "automate evals", "workflow generate", "CI setup", "generate pipelines", "create pipelines for my project". Install agentops-toolkit via pip. +description: Set up the full GenAIOps GitFlow CI/CD scaffold for an AgentOps project. Generates four GitHub Actions workflows (PR gate + Deploy DEV / QA / PROD) wired to GitHub Environments, OIDC auth, and AgentOps eval gating. Trigger on "CI", "CD", "pipeline", "workflow", "GitHub Actions", "PR gate", "deploy", "environments", "GitFlow", "release branch", "promote to prod", "DevOps", "GenAIOps pipeline". --- # AgentOps Workflow -Generate a complete CI/CD pipeline suite for AgentOps evaluations — tailored to the project's evaluation scenarios, bundles, and Foundry configuration. +Help the user wire AgentOps into a real GenAIOps GitFlow CI/CD setup with +three environments (`dev`, `qa`, `production`) and an automatic eval gate +on every change. -## Pipeline Types +This skill produces four workflow files via `agentops workflow generate` +and then walks the user through the GitHub-side configuration (OIDC, +environments, branch protection, deploy step). -`agentops workflow generate` auto-detects which pipelines to create: +## Branch model assumed -| Pipeline | File | When generated | Purpose | -|---|---|---|---| -| **PR Evaluation** | `agentops-eval.yml` | Always | Fast evaluation gate on pull requests | -| **CI Evaluation** | `agentops-eval-ci.yml` | Multiple bundles or run configs detected | Full evaluation on merge to develop/main | -| **CD Pipeline** | `agentops-eval-cd.yml` | Multiple bundles or run configs detected | Safety QA gate + deploy placeholder on merge to main | - -### Pipeline Flow (GenAIOps-inspired) - -``` -feature/* → PR to develop → agentops-eval.yml (PR gate) - merge to develop → agentops-eval-ci.yml (CI evaluation) - release/* → PR to main → agentops-eval.yml (PR gate) - merge to main → agentops-eval-cd.yml (safety QA → deploy) -``` - -## Step 0 — Prerequisites - -1. **AgentOps installed?** Check if `agentops` CLI is available. If not: `pip install agentops-toolkit`. -2. **Workspace exists?** Check for `.agentops/`. If missing: `agentops init`. -3. **Foundry endpoint configured?** Search for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` in environment variables, `.env`, `.env.local`, `.azure//.env`. If not found, ask the user for the endpoint URL. -4. **run.yaml ready?** A valid run config is required. If missing, delegate to `/agentops-config`. - -## Step 1 — Workspace Inspection - -Before generating, inspect the workspace to understand what pipelines are needed: - -1. **List bundles**: Read `.agentops/bundles/` — identify which evaluation scenarios are configured. -2. **List run configs**: Check `.agentops/` for `run*.yaml` files — if multiple configs exist, CI and CD pipelines are appropriate. -3. **Check Foundry endpoint**: Look for `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` or `project_endpoint` in run.yaml and env vars. -4. **Detect branches**: Run `git branch -a` to list local and remote branches. - - If `main` and `develop` exist → use them (default convention, no question needed). - - If branches don't exist yet → use `main`/`develop` convention (no question needed). - - If the repo uses different names (e.g. `master` instead of `main`, or no `develop`) → ask the user to confirm which branches to use for PR targets and push triggers. - -Present a summary: -``` -Detected: - Bundles: model_quality_baseline, rag_quality_baseline - Run configs: run.yaml - Foundry endpoint: ✓ (from .env) - Branches: main, develop - Pipelines: PR (always), CI + CD (multiple bundles detected) ``` - -## Step 2 — Ask Only What Cannot Be Inferred - -Only ask critical questions that workspace inspection cannot answer: - -1. If no Foundry endpoint found: *"What is your Azure AI Foundry project endpoint URL?"* -2. If branches differ from the `main`/`develop` convention: *"Your repo uses `master` instead of `main`. Should the pipelines target `master`, or do you plan to rename it to `main`?"* - -**DO NOT ask about**: -- Bundle selection (inferred from workspace) -- Evaluation scenarios (inferred from bundles) -- Authentication method (always OIDC / Workload Identity Federation) -- Workflow file locations (standard `.github/workflows/` paths) -- Which pipelines to generate (auto-detected) - -## Step 3 — Generate Workflows - -```bash -agentops workflow generate [--force] [--dir ] +feature/* ── PR ──▶ develop [agentops-pr.yml] gate + │ + └── merge ─▶ develop [agentops-deploy-dev.yml] build + eval + deploy DEV +release/* ── push [agentops-deploy-qa.yml] build + eval + deploy QA +release/* ── PR ──▶ main [agentops-pr.yml] gate + │ + └── merge ─▶ main [agentops-deploy-prod.yml] safety eval + build + deploy PROD ``` -Flags: -- `--force` — Overwrite existing workflow files. -- `--dir` — Target directory (default: current directory). - -After generation, explain what was created and why: -- `agentops-eval.yml` — Runs on PRs to main/develop. Gates merges on evaluation thresholds. -- `agentops-eval-ci.yml` — (if generated) Runs on push to develop/main when `.agentops/`, `src/`, or `pyproject.toml` change. Comprehensive post-merge evaluation with commented-out matrix strategy and baseline comparison. -- `agentops-eval-cd.yml` — (if generated) Runs on push to main. Two-job pipeline: safety QA evaluation gate → deploy placeholder. The deploy job is a TODO for the team to fill in with their deployment commands. - -## Step 4 — Configure Authentication - -All pipelines use **Workload Identity Federation (OIDC)** — no client secrets to manage or rotate. - -### Azure Setup (one-time) - -1. **Create or reuse an App Registration** in Microsoft Entra ID (Azure AD). -2. **Add a Federated Credential**: - - Go to App Registration → Certificates & secrets → Federated credentials → Add credential - - Organization: your GitHub org/user - - Repository: your repo name - - Entity type: select **Pull Request** (for PR pipeline) AND **Branch** (for CI and CD pipelines) - - Name: e.g. `github-agentops-eval` -3. **Grant the app required roles** on the Foundry project resource group: - - `Cognitive Services User` — invoke agents and evaluator models - - `Azure AI Developer` — access evaluation APIs and Foundry features +If the user is on trunk-based development, omit `qa` and `release/**` +and have them generate `--kinds pr,dev,prod`. -### GitHub Setup - -Set these as **repository variables** (Settings → Secrets and variables → Actions → Variables tab): - -| Variable | Value | -|---|---| -| `AZURE_CLIENT_ID` | Application (client) ID from App Registration | -| `AZURE_TENANT_ID` | Directory (tenant) ID | -| `AZURE_SUBSCRIPTION_ID` | Azure subscription ID | - -Set this as a **repository secret** (Secrets tab): +## Step 0 — Prerequisites -| Secret | Value | -|---|---| -| `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` | Foundry project endpoint URL | +1. `pip install agentops-toolkit` if `agentops` is missing. +2. `.agentops/run.yaml` exists and `agentops eval run` works locally. +3. The user's repo follows GitFlow (or is willing to). If not, ask which + branches map to dev/qa/prod and adjust the `on:` triggers after + generation. -### Verify Auth Locally +## Step 1 — Generate the workflows ```bash -az login -az account show --query "{sub:id, tenant:tenantId}" -o json -az account get-access-token --resource "https://cognitiveservices.azure.com" --query accessToken -o tsv +agentops workflow generate ``` -## Step 5 — Verify Pipelines - -1. **PR pipeline**: Push a branch and open a PR → check the Actions tab for `AgentOps Evaluation`. -2. **CI pipeline**: Merge to develop → check Actions tab for `AgentOps CI Evaluation`. -3. **CD pipeline**: Merge to main → check Actions tab for `AgentOps CD Pipeline`. The safety-qa job runs evaluation; the deploy job prints a placeholder notice. -4. **Check results**: Download artifacts, review PR comments, inspect job summaries. - -If any pipeline fails with authentication errors: -- Verify federated credential entity types match (Pull Request for PRs, Branch for push) -- Confirm the App Registration has `Cognitive Services User` role on the Foundry resource -- Check that variables and secrets are set at the repository level (not organization) - -## Exit Code Gating - -All pipelines use the same exit code contract: +This writes **four** files into `.github/workflows/`: -| Exit code | CI result | Meaning | +| File | Trigger | Environment | |---|---|---| -| `0` | ✅ Pass | All thresholds met | -| `2` | ❌ Fail | Threshold(s) failed — blocks merge / blocks deploy | -| `1` | ❌ Fail | Runtime or configuration error | - -## Customisation After Generation - -- **Change branch triggers**: Edit `on.pull_request.branches` or `on.push.branches` in the workflow files. -- **Enable matrix strategy**: Uncomment the `strategy.matrix` block in `agentops-eval-ci.yml` and list your run configs. -- **Enable baseline comparison**: Uncomment the comparison step in `agentops-eval-ci.yml`. -- **Add deployment steps**: Edit the `deploy` job in `agentops-eval-cd.yml` — replace the placeholder with your actual deployment commands. -- **Add environment approval**: Uncomment `environment: production` in the deploy job for manual approval gates. - -## Rules - -- Do not modify generated workflow files beyond user-requested customisation. -- Always recommend OIDC / Workload Identity Federation over client secrets. -- Delegate evaluation configuration to `/agentops-config`. -- Delegate dataset creation to `/agentops-dataset`. -- Do not fabricate endpoint URLs, agent IDs, or deployment names. -- Do not ask about bundle/scenario selection if it can be inferred from the workspace. +| `agentops-pr.yml` | PRs to `develop`, `release/**`, `main` | (none) | +| `agentops-deploy-dev.yml` | push to `develop` | `dev` | +| `agentops-deploy-qa.yml` | push to `release/**` | `qa` | +| `agentops-deploy-prod.yml` | push to `main` | `production` | + +Useful flags: + +- `--force` — overwrite existing workflow files. +- `--kinds pr,dev,qa,prod` — generate a subset (e.g. `--kinds pr,dev,prod` + for trunk-based teams). +- `--dir ` — non-default repo root. + +## Step 2 — Configure GitHub Environments + +Walk the user through Settings → Environments and create three: + +1. **`dev`** — no extra protection. Set any DEV-specific variables here + (e.g. `ACA_APP_NAME`, `AZURE_RESOURCE_GROUP` pointing at the dev RG). +2. **`qa`** — usually no required reviewers, but isolated variables for + the QA environment. +3. **`production`** — set: + - **Required reviewers**: at least one (deploys to PROD will pause + here until approved). + - (Optional) **Wait timer** for an extra delay. + - (Optional) **Deployment branches**: restrict to `main`. + - PROD-specific variables (e.g. production resource group). + +Tell the user that env-specific variables on the `production` environment +will override repo-level ones automatically inside the prod workflow. + +## Step 3 — Configure repository variables for OIDC + +At repository level (Settings → Secrets and variables → Actions → +**Variables** tab), set: + +- `AZURE_CLIENT_ID` — App registration / managed identity used for OIDC. +- `AZURE_TENANT_ID` +- `AZURE_SUBSCRIPTION_ID` +- `AZURE_AI_FOUNDRY_PROJECT_ENDPOINT` — Foundry project URL used by the + eval step. + +Then configure Workload Identity Federation on the Azure side +(`federated-credentials` on the app registration) for **each branch / +environment** the workflows will run from. See +`docs/ci-github-actions.md` for the exact `az` commands. + +## Step 4 — Fill in the Build and Deploy placeholders + +Each `agentops-deploy-*.yml` has a `Build (placeholder)` and a +`Deploy (placeholder)` step. The dev template includes commented +example snippets for the most common stacks. Replace them based on +the user's stack: + +- **Container Apps** — replace Build with `az acr build` and Deploy + with `az containerapp update --image ...`. +- **App Service** — replace Build with the package step, Deploy with + `azure/webapps-deploy@v3`. +- **Foundry hosted agent** — Build is typically empty; Deploy publishes + a new agent version (project-specific tooling). +- **azd-managed app** — replace Build with `azd package` and Deploy + with `azd deploy --no-prompt` (set `AZURE_ENV_NAME` per environment). + +Don't invent commands you can't see in the user's repo. If the stack +isn't obvious, ask. + +## Step 5 — Branch protection + +In Settings → Branches, add a rule for both `develop` and `main`: + +- Require a pull request before merging. +- Require status checks to pass: select **`AgentOps PR / Eval (PR gate)`** + (the job name from `agentops-pr.yml`). +- Optional: require linear history. + +This makes the eval gate a hard merge requirement. + +## Step 6 — Iterate + +Common follow-ups: + +- **Tighten thresholds for QA/PROD** — copy `.agentops/run.yaml` to + `.agentops/run-qa.yaml` / `.agentops/run-prod.yaml` and tighten the + bundle thresholds. Point each workflow at its own config via the + `inputs.config` default. +- **Scheduled runs** — add a `schedule:` entry in `agentops-pr.yml` (or a + new `agentops-nightly.yml`) to evaluate against `main` nightly. +- **Matrix per scenario** — if the user has multiple `runs/*.yaml` files, + extend the eval job with `strategy.matrix.config:` and reference + `${{ matrix.config }}`. +- **Regression baseline** — wire the deploy templates to download the + previous run's `results.json` artifact and call + `agentops eval compare`. + +## Guardrails + +- Do **not** invent CLI flags. The supported `workflow generate` flags + are `--force`, `--dir`, `--kinds`. +- Do **not** create parallel workflow files. Prefer editing the + generated ones. +- Do **not** auto-fill Build/Deploy with steps you can't justify from + the user's existing code. Ask before guessing. +- The four workflow names (`agentops-pr`, `agentops-deploy-dev`, + `agentops-deploy-qa`, `agentops-deploy-prod`) are fixed — don't rename + them or branch-protection wiring will break. diff --git a/src/agentops/templates/smoke.jsonl b/src/agentops/templates/smoke.jsonl new file mode 100644 index 00000000..b2246374 --- /dev/null +++ b/src/agentops/templates/smoke.jsonl @@ -0,0 +1,3 @@ +{"input": "What is AgentOps?", "expected": "AgentOps is a CLI for evaluating Foundry agents."} +{"input": "Which formats does it produce?", "expected": "It writes results.json and report.md."} +{"input": "How do I configure thresholds?", "expected": "Use the 'thresholds' map in agentops.yaml."} diff --git a/src/agentops/templates/workflows/agentops-deploy-dev.yml b/src/agentops/templates/workflows/agentops-deploy-dev.yml new file mode 100644 index 00000000..10e1a8dc --- /dev/null +++ b/src/agentops/templates/workflows/agentops-deploy-dev.yml @@ -0,0 +1,185 @@ +# AgentOps — Deploy to DEV +# +# Triggers on every push to `develop`. Runs the AgentOps eval as a +# quality gate, then builds and deploys to the `dev` GitHub Environment. +# +# To finish wiring this for your project: +# 1. Configure the `dev` GitHub Environment (Settings -> Environments). +# Add any environment-specific variables/secrets there. +# 2. Fill in the `Build` and `Deploy` steps below to match your stack +# (see commented examples). +# 3. Make sure the AgentOps eval passes locally first: +# agentops eval run +# +# Generated by `agentops workflow generate`. + +name: AgentOps Deploy (DEV) + +on: + push: + branches: + - develop + workflow_dispatch: + inputs: + config: + description: "Path to agentops.yaml (defaults to ./agentops.yaml at repo root)" + required: false + default: "agentops.yaml" + +permissions: + contents: read + id-token: write + packages: write + +concurrency: + group: agentops-deploy-dev-${{ github.ref }} + cancel-in-progress: false + +jobs: + eval: + name: Eval (gate) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install AgentOps Toolkit + run: | + python -m pip install --upgrade pip + # NOTE: pinned to develop branch until AgentOps 1.0 lands on PyPI. + # Switch to `pip install "agentops-toolkit[foundry]"` after the 1.0 release. + python -m pip install "agentops-toolkit[foundry] @ git+https://github.com/Azure/agentops.git@develop" + + - name: Run AgentOps eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + run: agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" + + - name: Upload AgentOps results + if: always() + uses: actions/upload-artifact@v4 + with: + name: agentops-dev-results + path: | + .agentops/results/latest/results.json + .agentops/results/latest/report.md + .agentops/results/latest/cloud_evaluation.json + if-no-files-found: warn + + - name: Step summary + if: always() + run: | + { + echo "## AgentOps Eval (DEV gate)" + echo + if [ -f .agentops/results/latest/report.md ]; then + cat .agentops/results/latest/report.md + fi + } >> "$GITHUB_STEP_SUMMARY" + + build: + name: Build + needs: eval + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + # --------------------------------------------------------------- + # TODO: replace this placeholder with your build step. + # + # Examples (uncomment one): + # + # # Python wheel: + # - uses: actions/setup-python@v5 + # with: { python-version: "3.11" } + # - run: pip install build && python -m build + # + # # Docker image to ACR (server-side build): + # - run: | + # az acr build \ + # --registry "${{ vars.ACR_NAME }}" \ + # --image "myapp:${{ github.sha }}" \ + # . + # + # # azd package: + # - uses: Azure/setup-azd@v2 + # - run: azd package --no-prompt + # --------------------------------------------------------------- + - name: Build (placeholder) + run: | + echo "TODO: replace this step with your build." + echo "See the commented examples in this workflow." + + deploy: + name: Deploy to DEV + needs: build + runs-on: ubuntu-latest + environment: dev + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + # --------------------------------------------------------------- + # TODO: replace this placeholder with your deploy step. + # + # Examples (uncomment one): + # + # # Azure Container Apps: + # - run: | + # az containerapp update \ + # --name "${{ vars.ACA_APP_NAME }}" \ + # --resource-group "${{ vars.AZURE_RESOURCE_GROUP }}" \ + # --image "${{ vars.ACR_NAME }}.azurecr.io/myapp:${{ github.sha }}" + # + # # Azure App Service: + # - uses: azure/webapps-deploy@v3 + # with: + # app-name: ${{ vars.WEBAPP_NAME }} + # package: ./dist + # + # # Foundry hosted agent (publish a new agent version): + # - run: | + # agentops agent publish \ + # --config agentops.yaml \ + # --tag dev-${{ github.sha }} + # + # # azd up / deploy: + # - uses: Azure/setup-azd@v2 + # - run: azd deploy --no-prompt + # env: + # AZURE_ENV_NAME: dev + # --------------------------------------------------------------- + - name: Deploy (placeholder) + run: | + echo "TODO: replace this step with your deploy." + echo "Target environment: dev" diff --git a/src/agentops/templates/workflows/agentops-deploy-prod.yml b/src/agentops/templates/workflows/agentops-deploy-prod.yml new file mode 100644 index 00000000..6373aac9 --- /dev/null +++ b/src/agentops/templates/workflows/agentops-deploy-prod.yml @@ -0,0 +1,149 @@ +# AgentOps — Deploy to PRODUCTION +# +# Triggers on every push to `main` (typically the merge of a release +# branch). Runs a safety/quality eval, builds, and deploys to the +# `production` GitHub Environment. +# +# IMPORTANT: configure the `production` environment with REQUIRED +# REVIEWERS so a deploy to prod always needs human approval: +# Repo Settings -> Environments -> production +# -> Required reviewers (add at least one) +# -> (optional) Wait timer +# -> (optional) Deployment branches: only `main` +# +# To finish wiring this for your project: +# 1. Configure the `production` environment as described above. +# 2. Optionally point the eval step at a hardened safety bundle +# (e.g. `.agentops/runs/safety.yaml`). +# 3. Fill in the `Build` and `Deploy` steps below to match your stack +# (see commented examples in agentops-deploy-dev.yml). +# +# Generated by `agentops workflow generate`. + +name: AgentOps Deploy (PROD) + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + config: + description: "Path to agentops.yaml (defaults to ./agentops.yaml at repo root)" + required: false + default: "agentops.yaml" + +permissions: + contents: read + id-token: write + packages: write + +concurrency: + group: agentops-deploy-prod-${{ github.ref }} + cancel-in-progress: false + +jobs: + safety-eval: + name: Safety eval (gate) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install AgentOps Toolkit + run: | + python -m pip install --upgrade pip + # NOTE: pinned to develop branch until AgentOps 1.0 lands on PyPI. + # Switch to `pip install "agentops-toolkit[foundry]"` after the 1.0 release. + python -m pip install "agentops-toolkit[foundry] @ git+https://github.com/Azure/agentops.git@develop" + + - name: Run AgentOps eval (production gate) + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + run: agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" + + - name: Upload AgentOps results + if: always() + uses: actions/upload-artifact@v4 + with: + name: agentops-prod-results + path: | + .agentops/results/latest/results.json + .agentops/results/latest/report.md + .agentops/results/latest/cloud_evaluation.json + if-no-files-found: warn + + - name: Step summary + if: always() + run: | + { + echo "## AgentOps Safety Eval (PROD gate)" + echo + if [ -f .agentops/results/latest/report.md ]; then + cat .agentops/results/latest/report.md + fi + } >> "$GITHUB_STEP_SUMMARY" + + build: + name: Build + needs: safety-eval + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + # --------------------------------------------------------------- + # TODO: replace this placeholder with your build step. + # See agentops-deploy-dev.yml for example snippets. + # --------------------------------------------------------------- + - name: Build (placeholder) + run: | + echo "TODO: replace this step with your build." + + deploy: + name: Deploy to PROD + needs: build + runs-on: ubuntu-latest + environment: production + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + # --------------------------------------------------------------- + # TODO: replace this placeholder with your deploy step. + # See agentops-deploy-dev.yml for example snippets (ACA, App + # Service, Foundry hosted agent, azd deploy). + # --------------------------------------------------------------- + - name: Deploy (placeholder) + run: | + echo "TODO: replace this step with your deploy." + echo "Target environment: production" diff --git a/src/agentops/templates/workflows/agentops-deploy-qa.yml b/src/agentops/templates/workflows/agentops-deploy-qa.yml new file mode 100644 index 00000000..ecec9c61 --- /dev/null +++ b/src/agentops/templates/workflows/agentops-deploy-qa.yml @@ -0,0 +1,143 @@ +# AgentOps — Deploy to QA +# +# Triggers on every push to a `release/**` branch. Runs the AgentOps +# eval as a quality gate, then builds and deploys to the `qa` GitHub +# Environment. +# +# To finish wiring this for your project: +# 1. Configure the `qa` GitHub Environment (Settings -> Environments). +# Add any environment-specific variables/secrets there. +# 2. Fill in the `Build` and `Deploy` steps below to match your stack +# (see commented examples). +# 3. Recommended: tighten thresholds in your bundle for QA-grade gating. +# +# Generated by `agentops workflow generate`. + +name: AgentOps Deploy (QA) + +on: + push: + branches: + - "release/**" + workflow_dispatch: + inputs: + config: + description: "Path to agentops.yaml (defaults to ./agentops.yaml at repo root)" + required: false + default: "agentops.yaml" + +permissions: + contents: read + id-token: write + packages: write + +concurrency: + group: agentops-deploy-qa-${{ github.ref }} + cancel-in-progress: false + +jobs: + eval: + name: Eval (gate) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install AgentOps Toolkit + run: | + python -m pip install --upgrade pip + # NOTE: pinned to develop branch until AgentOps 1.0 lands on PyPI. + # Switch to `pip install "agentops-toolkit[foundry]"` after the 1.0 release. + python -m pip install "agentops-toolkit[foundry] @ git+https://github.com/Azure/agentops.git@develop" + + - name: Run AgentOps eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + run: agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" + + - name: Upload AgentOps results + if: always() + uses: actions/upload-artifact@v4 + with: + name: agentops-qa-results + path: | + .agentops/results/latest/results.json + .agentops/results/latest/report.md + .agentops/results/latest/cloud_evaluation.json + if-no-files-found: warn + + - name: Step summary + if: always() + run: | + { + echo "## AgentOps Eval (QA gate)" + echo + if [ -f .agentops/results/latest/report.md ]; then + cat .agentops/results/latest/report.md + fi + } >> "$GITHUB_STEP_SUMMARY" + + build: + name: Build + needs: eval + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + # --------------------------------------------------------------- + # TODO: replace this placeholder with your build step. + # See agentops-deploy-dev.yml for example snippets (wheel, + # docker, az acr build, azd package). + # --------------------------------------------------------------- + - name: Build (placeholder) + run: | + echo "TODO: replace this step with your build." + + deploy: + name: Deploy to QA + needs: build + runs-on: ubuntu-latest + environment: qa + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + # --------------------------------------------------------------- + # TODO: replace this placeholder with your deploy step. + # See agentops-deploy-dev.yml for example snippets (ACA, App + # Service, Foundry hosted agent, azd deploy). + # --------------------------------------------------------------- + - name: Deploy (placeholder) + run: | + echo "TODO: replace this step with your deploy." + echo "Target environment: qa" diff --git a/src/agentops/templates/workflows/agentops-eval-cd.yml b/src/agentops/templates/workflows/agentops-eval-cd.yml deleted file mode 100644 index 11d675aa..00000000 --- a/src/agentops/templates/workflows/agentops-eval-cd.yml +++ /dev/null @@ -1,160 +0,0 @@ -# AgentOps CD Pipeline — GitHub Actions Workflow -# -# Generated by: agentops workflow generate -# -# Triggered after merges to main (production release path). -# Runs safety evaluation in a QA gate before deployment. -# Includes a placeholder deploy job to be filled in by the team. -# -# Pipeline flow (GenAIOps-inspired): -# -# feature/* → PR to develop → agentops-eval.yml (PR gate) -# merge to develop → agentops-eval-ci.yml (CI evaluation) -# release/* → PR to main → agentops-eval.yml (PR gate) -# merge to main → THIS PIPELINE (CD: safety QA + deploy) -# -# Authentication: -# Uses Workload Identity Federation (OIDC) — no secrets to rotate. -# Set AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_SUBSCRIPTION_ID as GitHub -# repository variables (not secrets). See docs/ci-github-actions.md for setup. -# -# Prerequisites: -# 1. An initialised .agentops/ workspace in your repo (run `agentops init`) -# 2. A valid .agentops/run.yaml pointing to your bundle and dataset -# 3. Azure federated credential configured for your GitHub repo - -name: AgentOps CD Pipeline - -on: - push: - branches: [main] - workflow_dispatch: - inputs: - config: - description: "Path to run.yaml (default: .agentops/run.yaml)" - required: false - default: ".agentops/run.yaml" - skip_safety: - description: "Skip safety evaluation (use with caution)" - required: false - type: boolean - default: false - -permissions: - contents: read - id-token: write # Required for OIDC / Workload Identity Federation - -env: - PYTHON_VERSION: "3.11" - -jobs: - # ------------------------------------------------------------------ - # Job 1: Safety QA Gate - # Runs the full evaluation suite as a quality gate before deployment. - # This job MUST pass before the deploy job can proceed. - # ------------------------------------------------------------------ - safety-qa: - name: Safety QA Evaluation - runs-on: ubuntu-latest - if: ${{ github.event.inputs.skip_safety != 'true' }} - - env: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Azure login (OIDC) - uses: azure/login@v2 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - - - name: Install agentops-toolkit - run: pip install agentops-toolkit - - - name: Run safety evaluation - id: eval - run: | - set +e - CONFIG="${{ github.event.inputs.config || '.agentops/run.yaml' }}" - agentops eval run --config "$CONFIG" - EXIT_CODE=$? - echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" - - if [ $EXIT_CODE -eq 0 ]; then - echo "## ✅ Safety QA Passed" >> "$GITHUB_STEP_SUMMARY" - echo "All thresholds met — safe to deploy." >> "$GITHUB_STEP_SUMMARY" - elif [ $EXIT_CODE -eq 2 ]; then - echo "## ❌ Safety QA Failed — Threshold(s) Not Met" >> "$GITHUB_STEP_SUMMARY" - echo "One or more evaluation thresholds were not satisfied. Deployment blocked." >> "$GITHUB_STEP_SUMMARY" - else - echo "## ⚠️ Safety QA Error" >> "$GITHUB_STEP_SUMMARY" - echo "A runtime or configuration error occurred (exit code $EXIT_CODE)." >> "$GITHUB_STEP_SUMMARY" - fi - - REPORT=".agentops/results/latest/report.md" - if [ -f "$REPORT" ]; then - echo "" >> "$GITHUB_STEP_SUMMARY" - cat "$REPORT" >> "$GITHUB_STEP_SUMMARY" - fi - - exit $EXIT_CODE - - - name: Upload evaluation artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: agentops-cd-safety-results - path: | - .agentops/results/latest/results.json - .agentops/results/latest/report.md - .agentops/results/latest/backend_metrics.json - .agentops/results/latest/cloud_evaluation.json - .agentops/results/latest/backend.stdout.log - .agentops/results/latest/backend.stderr.log - if-no-files-found: warn - - # ------------------------------------------------------------------ - # Job 2: Deploy - # Placeholder — fill in with your deployment steps. - # This job runs ONLY after safety-qa passes (or if safety is skipped). - # ------------------------------------------------------------------ - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: [safety-qa] - if: always() && (needs.safety-qa.result == 'success' || needs.safety-qa.result == 'skipped') - # Optional: use a GitHub Environment for manual approval gates - # environment: production - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # ================================================================ - # TODO: Add your deployment steps here. - # - # Examples: - # - Deploy to Azure Container Apps (az containerapp update) - # - Deploy to Azure App Service (az webapp deploy) - # - Deploy a Foundry agent (az ml agent deploy) - # - Run azd deploy - # - Push a container image - # - # The safety-qa job has already validated the evaluation thresholds - # at this point, so it's safe to proceed with deployment. - # ================================================================ - - name: Deploy placeholder - run: | - echo "::notice::Deploy step is a placeholder. Add your deployment commands here." - echo "## 🚀 Deploy" >> "$GITHUB_STEP_SUMMARY" - echo "No deployment configured yet. Edit the deploy job in this workflow to add your deployment steps." >> "$GITHUB_STEP_SUMMARY" diff --git a/src/agentops/templates/workflows/agentops-eval-ci.yml b/src/agentops/templates/workflows/agentops-eval-ci.yml deleted file mode 100644 index 2c26da87..00000000 --- a/src/agentops/templates/workflows/agentops-eval-ci.yml +++ /dev/null @@ -1,168 +0,0 @@ -# AgentOps CI Evaluation — GitHub Actions Workflow -# -# Generated by: agentops workflow generate -# -# Runs comprehensive `agentops eval run` after merges to develop/main. -# Unlike the PR pipeline, this runs the full evaluation suite and can -# optionally publish results to Azure AI Foundry. -# -# Authentication: -# Uses Workload Identity Federation (OIDC) — no secrets to rotate. -# Set AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_SUBSCRIPTION_ID as GitHub -# repository variables (not secrets). See docs/ci-github-actions.md for setup. -# -# Prerequisites: -# 1. An initialised .agentops/ workspace in your repo (run `agentops init`) -# 2. A valid .agentops/run.yaml pointing to your bundle and dataset -# 3. Azure federated credential configured for your GitHub repo -# -# Multi-config runs: -# Uncomment the matrix strategy block to evaluate multiple run configs -# in a single workflow run (e.g. model-direct + RAG + agent-tools). -# -# Baseline comparison: -# Uncomment the comparison step to compare the current run against a -# baseline and detect regressions automatically. - -name: AgentOps CI Evaluation - -on: - push: - branches: [develop, main] - paths: - - ".agentops/**" - - "src/**" - - "pyproject.toml" - workflow_dispatch: - inputs: - config: - description: "Path to run.yaml (default: .agentops/run.yaml)" - required: false - default: ".agentops/run.yaml" - output: - description: "Output directory for results" - required: false - default: "" - -permissions: - contents: read - id-token: write # Required for OIDC / Workload Identity Federation - -env: - PYTHON_VERSION: "3.11" - -jobs: - evaluate: - name: Run AgentOps CI Evaluation - runs-on: ubuntu-latest - - # ---------------------------------------------------------------- - # Matrix strategy (uncomment to evaluate multiple configs) - # ---------------------------------------------------------------- - # strategy: - # fail-fast: false - # matrix: - # config: - # - .agentops/run.yaml - # - .agentops/runs/rag-retrieval.yaml - # - .agentops/runs/agent-tools.yaml - - env: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # ---------------------------------------------------------------- - # Azure login via Workload Identity Federation (OIDC) - # ---------------------------------------------------------------- - - name: Azure login (OIDC) - uses: azure/login@v2 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - - - name: Install agentops-toolkit - run: pip install agentops-toolkit - - - name: Resolve config path - id: config - run: | - # Use matrix config if available, otherwise use input or default - # CONFIG="${{ matrix.config || github.event.inputs.config || '.agentops/run.yaml' }}" - CONFIG="${{ github.event.inputs.config || '.agentops/run.yaml' }}" - echo "path=$CONFIG" >> "$GITHUB_OUTPUT" - - - name: Resolve output directory - id: output - run: | - OUTPUT="${{ github.event.inputs.output }}" - if [ -n "$OUTPUT" ]; then - echo "flag=--output $OUTPUT" >> "$GITHUB_OUTPUT" - else - echo "flag=" >> "$GITHUB_OUTPUT" - fi - - - name: Run evaluation - id: eval - run: | - set +e - agentops eval run --config "${{ steps.config.outputs.path }}" ${{ steps.output.outputs.flag }} - EXIT_CODE=$? - echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" - - if [ $EXIT_CODE -eq 0 ]; then - echo "## ✅ CI Evaluation Passed" >> "$GITHUB_STEP_SUMMARY" - echo "All thresholds met on **${{ github.ref_name }}**." >> "$GITHUB_STEP_SUMMARY" - elif [ $EXIT_CODE -eq 2 ]; then - echo "## ❌ CI Evaluation Failed — Threshold(s) Not Met" >> "$GITHUB_STEP_SUMMARY" - echo "One or more evaluation thresholds were not satisfied on **${{ github.ref_name }}**." >> "$GITHUB_STEP_SUMMARY" - else - echo "## ⚠️ CI Evaluation Error" >> "$GITHUB_STEP_SUMMARY" - echo "A runtime or configuration error occurred (exit code $EXIT_CODE)." >> "$GITHUB_STEP_SUMMARY" - fi - - REPORT=".agentops/results/latest/report.md" - if [ -f "$REPORT" ]; then - echo "" >> "$GITHUB_STEP_SUMMARY" - cat "$REPORT" >> "$GITHUB_STEP_SUMMARY" - fi - - exit $EXIT_CODE - - # ---------------------------------------------------------------- - # Baseline comparison (uncomment to detect regressions) - # Requires a previous run ID stored as a file or variable. - # ---------------------------------------------------------------- - # - name: Compare against baseline - # if: always() && steps.eval.outputs.exit_code != '1' - # run: | - # BASELINE=$(cat .agentops/results/baseline_id.txt 2>/dev/null || echo "") - # if [ -n "$BASELINE" ]; then - # CURRENT=$(jq -r '.run_id' .agentops/results/latest/results.json 2>/dev/null || echo "") - # if [ -n "$CURRENT" ]; then - # agentops eval compare --runs "$BASELINE,$CURRENT" -f md - # fi - # fi - - - name: Upload evaluation artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: agentops-ci-eval-results - path: | - .agentops/results/latest/results.json - .agentops/results/latest/report.md - .agentops/results/latest/backend_metrics.json - .agentops/results/latest/cloud_evaluation.json - .agentops/results/latest/backend.stdout.log - .agentops/results/latest/backend.stderr.log - if-no-files-found: warn diff --git a/src/agentops/templates/workflows/agentops-eval.yml b/src/agentops/templates/workflows/agentops-eval.yml deleted file mode 100644 index 580e4315..00000000 --- a/src/agentops/templates/workflows/agentops-eval.yml +++ /dev/null @@ -1,178 +0,0 @@ -# AgentOps Evaluation — GitHub Actions Workflow -# -# Generated by: agentops workflow generate -# -# Runs `agentops eval run` on pull requests and manual dispatch. -# Uploads evaluation artifacts (results.json, report.md, logs). -# Fails the job when thresholds are not met (exit code 2) or on errors (exit code 1). -# -# Authentication: -# Uses Workload Identity Federation (OIDC) — no secrets to rotate. -# Set AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_SUBSCRIPTION_ID as GitHub -# repository variables (not secrets). See docs/ci-github-actions.md for setup. -# -# Prerequisites: -# 1. An initialised .agentops/ workspace in your repo (run `agentops init`) -# 2. A valid .agentops/run.yaml pointing to your bundle and dataset -# 3. Azure federated credential configured for your GitHub repo - -name: AgentOps Evaluation - -on: - pull_request: - branches: [main, develop] - workflow_dispatch: - inputs: - config: - description: "Path to run.yaml (default: .agentops/run.yaml)" - required: false - default: ".agentops/run.yaml" - output: - description: "Output directory for results" - required: false - default: "" - -permissions: - contents: read - id-token: write # Required for OIDC / Workload Identity Federation - pull-requests: write # Required for optional PR comment step - -env: - PYTHON_VERSION: "3.11" - -jobs: - evaluate: - name: Run AgentOps Evaluation - runs-on: ubuntu-latest - - env: - # Foundry project endpoint — set as a GitHub repository secret - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ secrets.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # ---------------------------------------------------------------- - # Azure login via Workload Identity Federation (OIDC) - # Set AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID as - # GitHub repository variables (Settings > Secrets > Variables). - # ---------------------------------------------------------------- - - name: Azure login (OIDC) - uses: azure/login@v2 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - - - name: Install agentops-toolkit - run: pip install agentops-toolkit - - - name: Resolve config path - id: config - run: | - CONFIG="${{ github.event.inputs.config || '.agentops/run.yaml' }}" - echo "path=$CONFIG" >> "$GITHUB_OUTPUT" - - - name: Resolve output directory - id: output - run: | - OUTPUT="${{ github.event.inputs.output }}" - if [ -n "$OUTPUT" ]; then - echo "flag=--output $OUTPUT" >> "$GITHUB_OUTPUT" - else - echo "flag=" >> "$GITHUB_OUTPUT" - fi - - - name: Run evaluation - id: eval - run: | - set +e - agentops eval run --config "${{ steps.config.outputs.path }}" ${{ steps.output.outputs.flag }} - EXIT_CODE=$? - echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" - - # Surface the exit code meaning - if [ $EXIT_CODE -eq 0 ]; then - echo "## ✅ Evaluation Passed" >> "$GITHUB_STEP_SUMMARY" - echo "All thresholds met." >> "$GITHUB_STEP_SUMMARY" - elif [ $EXIT_CODE -eq 2 ]; then - echo "## ❌ Evaluation Failed — Threshold(s) Not Met" >> "$GITHUB_STEP_SUMMARY" - echo "One or more evaluation thresholds were not satisfied." >> "$GITHUB_STEP_SUMMARY" - else - echo "## ⚠️ Evaluation Error" >> "$GITHUB_STEP_SUMMARY" - echo "A runtime or configuration error occurred (exit code $EXIT_CODE)." >> "$GITHUB_STEP_SUMMARY" - fi - - # Append report.md to job summary if it exists - REPORT=".agentops/results/latest/report.md" - if [ -f "$REPORT" ]; then - echo "" >> "$GITHUB_STEP_SUMMARY" - cat "$REPORT" >> "$GITHUB_STEP_SUMMARY" - fi - - exit $EXIT_CODE - - - name: Upload evaluation artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: agentops-eval-results - path: | - .agentops/results/latest/results.json - .agentops/results/latest/report.md - .agentops/results/latest/backend_metrics.json - .agentops/results/latest/cloud_evaluation.json - .agentops/results/latest/backend.stdout.log - .agentops/results/latest/backend.stderr.log - if-no-files-found: warn - - - name: Post report as PR comment - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const reportPath = '.agentops/results/latest/report.md'; - - if (!fs.existsSync(reportPath)) { - console.log('No report.md found — skipping PR comment.'); - return; - } - - const body = fs.readFileSync(reportPath, 'utf8'); - const marker = ''; - const commentBody = `${marker}\n${body}`; - - // Find existing comment to update (avoid duplicates) - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const existing = comments.find(c => c.body.includes(marker)); - - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body: commentBody, - }); - console.log(`Updated existing PR comment #${existing.id}`); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: commentBody, - }); - console.log('Created new PR comment with evaluation report.'); - } diff --git a/src/agentops/templates/workflows/agentops-pr.yml b/src/agentops/templates/workflows/agentops-pr.yml new file mode 100644 index 00000000..b8bfd595 --- /dev/null +++ b/src/agentops/templates/workflows/agentops-pr.yml @@ -0,0 +1,132 @@ +# AgentOps — PR evaluation gate +# +# Runs on pull requests targeting develop, release/**, or main and gates +# the merge on the AgentOps eval result. Posts the rendered Markdown +# report as an idempotent PR comment. +# +# Edit by: +# - Changing the run config under inputs.config (add a matrix if you +# have multiple scenarios). +# - Adjusting thresholds in the bundle, not here. +# +# Generated by `agentops workflow generate`. + +name: AgentOps PR + +on: + pull_request: + branches: + - develop + - "release/**" + - main + workflow_dispatch: + inputs: + config: + description: "Path to agentops.yaml (defaults to ./agentops.yaml at repo root)" + required: false + default: "agentops.yaml" + +permissions: + contents: read + pull-requests: write + id-token: write + +concurrency: + group: agentops-pr-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + eval: + name: AgentOps eval (PR gate) + runs-on: ubuntu-latest + # Run the PR gate against the dev environment so the OIDC token + # subject is `environment:dev` and `vars.*` resolves from there. + environment: dev + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install AgentOps Toolkit + run: | + python -m pip install --upgrade pip + # NOTE: pinned to develop branch until AgentOps 1.0 lands on PyPI. + # Switch to `pip install "agentops-toolkit[foundry]"` after the 1.0 release. + python -m pip install "agentops-toolkit[foundry] @ git+https://github.com/Azure/agentops.git@develop" + + - name: Run AgentOps eval + id: eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + run: | + set +e + agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" + ec=$? + echo "exit_code=$ec" >> "$GITHUB_OUTPUT" + if [ $ec -eq 0 ]; then + echo "result=pass" >> "$GITHUB_OUTPUT" + elif [ $ec -eq 2 ]; then + echo "result=threshold_failed" >> "$GITHUB_OUTPUT" + else + echo "result=error" >> "$GITHUB_OUTPUT" + fi + exit $ec + + - name: Upload AgentOps results + if: always() + uses: actions/upload-artifact@v4 + with: + name: agentops-pr-results + path: | + .agentops/results/latest/results.json + .agentops/results/latest/report.md + .agentops/results/latest/cloud_evaluation.json + if-no-files-found: warn + + - name: Step summary + if: always() + run: | + { + echo "## AgentOps PR Eval" + echo + echo "Result: \`${{ steps.eval.outputs.result }}\` (exit code \`${{ steps.eval.outputs.exit_code }}\`)" + echo + if [ -f .agentops/results/latest/report.md ]; then + cat .agentops/results/latest/report.md + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Comment AgentOps report on PR + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = '.agentops/results/latest/report.md'; + const marker = ''; + let body = `${marker}\n## AgentOps PR Eval\n\nNo report produced.`; + if (fs.existsSync(path)) { + const md = fs.readFileSync(path, 'utf8'); + body = `${marker}\n${md}`; + } + const { owner, repo } = context.repo; + const issue_number = context.payload.pull_request.number; + const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number }); + const existing = comments.find(c => c.body && c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } diff --git a/src/agentops/utils/colors.py b/src/agentops/utils/colors.py new file mode 100644 index 00000000..cbc49514 --- /dev/null +++ b/src/agentops/utils/colors.py @@ -0,0 +1,47 @@ +"""Tiny ANSI color helpers for CLI progress output. + +Colors are automatically disabled when stdout is not a TTY, when the +``NO_COLOR`` environment variable is set (https://no-color.org/), or when +``AGENTOPS_NO_COLOR`` is set. No emojis, no extended unicode — only plain +ASCII text wrapped in standard ANSI SGR escape codes that all modern +terminals (Windows Terminal, ConEmu, VS Code, macOS, Linux) understand. +""" + +from __future__ import annotations + +import os +import sys + +_RESET = "\x1b[0m" +_CODES = { + "dim": "\x1b[2m", + "bold": "\x1b[1m", + "red": "\x1b[31m", + "green": "\x1b[32m", + "yellow": "\x1b[33m", + "blue": "\x1b[34m", + "magenta": "\x1b[35m", + "cyan": "\x1b[36m", +} + + +def _enabled() -> bool: + if os.environ.get("NO_COLOR"): + return False + if os.environ.get("AGENTOPS_NO_COLOR"): + return False + stream = sys.stdout + try: + return bool(stream.isatty()) + except Exception: # noqa: BLE001 + return False + + +def style(text: str, *names: str) -> str: + """Wrap ``text`` in the given ANSI styles (e.g. ``"green"``, ``"bold"``).""" + if not _enabled() or not names: + return text + prefix = "".join(_CODES.get(name, "") for name in names) + if not prefix: + return text + return f"{prefix}{text}{_RESET}" diff --git a/src/agentops/utils/logging.py b/src/agentops/utils/logging.py index 52bde034..e7fca4d5 100644 --- a/src/agentops/utils/logging.py +++ b/src/agentops/utils/logging.py @@ -32,7 +32,12 @@ def setup_logging(verbose: bool = False) -> None: if not verbose: logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("azure").setLevel(logging.WARNING) - logging.getLogger("azure.identity").setLevel(logging.WARNING) + # azure.identity emits WARNING when individual credential sources + # in DefaultAzureCredential fail (e.g. the Azure CLI is locked or + # times out). Those failures are usually transient and the chain + # still succeeds via another source, so we hide them at the user + # level. They are still surfaced if the run fails outright. + logging.getLogger("azure.identity").setLevel(logging.ERROR) logging.getLogger("azure.core").setLevel(logging.WARNING) logging.getLogger("azure.core.pipeline").setLevel(logging.WARNING) logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel( diff --git a/tests/integration/test_cli_flat_schema.py b/tests/integration/test_cli_flat_schema.py new file mode 100644 index 00000000..54e401bb --- /dev/null +++ b/tests/integration/test_cli_flat_schema.py @@ -0,0 +1,124 @@ +"""CLI tests for the 1.0 flat schema path on ``agentops eval run``.""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +pytest.importorskip( + "azure.ai.evaluation", + reason="azure-ai-evaluation is required to instantiate evaluators in the pipeline runtime", +) + +from agentops.cli.app import app + +runner = CliRunner() + + +class _EchoHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8")) + message = body.get("message", "") + payload = json.dumps({"text": f"echo: {message}"}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args, **kwargs) -> None: # noqa: D401 + pass + + +@pytest.fixture() +def echo_server(): + server = HTTPServer(("127.0.0.1", 0), _EchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}/chat" + finally: + server.shutdown() + thread.join(timeout=1) + + +def _write_dataset(path: Path) -> None: + rows = [ + {"input": "say hi", "expected": "hi"}, + {"input": "say bye", "expected": "bye"}, + ] + path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + + +def _write_flat_config(path: Path, *, agent: str, dataset: Path) -> None: + payload = { + "version": 1, + "agent": agent, + "dataset": str(dataset), + "evaluators": [{"name": "F1ScoreEvaluator"}], + } + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_eval_run_routes_flat_schema_to_pipeline( + tmp_path: Path, echo_server: str +) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + config = tmp_path / "agentops.yaml" + _write_flat_config(config, agent=echo_server, dataset=dataset) + output = tmp_path / "out" + + result = runner.invoke( + app, + [ + "eval", + "run", + "--config", + str(config), + "--output", + str(output), + ], + ) + + assert result.exit_code in (0, 2), result.output + assert (output / "results.json").exists() + assert (output / "report.md").exists() + + +def test_eval_run_supports_baseline_flag(tmp_path: Path, echo_server: str) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + config = tmp_path / "agentops.yaml" + _write_flat_config(config, agent=echo_server, dataset=dataset) + + baseline_dir = tmp_path / "baseline" + runner.invoke( + app, + ["eval", "run", "--config", str(config), "--output", str(baseline_dir)], + ) + current = tmp_path / "current" + result = runner.invoke( + app, + [ + "eval", + "run", + "--config", + str(config), + "--output", + str(current), + "--baseline", + str(baseline_dir / "results.json"), + ], + ) + assert result.exit_code in (0, 2), result.output + payload = json.loads((current / "results.json").read_text(encoding="utf-8")) + assert payload["comparison"] is not None + assert payload["comparison"]["baseline_path"].endswith("results.json") diff --git a/tests/integration/test_eval_run_integration.py b/tests/integration/test_eval_run_integration.py deleted file mode 100644 index a593cbc6..00000000 --- a/tests/integration/test_eval_run_integration.py +++ /dev/null @@ -1,303 +0,0 @@ -from __future__ import annotations - -import json -import sys -from pathlib import Path - -from typer.testing import CliRunner - -from agentops.cli.app import app -from agentops.utils.yaml import save_yaml - - -def _fixture_adapter_script() -> Path: - return Path(__file__).resolve().parents[1] / "fixtures" / "fake_adapter.py" - - -_CALLABLE_PATH = "tests.fixtures.fake_adapter:main_callable" - - -def _write_project_files(tmp_path: Path, *, fail_thresholds: bool) -> Path: - agentops_dir = tmp_path / ".agentops" - bundles_dir = agentops_dir / "bundles" - datasets_dir = agentops_dir / "datasets" - data_dir = agentops_dir / "data" - - bundles_dir.mkdir(parents=True, exist_ok=True) - datasets_dir.mkdir(parents=True, exist_ok=True) - data_dir.mkdir(parents=True, exist_ok=True) - - threshold_value = 0.95 if fail_thresholds else 0.8 - - # For the fail case, use data where adapter response (= input) won't match expected - if fail_thresholds: - dataset_rows = [ - '{"id":"1","input":"hello","expected":"goodbye"}', - '{"id":"2","input":"world","expected":"earth"}', - ] - else: - dataset_rows = [ - '{"id":"1","input":"hello","expected":"hello"}', - '{"id":"2","input":"world","expected":"world"}', - ] - - save_yaml( - bundles_dir / "rag_baseline.yaml", - { - "version": 1, - "name": "rag_baseline", - "description": "Integration test bundle", - "evaluators": [ - {"name": "exact_match", "source": "local", "enabled": True}, - ], - "thresholds": [ - { - "evaluator": "exact_match", - "criteria": ">=", - "value": threshold_value, - }, - ], - "metadata": {"category": "integration"}, - }, - ) - - save_yaml( - datasets_dir / "smoke-agent.yaml", - { - "version": 1, - "name": "smoke", - "description": "Integration dataset", - "source": {"type": "file", "path": "../data/smoke.jsonl"}, - "format": { - "type": "jsonl", - "input_field": "input", - "expected_field": "expected", - }, - "metadata": {"owner": "tests"}, - }, - ) - - (data_dir / "smoke.jsonl").write_text( - "\n".join(dataset_rows) + "\n", - encoding="utf-8", - ) - - adapter_cmd = f"{sys.executable} {_fixture_adapter_script()}" - - run_payload = { - "version": 1, - "target": { - "type": "model", - "hosting": "local", - "execution_mode": "local", - "local": {"adapter": adapter_cmd}, - }, - "bundle": {"path": ".agentops/bundles/rag_baseline.yaml"}, - "dataset": {"path": ".agentops/datasets/smoke-agent.yaml"}, - "execution": {"timeout_seconds": 30}, - "output": {"write_report": True}, - } - - run_path = tmp_path / "run.fake.yaml" - save_yaml( - run_path, - run_payload, - ) - - save_yaml( - agentops_dir / "run.yaml", - run_payload, - ) - return run_path - - -def test_eval_run_integration_success(tmp_path: Path, monkeypatch) -> None: - run_path = _write_project_files(tmp_path, fail_thresholds=False) - output_dir = tmp_path / "out-pass" - - monkeypatch.chdir(tmp_path) - runner = CliRunner() - result = runner.invoke( - app, - ["eval", "run", "--config", str(run_path), "--output", str(output_dir)], - ) - - assert result.exit_code == 0 - assert (output_dir / "results.json").is_file() - assert (output_dir / "report.md").is_file() - assert (output_dir / "backend.stdout.log").is_file() - assert (output_dir / "backend.stderr.log").is_file() - - payload = json.loads((output_dir / "results.json").read_text(encoding="utf-8")) - assert payload["summary"]["overall_passed"] is True - assert payload["execution"]["exit_code"] == 0 - assert len(payload["row_metrics"]) == 2 - assert len(payload["item_evaluations"]) == 2 - run_metrics = {item["name"]: item["value"] for item in payload["run_metrics"]} - assert run_metrics["run_pass"] == 1.0 - assert run_metrics["threshold_pass_rate"] == 1.0 - assert run_metrics["items_total"] == 2.0 - assert run_metrics["items_passed_all"] == 2.0 - assert run_metrics["items_pass_rate"] == 1.0 - assert "exact_match_avg" in run_metrics - assert "exact_match_stddev" in run_metrics - - -def test_eval_run_integration_threshold_fail(tmp_path: Path, monkeypatch) -> None: - run_path = _write_project_files(tmp_path, fail_thresholds=True) - output_dir = tmp_path / "out-fail" - - monkeypatch.chdir(tmp_path) - runner = CliRunner() - result = runner.invoke( - app, - ["eval", "run", "--config", str(run_path), "--output", str(output_dir)], - ) - - assert result.exit_code == 2 - assert (output_dir / "results.json").is_file() - assert (output_dir / "report.md").is_file() - - payload = json.loads((output_dir / "results.json").read_text(encoding="utf-8")) - assert payload["summary"]["overall_passed"] is False - assert payload["summary"]["thresholds_failed"] > 0 - assert len(payload["row_metrics"]) == 2 - assert len(payload["item_evaluations"]) == 2 - run_metrics = {item["name"]: item["value"] for item in payload["run_metrics"]} - assert run_metrics["run_pass"] == 0.0 - assert run_metrics["threshold_pass_rate"] < 1.0 - assert run_metrics["items_total"] == 2.0 - assert run_metrics["items_passed_all"] == 0.0 - assert run_metrics["items_pass_rate"] == 0.0 - - -def test_eval_run_integration_uses_default_run_yaml_and_updates_latest( - tmp_path: Path, monkeypatch -) -> None: - _write_project_files(tmp_path, fail_thresholds=False) - - monkeypatch.chdir(tmp_path) - runner = CliRunner() - result = runner.invoke(app, ["eval", "run"]) - - assert result.exit_code == 0 - - results_root = tmp_path / ".agentops" / "results" - latest_dir = results_root / "latest" - assert latest_dir.is_dir() - assert (latest_dir / "results.json").is_file() - assert (latest_dir / "report.md").is_file() - - timestamp_dirs = [ - path - for path in results_root.iterdir() - if path.is_dir() and path.name != "latest" - ] - assert len(timestamp_dirs) == 1 - assert (timestamp_dirs[0] / "results.json").is_file() - assert (timestamp_dirs[0] / "report.md").is_file() - - -def _write_callable_project_files(tmp_path: Path) -> Path: - """Write project files that use callable mode instead of subprocess.""" - agentops_dir = tmp_path / ".agentops" - bundles_dir = agentops_dir / "bundles" - datasets_dir = agentops_dir / "datasets" - data_dir = agentops_dir / "data" - - bundles_dir.mkdir(parents=True, exist_ok=True) - datasets_dir.mkdir(parents=True, exist_ok=True) - data_dir.mkdir(parents=True, exist_ok=True) - - # Write a local callable adapter into the tmp project directory so it is - # importable after chdir(tmp_path) without relying on the repo root. - (tmp_path / "fake_callable.py").write_text( - "def main_callable(input_text: str, context: dict) -> dict:\n" - ' return {"response": input_text}\n', - encoding="utf-8", - ) - - save_yaml( - bundles_dir / "rag_baseline.yaml", - { - "version": 1, - "name": "rag_baseline", - "description": "Callable integration test bundle", - "evaluators": [ - {"name": "exact_match", "source": "local", "enabled": True}, - ], - "thresholds": [ - {"evaluator": "exact_match", "criteria": ">=", "value": 0.8}, - ], - "metadata": {"category": "integration"}, - }, - ) - - save_yaml( - datasets_dir / "smoke-agent.yaml", - { - "version": 1, - "name": "smoke", - "description": "Integration dataset", - "source": {"type": "file", "path": "../data/smoke.jsonl"}, - "format": { - "type": "jsonl", - "input_field": "input", - "expected_field": "expected", - }, - "metadata": {"owner": "tests"}, - }, - ) - - (data_dir / "smoke.jsonl").write_text( - '{"id":"1","input":"hello","expected":"hello"}\n' - '{"id":"2","input":"world","expected":"world"}\n', - encoding="utf-8", - ) - - run_path = tmp_path / "run-callable.yaml" - save_yaml( - run_path, - { - "version": 1, - "target": { - "type": "model", - "hosting": "local", - "execution_mode": "local", - "local": {"callable": "fake_callable:main_callable"}, - }, - "bundle": {"path": ".agentops/bundles/rag_baseline.yaml"}, - "dataset": {"path": ".agentops/datasets/smoke-agent.yaml"}, - "execution": {"timeout_seconds": 30}, - "output": {"write_report": True}, - }, - ) - return run_path - - -def test_eval_run_integration_callable_success( - tmp_path: Path, monkeypatch -) -> None: - run_path = _write_callable_project_files(tmp_path) - output_dir = tmp_path / "out-callable" - - monkeypatch.chdir(tmp_path) - runner = CliRunner() - result = runner.invoke( - app, - ["eval", "run", "--config", str(run_path), "--output", str(output_dir)], - ) - - assert result.exit_code == 0, f"Unexpected failure:\n{result.output}" - assert (output_dir / "results.json").is_file() - assert (output_dir / "report.md").is_file() - - payload = json.loads((output_dir / "results.json").read_text(encoding="utf-8")) - assert payload["summary"]["overall_passed"] is True - assert payload["execution"]["exit_code"] == 0 - assert len(payload["row_metrics"]) == 2 - assert len(payload["item_evaluations"]) == 2 - run_metrics = {item["name"]: item["value"] for item in payload["run_metrics"]} - assert run_metrics["run_pass"] == 1.0 - assert run_metrics["items_total"] == 2.0 - assert run_metrics["items_pass_rate"] == 1.0 diff --git a/tests/integration/test_pipeline_smoke.py b/tests/integration/test_pipeline_smoke.py new file mode 100644 index 00000000..f25bc556 --- /dev/null +++ b/tests/integration/test_pipeline_smoke.py @@ -0,0 +1,144 @@ +"""End-to-end smoke test for the AgentOps 1.0 pipeline. + +Spins up a tiny HTTP server, points an ``agentops.yaml`` at it, runs the +orchestrator without any Azure dependencies (no AI-assisted evaluators), and +asserts the resulting ``results.json`` and ``report.md``. +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest + +pytest.importorskip( + "azure.ai.evaluation", + reason="azure-ai-evaluation is required to instantiate evaluators in the pipeline runtime", +) + +from agentops.core.agentops_config import AgentOpsConfig +from agentops.core.config_loader import load_agentops_config +from agentops.pipeline.orchestrator import ( + RunOptions, + exit_code_from, + run_evaluation, +) + + +class _EchoHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length).decode("utf-8")) + message = body.get("message", "") + payload = json.dumps({"text": f"echo: {message}"}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args, **kwargs) -> None: # noqa: D401 + pass + + +@pytest.fixture() +def echo_server(): + server = HTTPServer(("127.0.0.1", 0), _EchoHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}/chat" + finally: + server.shutdown() + thread.join(timeout=1) + + +def _write_dataset(path: Path) -> None: + rows = [ + {"input": "say hi", "expected": "hi"}, + {"input": "say bye", "expected": "bye"}, + ] + path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + + +def _write_config(path: Path, *, agent_url: str, dataset: Path) -> None: + payload = { + "version": 1, + "agent": agent_url, + "dataset": str(dataset), + "evaluators": [{"name": "F1ScoreEvaluator"}], # avoids Azure model dependency + } + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_http_pipeline_end_to_end(tmp_path: Path, echo_server: str) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + + config_path = tmp_path / "agentops.yaml" + _write_config(config_path, agent_url=echo_server, dataset=dataset) + + config = load_agentops_config(config_path) + assert isinstance(config, AgentOpsConfig) + assert config.agent == echo_server + + output_dir = tmp_path / "results" + options = RunOptions( + config_path=config_path, + output_dir=output_dir, + timeout_seconds=10.0, + ) + + result = run_evaluation(config, options=options) + + assert (output_dir / "results.json").exists() + assert (output_dir / "report.md").exists() + assert result.summary.items_total == 2 + assert result.target.kind == "http_json" + assert "f1_score" in result.aggregate_metrics + assert result.rows[0].response.startswith("echo:") + + payload = json.loads((output_dir / "results.json").read_text(encoding="utf-8")) + assert payload["version"] == 1 + assert payload["target"]["url"] == echo_server + + code = exit_code_from(result) + assert code in (0, 2) + + +def test_http_pipeline_with_baseline(tmp_path: Path, echo_server: str) -> None: + dataset = tmp_path / "dataset.jsonl" + _write_dataset(dataset) + config_path = tmp_path / "agentops.yaml" + _write_config(config_path, agent_url=echo_server, dataset=dataset) + config = load_agentops_config(config_path) + + baseline_dir = tmp_path / "baseline" + run_evaluation( + config, + options=RunOptions( + config_path=config_path, + output_dir=baseline_dir, + timeout_seconds=10.0, + ), + ) + + current_dir = tmp_path / "current" + result = run_evaluation( + config, + options=RunOptions( + config_path=config_path, + output_dir=current_dir, + baseline_path=baseline_dir / "results.json", + timeout_seconds=10.0, + ), + ) + + assert result.comparison is not None + assert any(metric.metric == "f1_score" for metric in result.comparison.metrics) + report_text = (current_dir / "report.md").read_text(encoding="utf-8") + assert "Comparison vs Baseline" in report_text diff --git a/tests/unit/test_agent_analyzer.py b/tests/unit/test_agent_analyzer.py new file mode 100644 index 00000000..67f811c4 --- /dev/null +++ b/tests/unit/test_agent_analyzer.py @@ -0,0 +1,77 @@ +"""Tests for the analyzer + Markdown report renderer.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from agentops.agent.analyzer import analyze +from agentops.agent.config import AgentConfig, ResultsHistorySourceConfig, SourcesConfig +from agentops.agent.findings import Severity +from agentops.agent.report import render_report, short_chat_summary + + +def _seed_runs(workspace: Path) -> None: + root = workspace / ".agentops" / "results" + root.mkdir(parents=True, exist_ok=True) + for idx, (run_id, ts, coh) in enumerate( + [ + ("run-1", "2024-05-01T10:00:00Z", 4.5), + ("run-2", "2024-05-02T10:00:00Z", 4.5), + ("run-3", "2024-05-03T10:00:00Z", 3.0), + ] + ): + run_dir = root / run_id + run_dir.mkdir(exist_ok=True) + (run_dir / "results.json").write_text( + json.dumps( + { + "run_id": run_id, + "timestamp": ts, + "metrics": {"coherence": coh}, + "summary": { + "run_pass": idx < 2, + "items_total": 5, + "items_passed_all": 4 if idx < 2 else 2, + }, + } + ), + encoding="utf-8", + ) + + +def _config_with_disabled_remote_sources() -> AgentConfig: + sources = SourcesConfig() + sources.results_history = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + sources.azure_monitor.enabled = False + sources.foundry_control.enabled = False + return AgentConfig(sources=sources) + + +def test_analyzer_produces_regression_finding(tmp_path: Path) -> None: + _seed_runs(tmp_path) + result = analyze(tmp_path, _config_with_disabled_remote_sources()) + + ids = [f.id for f in result.findings] + assert "regression.coherence" in ids + assert result.max_severity == Severity.CRITICAL + + +def test_render_report_contains_verdict_and_findings(tmp_path: Path) -> None: + _seed_runs(tmp_path) + result = analyze(tmp_path, _config_with_disabled_remote_sources()) + + report = render_report(result) + assert "AgentOps Watchdog Report" in report + assert "Verdict:" in report + assert "regression.coherence" in report + assert "Recent runs" in report + + +def test_short_chat_summary_no_findings_path() -> None: + from agentops.agent.analyzer import AnalysisResult + + summary = short_chat_summary(AnalysisResult()) + assert "No issues" in summary diff --git a/tests/unit/test_agent_categories.py b/tests/unit/test_agent_categories.py new file mode 100644 index 00000000..2599a852 --- /dev/null +++ b/tests/unit/test_agent_categories.py @@ -0,0 +1,189 @@ +"""Tests for category filtering and rule exclusion in `analyze`.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import patch + +from agentops.agent.analyzer import analyze +from agentops.agent.config import ( + AgentConfig, + AzureResourcesSourceConfig, + PostureCheckConfig, + ResultsHistorySourceConfig, + SourcesConfig, + ChecksConfig, +) +from agentops.agent.findings import Category +from agentops.agent.sources.azure_resources import ( + AzureResourcesPayload, + CognitiveAccountSnapshot, + DeploymentSnapshot, +) + + +def _seed_runs(workspace: Path) -> None: + """Seed three runs that trigger a coherence regression.""" + root = workspace / ".agentops" / "results" + root.mkdir(parents=True, exist_ok=True) + for idx, (run_id, ts, coh) in enumerate( + [ + ("run-1", "2024-05-01T10:00:00Z", 4.5), + ("run-2", "2024-05-02T10:00:00Z", 4.5), + ("run-3", "2024-05-03T10:00:00Z", 3.0), + ] + ): + run_dir = root / run_id + run_dir.mkdir(exist_ok=True) + (run_dir / "results.json").write_text( + json.dumps( + { + "run_id": run_id, + "timestamp": ts, + "metrics": {"coherence": coh}, + "summary": { + "run_pass": idx < 2, + "items_total": 5, + "items_passed_all": 4 if idx < 2 else 2, + }, + } + ), + encoding="utf-8", + ) + + +def _insecure_resources_payload() -> AzureResourcesPayload: + """A payload that triggers all 5 WAF security rules.""" + return AzureResourcesPayload( + account=CognitiveAccountSnapshot( + name="ai-test", + disable_local_auth=False, + public_network_access="Enabled", + private_endpoint_count=0, + network_acls_default_action="Allow", + identity_type=None, + ), + deployments=[DeploymentSnapshot(name="legacy", model="gpt-3.5", rai_policy_name=None)], + diagnostic_settings=[], + diagnostics={"status": "ok"}, + ) + + +def _agent_config_with_posture() -> AgentConfig: + sources = SourcesConfig() + sources.results_history = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + sources.azure_monitor.enabled = False + sources.foundry_control.enabled = False + sources.azure_resources = AzureResourcesSourceConfig( + enabled=True, + subscription_id="sub-1", + resource_group="rg-1", + cognitive_services_account="ai-test", + ) + checks = ChecksConfig() + checks.posture = PostureCheckConfig(enabled=True, pillar="security") + return AgentConfig(sources=sources, checks=checks) + + +def test_categories_filter_keeps_only_security(tmp_path: Path) -> None: + _seed_runs(tmp_path) + config = _agent_config_with_posture() + with patch( + "agentops.agent.analyzer.collect_azure_resources", + return_value=_insecure_resources_payload(), + ): + result = analyze(tmp_path, config, categories=["security"]) + + assert result.findings, "expected at least one security finding" + assert {f.category for f in result.findings} == {Category.SECURITY} + # The regression finding (quality) must have been filtered out. + assert all(not f.id.startswith("regression.") for f in result.findings) + + +def test_categories_filter_keeps_only_quality(tmp_path: Path) -> None: + _seed_runs(tmp_path) + config = _agent_config_with_posture() + with patch( + "agentops.agent.analyzer.collect_azure_resources", + return_value=_insecure_resources_payload(), + ): + result = analyze(tmp_path, config, categories=["quality"]) + + assert {f.category for f in result.findings} == {Category.QUALITY} + + +def test_invalid_categories_are_ignored(tmp_path: Path) -> None: + _seed_runs(tmp_path) + config = _agent_config_with_posture() + with patch( + "agentops.agent.analyzer.collect_azure_resources", + return_value=_insecure_resources_payload(), + ): + # All-invalid input → behave like no filter. + result = analyze(tmp_path, config, categories=["bogus", " "]) + + categories = {f.category for f in result.findings} + assert Category.SECURITY in categories + assert Category.QUALITY in categories + + +def test_exclude_rules_skips_specific_posture_rule(tmp_path: Path) -> None: + _seed_runs(tmp_path) + config = _agent_config_with_posture() + with patch( + "agentops.agent.analyzer.collect_azure_resources", + return_value=_insecure_resources_payload(), + ): + result = analyze( + tmp_path, + config, + categories=["security"], + exclude_rules=["waf.security.local_auth_disabled"], + ) + + ids = {f.id for f in result.findings} + assert "waf.security.local_auth_disabled" not in ids + assert "waf.security.managed_identity" in ids + + +def test_exclude_rules_merges_with_config(tmp_path: Path) -> None: + _seed_runs(tmp_path) + config = _agent_config_with_posture() + config.checks.posture = PostureCheckConfig( + enabled=True, + pillar="security", + exclude_rules=["waf.security.diagnostic_settings"], + ) + with patch( + "agentops.agent.analyzer.collect_azure_resources", + return_value=_insecure_resources_payload(), + ): + result = analyze( + tmp_path, + config, + categories=["security"], + exclude_rules=["waf.security.local_auth_disabled"], + ) + + ids = {f.id for f in result.findings} + # Both the YAML-configured exclude AND the CLI exclude must apply. + assert "waf.security.diagnostic_settings" not in ids + assert "waf.security.local_auth_disabled" not in ids + assert "waf.security.managed_identity" in ids + + +def test_resources_field_present_on_analysis_result(tmp_path: Path) -> None: + _seed_runs(tmp_path) + config = _agent_config_with_posture() + payload = _insecure_resources_payload() + with patch( + "agentops.agent.analyzer.collect_azure_resources", + return_value=payload, + ): + result = analyze(tmp_path, config) + + assert result.resources is payload + assert result.diagnostics["azure_resources"] == {"status": "ok"} diff --git a/tests/unit/test_agent_checks_regression.py b/tests/unit/test_agent_checks_regression.py new file mode 100644 index 00000000..70adccf8 --- /dev/null +++ b/tests/unit/test_agent_checks_regression.py @@ -0,0 +1,65 @@ +"""Tests for the regression check.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from agentops.agent.checks.regression import run_regression_check +from agentops.agent.config import RegressionCheckConfig +from agentops.agent.findings import Severity +from agentops.agent.sources.results_history import ResultsHistory, RunSummary + + +def _run(metrics: dict, run_id: str = "r", offset_days: int = 0) -> RunSummary: + return RunSummary( + run_id=run_id, + timestamp=datetime.now(timezone.utc) + timedelta(days=offset_days), + metrics=metrics, + run_pass=True, + items_total=1, + items_passed_all=1, + raw_path=Path("dummy"), + ) + + +def test_regression_check_flags_drop_above_threshold() -> None: + history = ResultsHistory( + runs=[ + _run({"coherence": 4.5}, run_id="b1", offset_days=-3), + _run({"coherence": 4.5}, run_id="b2", offset_days=-2), + _run({"coherence": 3.0}, run_id="latest", offset_days=0), + ] + ) + config = RegressionCheckConfig( + metrics=["coherence"], threshold_drop=0.10, min_runs=3 + ) + findings = run_regression_check(history, config) + + assert len(findings) == 1 + assert findings[0].id == "regression.coherence" + # Drop is ~33% which is >= 2*threshold (20%) -> CRITICAL. + assert findings[0].severity == Severity.CRITICAL + assert findings[0].evidence["latest_run_id"] == "latest" + + +def test_regression_check_ignores_small_drops() -> None: + history = ResultsHistory( + runs=[ + _run({"coherence": 4.5}, run_id="b1", offset_days=-3), + _run({"coherence": 4.5}, run_id="b2", offset_days=-2), + _run({"coherence": 4.4}, run_id="latest", offset_days=0), + ] + ) + config = RegressionCheckConfig( + metrics=["coherence"], threshold_drop=0.10, min_runs=3 + ) + findings = run_regression_check(history, config) + assert findings == [] + + +def test_regression_check_skips_when_baseline_too_small() -> None: + history = ResultsHistory(runs=[_run({"coherence": 4.5}, run_id="only")]) + config = RegressionCheckConfig(metrics=["coherence"], min_runs=3) + findings = run_regression_check(history, config) + assert findings == [] diff --git a/tests/unit/test_agent_cli.py b/tests/unit/test_agent_cli.py new file mode 100644 index 00000000..c98d2461 --- /dev/null +++ b/tests/unit/test_agent_cli.py @@ -0,0 +1,98 @@ +"""CLI tests for `agentops agent analyze`.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from agentops.cli.app import app + +runner = CliRunner() + + +def _seed_regression(workspace: Path) -> None: + root = workspace / ".agentops" / "results" + root.mkdir(parents=True, exist_ok=True) + for run_id, ts, coh in [ + ("run-1", "2024-05-01T10:00:00Z", 4.5), + ("run-2", "2024-05-02T10:00:00Z", 4.5), + ("run-3", "2024-05-03T10:00:00Z", 2.5), + ]: + run_dir = root / run_id + run_dir.mkdir(exist_ok=True) + (run_dir / "results.json").write_text( + json.dumps( + { + "run_id": run_id, + "timestamp": ts, + "metrics": {"coherence": coh}, + "summary": { + "run_pass": coh >= 4.0, + "items_total": 3, + "items_passed_all": 3 if coh >= 4.0 else 1, + }, + } + ), + encoding="utf-8", + ) + + +def _agent_yaml(disable_remote: bool = True) -> str: + return ( + "version: 1\n" + "sources:\n" + " results_history:\n" + " enabled: true\n" + " path: .agentops/results\n" + " azure_monitor:\n" + f" enabled: {'false' if disable_remote else 'true'}\n" + " foundry_control:\n" + f" enabled: {'false' if disable_remote else 'true'}\n" + ) + + +def test_agent_analyze_reports_regression_and_exits_two(tmp_path: Path) -> None: + _seed_regression(tmp_path) + (tmp_path / ".agentops" / "agent.yaml").write_text( + _agent_yaml(), encoding="utf-8" + ) + + result = runner.invoke( + app, + ["agent", "analyze", "--workspace", str(tmp_path), "--severity-fail", "warning"], + ) + + assert result.exit_code == 2, result.stdout + report_path = tmp_path / ".agentops" / "agent" / "report.md" + assert report_path.exists() + body = report_path.read_text(encoding="utf-8") + assert "regression.coherence" in body + + +def test_agent_analyze_no_findings_exits_zero(tmp_path: Path) -> None: + # Empty workspace -> no runs -> no findings. + (tmp_path / ".agentops").mkdir() + (tmp_path / ".agentops" / "agent.yaml").write_text( + _agent_yaml(), encoding="utf-8" + ) + result = runner.invoke( + app, ["agent", "analyze", "--workspace", str(tmp_path)] + ) + assert result.exit_code == 0, result.stdout + + +def test_agent_analyze_rejects_invalid_severity(tmp_path: Path) -> None: + result = runner.invoke( + app, + [ + "agent", + "analyze", + "--workspace", + str(tmp_path), + "--severity-fail", + "wat", + ], + ) + assert result.exit_code == 1 diff --git a/tests/unit/test_agent_config.py b/tests/unit/test_agent_config.py new file mode 100644 index 00000000..d9d563f2 --- /dev/null +++ b/tests/unit/test_agent_config.py @@ -0,0 +1,45 @@ +"""Tests for the watchdog agent config loader.""" + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from agentops.agent.config import AgentConfig, load_agent_config + + +def test_load_agent_config_returns_defaults_when_missing(tmp_path: Path) -> None: + config = load_agent_config(tmp_path / "missing.yaml") + assert isinstance(config, AgentConfig) + assert config.version == 1 + assert config.sources.results_history.enabled is True + + +def test_load_agent_config_parses_yaml(tmp_path: Path) -> None: + cfg = tmp_path / "agent.yaml" + cfg.write_text( + """ +version: 1 +lookback_days: 14 +sources: + results_history: + enabled: false + path: custom/results +checks: + regression: + threshold_drop: 0.25 +""", + encoding="utf-8", + ) + config = load_agent_config(cfg) + assert config.lookback_days == 14 + assert config.sources.results_history.enabled is False + assert config.sources.results_history.path == "custom/results" + assert config.checks.regression.threshold_drop == 0.25 + + +def test_load_agent_config_rejects_unknown_keys(tmp_path: Path) -> None: + cfg = tmp_path / "agent.yaml" + cfg.write_text("version: 1\nbogus_field: 1\n", encoding="utf-8") + with pytest.raises(ValidationError): + load_agent_config(cfg) diff --git a/tests/unit/test_agent_findings.py b/tests/unit/test_agent_findings.py new file mode 100644 index 00000000..42234a59 --- /dev/null +++ b/tests/unit/test_agent_findings.py @@ -0,0 +1,30 @@ +"""Tests for the watchdog agent findings model.""" + +from agentops.agent.findings import Finding, Severity, severity_emoji + + +def test_severity_ordering() -> None: + assert Severity.INFO < Severity.WARNING < Severity.CRITICAL + assert Severity.CRITICAL > Severity.WARNING + assert Severity.WARNING <= Severity.WARNING + assert Severity.CRITICAL >= Severity.INFO + + +def test_finding_to_dict_roundtrip() -> None: + finding = Finding( + id="x.y", + severity=Severity.WARNING, + title="t", + summary="s", + recommendation="r", + source="results_history", + evidence={"k": 1}, + ) + payload = finding.to_dict() + assert payload["severity"] == "warning" + assert payload["evidence"] == {"k": 1} + + +def test_severity_emoji_mapping() -> None: + for sev in Severity: + assert severity_emoji(sev) diff --git a/tests/unit/test_agent_posture_rules.py b/tests/unit/test_agent_posture_rules.py new file mode 100644 index 00000000..759018d4 --- /dev/null +++ b/tests/unit/test_agent_posture_rules.py @@ -0,0 +1,295 @@ +"""Unit tests for the WAF-AI posture rule registry.""" + +from __future__ import annotations + +from agentops.agent.checks.posture import run_posture_check +from agentops.agent.checks.posture_rules import RULE_REGISTRY +from agentops.agent.checks.posture_rules.content_filter import ( + evaluate as content_filter_rule, +) +from agentops.agent.checks.posture_rules.diagnostics import ( + evaluate as diagnostics_rule, +) +from agentops.agent.checks.posture_rules.local_auth import ( + evaluate as local_auth_rule, +) +from agentops.agent.checks.posture_rules.managed_identity import ( + evaluate as managed_identity_rule, +) +from agentops.agent.checks.posture_rules.network import evaluate as network_rule +from agentops.agent.config import PostureCheckConfig +from agentops.agent.findings import Category, Severity +from agentops.agent.sources.azure_resources import ( + AzureResourcesPayload, + CognitiveAccountSnapshot, + DeploymentSnapshot, + DiagnosticSettingSnapshot, +) + + +_SENTINEL = object() + + +def _payload( + *, + disable_local_auth: bool = True, + public_network_access: str = "Disabled", + private_endpoint_count: int = 0, + network_acls_default_action: str | None = None, + identity_type: str | None = "SystemAssigned", + deployments=_SENTINEL, + diagnostic_settings=_SENTINEL, +) -> AzureResourcesPayload: + if deployments is _SENTINEL: + deployments = [ + DeploymentSnapshot(name="gpt-4o", model="gpt-4o", rai_policy_name="Microsoft.Default") + ] + if diagnostic_settings is _SENTINEL: + diagnostic_settings = [ + DiagnosticSettingSnapshot( + name="default", + workspace_id="/subscriptions/.../workspaces/log", + enabled_log_categories=["Audit", "RequestResponse"], + ) + ] + return AzureResourcesPayload( + account=CognitiveAccountSnapshot( + name="ai-test", + disable_local_auth=disable_local_auth, + public_network_access=public_network_access, + private_endpoint_count=private_endpoint_count, + network_acls_default_action=network_acls_default_action, + identity_type=identity_type, + ), + deployments=deployments, + diagnostic_settings=diagnostic_settings, + diagnostics={"status": "ok"}, + ) + + +# --------------------------------------------------------------------------- +# local_auth_disabled +# --------------------------------------------------------------------------- + + +def test_local_auth_rule_passes_when_disabled() -> None: + assert local_auth_rule(_payload(disable_local_auth=True), "azure_resources") == [] + + +def test_local_auth_rule_fires_when_enabled() -> None: + findings = local_auth_rule(_payload(disable_local_auth=False), "azure_resources") + assert len(findings) == 1 + assert findings[0].id == "waf.security.local_auth_disabled" + assert findings[0].severity is Severity.CRITICAL + assert findings[0].category is Category.SECURITY + assert findings[0].evidence["disable_local_auth"] is False + + +def test_local_auth_rule_fires_when_unknown() -> None: + # `None` means "we don't know" — treat as a finding so the user investigates. + findings = local_auth_rule(_payload(disable_local_auth=None), "azure_resources") + assert len(findings) == 1 + + +# --------------------------------------------------------------------------- +# public_network_access +# --------------------------------------------------------------------------- + + +def test_network_rule_passes_when_disabled() -> None: + assert network_rule(_payload(public_network_access="Disabled"), "azure_resources") == [] + + +def test_network_rule_passes_when_private_endpoint() -> None: + assert ( + network_rule( + _payload(public_network_access="Enabled", private_endpoint_count=1), + "azure_resources", + ) + == [] + ) + + +def test_network_rule_passes_when_acl_deny() -> None: + assert ( + network_rule( + _payload( + public_network_access="Enabled", + network_acls_default_action="Deny", + ), + "azure_resources", + ) + == [] + ) + + +def test_network_rule_fires_when_open() -> None: + findings = network_rule( + _payload( + public_network_access="Enabled", + private_endpoint_count=0, + network_acls_default_action="Allow", + ), + "azure_resources", + ) + assert len(findings) == 1 + assert findings[0].id == "waf.security.public_network_access" + assert findings[0].severity is Severity.WARNING + + +# --------------------------------------------------------------------------- +# managed_identity +# --------------------------------------------------------------------------- + + +def test_managed_identity_rule_passes_when_system_assigned() -> None: + assert ( + managed_identity_rule(_payload(identity_type="SystemAssigned"), "azure_resources") + == [] + ) + + +def test_managed_identity_rule_passes_when_user_assigned() -> None: + assert ( + managed_identity_rule(_payload(identity_type="UserAssigned"), "azure_resources") + == [] + ) + + +def test_managed_identity_rule_fires_when_none() -> None: + findings = managed_identity_rule(_payload(identity_type="None"), "azure_resources") + assert len(findings) == 1 + + +def test_managed_identity_rule_fires_when_missing() -> None: + findings = managed_identity_rule(_payload(identity_type=None), "azure_resources") + assert len(findings) == 1 + + +# --------------------------------------------------------------------------- +# diagnostic_settings +# --------------------------------------------------------------------------- + + +def test_diagnostics_rule_passes_when_workspace_and_categories() -> None: + assert diagnostics_rule(_payload(), "azure_resources") == [] + + +def test_diagnostics_rule_fires_when_no_destination() -> None: + findings = diagnostics_rule( + _payload( + diagnostic_settings=[ + DiagnosticSettingSnapshot( + name="default", + workspace_id=None, + enabled_log_categories=["Audit"], + ) + ] + ), + "azure_resources", + ) + assert len(findings) == 1 + assert findings[0].id == "waf.security.diagnostic_settings" + + +def test_diagnostics_rule_fires_when_no_categories() -> None: + findings = diagnostics_rule( + _payload( + diagnostic_settings=[ + DiagnosticSettingSnapshot( + name="default", + workspace_id="/some/workspace", + enabled_log_categories=[], + ) + ] + ), + "azure_resources", + ) + assert len(findings) == 1 + + +def test_diagnostics_rule_fires_when_empty() -> None: + findings = diagnostics_rule( + _payload(diagnostic_settings=[]), "azure_resources" + ) + assert len(findings) == 1 + + +# --------------------------------------------------------------------------- +# content_filter +# --------------------------------------------------------------------------- + + +def test_content_filter_rule_passes_when_all_have_policy() -> None: + assert content_filter_rule(_payload(), "azure_resources") == [] + + +def test_content_filter_rule_fires_when_any_missing() -> None: + findings = content_filter_rule( + _payload( + deployments=[ + DeploymentSnapshot( + name="gpt-4o", model="gpt-4o", rai_policy_name="Microsoft.Default" + ), + DeploymentSnapshot(name="legacy", model="gpt-3.5", rai_policy_name=None), + ] + ), + "azure_resources", + ) + assert len(findings) == 1 + assert findings[0].evidence["deployments_missing_filter"] == [ + {"name": "legacy", "model": "gpt-3.5"} + ] + assert findings[0].severity is Severity.CRITICAL + + +# --------------------------------------------------------------------------- +# dispatcher +# --------------------------------------------------------------------------- + + +def test_run_posture_check_returns_nothing_when_disabled() -> None: + payload = _payload(disable_local_auth=False) + findings = run_posture_check(payload, PostureCheckConfig(enabled=False)) + assert findings == [] + + +def test_run_posture_check_returns_nothing_when_source_skipped() -> None: + payload = AzureResourcesPayload(diagnostics={"status": "skipped"}) + findings = run_posture_check(payload, PostureCheckConfig(enabled=True)) + assert findings == [] + + +def test_run_posture_check_aggregates_rules() -> None: + payload = _payload( + disable_local_auth=False, + public_network_access="Enabled", + identity_type=None, + ) + findings = run_posture_check(payload, PostureCheckConfig(enabled=True)) + ids = {f.id for f in findings} + assert "waf.security.local_auth_disabled" in ids + assert "waf.security.public_network_access" in ids + assert "waf.security.managed_identity" in ids + assert all(f.category is Category.SECURITY for f in findings) + + +def test_run_posture_check_honours_excluded_rules() -> None: + payload = _payload(disable_local_auth=False, identity_type=None) + config = PostureCheckConfig( + enabled=True, exclude_rules=["waf.security.local_auth_disabled"] + ) + findings = run_posture_check(payload, config) + ids = {f.id for f in findings} + assert "waf.security.local_auth_disabled" not in ids + assert "waf.security.managed_identity" in ids + + +def test_rule_registry_has_five_mvp_rules() -> None: + assert set(RULE_REGISTRY.keys()) == { + "waf.security.local_auth_disabled", + "waf.security.public_network_access", + "waf.security.managed_identity", + "waf.security.diagnostic_settings", + "waf.security.content_filter", + } diff --git a/tests/unit/test_agent_results_history.py b/tests/unit/test_agent_results_history.py new file mode 100644 index 00000000..071ad15f --- /dev/null +++ b/tests/unit/test_agent_results_history.py @@ -0,0 +1,67 @@ +"""Tests for the results-history source.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from agentops.agent.config import ResultsHistorySourceConfig +from agentops.agent.sources.results_history import collect_results_history + + +def _write_run( + results_root: Path, + run_id: str, + timestamp: str, + metrics: dict, + *, + items_total: int = 3, + items_passed_all: int = 3, + run_pass: bool = True, +) -> None: + run_dir = results_root / run_id + run_dir.mkdir(parents=True, exist_ok=True) + payload = { + "run_id": run_id, + "timestamp": timestamp, + "metrics": metrics, + "summary": { + "run_pass": run_pass, + "items_total": items_total, + "items_passed_all": items_passed_all, + }, + } + (run_dir / "results.json").write_text(json.dumps(payload), encoding="utf-8") + + +def test_collect_results_history_orders_oldest_to_newest(tmp_path: Path) -> None: + workspace = tmp_path + results = workspace / ".agentops" / "results" + _write_run(results, "run-1", "2024-05-01T10:00:00Z", {"coherence": 4.5}) + _write_run(results, "run-2", "2024-05-02T10:00:00Z", {"coherence": 4.0}) + _write_run(results, "latest", "2024-06-01T10:00:00Z", {"coherence": 1.0}) + + config = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + history = collect_results_history(workspace, config) + + assert [r.run_id for r in history.runs] == ["run-1", "run-2"] + assert history.runs[-1].metrics["coherence"] == 4.0 + assert history.diagnostics["status"] == "ok" + + +def test_collect_results_history_handles_missing_dir(tmp_path: Path) -> None: + config = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + history = collect_results_history(tmp_path, config) + assert history.runs == [] + assert history.diagnostics["status"] == "missing" + + +def test_collect_results_history_disabled(tmp_path: Path) -> None: + config = ResultsHistorySourceConfig(enabled=False) + history = collect_results_history(tmp_path, config) + assert history.runs == [] + assert history.diagnostics["status"] == "disabled" diff --git a/tests/unit/test_agent_server.py b/tests/unit/test_agent_server.py new file mode 100644 index 00000000..09699df5 --- /dev/null +++ b/tests/unit/test_agent_server.py @@ -0,0 +1,58 @@ +"""Tests for the FastAPI Copilot Extension server (agent extras).""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from agentops.agent.config import AgentConfig, ResultsHistorySourceConfig, SourcesConfig + +if importlib.util.find_spec("fastapi") is None: + pytest.skip("FastAPI not installed; agent extras unavailable", allow_module_level=True) + +from fastapi.testclient import TestClient # noqa: E402 + +from agentops.agent.server.app import create_app # noqa: E402 + + +def _config() -> AgentConfig: + sources = SourcesConfig() + sources.results_history = ResultsHistorySourceConfig( + enabled=True, path=".agentops/results", lookback_runs=10 + ) + sources.azure_monitor.enabled = False + sources.foundry_control.enabled = False + return AgentConfig(sources=sources) + + +def test_healthz(tmp_path: Path) -> None: + app = create_app(workspace=tmp_path, config=_config(), verify_signature=False) + client = TestClient(app) + response = client.get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_messages_streams_sse(tmp_path: Path) -> None: + app = create_app(workspace=tmp_path, config=_config(), verify_signature=False) + client = TestClient(app) + response = client.post( + "/agents/messages", + json={"messages": [{"role": "user", "content": "Run the watchdog"}]}, + ) + assert response.status_code == 200 + body = response.text + assert "data:" in body + assert "[DONE]" in body + + +def test_messages_requires_signature_when_enabled(tmp_path: Path) -> None: + app = create_app(workspace=tmp_path, config=_config(), verify_signature=True) + client = TestClient(app) + response = client.post( + "/agents/messages", + json={"messages": [{"role": "user", "content": "hi"}]}, + ) + assert response.status_code == 401 diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py new file mode 100644 index 00000000..0023c705 --- /dev/null +++ b/tests/unit/test_agentops_config.py @@ -0,0 +1,254 @@ +"""Tests for the flat ``agentops.yaml`` schema and agent classifier.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from agentops.core.agentops_config import ( + AgentOpsConfig, + Threshold, + classify_agent, +) + + +# --------------------------------------------------------------------------- +# classify_agent +# --------------------------------------------------------------------------- + + +class TestClassifyAgent: + def test_foundry_prompt_name_version(self) -> None: + result = classify_agent("my-rag:3") + assert result.kind == "foundry_prompt" + assert result.name == "my-rag" + assert result.version == "3" + assert result.protocol is None + + def test_foundry_prompt_rejects_empty_parts(self) -> None: + with pytest.raises(ValueError, match="name:version"): + classify_agent(":3") + with pytest.raises(ValueError, match="name:version"): + classify_agent("foo:") + + def test_model_direct(self) -> None: + result = classify_agent("model:gpt-4o-mini") + assert result.kind == "model_direct" + assert result.deployment == "gpt-4o-mini" + assert result.protocol is None + + def test_model_direct_rejects_empty_deployment(self) -> None: + with pytest.raises(ValueError, match="deployment name"): + classify_agent("model:") + + def test_foundry_hosted_default_protocol_responses(self) -> None: + url = "https://my-project.services.ai.azure.com/agents/foo" + result = classify_agent(url) + assert result.kind == "foundry_hosted" + assert result.protocol == "responses" + assert result.url == url + + def test_foundry_hosted_invocations(self) -> None: + url = "https://my-project.services.ai.azure.com/agents/foo" + result = classify_agent(url, protocol="invocations") + assert result.kind == "foundry_hosted" + assert result.protocol == "invocations" + + def test_foundry_hosted_rejects_http_json_protocol(self) -> None: + url = "https://my-project.services.ai.azure.com/agents/foo" + with pytest.raises(ValueError, match="responses"): + classify_agent(url, protocol="http-json") + + def test_http_json_default_protocol(self) -> None: + url = "https://my-app.azurecontainerapps.io/chat" + result = classify_agent(url) + assert result.kind == "http_json" + assert result.protocol == "http-json" + + def test_http_json_rejects_responses_protocol(self) -> None: + url = "https://my-app.azurecontainerapps.io/chat" + with pytest.raises(ValueError, match="http-json"): + classify_agent(url, protocol="responses") + + def test_unrecognized_value(self) -> None: + with pytest.raises(ValueError, match="unrecognized"): + classify_agent("just-a-name") + + +# --------------------------------------------------------------------------- +# Threshold parser +# --------------------------------------------------------------------------- + + +class TestThresholdFromExpression: + @pytest.mark.parametrize( + "expression, expected_criteria, expected_value", + [ + (">=3", ">=", 3.0), + ("<=10", "<=", 10.0), + (">2.5", ">", 2.5), + ("<0.7", "<", 0.7), + ("==1", "==", 1.0), + (" >= 3 ", ">=", 3.0), + ], + ) + def test_comparison( + self, expression: str, expected_criteria: str, expected_value: float + ) -> None: + threshold = Threshold.from_expression("metric", expression) + assert threshold.criteria == expected_criteria + assert threshold.value == expected_value + + def test_bool_true(self) -> None: + threshold = Threshold.from_expression("metric", True) + assert threshold.criteria == "true" + assert threshold.value is None + + def test_bool_false_string(self) -> None: + threshold = Threshold.from_expression("metric", "false") + assert threshold.criteria == "false" + + def test_number_shorthand(self) -> None: + # bare number defaults to >= + threshold = Threshold.from_expression("metric", 3) + assert threshold.criteria == ">=" + assert threshold.value == 3.0 + + def test_invalid_expression(self) -> None: + with pytest.raises(ValueError, match="expected"): + Threshold.from_expression("metric", "approximately 3") + + def test_invalid_number(self) -> None: + with pytest.raises(ValueError, match="cannot parse"): + Threshold.from_expression("metric", ">=abc") + + +# --------------------------------------------------------------------------- +# AgentOpsConfig +# --------------------------------------------------------------------------- + + +class TestAgentOpsConfig: + def test_minimal_config(self, tmp_path) -> None: + cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") + assert cfg.version == 1 + assert cfg.agent == "my-rag:3" + assert cfg.thresholds == {} + + def test_resolved_target(self) -> None: + cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") + target = cfg.resolved_target() + assert target.kind == "foundry_prompt" + + def test_rejects_legacy_keys(self) -> None: + with pytest.raises(ValidationError) as exc_info: + AgentOpsConfig.model_validate( + { + "version": 1, + "agent": "my-rag:3", + "dataset": "./qa.jsonl", + "scenario": "rag", + } + ) + assert "legacy" in str(exc_info.value).lower() + + def test_rejects_extra_fields(self) -> None: + with pytest.raises(ValidationError): + AgentOpsConfig.model_validate( + { + "version": 1, + "agent": "my-rag:3", + "dataset": "./qa.jsonl", + "unknown_key": "x", + } + ) + + def test_rejects_wrong_version(self) -> None: + with pytest.raises(ValidationError, match="version must be 1"): + AgentOpsConfig(version=2, agent="my-rag:3", dataset="./qa.jsonl") + + def test_thresholds_parsed(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + thresholds={"groundedness": ">=3", "coherence": ">=3.5"}, + ) + parsed = {t.metric: t for t in cfg.parsed_thresholds()} + assert parsed["groundedness"].criteria == ">=" + assert parsed["groundedness"].value == 3.0 + assert parsed["coherence"].value == 3.5 + + def test_publish_foundry_accepted(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + publish="foundry", + project_endpoint="https://x.services.ai.azure.com/api/projects/p", + ) + assert cfg.publish == "foundry" + assert cfg.project_endpoint.endswith("/projects/p") + + def test_publish_defaults_to_none(self) -> None: + cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") + assert cfg.publish is None + assert cfg.project_endpoint is None + + def test_publish_rejects_unknown_target(self) -> None: + with pytest.raises(ValidationError): + AgentOpsConfig.model_validate( + { + "version": 1, + "agent": "my-rag:3", + "dataset": "./qa.jsonl", + "publish": "datadog", + } + ) + + def test_protocol_rejected_for_prompt_agent(self) -> None: + with pytest.raises(ValidationError, match="prompt agent"): + AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + protocol="responses", + ) + + def test_protocol_rejected_for_model_direct(self) -> None: + with pytest.raises(ValidationError, match="protocol"): + AgentOpsConfig( + version=1, + agent="model:gpt-4o", + dataset="./qa.jsonl", + protocol="http-json", + ) + + def test_http_fields_allowed_for_http_target(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="https://my-app.azurecontainerapps.io/chat", + dataset="./qa.jsonl", + request_field="message", + response_field="text", + ) + assert cfg.request_field == "message" + + def test_http_fields_rejected_for_prompt_agent(self) -> None: + with pytest.raises(ValidationError, match="HTTP/JSON"): + AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + request_field="message", + ) + + def test_evaluators_override(self) -> None: + cfg = AgentOpsConfig( + version=1, + agent="my-rag:3", + dataset="./qa.jsonl", + evaluators=[{"name": "GroundednessEvaluator"}], # type: ignore[list-item] + ) + assert cfg.evaluators is not None + assert cfg.evaluators[0].name == "GroundednessEvaluator" diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py deleted file mode 100644 index 1710dd6e..00000000 --- a/tests/unit/test_browse.py +++ /dev/null @@ -1,342 +0,0 @@ -"""Tests for browse services (bundle list/show, run list/show).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest -from typer.testing import CliRunner - -from agentops.cli.app import app -from agentops.services.browse import ( - list_bundles, - list_runs, - show_bundle, - show_run, -) -from agentops.utils.yaml import save_yaml - -runner = CliRunner() - - -def _create_workspace(tmp_path: Path) -> Path: - """Create a minimal .agentops workspace.""" - ws = tmp_path / ".agentops" - ws.mkdir() - (ws / "bundles").mkdir() - (ws / "results").mkdir() - return ws - - -def _create_workspace_without_results(tmp_path: Path) -> Path: - """Create a .agentops workspace that has no results directory.""" - ws = tmp_path / ".agentops" - ws.mkdir() - (ws / "bundles").mkdir() - return ws - - -def _write_bundle(ws: Path, name: str, evaluators: list, thresholds: list) -> Path: - bundle_path = ws / "bundles" / f"{name}.yaml" - save_yaml( - bundle_path, - { - "version": 1, - "name": name, - "description": f"Test bundle {name}", - "evaluators": evaluators, - "thresholds": thresholds, - "metadata": {"category": "test"}, - }, - ) - return bundle_path - - -def _write_run(ws: Path, run_id: str, *, passed: bool = True) -> Path: - run_dir = ws / "results" / run_id - run_dir.mkdir(parents=True) - results = { - "version": 1, - "status": "completed", - "bundle": {"name": "test_bundle", "path": "bundles/test.yaml"}, - "dataset": {"name": "test_dataset", "path": "datasets/test.yaml"}, - "execution": { - "backend": "foundry", - "command": "test", - "started_at": "2026-04-07T10:00:00Z", - "finished_at": "2026-04-07T10:01:00Z", - "duration_seconds": 60.0, - "exit_code": 0, - }, - "metrics": [ - {"name": "CoherenceEvaluator", "value": 4.5}, - {"name": "samples_evaluated", "value": 3.0}, - ], - "row_metrics": [], - "item_evaluations": [ - {"row_index": 1, "passed_all": True, "thresholds": []}, - {"row_index": 2, "passed_all": passed, "thresholds": []}, - ], - "thresholds": [ - { - "evaluator": "CoherenceEvaluator", - "criteria": ">=", - "expected": "3.000000", - "actual": "2/2 items", - "passed": passed, - } - ], - "summary": { - "metrics_count": 2, - "thresholds_count": 1, - "thresholds_passed": 1 if passed else 0, - "thresholds_failed": 0 if passed else 1, - "overall_passed": passed, - }, - } - (run_dir / "results.json").write_text( - json.dumps(results, indent=2), encoding="utf-8" - ) - (run_dir / "report.md").write_text("# Report", encoding="utf-8") - return run_dir - - -# --------------------------------------------------------------------------- -# Service tests -# --------------------------------------------------------------------------- - - -class TestListBundles: - def test_empty_workspace(self, tmp_path: Path) -> None: - _create_workspace(tmp_path) - result = list_bundles(directory=tmp_path) - assert result.bundles == [] - - def test_lists_bundles(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_bundle( - ws, - "baseline", - [{"name": "CoherenceEvaluator", "source": "foundry", "enabled": True}], - [{"evaluator": "CoherenceEvaluator", "criteria": ">=", "value": 3}], - ) - result = list_bundles(directory=tmp_path) - assert len(result.bundles) == 1 - assert result.bundles[0].name == "baseline" - assert result.bundles[0].evaluators == ["CoherenceEvaluator"] - assert result.bundles[0].thresholds == 1 - - def test_no_workspace_raises(self, tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError, match="No .agentops workspace"): - list_bundles(directory=tmp_path) - - -class TestShowBundle: - def test_by_name(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_bundle( - ws, - "my_bundle", - [{"name": "FluencyEvaluator", "source": "foundry", "enabled": True}], - [{"evaluator": "FluencyEvaluator", "criteria": ">=", "value": 4}], - ) - detail = show_bundle("my_bundle", directory=tmp_path) - assert detail.name == "my_bundle" - assert len(detail.evaluators) == 1 - assert detail.evaluators[0]["name"] == "FluencyEvaluator" - - def test_not_found(self, tmp_path: Path) -> None: - _create_workspace(tmp_path) - with pytest.raises(FileNotFoundError, match="not found"): - show_bundle("nonexistent", directory=tmp_path) - - -class TestListRuns: - def test_empty(self, tmp_path: Path) -> None: - _create_workspace(tmp_path) - result = list_runs(directory=tmp_path) - assert result.runs == [] - - def test_missing_results_dir_returns_empty(self, tmp_path: Path) -> None: - _create_workspace_without_results(tmp_path) - result = list_runs(directory=tmp_path) - assert result.runs == [] - - def test_lists_runs(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "2026-04-07_100000", passed=True) - _write_run(ws, "2026-04-07_110000", passed=False) - result = list_runs(directory=tmp_path) - assert len(result.runs) == 2 - # Sorted reverse (newest first) - assert result.runs[0].run_id == "2026-04-07_110000" - assert result.runs[0].overall_passed is False - assert result.runs[1].run_id == "2026-04-07_100000" - assert result.runs[1].overall_passed is True - - def test_skips_latest_when_history_runs_exist(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "2026-04-07_100000") - _write_run(ws, "2026-04-07_110000") - _write_run(ws, "latest") - result = list_runs(directory=tmp_path) - assert [run.run_id for run in result.runs] == [ - "2026-04-07_110000", - "2026-04-07_100000", - ] - - def test_skips_empty_latest_when_no_history_runs(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - (ws / "results" / "latest").mkdir() - result = list_runs(directory=tmp_path) - assert result.runs == [] - - def test_lists_malformed_history_run_and_skips_latest_mirror( - self, tmp_path: Path - ) -> None: - ws = _create_workspace(tmp_path) - malformed_run = ws / "results" / "2026-04-07_100000" - malformed_run.mkdir() - (malformed_run / "results.json").write_text("{", encoding="utf-8") - _write_run(ws, "latest") - result = list_runs(directory=tmp_path) - assert len(result.runs) == 1 - assert result.runs[0].run_id == "2026-04-07_100000" - assert result.runs[0].status == "error" - - def test_lists_malformed_latest_when_no_history_runs( - self, tmp_path: Path - ) -> None: - ws = _create_workspace(tmp_path) - latest_run = ws / "results" / "latest" - latest_run.mkdir() - (latest_run / "results.json").write_text("{", encoding="utf-8") - result = list_runs(directory=tmp_path) - assert len(result.runs) == 1 - assert result.runs[0].run_id == "latest" - assert result.runs[0].status == "error" - - def test_lists_latest_when_no_history_runs(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "latest", passed=False) - result = list_runs(directory=tmp_path) - assert len(result.runs) == 1 - assert result.runs[0].run_id == "latest" - assert result.runs[0].overall_passed is False - - -class TestShowRun: - def test_shows_run(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "2026-04-07_100000", passed=True) - detail = show_run("2026-04-07_100000", directory=tmp_path) - assert detail.run_id == "2026-04-07_100000" - assert detail.bundle_name == "test_bundle" - assert detail.overall_passed is True - assert detail.items_total == 2 - assert detail.items_passed == 2 - - def test_not_found(self, tmp_path: Path) -> None: - _create_workspace(tmp_path) - with pytest.raises(FileNotFoundError, match="not found"): - show_run("nonexistent", directory=tmp_path) - - def test_not_found_hints_latest_when_latest_is_only_listable_run( - self, tmp_path: Path - ) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "latest") - with pytest.raises(FileNotFoundError) as exc_info: - show_run("nonexistent", directory=tmp_path) - - assert "Recent runs: latest" in str(exc_info.value) - - def test_not_found_with_missing_results_dir_has_empty_recent_hint( - self, tmp_path: Path - ) -> None: - _create_workspace_without_results(tmp_path) - with pytest.raises(FileNotFoundError) as exc_info: - show_run("nonexistent", directory=tmp_path) - - assert "Recent runs: (none)" in str(exc_info.value) - - -# --------------------------------------------------------------------------- -# CLI tests -# --------------------------------------------------------------------------- - - -class TestBundleListCLI: - def test_lists_bundles(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_bundle( - ws, - "baseline", - [{"name": "CoherenceEvaluator", "source": "foundry", "enabled": True}], - [{"evaluator": "CoherenceEvaluator", "criteria": ">=", "value": 3}], - ) - result = runner.invoke(app, ["bundle", "list", "--dir", str(tmp_path)]) - assert result.exit_code == 0 - assert "baseline" in result.stdout - assert "CoherenceEvaluator" in result.stdout - - def test_no_workspace(self, tmp_path: Path) -> None: - result = runner.invoke(app, ["bundle", "list", "--dir", str(tmp_path)]) - assert result.exit_code == 1 - assert "No .agentops workspace" in (result.stdout + result.stderr) - - -class TestBundleShowCLI: - def test_shows_bundle(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_bundle( - ws, - "my_bundle", - [{"name": "FluencyEvaluator", "source": "foundry", "enabled": True}], - [{"evaluator": "FluencyEvaluator", "criteria": ">=", "value": 4}], - ) - result = runner.invoke( - app, ["bundle", "show", "my_bundle", "--dir", str(tmp_path)] - ) - assert result.exit_code == 0 - assert "my_bundle" in result.stdout - assert "FluencyEvaluator" in result.stdout - - -class TestRunListCLI: - def test_lists_runs(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "2026-04-07_100000", passed=True) - result = runner.invoke(app, ["run", "list", "--dir", str(tmp_path)]) - assert result.exit_code == 0 - assert "2026-04-07_100000" in result.stdout - assert "PASS" in result.stdout - - def test_lists_latest_when_no_history_runs(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "latest", passed=True) - result = runner.invoke(app, ["run", "list", "--dir", str(tmp_path)]) - assert result.exit_code == 0 - assert "latest" in result.stdout - assert "No runs found" not in result.stdout - - -class TestRunShowCLI: - def test_shows_run(self, tmp_path: Path) -> None: - ws = _create_workspace(tmp_path) - _write_run(ws, "2026-04-07_100000") - result = runner.invoke( - app, ["run", "show", "2026-04-07_100000", "--dir", str(tmp_path)] - ) - assert result.exit_code == 0 - assert "test_bundle" in result.stdout - assert "CoherenceEvaluator" in result.stdout - - def test_not_found(self, tmp_path: Path) -> None: - _create_workspace(tmp_path) - result = runner.invoke( - app, ["run", "show", "nonexistent", "--dir", str(tmp_path)] - ) - assert result.exit_code == 1 - assert "not found" in (result.stdout + result.stderr) diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index 894c919d..2c88fa56 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -1,327 +1,261 @@ +"""Tests for `agentops workflow generate` (4-template GenAIOps GitFlow scaffold).""" + from pathlib import Path +import yaml from typer.testing import CliRunner from agentops.cli.app import app -from agentops.services.cicd import generate_cicd_workflow, generate_cicd_workflows +from agentops.services.cicd import ( + ALL_KINDS, + generate_cicd_workflow, + generate_cicd_workflows, +) runner = CliRunner() -_WORKFLOW_PATH = ".github/workflows/agentops-eval.yml" -_CI_WORKFLOW_PATH = ".github/workflows/agentops-eval-ci.yml" -_CD_WORKFLOW_PATH = ".github/workflows/agentops-eval-cd.yml" +_WORKFLOW_DIR = ".github/workflows" +_PR_PATH = f"{_WORKFLOW_DIR}/agentops-pr.yml" +_DEV_PATH = f"{_WORKFLOW_DIR}/agentops-deploy-dev.yml" +_QA_PATH = f"{_WORKFLOW_DIR}/agentops-deploy-qa.yml" +_PROD_PATH = f"{_WORKFLOW_DIR}/agentops-deploy-prod.yml" +ALL_PATHS = (_PR_PATH, _DEV_PATH, _QA_PATH, _PROD_PATH) -def _scaffold_agentops_workspace(tmp_path: Path, bundles: list[str] | None = None, run_configs: list[str] | None = None) -> None: - """Create a minimal .agentops/ workspace with optional bundles and run configs.""" - agentops_dir = tmp_path / ".agentops" - agentops_dir.mkdir(parents=True, exist_ok=True) - # Default run.yaml - (agentops_dir / "run.yaml").write_text("version: 1\n", encoding="utf-8") +# --------------------------------------------------------------------------- +# generate_cicd_workflows — defaults to all four +# --------------------------------------------------------------------------- - if bundles: - bundles_dir = agentops_dir / "bundles" - bundles_dir.mkdir(parents=True, exist_ok=True) - for name in bundles: - (bundles_dir / name).write_text("version: 1\n", encoding="utf-8") - if run_configs: - for name in run_configs: - (agentops_dir / name).write_text("version: 1\n", encoding="utf-8") +def test_default_generates_all_four_templates(tmp_path: Path) -> None: + result = generate_cicd_workflows(directory=tmp_path) + assert {p.name for p in result.created_files} == { + "agentops-pr.yml", + "agentops-deploy-dev.yml", + "agentops-deploy-qa.yml", + "agentops-deploy-prod.yml", + } + for rel in ALL_PATHS: + assert (tmp_path / rel).exists() -def test_generate_cicd_creates_workflow(tmp_path: Path) -> None: - result = generate_cicd_workflow(directory=tmp_path) - workflow = tmp_path / _WORKFLOW_PATH - assert workflow.exists() - assert len(result.created_files) == 1 - assert len(result.skipped_files) == 0 +def test_kinds_filter_subset(tmp_path: Path) -> None: + result = generate_cicd_workflows(directory=tmp_path, kinds=["pr", "dev"]) - content = workflow.read_text(encoding="utf-8") - assert "agentops eval run" in content - assert "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT" in content - - -def test_generate_cicd_skips_existing(tmp_path: Path) -> None: - workflow = tmp_path / _WORKFLOW_PATH - workflow.parent.mkdir(parents=True, exist_ok=True) - workflow.write_text("existing content", encoding="utf-8") + assert {p.name for p in result.created_files} == { + "agentops-pr.yml", + "agentops-deploy-dev.yml", + } + assert (tmp_path / _PR_PATH).exists() + assert (tmp_path / _DEV_PATH).exists() + assert not (tmp_path / _QA_PATH).exists() + assert not (tmp_path / _PROD_PATH).exists() - result = generate_cicd_workflow(directory=tmp_path, force=False) - assert len(result.skipped_files) == 1 - assert len(result.created_files) == 0 - assert workflow.read_text(encoding="utf-8") == "existing content" +def test_kinds_unknown_value_is_ignored(tmp_path: Path) -> None: + result = generate_cicd_workflows(directory=tmp_path, kinds=["pr", "bogus"]) + assert {p.name for p in result.created_files} == {"agentops-pr.yml"} -def test_generate_cicd_overwrites_with_force(tmp_path: Path) -> None: - workflow = tmp_path / _WORKFLOW_PATH - workflow.parent.mkdir(parents=True, exist_ok=True) - workflow.write_text("old content", encoding="utf-8") +def test_kinds_dedupes(tmp_path: Path) -> None: + result = generate_cicd_workflows(directory=tmp_path, kinds=["pr", "pr", "dev"]) + assert len(result.created_files) == 2 - result = generate_cicd_workflow(directory=tmp_path, force=True) - assert len(result.overwritten_files) == 1 - assert len(result.skipped_files) == 0 +def test_skips_existing_without_force(tmp_path: Path) -> None: + pr = tmp_path / _PR_PATH + pr.parent.mkdir(parents=True, exist_ok=True) + pr.write_text("existing", encoding="utf-8") - content = workflow.read_text(encoding="utf-8") - assert "agentops eval run" in content - assert content != "old content" + result = generate_cicd_workflows(directory=tmp_path, kinds=["pr", "dev"]) + assert len(result.skipped_files) == 1 + assert len(result.created_files) == 1 + assert pr.read_text(encoding="utf-8") == "existing" + assert (tmp_path / _DEV_PATH).exists() -def test_cli_workflow_generate_creates_workflow(tmp_path: Path) -> None: - result = runner.invoke(app, ["workflow", "generate", "--dir", str(tmp_path)]) - assert result.exit_code == 0 - assert "created" in result.stdout +def test_force_overwrites_all(tmp_path: Path) -> None: + for rel in ALL_PATHS: + wf = tmp_path / rel + wf.parent.mkdir(parents=True, exist_ok=True) + wf.write_text("old", encoding="utf-8") - workflow = tmp_path / _WORKFLOW_PATH - assert workflow.exists() + result = generate_cicd_workflows(directory=tmp_path, force=True) + assert len(result.overwritten_files) == 4 + assert len(result.skipped_files) == 0 + for rel in ALL_PATHS: + assert (tmp_path / rel).read_text(encoding="utf-8") != "old" -def test_cli_workflow_generate_skips_existing(tmp_path: Path) -> None: - workflow = tmp_path / _WORKFLOW_PATH - workflow.parent.mkdir(parents=True, exist_ok=True) - workflow.write_text("existing", encoding="utf-8") - result = runner.invoke(app, ["workflow", "generate", "--dir", str(tmp_path)]) +def test_legacy_generate_cicd_workflow_writes_pr_only(tmp_path: Path) -> None: + result = generate_cicd_workflow(directory=tmp_path) + assert {p.name for p in result.created_files} == {"agentops-pr.yml"} + assert (tmp_path / _PR_PATH).exists() + assert not (tmp_path / _DEV_PATH).exists() - assert result.exit_code == 0 - assert "skipped" in result.stdout +# --------------------------------------------------------------------------- +# Template content checks +# --------------------------------------------------------------------------- -def test_cli_workflow_generate_force_overwrites(tmp_path: Path) -> None: - workflow = tmp_path / _WORKFLOW_PATH - workflow.parent.mkdir(parents=True, exist_ok=True) - workflow.write_text("old", encoding="utf-8") - result = runner.invoke(app, ["workflow", "generate", "--force", "--dir", str(tmp_path)]) +def _read_yaml(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) - assert result.exit_code == 0 - assert "overwritten" in result.stdout +def test_all_templates_are_valid_yaml(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path) + for rel in ALL_PATHS: + data = _read_yaml(tmp_path / rel) + assert isinstance(data, dict) + assert "jobs" in data + # `on:` is parsed as the boolean True by yaml.safe_load when the key + # is bare; just check the raw text contains the trigger block. + text = (tmp_path / rel).read_text(encoding="utf-8") + assert "\non:" in text -def test_workflow_template_has_required_features(tmp_path: Path) -> None: - """Verify the generated workflow meets issue #20 acceptance criteria.""" - generate_cicd_workflow(directory=tmp_path) - content = (tmp_path / _WORKFLOW_PATH).read_text(encoding="utf-8") +def test_pr_template_triggers_and_no_environment(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path, kinds=["pr"]) + content = (tmp_path / _PR_PATH).read_text(encoding="utf-8") - # Triggers assert "pull_request" in content - assert "workflow_dispatch" in content - - # Python 3.11 - assert "3.11" in content - - # Install - assert "agentops-toolkit" in content + # PR fires for develop, release/**, and main + assert "develop" in content + assert "release/**" in content + assert "main" in content - # Eval command assert "agentops eval run" in content - - # Artifacts - assert "results.json" in content - assert "report.md" in content - - # Exit code handling - assert "EXIT_CODE" in content - assert "exit $EXIT_CODE" in content - - # OIDC auth + assert "agentops-toolkit" in content assert "azure/login@v2" in content + assert "actions/setup-python@v5" in content + assert "3.11" in content + # PR template runs inside the dev environment so the OIDC token subject + # is `repo:/:environment:dev` and `vars.AZURE_*` resolve from + # the dev environment scope. Without this the gate fails on `azure/login`. + assert "environment: dev" in content -# --------------------------------------------------------------------------- -# generate_cicd_workflows — multi-template generation -# --------------------------------------------------------------------------- + # PR comment idempotency marker + assert "" in content -def test_generate_workflows_pr_only_default(tmp_path: Path) -> None: - """Minimal workspace (no extra bundles) → only PR template generated.""" - _scaffold_agentops_workspace(tmp_path, bundles=["model_quality_baseline.yaml"]) +def test_dev_template_triggers_and_environment(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path, kinds=["dev"]) + content = (tmp_path / _DEV_PATH).read_text(encoding="utf-8") - result = generate_cicd_workflows(directory=tmp_path) + assert "push" in content + assert "develop" in content + assert "environment: dev" in content + assert "agentops eval run" in content + # Has eval, build, deploy jobs + assert "needs: eval" in content + assert "needs: build" in content - assert len(result.created_files) == 1 - assert (tmp_path / _WORKFLOW_PATH).exists() - assert not (tmp_path / _CI_WORKFLOW_PATH).exists() - assert not (tmp_path / _CD_WORKFLOW_PATH).exists() +def test_qa_template_triggers_and_environment(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path, kinds=["qa"]) + content = (tmp_path / _QA_PATH).read_text(encoding="utf-8") -def test_generate_workflows_auto_detects_ci_and_cd(tmp_path: Path) -> None: - """Multiple bundles in workspace → PR + CI + CD templates generated.""" - _scaffold_agentops_workspace( - tmp_path, - bundles=["model_quality_baseline.yaml", "safe_agent_baseline.yaml"], - ) + assert "push" in content + assert "release/**" in content + assert "environment: qa" in content + assert "agentops eval run" in content + assert "needs: eval" in content + assert "needs: build" in content - result = generate_cicd_workflows(directory=tmp_path) - created_names = {p.name for p in result.created_files} - assert "agentops-eval.yml" in created_names - assert "agentops-eval-ci.yml" in created_names - assert "agentops-eval-cd.yml" in created_names - assert (tmp_path / _WORKFLOW_PATH).exists() - assert (tmp_path / _CI_WORKFLOW_PATH).exists() - assert (tmp_path / _CD_WORKFLOW_PATH).exists() +def test_prod_template_triggers_and_environment_with_reviewer_hint(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path, kinds=["prod"]) + content = (tmp_path / _PROD_PATH).read_text(encoding="utf-8") + assert "push" in content + assert "main" in content + assert "environment: production" in content + assert "agentops eval run" in content + # Prod uses safety-eval as the gate name and warns about reviewers + assert "safety-eval" in content + assert "Required reviewers" in content or "REQUIRED REVIEWERS" in content -def test_generate_workflows_auto_detects_ci(tmp_path: Path) -> None: - """Multiple bundles → PR + CI + CD templates generated.""" - _scaffold_agentops_workspace( - tmp_path, - bundles=["model_quality_baseline.yaml", "rag_quality_baseline.yaml"], - ) - result = generate_cicd_workflows(directory=tmp_path) +def test_all_kinds_constant_matches_documented_set() -> None: + assert set(ALL_KINDS) == {"pr", "dev", "qa", "prod"} - created_names = {p.name for p in result.created_files} - assert "agentops-eval.yml" in created_names - assert "agentops-eval-ci.yml" in created_names - assert "agentops-eval-cd.yml" in created_names +# --------------------------------------------------------------------------- +# CLI surface +# --------------------------------------------------------------------------- -def test_generate_workflows_creates_all_templates(tmp_path: Path) -> None: - """Explicit kinds=all → all three templates generated.""" - result = generate_cicd_workflows( - directory=tmp_path, - kinds=["pr", "ci", "cd"], - ) - assert len(result.created_files) == 3 - assert (tmp_path / _WORKFLOW_PATH).exists() - assert (tmp_path / _CI_WORKFLOW_PATH).exists() - assert (tmp_path / _CD_WORKFLOW_PATH).exists() +def test_cli_default_creates_all_four(tmp_path: Path) -> None: + result = runner.invoke(app, ["workflow", "generate", "--dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + assert result.stdout.count("+ created") == 4 + for rel in ALL_PATHS: + assert (tmp_path / rel).exists() -def test_generate_workflows_skips_existing_without_force(tmp_path: Path) -> None: - """Existing files are skipped without --force, per file.""" - # Pre-create the PR workflow - pr_workflow = tmp_path / _WORKFLOW_PATH - pr_workflow.parent.mkdir(parents=True, exist_ok=True) - pr_workflow.write_text("existing", encoding="utf-8") - result = generate_cicd_workflows( - directory=tmp_path, - kinds=["pr", "ci"], - force=False, +def test_cli_kinds_subset(tmp_path: Path) -> None: + result = runner.invoke( + app, + ["workflow", "generate", "--dir", str(tmp_path), "--kinds", "pr,prod"], ) - assert len(result.skipped_files) == 1 - assert len(result.created_files) == 1 - assert pr_workflow.read_text(encoding="utf-8") == "existing" - assert (tmp_path / _CI_WORKFLOW_PATH).exists() + assert result.exit_code == 0, result.stdout + assert (tmp_path / _PR_PATH).exists() + assert (tmp_path / _PROD_PATH).exists() + assert not (tmp_path / _DEV_PATH).exists() + assert not (tmp_path / _QA_PATH).exists() -def test_generate_workflows_force_overwrites_all(tmp_path: Path) -> None: - """Force overwrites all existing files.""" - for rel in (_WORKFLOW_PATH, _CI_WORKFLOW_PATH, _CD_WORKFLOW_PATH): - wf = tmp_path / rel - wf.parent.mkdir(parents=True, exist_ok=True) - wf.write_text("old", encoding="utf-8") - - result = generate_cicd_workflows( - directory=tmp_path, - kinds=["pr", "ci", "cd"], - force=True, +def test_cli_kinds_invalid_value_fails(tmp_path: Path) -> None: + result = runner.invoke( + app, + ["workflow", "generate", "--dir", str(tmp_path), "--kinds", "pr,nonsense"], ) - assert len(result.overwritten_files) == 3 - assert len(result.skipped_files) == 0 - - for rel in (_WORKFLOW_PATH, _CI_WORKFLOW_PATH, _CD_WORKFLOW_PATH): - content = (tmp_path / rel).read_text(encoding="utf-8") - assert content != "old" - assert "agentops" in content.lower() - - -def test_ci_template_has_required_features(tmp_path: Path) -> None: - """Verify the CI template has expected structure.""" - generate_cicd_workflows(directory=tmp_path, kinds=["ci"]) - - content = (tmp_path / _CI_WORKFLOW_PATH).read_text(encoding="utf-8") - - # Triggers - assert "push" in content - assert "workflow_dispatch" in content - - # Branches - assert "develop" in content - assert "main" in content - - # Core features - assert "3.11" in content - assert "agentops-toolkit" in content - assert "agentops eval run" in content - assert "EXIT_CODE" in content - assert "azure/login@v2" in content - - # Artifacts - assert "results.json" in content - assert "report.md" in content + assert result.exit_code == 1 + assert "unknown" in result.stdout.lower() or "unknown" in (result.stderr or "").lower() - # Matrix strategy (commented out but present) - assert "matrix" in content +def test_cli_skips_existing_without_force(tmp_path: Path) -> None: + pr = tmp_path / _PR_PATH + pr.parent.mkdir(parents=True, exist_ok=True) + pr.write_text("existing", encoding="utf-8") -def test_cd_template_has_required_features(tmp_path: Path) -> None: - """Verify the CD template has expected structure.""" - generate_cicd_workflows(directory=tmp_path, kinds=["cd"]) - - content = (tmp_path / _CD_WORKFLOW_PATH).read_text(encoding="utf-8") - - # Triggers - assert "push" in content - assert "workflow_dispatch" in content - - # Branches - assert "main" in content - - # Core features - assert "3.11" in content - assert "agentops-toolkit" in content - assert "agentops eval run" in content - assert "EXIT_CODE" in content - assert "azure/login@v2" in content + result = runner.invoke(app, ["workflow", "generate", "--dir", str(tmp_path)]) - # Two-job structure - assert "safety-qa" in content - assert "deploy" in content - assert "needs: [safety-qa]" in content + assert result.exit_code == 0 + assert "skipped" in result.stdout + assert pr.read_text(encoding="utf-8") == "existing" - # Artifacts - assert "results.json" in content - assert "report.md" in content +def test_cli_force_overwrites(tmp_path: Path) -> None: + pr = tmp_path / _PR_PATH + pr.parent.mkdir(parents=True, exist_ok=True) + pr.write_text("old", encoding="utf-8") -def test_cli_workflow_generate_creates_multiple(tmp_path: Path) -> None: - """CLI generates multiple workflows when workspace triggers auto-detection.""" - _scaffold_agentops_workspace( - tmp_path, - bundles=["model_quality_baseline.yaml", "safe_agent_baseline.yaml"], + result = runner.invoke( + app, + ["workflow", "generate", "--dir", str(tmp_path), "--force"], ) - result = runner.invoke(app, ["workflow", "generate", "--dir", str(tmp_path)]) - assert result.exit_code == 0 - assert "created" in result.stdout - assert (tmp_path / _WORKFLOW_PATH).exists() - assert (tmp_path / _CD_WORKFLOW_PATH).exists() - + assert "overwritten" in result.stdout -def test_cli_workflow_generate_shows_all_files(tmp_path: Path) -> None: - """CLI output lists all generated files.""" - _scaffold_agentops_workspace( - tmp_path, - bundles=["model_quality_baseline.yaml", "rag_quality_baseline.yaml"], - ) +def test_cli_next_steps_mention_environments(tmp_path: Path) -> None: result = runner.invoke(app, ["workflow", "generate", "--dir", str(tmp_path)]) assert result.exit_code == 0 - # Should mention all created files - assert result.stdout.count("+ created") >= 2 + out = result.stdout + assert "Next steps" in out + assert "dev" in out and "qa" in out and "production" in out + assert "OIDC" in out or "Workload Identity Federation" in out + assert "branch" in out.lower() diff --git a/tests/unit/test_cli_commands.py b/tests/unit/test_cli_commands.py index cf77e6ce..86617009 100644 --- a/tests/unit/test_cli_commands.py +++ b/tests/unit/test_cli_commands.py @@ -20,35 +20,42 @@ def test_init_help_exposes_path_alias() -> None: assert "--path" in _strip_ansi(result.stdout) -def test_eval_compare_rejects_wrong_run_count() -> None: - result = runner.invoke(app, ["eval", "compare", "--runs", "only_one"]) +def test_version_flag() -> None: + result = runner.invoke(app, ["--version"]) - assert result.exit_code == 1 - assert ( - "at least two" in result.stdout.lower() - or "at least two" in (result.stderr or "").lower() - ) + assert result.exit_code == 0 + assert "agentops" in result.stdout.lower() -def test_model_list_is_planned_stub() -> None: - result = runner.invoke(app, ["model", "list"]) +def test_report_help_only_exposes_generate() -> None: + result = runner.invoke(app, ["report", "--help"]) - assert result.exit_code == 1 - assert "planned but not implemented" in result.stdout.lower() + assert result.exit_code == 0 + stripped = _strip_ansi(result.stdout) + assert "generate" in stripped + assert "show" not in stripped + assert "export" not in stripped -def test_version_flag() -> None: - result = runner.invoke(app, ["--version"]) +def test_eval_help_does_not_expose_compare_subcommand() -> None: + result = runner.invoke(app, ["eval", "--help"]) assert result.exit_code == 0 - assert "agentops" in result.stdout.lower() + stripped = _strip_ansi(result.stdout) + assert "compare" not in stripped -def test_report_help_exposes_available_and_planned_commands() -> None: - result = runner.invoke(app, ["report", "--help"]) +def test_planned_command_groups_removed() -> None: + """Stub command groups (monitor/model/dataset/config) are gone in 1.0.""" + for group in ("monitor", "model", "dataset", "config"): + result = runner.invoke(app, [group, "--help"]) + assert result.exit_code != 0, f"unexpected: 'agentops {group}' is still wired" + +def test_agent_command_group_wired() -> None: + """`agentops agent` exposes the watchdog subcommands.""" + result = runner.invoke(app, ["agent", "--help"]) assert result.exit_code == 0 stripped = _strip_ansi(result.stdout) - assert "generate" in stripped - assert "show" in stripped - assert "export" in stripped + assert "analyze" in stripped + assert "serve" in stripped diff --git a/tests/unit/test_cloud_publisher.py b/tests/unit/test_cloud_publisher.py new file mode 100644 index 00000000..2669c521 --- /dev/null +++ b/tests/unit/test_cloud_publisher.py @@ -0,0 +1,425 @@ +"""Unit tests for the cloud (New Foundry) publisher.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pytest + +from agentops.core.results import ( + RowMetric, + RowResult, + RunResult, + RunSummary, + TargetInfo, +) +from agentops.pipeline import cloud_publisher + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _make_result(*, kind: str = "foundry_prompt", name: str = "support-bot", + version: str = "1") -> RunResult: + return RunResult( + started_at="2026-05-06T10:00:00+00:00", + finished_at="2026-05-06T10:00:05+00:00", + duration_seconds=5.0, + target=TargetInfo( + kind=kind, raw=f"{name}:{version}", name=name, version=version, + ), + dataset_path="dataset.jsonl", + evaluators=["CoherenceEvaluator", "FluencyEvaluator"], + rows=[ + RowResult( + row_index=0, + input="hi", + expected="hello", + response="hello", + metrics=[ + RowMetric(name="coherence", value=4.0), + RowMetric(name="fluency", value=4.0), + RowMetric(name="avg_latency_seconds", value=1.2), + ], + ), + ], + aggregate_metrics={ + "coherence": 4.0, + "fluency": 4.0, + "avg_latency_seconds": 1.2, + }, + summary=RunSummary( + items_total=1, + items_passed_all=1, + items_pass_rate=1.0, + thresholds_total=0, + thresholds_passed=0, + threshold_pass_rate=1.0, + overall_passed=True, + ), + ) + + +@pytest.fixture +def dataset_file(tmp_path: Path) -> Path: + """A two-row JSONL dataset on disk.""" + path = tmp_path / "dataset.jsonl" + path.write_text( + "\n".join([ + json.dumps({"input": "hi", "expected": "hello"}), + json.dumps({"input": "bye", "expected": "goodbye"}), + ]) + "\n", + encoding="utf-8", + ) + return path + + +# --------------------------------------------------------------------------- +# _build_testing_criteria +# --------------------------------------------------------------------------- + + +def test_build_testing_criteria_maps_quality_evaluators(): + result = _make_result() + criteria = cloud_publisher._build_testing_criteria(result) + + azure_names = {c["evaluator_name"] for c in criteria} + assert "builtin.coherence" in azure_names + assert "builtin.fluency" in azure_names + # Every criterion is an azure_ai_evaluator entry with a stable name. + for c in criteria: + assert c["type"] == "azure_ai_evaluator" + assert c["name"] in {"coherence", "fluency"} + + +def test_build_testing_criteria_skips_latency(): + """avg_latency_seconds is a runtime-only metric and must NOT become an + azure_ai_evaluator (Foundry has its own server-side latency view).""" + result = _make_result() + criteria = cloud_publisher._build_testing_criteria(result) + names = {c["name"] for c in criteria} + assert "avg_latency_seconds" not in names + + +def test_build_testing_criteria_warns_on_unknown_evaluator(caplog): + """Metrics whose preset has no azure_ai_evaluator mapping are logged + and skipped, never raised — local results.json remains canonical.""" + result = _make_result() + # Drop into an unknown evaluator class via monkey-patch-style override: + # we add a synthetic preset to CATALOG so the lookup hits, but with a + # class_name that is not in _AZURE_AI_EVALUATOR_NAMES. + from agentops.core import evaluators as _ev + + fake = _ev.EvaluatorPreset( + name="MyCustomEvaluator", + class_name="MyCustomEvaluator", + score_key="my_custom", + input_mapping={"query": "$prompt"}, + default_threshold=None, + categories=frozenset({"agent"}), + ) + result.aggregate_metrics["my_custom"] = 0.42 + with mock.patch.dict(_ev.CATALOG, {"MyCustomEvaluator": fake}): + with caplog.at_level("WARNING"): + criteria = cloud_publisher._build_testing_criteria(result) + assert all(c["name"] != "my_custom" for c in criteria) + assert any("no azure_ai_evaluator mapping" in rec.message for rec in caplog.records) + + +# --------------------------------------------------------------------------- +# _build_item_schema +# --------------------------------------------------------------------------- + + +def test_build_item_schema_uses_first_row_keys(dataset_file: Path): + schema = cloud_publisher._build_item_schema(dataset_file) + assert schema["type"] == "object" + assert set(schema["properties"].keys()) == {"input", "expected"} + assert set(schema["required"]) == {"input", "expected"} + + +def test_build_item_schema_empty_file_falls_back(tmp_path: Path): + empty = tmp_path / "empty.jsonl" + empty.write_text("", encoding="utf-8") + schema = cloud_publisher._build_item_schema(empty) + assert "input" in schema["properties"] + + +# --------------------------------------------------------------------------- +# Validation guards +# --------------------------------------------------------------------------- + + +def test_publish_rejects_non_foundry_targets(dataset_file: Path): + result = _make_result(kind="http_json") + with pytest.raises(ValueError, match="foundry_cloud only supports"): + cloud_publisher.publish_to_foundry_cloud( + result, + dataset_path=dataset_file, + project_endpoint="https://x.example/api/projects/p", + ) + + +def test_publish_requires_dataset_to_exist(tmp_path: Path): + result = _make_result() + missing = tmp_path / "does_not_exist.jsonl" + with pytest.raises(ValueError, match="dataset file not found"): + cloud_publisher.publish_to_foundry_cloud( + result, + dataset_path=missing, + project_endpoint="https://x.example/api/projects/p", + ) + + +def test_publish_requires_name_and_version(dataset_file: Path): + """Even if target.kind is foundry_prompt, missing name/version (could + happen with a mistakenly hand-built TargetInfo) is rejected.""" + result = _make_result() + # Bypass model validation by reconstructing the info directly. + result.target = TargetInfo( + kind="foundry_prompt", raw="bot:1", name=None, version="1", + ) + with pytest.raises(ValueError, match="fully qualified"): + cloud_publisher.publish_to_foundry_cloud( + result, + dataset_path=dataset_file, + project_endpoint="https://x.example/api/projects/p", + ) + + +# --------------------------------------------------------------------------- +# _poll_until_terminal +# --------------------------------------------------------------------------- + + +def test_poll_returns_when_status_completed(): + """Polling stops as soon as the run hits a terminal status.""" + statuses = iter(["queued", "in_progress", "completed"]) + + def _retrieve(eval_id: str, run_id: str): + return SimpleNamespace(status=next(statuses)) + + fake_runs = SimpleNamespace(retrieve=_retrieve) + fake_evals = SimpleNamespace(runs=fake_runs) + fake_client = SimpleNamespace(evals=fake_evals) + + with mock.patch("agentops.pipeline.cloud_publisher.time.sleep") as sleep: + run = cloud_publisher._poll_until_terminal( + fake_client, + eval_id="e1", run_id="r1", + interval_seconds=0.0, max_attempts=10, + progress=lambda _msg: None, + ) + assert run.status == "completed" + # We slept after each non-terminal poll (queued + in_progress). + assert sleep.call_count == 2 + + +def test_poll_times_out_when_never_terminal(): + """Hitting max_attempts raises a clear RuntimeError.""" + + def _retrieve(eval_id: str, run_id: str): + return SimpleNamespace(status="queued") + + fake_runs = SimpleNamespace(retrieve=_retrieve) + fake_evals = SimpleNamespace(runs=fake_runs) + fake_client = SimpleNamespace(evals=fake_evals) + + with mock.patch("agentops.pipeline.cloud_publisher.time.sleep"): + with pytest.raises(RuntimeError, match="did not finish"): + cloud_publisher._poll_until_terminal( + fake_client, + eval_id="e1", run_id="r1", + interval_seconds=0.0, max_attempts=3, + progress=lambda _msg: None, + ) + + +# --------------------------------------------------------------------------- +# _extract_report_url +# --------------------------------------------------------------------------- + + +def test_extract_report_url_from_attribute(): + run = SimpleNamespace(report_url="https://portal/x", status="completed") + assert cloud_publisher._extract_report_url(run) == "https://portal/x" + + +def test_extract_report_url_from_metadata(): + run = SimpleNamespace( + status="completed", + metadata={"report_url": "https://portal/y"}, + ) + assert cloud_publisher._extract_report_url(run) == "https://portal/y" + + +def test_extract_report_url_returns_none_when_absent(): + run = SimpleNamespace(status="completed") + assert cloud_publisher._extract_report_url(run) is None + + +# --------------------------------------------------------------------------- +# End-to-end happy path with a fully mocked OpenAI/Foundry client +# --------------------------------------------------------------------------- + + +class _FakeFiles: + def __init__(self) -> None: + self.uploaded: list = [] + + def create(self, *, file, purpose): + self.uploaded.append((file.name if hasattr(file, "name") else None, purpose)) + return SimpleNamespace(id="file-abc") + + +class _FakeRuns: + def __init__(self, statuses): + self._statuses = list(statuses) + self.created_with: dict = {} + + def create(self, *, eval_id, name, data_source): + self.created_with = { + "eval_id": eval_id, "name": name, "data_source": data_source, + } + return SimpleNamespace(id="run-xyz") + + def retrieve(self, *, eval_id, run_id): + status = self._statuses.pop(0) if self._statuses else "completed" + return SimpleNamespace( + id=run_id, + status=status, + report_url="https://ai.azure.com/foundry/runs/run-xyz", + ) + + +class _FakeEvals: + def __init__(self, statuses): + self.runs = _FakeRuns(statuses) + self.created_with: dict = {} + + def create(self, *, name, data_source_config, testing_criteria): + self.created_with = { + "name": name, + "data_source_config": data_source_config, + "testing_criteria": testing_criteria, + } + return SimpleNamespace(id="eval-123") + + +class _FakeOpenAIClient: + def __init__(self, statuses): + self.files = _FakeFiles() + self.evals = _FakeEvals(statuses) + + +class _FakeProjectClient: + def __init__(self, openai_client): + self._openai = openai_client + + def get_openai_client(self): + # NB: must be callable with NO arguments — we never want callers + # to pass api_version (regression guard). + return self._openai + + +def test_publish_to_foundry_cloud_happy_path(dataset_file: Path): + """End-to-end happy path with all Azure SDKs mocked. + + Verifies: + - dataset is uploaded with purpose='evals' + - testing_criteria contain only mappable evaluators (coherence + fluency) + - data_source carries an agent_reference with name + version + - agent_reference is built from result.target (not the raw string) + - polling runs to completion and the result captures the portal URL + """ + fake_openai = _FakeOpenAIClient(statuses=["queued", "completed"]) + fake_project = _FakeProjectClient(fake_openai) + + fake_projects_module = mock.MagicMock() + fake_projects_module.AIProjectClient = mock.MagicMock(return_value=fake_project) + fake_identity_module = mock.MagicMock() + + progress_messages: list = [] + + with mock.patch.dict( + "sys.modules", + { + "azure.ai.projects": fake_projects_module, + "azure.identity": fake_identity_module, + }, + ): + with mock.patch("agentops.pipeline.cloud_publisher.time.sleep"): + published = cloud_publisher.publish_to_foundry_cloud( + _make_result(), + dataset_path=dataset_file, + project_endpoint="https://contoso.services.ai.azure.com/api/projects/p", + poll_interval_seconds=0.0, + max_poll_attempts=5, + progress=progress_messages.append, + ) + + # The SDK was called with the right project endpoint. + fake_projects_module.AIProjectClient.assert_called_once() + _, kwargs = fake_projects_module.AIProjectClient.call_args + assert kwargs["endpoint"].endswith("/api/projects/p") + + # The dataset was uploaded for evals. + assert fake_openai.files.uploaded + assert fake_openai.files.uploaded[0][1] == "evals" + + # Testing criteria contain only the mappable evaluators. + criteria = fake_openai.evals.created_with["testing_criteria"] + azure_names = {c["evaluator_name"] for c in criteria} + assert "builtin.coherence" in azure_names + assert "builtin.fluency" in azure_names + + # The data_source uses azure_ai_target_completions with the right agent. + data_source = fake_openai.evals.runs.created_with["data_source"] + assert data_source["type"] == "azure_ai_target_completions" + ref = data_source["agent_reference"] + assert ref["name"] == "support-bot" + assert ref["version"] == "1" + assert data_source["source"] == {"type": "file_id", "id": "file-abc"} + + # Result captures status + portal URL. + assert published.status == "completed" + assert published.eval_id == "eval-123" + assert published.run_id == "run-xyz" + assert published.report_url == "https://ai.azure.com/foundry/runs/run-xyz" + + # Progress messages went through. + assert any("uploading" in m for m in progress_messages) + assert any("status -> completed" in m for m in progress_messages) + + +def test_publish_to_foundry_cloud_raises_when_run_fails(dataset_file: Path): + """A non-completed terminal status surfaces as a RuntimeError so the + orchestrator can downgrade it to a warning + best-effort log.""" + fake_openai = _FakeOpenAIClient(statuses=["failed"]) + fake_project = _FakeProjectClient(fake_openai) + + fake_projects_module = mock.MagicMock() + fake_projects_module.AIProjectClient = mock.MagicMock(return_value=fake_project) + fake_identity_module = mock.MagicMock() + + with mock.patch.dict( + "sys.modules", + { + "azure.ai.projects": fake_projects_module, + "azure.identity": fake_identity_module, + }, + ): + with mock.patch("agentops.pipeline.cloud_publisher.time.sleep"): + with pytest.raises(RuntimeError, match="status 'failed'"): + cloud_publisher.publish_to_foundry_cloud( + _make_result(), + dataset_path=dataset_file, + project_endpoint="https://x.example/api/projects/p", + poll_interval_seconds=0.0, + max_poll_attempts=2, + ) diff --git a/tests/unit/test_comparison.py b/tests/unit/test_comparison.py deleted file mode 100644 index 22588295..00000000 --- a/tests/unit/test_comparison.py +++ /dev/null @@ -1,486 +0,0 @@ -"""Unit tests for the unified comparison service and models.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from agentops.core.models import ( - ComparisonItemRow, - ComparisonMetricRow, - ComparisonResult, - ComparisonSummary, - ComparisonThresholdRow, - RunReference, - RunResult, -) -from agentops.core.reporter import generate_comparison_markdown -from agentops.services.comparison import ( - _compute_metric_direction, - _resolve_run_path, - compare_runs, -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _sample_result( - *, - groundedness: float = 0.84, - relevance: float = 0.83, - overall_passed: bool = True, - row1_groundedness: float = 0.90, - row2_groundedness: float = 0.78, -) -> RunResult: - return RunResult.model_validate( - { - "version": 1, - "status": "completed", - "bundle": { - "name": "rag_baseline", - "path": ".agentops/bundles/rag_baseline.yaml", - }, - "dataset": {"name": "smoke", "path": ".agentops/datasets/smoke.yaml"}, - "execution": { - "backend": "subprocess", - "command": "python -m fake_eval_runner", - "started_at": "2026-03-01T10:00:00Z", - "finished_at": "2026-03-01T10:00:05Z", - "duration_seconds": 5.0, - "exit_code": 0, - }, - "metrics": [ - {"name": "groundedness", "value": groundedness}, - {"name": "relevance", "value": relevance}, - ], - "row_metrics": [ - { - "row_index": 1, - "metrics": [{"name": "groundedness", "value": row1_groundedness}], - }, - { - "row_index": 2, - "metrics": [{"name": "groundedness", "value": row2_groundedness}], - }, - ], - "item_evaluations": [ - { - "row_index": 1, - "passed_all": True, - "thresholds": [ - { - "row_index": 1, - "evaluator": "groundedness", - "criteria": ">=", - "expected": "0.800000", - "actual": str(row1_groundedness), - "passed": row1_groundedness >= 0.8, - }, - ], - }, - { - "row_index": 2, - "passed_all": overall_passed, - "thresholds": [ - { - "row_index": 2, - "evaluator": "groundedness", - "criteria": ">=", - "expected": "0.800000", - "actual": str(row2_groundedness), - "passed": row2_groundedness >= 0.8, - }, - ], - }, - ], - "thresholds": [ - { - "evaluator": "groundedness", - "criteria": ">=", - "expected": "0.800000", - "actual": f"{groundedness:.6f}", - "passed": groundedness >= 0.8, - }, - { - "evaluator": "relevance", - "criteria": ">=", - "expected": "0.800000", - "actual": f"{relevance:.6f}", - "passed": relevance >= 0.8, - }, - ], - "summary": { - "metrics_count": 2, - "thresholds_count": 2, - "thresholds_passed": 2 if overall_passed else 1, - "thresholds_failed": 0 if overall_passed else 1, - "overall_passed": overall_passed, - }, - } - ) - - -def _sample_result_with_latency( - *, similarity: float = 5.0, latency: float = 5.0 -) -> RunResult: - return RunResult.model_validate( - { - "version": 1, - "status": "completed", - "bundle": { - "name": "model_direct", - "path": ".agentops/bundles/model_direct.yaml", - }, - "dataset": {"name": "smoke", "path": ".agentops/datasets/smoke.yaml"}, - "execution": { - "backend": "foundry", - "command": "foundry.cloud_evaluation", - "started_at": "2026-03-01T10:00:00Z", - "finished_at": "2026-03-01T10:00:05Z", - "duration_seconds": 5.0, - "exit_code": 0, - }, - "metrics": [ - {"name": "SimilarityEvaluator", "value": similarity}, - {"name": "avg_latency_seconds", "value": latency}, - ], - "row_metrics": [ - { - "row_index": 1, - "metrics": [ - {"name": "SimilarityEvaluator", "value": similarity}, - {"name": "avg_latency_seconds", "value": latency}, - ], - }, - ], - "item_evaluations": [ - { - "row_index": 1, - "passed_all": True, - "thresholds": [ - { - "row_index": 1, - "evaluator": "SimilarityEvaluator", - "criteria": ">=", - "expected": "3.000000", - "actual": str(similarity), - "passed": similarity >= 3, - }, - { - "row_index": 1, - "evaluator": "avg_latency_seconds", - "criteria": "<=", - "expected": "10.000000", - "actual": str(latency), - "passed": latency <= 10, - }, - ], - }, - ], - "thresholds": [ - { - "evaluator": "SimilarityEvaluator", - "criteria": ">=", - "expected": "3.000000", - "actual": f"{similarity:.6f}", - "passed": similarity >= 3, - }, - { - "evaluator": "avg_latency_seconds", - "criteria": "<=", - "expected": "10.000000", - "actual": f"{latency:.6f}", - "passed": latency <= 10, - }, - ], - "summary": { - "metrics_count": 2, - "thresholds_count": 2, - "thresholds_passed": 2, - "thresholds_failed": 0, - "overall_passed": True, - }, - } - ) - - -def _write_result(path: Path, result: RunResult) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(result.model_dump(mode="json"), indent=2)) - return path - - -# --------------------------------------------------------------------------- -# Model tests -# --------------------------------------------------------------------------- - - -class TestComparisonModels: - def test_comparison_result_roundtrip(self) -> None: - result = ComparisonResult( - version=1, - runs=[ - RunReference( - run_id="run1", bundle_name="b", dataset_name="d", started_at="t1" - ), - RunReference( - run_id="run2", bundle_name="b", dataset_name="d", started_at="t2" - ), - ], - metric_rows=[], - threshold_rows=[], - item_rows=[], - summary=ComparisonSummary( - run_count=2, any_regressions=False, runs_with_regressions=[] - ), - ) - payload = json.loads(result.model_dump_json()) - restored = ComparisonResult.model_validate(payload) - assert restored.version == 1 - assert restored.summary.any_regressions is False - assert len(restored.runs) == 2 - - -# --------------------------------------------------------------------------- -# Direction helpers -# --------------------------------------------------------------------------- - - -class TestComputeMetricDirection: - def test_higher_is_better_positive_delta(self) -> None: - assert _compute_metric_direction(0.05, lower_is_better=False) == "improved" - - def test_higher_is_better_negative_delta(self) -> None: - assert _compute_metric_direction(-0.05, lower_is_better=False) == "regressed" - - def test_lower_is_better_negative_delta_is_improved(self) -> None: - assert _compute_metric_direction(-0.05, lower_is_better=True) == "improved" - - def test_lower_is_better_positive_delta_is_regressed(self) -> None: - assert _compute_metric_direction(0.05, lower_is_better=True) == "regressed" - - def test_zero_is_unchanged(self) -> None: - assert _compute_metric_direction(0.0, lower_is_better=False) == "unchanged" - assert _compute_metric_direction(0.0, lower_is_better=True) == "unchanged" - - -# --------------------------------------------------------------------------- -# compare_runs (2 runs) -# --------------------------------------------------------------------------- - - -class TestCompareRunsTwoRuns: - def test_regression_detected(self, tmp_path: Path) -> None: - baseline = _sample_result( - groundedness=0.90, relevance=0.90, overall_passed=True - ) - current = _sample_result( - groundedness=0.70, relevance=0.95, overall_passed=False - ) - - bp = _write_result(tmp_path / "baseline" / "results.json", baseline) - cp = _write_result(tmp_path / "current" / "results.json", current) - - result = compare_runs([bp, cp], ["baseline", "current"]) - - assert result.summary.any_regressions is True - assert len(result.summary.runs_with_regressions) >= 1 - g_row = next(r for r in result.metric_rows if r.name == "groundedness") - assert g_row.directions[1] == "regressed" - r_row = next(r for r in result.metric_rows if r.name == "relevance") - assert r_row.directions[1] == "improved" - - def test_no_regression(self, tmp_path: Path) -> None: - baseline = _sample_result( - groundedness=0.80, relevance=0.80, overall_passed=True - ) - current = _sample_result(groundedness=0.90, relevance=0.90, overall_passed=True) - - bp = _write_result(tmp_path / "baseline" / "results.json", baseline) - cp = _write_result(tmp_path / "current" / "results.json", current) - - result = compare_runs([bp, cp], ["baseline", "current"]) - - assert result.summary.any_regressions is False - - def test_lower_is_better_latency(self, tmp_path: Path) -> None: - baseline = _sample_result_with_latency(similarity=5.0, latency=6.0) - current = _sample_result_with_latency(similarity=5.0, latency=4.0) - - bp = _write_result(tmp_path / "baseline" / "results.json", baseline) - cp = _write_result(tmp_path / "current" / "results.json", current) - - result = compare_runs([bp, cp], ["baseline", "current"]) - - lat = next(r for r in result.metric_rows if r.name == "avg_latency_seconds") - assert lat.directions[1] == "improved" - assert lat.deltas[1] == pytest.approx(-2.0, abs=1e-6) - - -# --------------------------------------------------------------------------- -# compare_runs (3+ runs) -# --------------------------------------------------------------------------- - - -class TestCompareRunsMultiple: - def test_three_runs(self, tmp_path: Path) -> None: - r1 = _sample_result(groundedness=0.80, relevance=0.80, overall_passed=True) - r2 = _sample_result(groundedness=0.90, relevance=0.85, overall_passed=True) - r3 = _sample_result(groundedness=0.70, relevance=0.95, overall_passed=False) - - p1 = _write_result(tmp_path / "run1" / "results.json", r1) - p2 = _write_result(tmp_path / "run2" / "results.json", r2) - p3 = _write_result(tmp_path / "run3" / "results.json", r3) - - result = compare_runs([p1, p2, p3], ["run1", "run2", "run3"]) - - assert result.summary.run_count == 3 - assert len(result.runs) == 3 - - # run2 improved groundedness, run3 regressed - g_row = next(r for r in result.metric_rows if r.name == "groundedness") - assert g_row.directions[1] == "improved" - assert g_row.directions[2] == "regressed" - - # run3 should be in regressions list - assert result.summary.any_regressions is True - assert 2 in result.summary.runs_with_regressions - # run2 should not have regressions - assert 1 not in result.summary.runs_with_regressions - - def test_best_run_index(self, tmp_path: Path) -> None: - r1 = _sample_result(groundedness=0.80, relevance=0.80) - r2 = _sample_result(groundedness=0.95, relevance=0.70) - r3 = _sample_result(groundedness=0.85, relevance=0.90) - - p1 = _write_result(tmp_path / "run1" / "results.json", r1) - p2 = _write_result(tmp_path / "run2" / "results.json", r2) - p3 = _write_result(tmp_path / "run3" / "results.json", r3) - - result = compare_runs([p1, p2, p3], ["run1", "run2", "run3"]) - - g_row = next(r for r in result.metric_rows if r.name == "groundedness") - assert g_row.best_run_index == 1 # run2 has 0.95 - - r_row = next(r for r in result.metric_rows if r.name == "relevance") - assert r_row.best_run_index == 2 # run3 has 0.90 - - -# --------------------------------------------------------------------------- -# Run path resolution -# --------------------------------------------------------------------------- - - -class TestResolveRunPath: - def test_resolve_absolute_file(self, tmp_path: Path) -> None: - f = tmp_path / "results.json" - f.write_text("{}") - assert _resolve_run_path(str(f)) == f - - def test_resolve_absolute_dir(self, tmp_path: Path) -> None: - d = tmp_path / "run1" - d.mkdir() - f = d / "results.json" - f.write_text("{}") - assert _resolve_run_path(str(d)) == f - - def test_resolve_by_run_id(self, tmp_path: Path) -> None: - results_dir = tmp_path / "results" / "2026-03-01_100000" - results_dir.mkdir(parents=True) - f = results_dir / "results.json" - f.write_text("{}") - resolved = _resolve_run_path("2026-03-01_100000", workspace_dir=tmp_path) - assert resolved == f.resolve() - - def test_resolve_missing_raises(self, tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError): - _resolve_run_path("nonexistent_run", workspace_dir=tmp_path) - - -# --------------------------------------------------------------------------- -# Comparison report markdown -# --------------------------------------------------------------------------- - - -class TestComparisonReport: - def test_report_contains_required_sections(self) -> None: - result = ComparisonResult( - version=1, - runs=[ - RunReference( - run_id="run1", - bundle_name="rag_baseline", - dataset_name="smoke", - started_at="t1", - ), - RunReference( - run_id="run2", - bundle_name="rag_baseline", - dataset_name="smoke", - started_at="t2", - ), - ], - metric_rows=[ - ComparisonMetricRow( - name="groundedness", - values=[0.9, 0.7], - deltas=[None, -0.2], - delta_percents=[None, -22.22], - directions=["unchanged", "regressed"], - best_run_index=0, - ), - ], - threshold_rows=[ - ComparisonThresholdRow( - evaluator="groundedness", criteria=">=", passed=[True, False] - ), - ], - item_rows=[ - ComparisonItemRow(row_index=1, passed_all=[True, False]), - ], - summary=ComparisonSummary( - run_count=2, any_regressions=True, runs_with_regressions=[1] - ), - ) - - md = generate_comparison_markdown(result) - - assert "# AgentOps Comparison Report" in md - assert "REGRESSIONS DETECTED" in md - assert "groundedness" in md - assert "regressed" in md - assert "FAIL" in md - - def test_report_no_regressions(self) -> None: - result = ComparisonResult( - version=1, - runs=[ - RunReference( - run_id="run1", bundle_name="b", dataset_name="d", started_at="t1" - ), - RunReference( - run_id="run2", bundle_name="b", dataset_name="d", started_at="t2" - ), - ], - metric_rows=[ - ComparisonMetricRow( - name="g", - values=[0.7, 0.9], - deltas=[None, 0.2], - directions=["unchanged", "improved"], - ), - ], - threshold_rows=[], - item_rows=[], - summary=ComparisonSummary( - run_count=2, any_regressions=False, runs_with_regressions=[] - ), - ) - - md = generate_comparison_markdown(result) - assert "NO REGRESSIONS" in md diff --git a/tests/unit/test_e2e_render.py b/tests/unit/test_e2e_render.py new file mode 100644 index 00000000..df36d3b2 --- /dev/null +++ b/tests/unit/test_e2e_render.py @@ -0,0 +1,132 @@ +"""Tests for ``scripts/e2e_render_config.py``. + +Locks in the contract that each rendered ``agentops.yaml`` parses with the +AgentOps loader and classifies into the expected target kind. Catches +schema drift before the live e2e workflow runs against real Azure. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = REPO_ROOT / "scripts" / "e2e_render_config.py" + + +@pytest.fixture +def render_module(monkeypatch, tmp_path): + """Import e2e_render_config with ROOT pointing at tmp_path.""" + + spec = importlib.util.spec_from_file_location("e2e_render_config", SCRIPT_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["e2e_render_config"] = module + spec.loader.exec_module(module) + monkeypatch.setattr(module, "ROOT", tmp_path) + monkeypatch.setattr( + module, "DATASET_BASIC", tmp_path / "scripts" / "e2e_data" / "basic.jsonl" + ) + monkeypatch.setattr( + module, "DATASET_RAG", tmp_path / "scripts" / "e2e_data" / "rag.jsonl" + ) + monkeypatch.setattr( + module, "DATASET_TOOLS", tmp_path / "scripts" / "e2e_data" / "tools.jsonl" + ) + yield module + sys.modules.pop("e2e_render_config", None) + + +@pytest.fixture +def all_scenarios_env(monkeypatch): + monkeypatch.setenv("AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT", "e2e-prompt:1") + monkeypatch.setenv("AGENTOPS_E2E_FOUNDRY_HOSTED_AGENT", "e2e-hosted-run42:1") + monkeypatch.setenv( + "AGENTOPS_E2E_ACA_URL", + "https://aca-echo-run123.icy.eastus2.azurecontainerapps.io", + ) + monkeypatch.setenv("AGENTOPS_E2E_MODEL_DEPLOYMENT", "gpt-4o-mini") + + +def test_render_writes_only_for_set_env_vars(render_module, monkeypatch, tmp_path): + for var in ( + "AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT", + "AGENTOPS_E2E_FOUNDRY_HOSTED_AGENT", + "AGENTOPS_E2E_ACA_URL", + "AGENTOPS_E2E_MODEL_DEPLOYMENT", + ): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AGENTOPS_E2E_MODEL_DEPLOYMENT", "gpt-4o-mini") + + written = render_module.render() + + assert written == ["model-direct"] + assert (tmp_path / "e2e-runs" / "model-direct" / "agentops.yaml").exists() + assert not (tmp_path / "e2e-runs" / "foundry-prompt").exists() + + +def test_render_all_scenarios_load_and_classify( + render_module, all_scenarios_env, tmp_path +): + from agentops.core.config_loader import load_agentops_config + + written = render_module.render() + assert set(written) == { + "foundry-prompt", + "foundry-hosted", + "http-aca", + "model-direct", + } + + expected_kinds = { + "foundry-prompt": ("foundry_prompt", None), + # The hosted agent is created dynamically and referenced as + # name:version, so it routes through the foundry_prompt + # (agent_reference) invocation path — same as the prompt scenario. + "foundry-hosted": ("foundry_prompt", None), + "http-aca": ("http_json", "http-json"), + "model-direct": ("model_direct", None), + } + for scenario, (kind, protocol) in expected_kinds.items(): + cfg_path = tmp_path / "e2e-runs" / scenario / "agentops.yaml" + cfg = load_agentops_config(cfg_path) + target = cfg.resolved_target() + assert target.kind == kind, ( + f"{scenario}: expected kind={kind}, got {target.kind}" + ) + assert target.protocol == protocol, ( + f"{scenario}: expected protocol={protocol}, got {target.protocol}" + ) + + # Each rendered scenario must also write a HEADER.md so the + # transcript script can produce a self-explanatory artifact. + header = cfg_path.parent / "HEADER.md" + assert header.exists(), f"{scenario}: HEADER.md is missing" + assert header.stat().st_size > 0 + + +def test_render_creates_datasets_when_missing(render_module, all_scenarios_env, tmp_path): + render_module.render() + basic = tmp_path / "scripts" / "e2e_data" / "basic.jsonl" + rag = tmp_path / "scripts" / "e2e_data" / "rag.jsonl" + assert basic.exists() and basic.stat().st_size > 0 + assert rag.exists() and rag.stat().st_size > 0 + + +def test_render_main_exits_nonzero_with_no_env(render_module, monkeypatch, capsys): + for var in ( + "AGENTOPS_E2E_FOUNDRY_PROMPT_AGENT", + "AGENTOPS_E2E_FOUNDRY_HOSTED_AGENT", + "AGENTOPS_E2E_ACA_URL", + "AGENTOPS_E2E_MODEL_DEPLOYMENT", + ): + monkeypatch.delenv(var, raising=False) + + rc = render_module.main() + err = capsys.readouterr().err + + assert rc == 1 + assert "no scenario env vars set" in err diff --git a/tests/unit/test_evaluators.py b/tests/unit/test_evaluators.py new file mode 100644 index 00000000..c14a2f2a --- /dev/null +++ b/tests/unit/test_evaluators.py @@ -0,0 +1,226 @@ +"""Tests for the evaluator catalog and auto-selection rules.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agentops.core.agentops_config import Threshold, classify_agent +from agentops.core.evaluators import ( + CATALOG, + DatasetShape, + detect_dataset_shape, + merge_thresholds, + select_evaluators, +) + + +# --------------------------------------------------------------------------- +# detect_dataset_shape +# --------------------------------------------------------------------------- + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + import json + + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row) + "\n") + + +class TestDetectDatasetShape: + def test_quality_dataset(self, tmp_path: Path) -> None: + path = tmp_path / "qa.jsonl" + _write_jsonl( + path, + [ + {"input": "hello", "expected": "hi"}, + {"input": "bye", "expected": "goodbye"}, + ], + ) + shape = detect_dataset_shape(path) + assert shape.row_count == 2 + assert not shape.looks_rag + assert not shape.looks_tool_use + + def test_rag_dataset(self, tmp_path: Path) -> None: + path = tmp_path / "rag.jsonl" + _write_jsonl( + path, + [ + {"input": "q", "expected": "a", "context": "Paris is the capital."}, + ], + ) + shape = detect_dataset_shape(path) + assert shape.looks_rag + + def test_tool_use_dataset(self, tmp_path: Path) -> None: + path = tmp_path / "tools.jsonl" + _write_jsonl( + path, + [ + { + "input": "weather?", + "expected": "sunny", + "tool_calls": [{"name": "get_weather", "args": {}}], + }, + ], + ) + shape = detect_dataset_shape(path) + assert shape.looks_tool_use + + def test_empty_context_does_not_count(self, tmp_path: Path) -> None: + path = tmp_path / "empty_ctx.jsonl" + _write_jsonl( + path, + [{"input": "q", "expected": "a", "context": ""}], + ) + shape = detect_dataset_shape(path) + assert not shape.looks_rag + + def test_empty_dataset_raises(self, tmp_path: Path) -> None: + path = tmp_path / "empty.jsonl" + path.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + detect_dataset_shape(path) + + def test_invalid_json_raises(self, tmp_path: Path) -> None: + path = tmp_path / "bad.jsonl" + path.write_text("not json\n", encoding="utf-8") + with pytest.raises(ValueError, match="invalid JSON"): + detect_dataset_shape(path) + + def test_missing_file(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + detect_dataset_shape(tmp_path / "missing.jsonl") + + +# --------------------------------------------------------------------------- +# select_evaluators +# --------------------------------------------------------------------------- + + +_PROMPT_AGENT = classify_agent("my-rag:3") +_MODEL_DIRECT = classify_agent("model:gpt-4o") +_HTTP_AGENT = classify_agent("https://my-app.azurecontainerapps.io/chat") + + +def _shape(*, context: bool = False, tool_calls: bool = False, tool_defs: bool = False) -> DatasetShape: + return DatasetShape( + has_context=context, + has_tool_calls=tool_calls, + has_tool_definitions=tool_defs, + row_count=10, + ) + + +class TestSelectEvaluators: + def test_quality_baseline_always_present(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape()) + names = [p.name for p in result] + assert "CoherenceEvaluator" in names + assert "FluencyEvaluator" in names + assert "SimilarityEvaluator" in names + assert "F1ScoreEvaluator" in names + assert "avg_latency_seconds" in names + + def test_quality_only_for_quality_dataset(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape()) + names = [p.name for p in result] + assert "GroundednessEvaluator" not in names + assert "ToolCallAccuracyEvaluator" not in names + + def test_rag_evaluators_added_with_context(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(context=True)) + names = [p.name for p in result] + for evaluator in [ + "GroundednessEvaluator", + "RelevanceEvaluator", + "RetrievalEvaluator", + "ResponseCompletenessEvaluator", + ]: + assert evaluator in names + + def test_tool_use_added_with_tool_calls(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(tool_calls=True)) + names = [p.name for p in result] + assert "ToolCallAccuracyEvaluator" in names + assert "ToolCallAccuracyEvaluator" in names + + def test_tool_use_added_with_tool_definitions(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(tool_defs=True)) + names = [p.name for p in result] + assert "ToolCallAccuracyEvaluator" in names + + def test_combined_rag_and_tools(self) -> None: + result = select_evaluators(_PROMPT_AGENT, _shape(context=True, tool_calls=True)) + names = [p.name for p in result] + assert "GroundednessEvaluator" in names + assert "ToolCallAccuracyEvaluator" in names + + def test_model_direct_skips_agent_evaluators(self) -> None: + # Even if the dataset has context/tool_calls, model targets stay quality-only. + result = select_evaluators( + _MODEL_DIRECT, _shape(context=True, tool_calls=True) + ) + names = [p.name for p in result] + assert "GroundednessEvaluator" not in names + assert "ToolCallAccuracyEvaluator" not in names + assert "CoherenceEvaluator" in names + + def test_http_agent_treated_like_agent(self) -> None: + result = select_evaluators(_HTTP_AGENT, _shape(context=True)) + names = [p.name for p in result] + assert "GroundednessEvaluator" in names + + def test_overrides_bypass_inference(self) -> None: + result = select_evaluators( + _PROMPT_AGENT, + _shape(context=True, tool_calls=True), + overrides=["CoherenceEvaluator"], + ) + names = [p.name for p in result] + assert names == ["CoherenceEvaluator"] + + def test_unknown_override_raises(self) -> None: + with pytest.raises(ValueError, match="unknown evaluator"): + select_evaluators(_PROMPT_AGENT, _shape(), overrides=["NotAnEvaluator"]) + + +# --------------------------------------------------------------------------- +# merge_thresholds +# --------------------------------------------------------------------------- + + +class TestMergeThresholds: + def test_user_override_wins(self) -> None: + presets = select_evaluators(_PROMPT_AGENT, _shape()) + user = [Threshold(metric="coherence", criteria=">=", value=4.0)] + merged = merge_thresholds(presets, user) + coherence = [t for t in merged if t.metric == "coherence"][0] + assert coherence.value == 4.0 + + def test_preset_default_used_when_no_override(self) -> None: + presets = select_evaluators(_PROMPT_AGENT, _shape()) + merged = merge_thresholds(presets, user_thresholds=[]) + # CoherenceEvaluator default is >=3.0 + coherence = [t for t in merged if t.metric == "coherence"][0] + assert coherence.value == 3.0 + + def test_user_only_metric_appended(self) -> None: + presets = select_evaluators(_PROMPT_AGENT, _shape()) + user = [Threshold(metric="custom_metric", criteria=">=", value=1.0)] + merged = merge_thresholds(presets, user) + names = [t.metric for t in merged] + assert "custom_metric" in names + + +# --------------------------------------------------------------------------- +# CATALOG +# --------------------------------------------------------------------------- + + +def test_catalog_keys_match_preset_names() -> None: + for name, preset in CATALOG.items(): + assert preset.name == name diff --git a/tests/unit/test_foundry_backend.py b/tests/unit/test_foundry_backend.py deleted file mode 100644 index c0fa5dfb..00000000 --- a/tests/unit/test_foundry_backend.py +++ /dev/null @@ -1,820 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -from agentops.backends.base import BackendRunContext -from agentops.backends.eval_engine import ( - FoundryEvaluatorRuntime, - _cloud_evaluator_data_mapping, - _cloud_evaluator_needs_model, - _default_foundry_input_mapping, -) -from agentops.backends.foundry_backend import ( - FoundryBackend, -) -from agentops.core.models import ( - BundleRef, - DatasetRef, - ExecutionConfig, - OutputConfig, - RunConfig, - TargetConfig, - TargetEndpointConfig, -) -from agentops.utils.yaml import save_yaml - - -def _foundry_context( - *, - bundle_path: Path, - dataset_path: Path, - output_dir: Path, - target_type: str = "agent", - agent_id: str | None = "asst_abc123", - model: str | None = None, - project_endpoint: str = "https://example.services.ai.azure.com/api/projects/proj-a", - api_version: str | None = "2025-05-01", - poll_interval_seconds: float | None = 0.01, - max_poll_attempts: int | None = 5, - timeout_seconds: int = 15, -) -> BackendRunContext: - endpoint = TargetEndpointConfig( - kind="foundry_agent", - agent_id=agent_id, - model=model, - project_endpoint=project_endpoint, - project_endpoint_env="AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", - api_version=api_version, - poll_interval_seconds=poll_interval_seconds, - max_poll_attempts=max_poll_attempts, - ) - run_config = RunConfig( - version=2, - target=TargetConfig( - type=target_type, - hosting="foundry", - execution_mode="remote", - endpoint=endpoint, - ), - bundle=BundleRef(path=bundle_path), - dataset=DatasetRef(path=dataset_path), - execution=ExecutionConfig(timeout_seconds=timeout_seconds), - output=OutputConfig(), - ) - return BackendRunContext( - run_config=run_config, - bundle_path=bundle_path, - dataset_path=dataset_path, - backend_output_dir=output_dir, - ) - - -class _FakeHttpResponse: - def __init__(self, payload: dict): - self._payload = payload - - def read(self) -> bytes: - return json.dumps(self._payload).encode("utf-8") - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return None - - -def _dataset_yaml(tmp_path: Path) -> Path: - dataset_file = tmp_path / "samples.jsonl" - dataset_file.write_text( - "\n".join( - [ - json.dumps({"input": "2 + 2", "expected": "4"}), - json.dumps({"input": "3 + 5", "expected": "8"}), - ] - ), - encoding="utf-8", - ) - - config_path = tmp_path / "dataset.yaml" - save_yaml( - config_path, - { - "version": 1, - "name": "smoke", - "source": {"type": "file", "path": str(dataset_file)}, - "format": { - "type": "jsonl", - "input_field": "input", - "expected_field": "expected", - }, - }, - ) - return config_path - - -def _bundle_yaml(tmp_path: Path, *, similarity_source: str | None = None) -> Path: - evaluators = [ - {"name": "exact_match", "source": "local", "enabled": True}, - {"name": "avg_latency_seconds", "source": "local", "enabled": True}, - ] - thresholds = [ - {"evaluator": "exact_match", "criteria": "true"}, - {"evaluator": "avg_latency_seconds", "criteria": "<=", "value": 10.0}, - ] - - if similarity_source is not None: - evaluators.insert( - 0, - { - "name": "SimilarityEvaluator", - "source": similarity_source, - "enabled": True, - }, - ) - thresholds.insert( - 0, {"evaluator": "SimilarityEvaluator", "criteria": ">=", "value": 3} - ) - - bundle_path = tmp_path / "bundle.yaml" - save_yaml( - bundle_path, - { - "version": 1, - "name": "qa_similarity_baseline", - "description": "Test bundle", - "evaluators": evaluators, - "thresholds": thresholds, - "metadata": {"category": "test"}, - }, - ) - return bundle_path - - -def test_foundry_backend_uses_default_azure_credential(tmp_path: Path) -> None: - """Verify the backend acquires a token via _acquire_token (DefaultAzureCredential).""" - dataset_path = _dataset_yaml(tmp_path) - bundle_path = _bundle_yaml(tmp_path) - context = _foundry_context( - bundle_path=bundle_path, - dataset_path=dataset_path, - output_dir=tmp_path / "out", - ) - - # When _acquire_token raises, the error should propagate clearly - with patch( - "agentops.backends.foundry_backend._acquire_token", - side_effect=RuntimeError("azure-identity not installed"), - ): - try: - FoundryBackend().execute(context) - assert False, "expected RuntimeError" - except RuntimeError as exc: - assert "azure-identity" in str(exc) - - -def test_foundry_backend_agent_service_target(tmp_path: Path) -> None: - dataset_path = _dataset_yaml(tmp_path) - bundle_path = _bundle_yaml(tmp_path) - context = _foundry_context( - bundle_path=bundle_path, - dataset_path=dataset_path, - output_dir=tmp_path / "out-agent", - ) - - responses = [ - _FakeHttpResponse({"id": "thread_1"}), - _FakeHttpResponse({"id": "msg_1"}), - _FakeHttpResponse({"id": "run_1"}), - _FakeHttpResponse({"status": "completed"}), - _FakeHttpResponse( - {"data": [{"role": "assistant", "content": [{"text": {"value": "4"}}]}]} - ), - _FakeHttpResponse({"id": "thread_2"}), - _FakeHttpResponse({"id": "msg_2"}), - _FakeHttpResponse({"id": "run_2"}), - _FakeHttpResponse({"status": "completed"}), - _FakeHttpResponse( - {"data": [{"role": "assistant", "content": [{"text": {"value": "8"}}]}]} - ), - ] - - with ( - patch( - "agentops.backends.foundry_backend._acquire_token", - return_value="fake-agent-token", - ), - patch( - "agentops.backends.foundry_backend.urllib.request.urlopen", - side_effect=responses, - ), - ): - result = FoundryBackend().execute(context) - - assert result.backend == "foundry" - assert result.exit_code == 0 - assert "foundry.agent_service" in result.command - - metrics_path = tmp_path / "out-agent" / "backend_metrics.json" - payload = json.loads(metrics_path.read_text(encoding="utf-8")) - metrics_by_name = {item["name"]: item["value"] for item in payload["metrics"]} - - assert metrics_by_name["exact_match"] == 1.0 - assert "SimilarityEvaluator" not in metrics_by_name - assert "GroundednessEvaluator" not in metrics_by_name - assert metrics_by_name["samples_evaluated"] == 2.0 - assert len(payload["row_metrics"]) == 2 - first_row_metrics = { - item["name"]: item["value"] for item in payload["row_metrics"][0]["metrics"] - } - assert "GroundednessEvaluator" not in first_row_metrics - assert first_row_metrics["exact_match"] == 1.0 - - -def test_foundry_backend_uses_similarity_evaluator_when_source_is_foundry( - tmp_path: Path, -) -> None: - dataset_path = _dataset_yaml(tmp_path) - bundle_path = _bundle_yaml(tmp_path, similarity_source="foundry") - context = _foundry_context( - bundle_path=bundle_path, - dataset_path=dataset_path, - output_dir=tmp_path / "out-agent-foundry-sim", - ) - - responses = [ - _FakeHttpResponse({"id": "thread_1"}), - _FakeHttpResponse({"id": "msg_1"}), - _FakeHttpResponse({"id": "run_1"}), - _FakeHttpResponse({"status": "completed"}), - _FakeHttpResponse( - {"data": [{"role": "assistant", "content": [{"text": {"value": "4"}}]}]} - ), - _FakeHttpResponse({"id": "thread_2"}), - _FakeHttpResponse({"id": "msg_2"}), - _FakeHttpResponse({"id": "run_2"}), - _FakeHttpResponse({"status": "completed"}), - _FakeHttpResponse( - {"data": [{"role": "assistant", "content": [{"text": {"value": "8"}}]}]} - ), - ] - - class _FakeSimilarityEvaluator: - def __call__(self, **kwargs): - assert "query" in kwargs - assert "response" in kwargs - assert "ground_truth" in kwargs - return {"similarity": 4.0} - - with ( - patch( - "agentops.backends.foundry_backend._acquire_token", - return_value="fake-agent-token", - ), - patch( - "agentops.backends.foundry_backend._build_foundry_evaluator_runtimes", - return_value=[ - FoundryEvaluatorRuntime( - name="SimilarityEvaluator", - evaluator=_FakeSimilarityEvaluator(), - input_mapping={ - "query": "$prompt", - "response": "$prediction", - "ground_truth": "$expected", - }, - score_keys=["similarity"], - ) - ], - ), - patch( - "agentops.backends.foundry_backend.urllib.request.urlopen", - side_effect=responses, - ), - ): - result = FoundryBackend().execute(context) - - assert result.backend == "foundry" - assert result.exit_code == 0 - - metrics_path = tmp_path / "out-agent-foundry-sim" / "backend_metrics.json" - payload = json.loads(metrics_path.read_text(encoding="utf-8")) - metrics_by_name = {item["name"]: item["value"] for item in payload["metrics"]} - assert metrics_by_name["SimilarityEvaluator"] == 4.0 - - -def test_foundry_backend_rejects_unsupported_local_evaluator(tmp_path: Path) -> None: - dataset_path = _dataset_yaml(tmp_path) - bundle_path = _bundle_yaml(tmp_path, similarity_source="local") - context = _foundry_context( - bundle_path=bundle_path, - dataset_path=dataset_path, - output_dir=tmp_path / "out-agent-unsupported-local", - ) - - with patch( - "agentops.backends.foundry_backend._acquire_token", - return_value="fake-agent-token", - ): - try: - FoundryBackend().execute(context) - assert False, "expected ValueError" - except ValueError as exc: - assert "Unsupported local evaluator(s): SimilarityEvaluator" in str(exc) - - -def test_foundry_backend_model_direct_target(tmp_path: Path) -> None: - """Verify model-direct target calls the model via chat completions.""" - dataset_path = _dataset_yaml(tmp_path) - bundle_path = _bundle_yaml(tmp_path) - context = _foundry_context( - bundle_path=bundle_path, - dataset_path=dataset_path, - output_dir=tmp_path / "out-model-direct", - target_type="model", - agent_id=None, - model="gpt-5-mini", - ) - - def _fake_invoke_model_direct(self_backend, settings, prompt): - if "2 + 2" in prompt: - return "4" - return "8" - - with ( - patch( - "agentops.backends.foundry_backend._acquire_token", - return_value="fake-token", - ), - patch.object(FoundryBackend, "_invoke_model_direct", _fake_invoke_model_direct), - ): - result = FoundryBackend().execute(context) - - assert result.backend == "foundry" - assert result.exit_code == 0 - assert "model_direct" in result.command - - metrics_path = tmp_path / "out-model-direct" / "backend_metrics.json" - payload = json.loads(metrics_path.read_text(encoding="utf-8")) - metrics_by_name = {item["name"]: item["value"] for item in payload["metrics"]} - assert metrics_by_name["exact_match"] == 1.0 - assert metrics_by_name["samples_evaluated"] == 2.0 - - -def test_foundry_backend_model_target_requires_explicit_model(tmp_path: Path) -> None: - dataset_path = _dataset_yaml(tmp_path) - bundle_path = _bundle_yaml(tmp_path) - context = _foundry_context( - bundle_path=bundle_path, - dataset_path=dataset_path, - output_dir=tmp_path / "out-model-missing", - target_type="model", - agent_id=None, - ) - - try: - FoundryBackend().execute(context) - assert False, "expected ValueError" - except ValueError as exc: - assert "model" in str(exc).lower() - assert "endpoint.model" in str(exc) or "deployment" in str(exc) - - -# --------------------------------------------------------------------------- -# Unit tests for _cloud_evaluator_data_mapping and _default_foundry_input_mapping -# --------------------------------------------------------------------------- - - -def test_cloud_evaluator_data_mapping_similarity() -> None: - mapping = _cloud_evaluator_data_mapping("similarity", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert mapping["ground_truth"] == "{{item.expected}}" - assert "context" not in mapping - - -def test_cloud_evaluator_data_mapping_groundedness_uses_expected_when_no_context_field() -> ( - None -): - mapping = _cloud_evaluator_data_mapping("groundedness", "input", "expected") - assert mapping["context"] == "{{item.expected}}" - - -def test_cloud_evaluator_data_mapping_groundedness_uses_context_field_when_set() -> ( - None -): - mapping = _cloud_evaluator_data_mapping( - "groundedness", "input", "expected", context_field="context" - ) - assert mapping["context"] == "{{item.context}}" - assert "ground_truth" not in mapping - - -def test_cloud_evaluator_data_mapping_task_completion() -> None: - mapping = _cloud_evaluator_data_mapping("task_completion", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert "ground_truth" not in mapping - assert "context" not in mapping - assert "tool_calls" not in mapping - - -def test_cloud_evaluator_data_mapping_tool_call_accuracy() -> None: - mapping = _cloud_evaluator_data_mapping("tool_call_accuracy", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert mapping["tool_calls"] == "{{sample.tool_calls}}" - assert mapping["tool_definitions"] == "{{item.tool_definitions}}" - - -def test_default_foundry_input_mapping_groundedness_uses_row_context() -> None: - mapping = _default_foundry_input_mapping("GroundednessEvaluator") - assert mapping["context"] == "$row.context" - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - - -def test_default_foundry_input_mapping_task_completion() -> None: - mapping = _default_foundry_input_mapping("TaskCompletionEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert "ground_truth" not in mapping - assert "context" not in mapping - - -def test_default_foundry_input_mapping_tool_call_accuracy() -> None: - mapping = _default_foundry_input_mapping("ToolCallAccuracyEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert mapping["tool_calls"] == "$row.tool_calls" - assert mapping["tool_definitions"] == "$row.tool_definitions" - - -# --------------------------------------------------------------------------- -# Extended evaluator coverage (issue #51) -# --------------------------------------------------------------------------- - - -def test_cloud_evaluator_data_mapping_response_completeness() -> None: - mapping = _cloud_evaluator_data_mapping( - "response_completeness", "input", "expected" - ) - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert mapping["ground_truth"] == "{{item.expected}}" - - -def test_cloud_evaluator_data_mapping_groundedness_pro() -> None: - mapping = _cloud_evaluator_data_mapping( - "groundedness_pro", "input", "expected", context_field="context" - ) - assert mapping["context"] == "{{item.context}}" - assert mapping["query"] == "{{item.input}}" - assert "ground_truth" not in mapping - - -def test_cloud_evaluator_data_mapping_retrieval() -> None: - mapping = _cloud_evaluator_data_mapping("retrieval", "input", "expected") - assert mapping["context"] == "{{item.expected}}" - assert mapping["query"] == "{{item.input}}" - - -def test_cloud_evaluator_data_mapping_tool_output_utilization() -> None: - mapping = _cloud_evaluator_data_mapping( - "tool_output_utilization", "input", "expected" - ) - assert mapping["query"] == "{{item.input}}" - assert mapping["tool_definitions"] == "{{item.tool_definitions}}" - assert "tool_calls" not in mapping - - -def test_cloud_evaluator_data_mapping_tool_call_success() -> None: - mapping = _cloud_evaluator_data_mapping("tool_call_success", "input", "expected") - assert mapping["tool_definitions"] == "{{item.tool_definitions}}" - assert "tool_calls" not in mapping - - -def test_cloud_evaluator_data_mapping_task_adherence_uses_output_items() -> None: - mapping = _cloud_evaluator_data_mapping("task_adherence", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_items}}" - assert "ground_truth" not in mapping - - -def test_cloud_evaluator_data_mapping_coherence_default_path() -> None: - mapping = _cloud_evaluator_data_mapping("coherence", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert "ground_truth" not in mapping - assert "context" not in mapping - assert "tool_calls" not in mapping - - -def test_cloud_evaluator_data_mapping_violence_default_path() -> None: - mapping = _cloud_evaluator_data_mapping("violence", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert "ground_truth" not in mapping - - -def test_cloud_evaluator_data_mapping_intent_resolution_default_path() -> None: - mapping = _cloud_evaluator_data_mapping("intent_resolution", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - - -def test_default_foundry_input_mapping_coherence() -> None: - mapping = _default_foundry_input_mapping("CoherenceEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert "ground_truth" not in mapping - assert "context" not in mapping - - -def test_default_foundry_input_mapping_fluency() -> None: - mapping = _default_foundry_input_mapping("FluencyEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - - -def test_default_foundry_input_mapping_f1_score() -> None: - mapping = _default_foundry_input_mapping("F1ScoreEvaluator") - assert mapping["response"] == "$prediction" - assert mapping["ground_truth"] == "$expected" - assert "query" not in mapping - - -def test_default_foundry_input_mapping_relevance() -> None: - mapping = _default_foundry_input_mapping("RelevanceEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert mapping["context"] == "$row.context" - - -def test_default_foundry_input_mapping_retrieval() -> None: - mapping = _default_foundry_input_mapping("RetrievalEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert mapping["context"] == "$row.context" - - -def test_default_foundry_input_mapping_response_completeness() -> None: - mapping = _default_foundry_input_mapping("ResponseCompletenessEvaluator") - assert mapping["response"] == "$prediction" - assert mapping["ground_truth"] == "$expected" - assert "query" not in mapping - - -def test_default_foundry_input_mapping_intent_resolution() -> None: - mapping = _default_foundry_input_mapping("IntentResolutionEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert "tool_calls" not in mapping - - -def test_default_foundry_input_mapping_task_adherence() -> None: - mapping = _default_foundry_input_mapping("TaskAdherenceEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - - -def test_default_foundry_input_mapping_tool_selection() -> None: - mapping = _default_foundry_input_mapping("ToolSelectionEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert mapping["tool_calls"] == "$row.tool_calls" - assert mapping["tool_definitions"] == "$row.tool_definitions" - - -def test_default_foundry_input_mapping_tool_input_accuracy() -> None: - mapping = _default_foundry_input_mapping("ToolInputAccuracyEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert mapping["tool_definitions"] == "$row.tool_definitions" - assert "tool_calls" not in mapping - - -def test_cloud_evaluator_data_mapping_relevance_uses_context() -> None: - mapping = _cloud_evaluator_data_mapping( - "relevance", "input", "expected", context_field="context" - ) - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert mapping["context"] == "{{item.context}}" - assert "ground_truth" not in mapping - - -def test_cloud_evaluator_data_mapping_retrieval_uses_context() -> None: - mapping = _cloud_evaluator_data_mapping( - "retrieval", "input", "expected", context_field="context" - ) - assert mapping["context"] == "{{item.context}}" - - -def test_cloud_evaluator_data_mapping_tool_selection() -> None: - mapping = _cloud_evaluator_data_mapping("tool_selection", "input", "expected") - assert mapping["tool_calls"] == "{{sample.tool_calls}}" - assert mapping["tool_definitions"] == "{{item.tool_definitions}}" - - -def test_cloud_evaluator_data_mapping_tool_input_accuracy() -> None: - mapping = _cloud_evaluator_data_mapping("tool_input_accuracy", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["tool_definitions"] == "{{item.tool_definitions}}" - assert "tool_calls" not in mapping - - -# --------------------------------------------------------------------------- -# Safety evaluator tests -# --------------------------------------------------------------------------- - - -def test_cloud_evaluator_data_mapping_violence() -> None: - mapping = _cloud_evaluator_data_mapping("violence", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert "ground_truth" not in mapping - assert "context" not in mapping - assert "tool_calls" not in mapping - - -def test_cloud_evaluator_data_mapping_sexual() -> None: - mapping = _cloud_evaluator_data_mapping("sexual", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert len(mapping) == 2 - - -def test_cloud_evaluator_data_mapping_self_harm() -> None: - mapping = _cloud_evaluator_data_mapping("self_harm", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert len(mapping) == 2 - - -def test_cloud_evaluator_data_mapping_hate_unfairness() -> None: - mapping = _cloud_evaluator_data_mapping("hate_unfairness", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert len(mapping) == 2 - - -def test_cloud_evaluator_data_mapping_protected_material() -> None: - mapping = _cloud_evaluator_data_mapping("protected_material", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert len(mapping) == 2 - - -def test_cloud_evaluator_data_mapping_content_safety() -> None: - mapping = _cloud_evaluator_data_mapping("content_safety", "input", "expected") - assert mapping["query"] == "{{item.input}}" - assert mapping["response"] == "{{sample.output_text}}" - assert len(mapping) == 2 - - -def test_cloud_evaluator_needs_model_safety_evaluators() -> None: - """Safety evaluators use azure_ai_project, not a judge model.""" - safety_builtins = [ - "violence", - "sexual", - "self_harm", - "hate_unfairness", - "content_safety", - "protected_material", - "code_vulnerability", - "ungrounded_attributes", - "indirect_attack", - ] - for name in safety_builtins: - assert not _cloud_evaluator_needs_model(name), f"{name} should not need a model" - - -def test_cloud_evaluator_needs_model_quality_evaluators() -> None: - """Quality evaluators still need a model.""" - quality_builtins = ["similarity", "coherence", "fluency", "groundedness"] - for name in quality_builtins: - assert _cloud_evaluator_needs_model(name), f"{name} should need a model" - - -def test_cloud_evaluator_needs_model_nlp_evaluators() -> None: - """NLP evaluators do not need a model.""" - nlp_builtins = [ - "f1_score", - "bleu_score", - "rouge_score", - "meteor_score", - "gleu_score", - ] - for name in nlp_builtins: - assert not _cloud_evaluator_needs_model(name), f"{name} should not need a model" - - -def test_default_foundry_input_mapping_violence() -> None: - mapping = _default_foundry_input_mapping("ViolenceEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert "ground_truth" not in mapping - assert "context" not in mapping - - -def test_default_foundry_input_mapping_sexual() -> None: - mapping = _default_foundry_input_mapping("SexualEvaluator") - assert mapping["query"] == "$prompt" - assert mapping["response"] == "$prediction" - assert len(mapping) == 2 - - -def test_default_foundry_input_mapping_self_harm() -> None: - mapping = _default_foundry_input_mapping("SelfHarmEvaluator") - assert mapping == {"query": "$prompt", "response": "$prediction"} - - -def test_default_foundry_input_mapping_hate_unfairness() -> None: - mapping = _default_foundry_input_mapping("HateUnfairnessEvaluator") - assert mapping == {"query": "$prompt", "response": "$prediction"} - - -def test_default_foundry_input_mapping_protected_material() -> None: - mapping = _default_foundry_input_mapping("ProtectedMaterialEvaluator") - assert mapping == {"query": "$prompt", "response": "$prediction"} - - -def test_default_foundry_input_mapping_content_safety() -> None: - mapping = _default_foundry_input_mapping("ContentSafetyEvaluator") - assert mapping == {"query": "$prompt", "response": "$prediction"} - - -def test_default_foundry_input_mapping_groundedness_pro() -> None: - mapping = _default_foundry_input_mapping("GroundednessProEvaluator") - assert mapping == {"query": "$prompt", "response": "$prediction"} - - -# --------------------------------------------------------------------------- -# model_config auto-injection tests -# --------------------------------------------------------------------------- - - -def test_model_config_injected_for_all_ai_assisted_evaluators() -> None: - """Verify model_config is auto-injected for ALL AI-assisted evaluators, not just 2.""" - import importlib as _real_importlib - - from agentops.backends.eval_engine import ( - _AI_ASSISTED_EVALUATORS, - _load_foundry_evaluator_callable, - ) - - # Capture a direct reference to the real import_module BEFORE patching - _orig_import_module = _real_importlib.import_module - - # Create a fake evaluator class that captures its kwargs - class FakeEvaluator: - def __init__(self, **kwargs): - self.init_kwargs = kwargs - - def __call__(self, **kwargs): - return {} - - # Create a fake module with all AI-assisted evaluator classes - fake_module = SimpleNamespace( - **{name: type(name, (FakeEvaluator,), {}) for name in _AI_ASSISTED_EVALUATORS} - ) - - # Only intercept "azure.ai.evaluation" imports, let everything else through - def _selective_import(name, *args, **kwargs): - if name == "azure.ai.evaluation": - return fake_module - return _orig_import_module(name, *args, **kwargs) - - for evaluator_name in _AI_ASSISTED_EVALUATORS: - with ( - patch.dict( - "os.environ", - { - "AZURE_OPENAI_ENDPOINT": "https://test.openai.azure.com/", - "AZURE_OPENAI_DEPLOYMENT": "gpt-4o-mini", - }, - ), - patch( - "agentops.backends.eval_engine.importlib.import_module", - side_effect=_selective_import, - ), - patch( - "agentops.backends.eval_engine._default_credential", - return_value="fake-cred", - ), - ): - evaluator = _load_foundry_evaluator_callable( - evaluator_name=evaluator_name, - evaluator_config={"kind": "builtin", "class_name": evaluator_name}, - ) - assert hasattr(evaluator, "init_kwargs"), ( - f"{evaluator_name}: expected FakeEvaluator instance" - ) - assert "model_config" in evaluator.init_kwargs, ( - f"{evaluator_name}: model_config was NOT auto-injected" - ) - mc = evaluator.init_kwargs["model_config"] - assert mc["azure_endpoint"] == "https://test.openai.azure.com/" - assert mc["azure_deployment"] == "gpt-4o-mini" diff --git a/tests/unit/test_http_backend.py b/tests/unit/test_http_backend.py deleted file mode 100644 index 769cbe11..00000000 --- a/tests/unit/test_http_backend.py +++ /dev/null @@ -1,583 +0,0 @@ -"""Unit tests for the HTTP backend.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock, patch - -import pytest - -from agentops.backends.base import BackendRunContext -from agentops.backends.http_backend import HttpBackend, _extract_dot_path -from agentops.core.models import ( - BundleRef, - DatasetRef, - ExecutionConfig, - OutputConfig, - RunConfig, - TargetConfig, - TargetEndpointConfig, -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -_BUNDLE_YAML = """\ -version: 1 -name: test_http_bundle -evaluators: - - name: exact_match - source: local - enabled: true - - name: avg_latency_seconds - source: local - enabled: true -thresholds: - - evaluator: exact_match - criteria: ">=" - value: 0.5 -""" - -_DATASET_YAML = """\ -version: 1 -name: test_http_dataset -source: - type: file - path: smoke.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -""" - -_DATASET_ROWS = [ - {"id": "1", "input": "What is 2+2?", "expected": "4"}, - {"id": "2", "input": "Capital of France?", "expected": "Paris"}, -] - - -def _write_fixtures(tmp_path: Path) -> tuple[Path, Path]: - bundle_path = tmp_path / "bundle.yaml" - dataset_path = tmp_path / "dataset.yaml" - data_path = tmp_path / "smoke.jsonl" - - bundle_path.write_text(_BUNDLE_YAML, encoding="utf-8") - dataset_path.write_text(_DATASET_YAML, encoding="utf-8") - data_path.write_text( - "\n".join(json.dumps(row) for row in _DATASET_ROWS), encoding="utf-8" - ) - return bundle_path, dataset_path - - -def _build_context( - tmp_path: Path, - *, - url: str = "http://localhost:8080/chat", - url_env: str | None = None, - request_field: str = "message", - response_field: str = "text", - auth_header_env: str | None = None, - headers: dict[str, str] | None = None, - tool_calls_field: str | None = None, - extra_fields: list[str] | None = None, - bundle_yaml: str | None = None, - dataset_yaml: str | None = None, - dataset_rows: list[dict] | None = None, -) -> BackendRunContext: - if bundle_yaml and dataset_yaml and dataset_rows is not None: - bundle_path = tmp_path / "bundle.yaml" - dataset_path = tmp_path / "dataset.yaml" - data_path = tmp_path / "smoke.jsonl" - bundle_path.write_text(bundle_yaml, encoding="utf-8") - dataset_path.write_text(dataset_yaml, encoding="utf-8") - data_path.write_text( - "\n".join(json.dumps(row) for row in dataset_rows), encoding="utf-8" - ) - else: - bundle_path, dataset_path = _write_fixtures(tmp_path) - endpoint = TargetEndpointConfig( - kind="http", - url=url if url_env is None else None, - url_env=url_env, - request_field=request_field, - response_field=response_field, - auth_header_env=auth_header_env, - headers=headers or {}, - tool_calls_field=tool_calls_field, - extra_fields=extra_fields, - ) - run_config = RunConfig( - version=2, - target=TargetConfig( - type="model", - hosting="local", - execution_mode="remote", - endpoint=endpoint, - ), - bundle=BundleRef(path=bundle_path), - dataset=DatasetRef(path=dataset_path), - execution=ExecutionConfig(timeout_seconds=30), - output=OutputConfig(), - ) - return BackendRunContext( - run_config=run_config, - bundle_path=bundle_path, - dataset_path=dataset_path, - backend_output_dir=tmp_path / "out", - ) - - -def _fake_urlopen(response_body: dict[str, Any]): - """Return a context-manager mock that yields a fake HTTP response.""" - mock_response = MagicMock() - mock_response.read.return_value = json.dumps(response_body).encode("utf-8") - mock_response.__enter__ = lambda self: self - mock_response.__exit__ = MagicMock(return_value=False) - return mock_response - - -# --------------------------------------------------------------------------- -# _extract_dot_path -# --------------------------------------------------------------------------- - - -def test_extract_dot_path_single_key() -> None: - assert _extract_dot_path({"text": "hello"}, "text") == "hello" - - -def test_extract_dot_path_nested() -> None: - assert _extract_dot_path({"output": {"text": "world"}}, "output.text") == "world" - - -def test_extract_dot_path_missing_key_raises() -> None: - with pytest.raises(ValueError, match="Response field 'missing'"): - _extract_dot_path({"text": "hi"}, "missing") - - -def test_extract_dot_path_non_dict_intermediate_raises() -> None: - with pytest.raises(ValueError, match="expected object at 'nested'"): - _extract_dot_path({"text": "flat"}, "text.nested") - - -# --------------------------------------------------------------------------- -# TargetEndpointConfig validation -# --------------------------------------------------------------------------- - - -def test_endpoint_config_accepts_http_with_url() -> None: - config = TargetEndpointConfig.model_validate( - { - "kind": "http", - "url": "http://localhost/chat", - } - ) - assert config.kind == "http" - assert config.url == "http://localhost/chat" - - -def test_endpoint_config_accepts_http_with_url_env() -> None: - config = TargetEndpointConfig.model_validate( - { - "kind": "http", - "url_env": "AGENT_HTTP_URL", - } - ) - assert config.kind == "http" - assert config.url_env == "AGENT_HTTP_URL" - - -def test_endpoint_config_http_requires_url_or_url_env() -> None: - with pytest.raises(Exception, match="url"): - TargetEndpointConfig.model_validate({"kind": "http"}) - - -# --------------------------------------------------------------------------- -# HttpBackend URL resolution -# --------------------------------------------------------------------------- - - -def test_resolve_url_from_config(tmp_path: Path) -> None: - context = _build_context(tmp_path, url="http://example.com/api") - backend = HttpBackend() - assert backend._resolve_url(context) == "http://example.com/api" - - -def test_resolve_url_from_env_var( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("MY_AGENT_URL", "http://agent.example.com/chat") - bundle_path, dataset_path = _write_fixtures(tmp_path) - endpoint = TargetEndpointConfig(kind="http", url_env="MY_AGENT_URL") - run_config = RunConfig( - version=2, - target=TargetConfig( - type="model", - hosting="local", - execution_mode="remote", - endpoint=endpoint, - ), - bundle=BundleRef(path=bundle_path), - dataset=DatasetRef(path=dataset_path), - execution=ExecutionConfig(timeout_seconds=30), - output=OutputConfig(), - ) - context = BackendRunContext( - run_config=run_config, - bundle_path=bundle_path, - dataset_path=dataset_path, - backend_output_dir=tmp_path / "out", - ) - backend = HttpBackend() - assert backend._resolve_url(context) == "http://agent.example.com/chat" - - -def test_resolve_url_env_missing_raises( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("MISSING_URL_VAR", raising=False) - bundle_path, dataset_path = _write_fixtures(tmp_path) - endpoint = TargetEndpointConfig(kind="http", url_env="MISSING_URL_VAR") - run_config = RunConfig( - version=2, - target=TargetConfig( - type="model", - hosting="local", - execution_mode="remote", - endpoint=endpoint, - ), - bundle=BundleRef(path=bundle_path), - dataset=DatasetRef(path=dataset_path), - execution=ExecutionConfig(timeout_seconds=30), - output=OutputConfig(), - ) - context = BackendRunContext( - run_config=run_config, - bundle_path=bundle_path, - dataset_path=dataset_path, - backend_output_dir=tmp_path / "out", - ) - backend = HttpBackend() - with pytest.raises(ValueError, match="MISSING_URL_VAR"): - backend._resolve_url(context) - - -# --------------------------------------------------------------------------- -# HttpBackend.execute — happy path -# --------------------------------------------------------------------------- - - -def test_execute_posts_to_url_and_writes_metrics(tmp_path: Path) -> None: - context = _build_context(tmp_path, request_field="message", response_field="text") - fake_response = {"text": "4"} - - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen(fake_response) - HttpBackend().execute(context) - - metrics_path = context.backend_output_dir / "backend_metrics.json" - assert metrics_path.exists() - payload = json.loads(metrics_path.read_text(encoding="utf-8")) - assert "metrics" in payload - assert "row_metrics" in payload - assert len(payload["row_metrics"]) == len(_DATASET_ROWS) - - -def test_execute_uses_correct_request_field(tmp_path: Path) -> None: - context = _build_context(tmp_path, request_field="query", response_field="answer") - calls: list[dict] = [] - - def fake_urlopen(request, timeout=None): - body = json.loads(request.data.decode("utf-8")) - calls.append(body) - mock = _fake_urlopen({"answer": "some answer"}) - return mock - - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=fake_urlopen, - ): - HttpBackend().execute(context) - - assert len(calls) == len(_DATASET_ROWS) - for call, row in zip(calls, _DATASET_ROWS): - assert "query" in call - assert call["query"] == row["input"] - assert "message" not in call - - -def test_execute_dot_path_response_extraction(tmp_path: Path) -> None: - context = _build_context(tmp_path, response_field="output.text") - fake_response = {"output": {"text": "Paris"}} - - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen(fake_response) - result = HttpBackend().execute(context) - - assert result.exit_code == 0 - payload = json.loads( - (context.backend_output_dir / "backend_metrics.json").read_text( - encoding="utf-8" - ) - ) - assert len(payload["row_metrics"]) == len(_DATASET_ROWS) - - -def test_execute_exact_match_scores(tmp_path: Path) -> None: - """Row 1: matches (2+2=4 → '4'), row 2: does not match ('Paris' vs 'Paris' — same).""" - responses = [{"text": "4"}, {"text": "Paris"}] - call_index = 0 - - def fake_urlopen(request, timeout=None): - nonlocal call_index - mock = _fake_urlopen(responses[call_index % len(responses)]) - call_index += 1 - return mock - - context = _build_context(tmp_path) - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=fake_urlopen, - ): - HttpBackend().execute(context) - - payload = json.loads( - (context.backend_output_dir / "backend_metrics.json").read_text( - encoding="utf-8" - ) - ) - row_metrics = payload["row_metrics"] - assert len(row_metrics) == 2 - - for rm in row_metrics: - names = {m["name"] for m in rm["metrics"]} - assert "exact_match" in names - assert "avg_latency_seconds" in names - - -def test_execute_sets_auth_header( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("MY_TOKEN", "secret-token-123") - context = _build_context(tmp_path, auth_header_env="MY_TOKEN") - captured_headers: list[dict] = [] - - def fake_urlopen(request, timeout=None): - captured_headers.append(dict(request.headers)) - return _fake_urlopen({"text": "4"}) - - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=fake_urlopen, - ): - HttpBackend().execute(context) - - for headers in captured_headers: - # urllib capitalizes the first letter of each header word - auth = headers.get("Authorization") or headers.get("authorization") - assert auth == "Bearer secret-token-123" - - -def test_execute_includes_extra_headers(tmp_path: Path) -> None: - context = _build_context(tmp_path, headers={"X-Custom-Header": "myvalue"}) - captured_headers: list[dict] = [] - - def fake_urlopen(request, timeout=None): - captured_headers.append(dict(request.headers)) - return _fake_urlopen({"text": "4"}) - - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=fake_urlopen, - ): - HttpBackend().execute(context) - - for headers in captured_headers: - custom = headers.get("X-custom-header") or headers.get("X-Custom-Header") - assert custom == "myvalue" - - -# --------------------------------------------------------------------------- -# HttpBackend.execute — error handling -# --------------------------------------------------------------------------- - - -def test_execute_returns_nonzero_exit_code_on_http_error(tmp_path: Path) -> None: - import urllib.error - - context = _build_context(tmp_path) - - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=urllib.error.URLError("connection refused"), - ): - result = HttpBackend().execute(context) - - assert result.exit_code == 1 - stderr = (context.backend_output_dir / "backend.stderr.log").read_text( - encoding="utf-8" - ) - assert "connection refused" in stderr.lower() or "row=1" in stderr - - -def test_execute_writes_stdout_log(tmp_path: Path) -> None: - context = _build_context(tmp_path) - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen({"text": "4"}) - HttpBackend().execute(context) - - stdout = (context.backend_output_dir / "backend.stdout.log").read_text( - encoding="utf-8" - ) - assert "row=1" in stdout - - -def test_execute_result_backend_label(tmp_path: Path) -> None: - context = _build_context(tmp_path) - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen({"text": "4"}) - result = HttpBackend().execute(context) - - assert result.backend == "http" - assert result.started_at.endswith("Z") - assert result.finished_at.endswith("Z") - assert result.duration_seconds >= 0.0 - - -# --------------------------------------------------------------------------- -# Extra fields forwarding -# --------------------------------------------------------------------------- - - -def test_execute_forwards_extra_fields_in_request(tmp_path: Path) -> None: - """When extra_fields is configured, those JSONL row fields appear in the request body.""" - dataset_rows = [ - { - "id": "1", - "input": "Hello", - "expected": "Hi", - "session_id": "s1", - "user_id": "u1", - }, - ] - dataset_yaml = """\ -version: 1 -name: test_extra -source: - type: file - path: smoke.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -""" - context = _build_context( - tmp_path, - extra_fields=["session_id", "user_id"], - bundle_yaml=_BUNDLE_YAML, - dataset_yaml=dataset_yaml, - dataset_rows=dataset_rows, - ) - calls: list[dict] = [] - - def fake_urlopen(request, timeout=None): - body = json.loads(request.data.decode("utf-8")) - calls.append(body) - return _fake_urlopen({"text": "Hi"}) - - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=fake_urlopen, - ): - HttpBackend().execute(context) - - assert len(calls) == 1 - assert calls[0]["message"] == "Hello" - assert calls[0]["session_id"] == "s1" - assert calls[0]["user_id"] == "u1" - - -def test_execute_extra_fields_skips_missing_row_fields(tmp_path: Path) -> None: - """Extra fields not present in a JSONL row are silently skipped.""" - dataset_rows = [ - {"id": "1", "input": "Hello", "expected": "Hi"}, - ] - dataset_yaml = """\ -version: 1 -name: test_extra_skip -source: - type: file - path: smoke.jsonl -format: - type: jsonl - input_field: input - expected_field: expected -""" - context = _build_context( - tmp_path, - extra_fields=["session_id"], - bundle_yaml=_BUNDLE_YAML, - dataset_yaml=dataset_yaml, - dataset_rows=dataset_rows, - ) - calls: list[dict] = [] - - def fake_urlopen(request, timeout=None): - body = json.loads(request.data.decode("utf-8")) - calls.append(body) - return _fake_urlopen({"text": "Hi"}) - - with patch( - "agentops.backends.http_backend.urllib.request.urlopen", - side_effect=fake_urlopen, - ): - HttpBackend().execute(context) - - assert "session_id" not in calls[0] - - -# --------------------------------------------------------------------------- -# Tool calls extraction -# --------------------------------------------------------------------------- - - -def test_execute_extracts_tool_calls_from_response(tmp_path: Path) -> None: - """When tool_calls_field is set, tool_calls are extracted from the HTTP response.""" - context = _build_context(tmp_path, tool_calls_field="tool_calls") - expected_tool_calls = [{"name": "get_weather", "arguments": {"city": "Seattle"}}] - fake_response = {"text": "The weather is sunny", "tool_calls": expected_tool_calls} - - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen(fake_response) - result = HttpBackend().execute(context) - - assert result.exit_code == 0 - - -def test_execute_tool_calls_field_nested_dot_path(tmp_path: Path) -> None: - """tool_calls_field supports dot-path notation.""" - context = _build_context(tmp_path, tool_calls_field="metadata.tool_calls") - expected_tool_calls = [{"name": "search", "arguments": {"q": "test"}}] - fake_response = {"text": "results", "metadata": {"tool_calls": expected_tool_calls}} - - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen(fake_response) - result = HttpBackend().execute(context) - - assert result.exit_code == 0 - - -def test_execute_tool_calls_field_missing_in_response_is_silently_skipped( - tmp_path: Path, -) -> None: - """If tool_calls_field is configured but not in the response, execution continues.""" - context = _build_context(tmp_path, tool_calls_field="tool_calls") - fake_response = {"text": "No tools used"} - - with patch("agentops.backends.http_backend.urllib.request.urlopen") as mock_urlopen: - mock_urlopen.return_value = _fake_urlopen(fake_response) - result = HttpBackend().execute(context) - - assert result.exit_code == 0 diff --git a/tests/unit/test_initializer.py b/tests/unit/test_initializer.py index 74daab3e..b50587ba 100644 --- a/tests/unit/test_initializer.py +++ b/tests/unit/test_initializer.py @@ -1,85 +1,49 @@ from pathlib import Path -from agentops.services.initializer import initialize_workspace -from agentops.utils.yaml import load_yaml, save_yaml - - -def test_init_creates_expected_files(tmp_path: Path) -> None: - result = initialize_workspace(tmp_path, force=False) - - assert (tmp_path / ".agentops").is_dir() - assert (tmp_path / ".agentops" / "bundles").is_dir() - assert (tmp_path / ".agentops" / "datasets").is_dir() - assert (tmp_path / ".agentops" / "data").is_dir() - assert (tmp_path / ".agentops" / "results").is_dir() - - assert (tmp_path / ".agentops" / "config.yaml").is_file() - assert (tmp_path / ".agentops" / "bundles" / "model_quality_baseline.yaml").is_file() - assert ( - tmp_path / ".agentops" / "bundles" / "rag_quality_baseline.yaml" - ).is_file() - assert ( - tmp_path / ".agentops" / "bundles" / "conversational_agent_baseline.yaml" - ).is_file() - assert (tmp_path / ".agentops" / "bundles" / "agent_workflow_baseline.yaml").is_file() - assert (tmp_path / ".agentops" / "bundles" / "safe_agent_baseline.yaml").is_file() - assert (tmp_path / ".agentops" / "datasets" / "smoke-model-direct.yaml").is_file() - assert (tmp_path / ".agentops" / "datasets" / "smoke-rag.yaml").is_file() - assert (tmp_path / ".agentops" / "datasets" / "smoke-agent-tools.yaml").is_file() - assert (tmp_path / ".agentops" / "data" / "smoke-model-direct.jsonl").is_file() - assert (tmp_path / ".agentops" / "data" / "smoke-rag.jsonl").is_file() - assert (tmp_path / ".agentops" / "data" / "smoke-agent-tools.jsonl").is_file() - assert (tmp_path / ".agentops" / "run.yaml").is_file() - assert (tmp_path / ".agentops" / "run-rag.yaml").is_file() - assert (tmp_path / ".agentops" / "run-agent.yaml").is_file() - assert (tmp_path / ".agentops" / "run-http-model.yaml").is_file() - assert (tmp_path / ".agentops" / "run-http-rag.yaml").is_file() - assert (tmp_path / ".agentops" / "run-http-agent-tools.yaml").is_file() - assert (tmp_path / ".agentops" / "run-callable.yaml").is_file() - assert (tmp_path / ".agentops" / "callable_adapter.py").is_file() - assert (tmp_path / ".agentops" / ".gitignore").is_file() - assert (tmp_path / ".agentops" / "datasets" / "smoke-conversational.yaml").is_file() - assert (tmp_path / ".agentops" / "data" / "smoke-conversational.jsonl").is_file() - assert (tmp_path / ".agentops" / "workflows" / "agentops-eval.yml").is_file() - - assert len(result.created_files) == 27 +from agentops.services.initializer import initialize_flat_workspace +from agentops.utils.yaml import load_yaml + + +def test_init_creates_flat_workspace(tmp_path: Path) -> None: + result = initialize_flat_workspace(tmp_path, force=False) + + agentops_yaml = tmp_path / "agentops.yaml" + smoke_jsonl = tmp_path / ".agentops" / "data" / "smoke.jsonl" + + assert agentops_yaml.is_file() + assert smoke_jsonl.is_file() + + assert agentops_yaml in result.created_files + assert smoke_jsonl in result.created_files assert len(result.overwritten_files) == 0 - run_config = load_yaml(tmp_path / ".agentops" / "run.yaml") - assert run_config["version"] == 1 - assert run_config["target"]["type"] == "model" - assert run_config["target"]["hosting"] == "foundry" - assert run_config["target"]["execution_mode"] == "remote" - assert run_config["bundle"]["name"] == "model_quality_baseline" - assert run_config["dataset"]["name"] == "smoke-model-direct" + config = load_yaml(agentops_yaml) + assert config["version"] == 1 + assert "agent" in config + assert "dataset" in config def test_init_does_not_overwrite_without_force(tmp_path: Path) -> None: - initialize_workspace(tmp_path, force=False) + initialize_flat_workspace(tmp_path, force=False) - config_path = tmp_path / ".agentops" / "config.yaml" - original = load_yaml(config_path) - original["defaults"]["timeout_seconds"] = 999 - save_yaml(config_path, original) + agentops_yaml = tmp_path / "agentops.yaml" + sentinel = "# user edit\n" + agentops_yaml.write_text(sentinel + agentops_yaml.read_text(encoding="utf-8"), encoding="utf-8") - result = initialize_workspace(tmp_path, force=False) - after = load_yaml(config_path) + result = initialize_flat_workspace(tmp_path, force=False) - assert after["defaults"]["timeout_seconds"] == 999 - assert config_path in result.skipped_files - assert config_path not in result.overwritten_files + assert agentops_yaml.read_text(encoding="utf-8").startswith("# user edit") + assert agentops_yaml in result.skipped_files + assert agentops_yaml not in result.overwritten_files def test_init_overwrites_with_force(tmp_path: Path) -> None: - initialize_workspace(tmp_path, force=False) + initialize_flat_workspace(tmp_path, force=False) - config_path = tmp_path / ".agentops" / "config.yaml" - modified = load_yaml(config_path) - modified["defaults"]["timeout_seconds"] = 999 - save_yaml(config_path, modified) + agentops_yaml = tmp_path / "agentops.yaml" + agentops_yaml.write_text("# tampered\n", encoding="utf-8") - result = initialize_workspace(tmp_path, force=True) - after = load_yaml(config_path) + result = initialize_flat_workspace(tmp_path, force=True) - assert after["defaults"]["timeout_seconds"] == 1800 - assert config_path in result.overwritten_files + assert "tampered" not in agentops_yaml.read_text(encoding="utf-8") + assert agentops_yaml in result.overwritten_files diff --git a/tests/unit/test_local_adapter_callable.py b/tests/unit/test_local_adapter_callable.py deleted file mode 100644 index 26c03610..00000000 --- a/tests/unit/test_local_adapter_callable.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Unit tests for callable adapter support in LocalAdapterBackend.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import pytest - -from agentops.backends.local_adapter_backend import _load_callable - - -def test_load_callable_resolves_valid_path( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # Write a small callable module in a temp dir and import from there. - (tmp_path / "echo_adapter.py").write_text( - "def echo(input_text: str, context: dict) -> dict:\n" - ' return {"response": input_text}\n', - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - fn = _load_callable("echo_adapter:echo") - assert callable(fn) - result = fn("hello", {"input": "hello"}) - assert result == {"response": "hello"} - - -def test_load_callable_bad_module() -> None: - with pytest.raises(ValueError, match="Could not import module"): - _load_callable("nonexistent_module_xyz:func") - - -def test_load_callable_bad_function( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - (tmp_path / "echo_adapter2.py").write_text( - "def echo(input_text, context):\n return {}\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - with pytest.raises(ValueError, match="has no function"): - _load_callable("echo_adapter2:nonexistent_function") - - -def test_load_callable_non_callable() -> None: - # json module has a constant we can use — __name__ is a str, not callable - with pytest.raises(ValueError, match="non-callable"): - _load_callable("json:__file__") - - -def test_load_callable_from_agentops_dir( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Verify _load_callable can import a module placed inside .agentops/ directory.""" - # Create a .agentops/ directory with a callable module - agentops_dir = tmp_path / ".agentops" - agentops_dir.mkdir() - adapter_file = agentops_dir / "my_test_adapter_in_agentops.py" - adapter_file.write_text( - "def run_evaluation(input_text, context):\n" - " return {'response': 'from-agentops-dir'}\n", - encoding="utf-8", - ) - - # Change cwd to tmp_path (the project root) and clean sys.path / modules - monkeypatch.chdir(tmp_path) - # Remove any stale entries that might interfere - monkeypatch.setattr("sys.path", [p for p in sys.path if str(tmp_path) not in p]) - - try: - fn = _load_callable("my_test_adapter_in_agentops:run_evaluation") - assert callable(fn) - result = fn("test", {}) - assert result == {"response": "from-agentops-dir"} - finally: - # Clean up imported module - sys.modules.pop("my_test_adapter_in_agentops", None) - - -def test_load_callable_error_message_mentions_agentops_dir() -> None: - """Verify the error message mentions .agentops/ as a valid location.""" - with pytest.raises(ValueError, match=r"\.agentops/"): - _load_callable("nonexistent_module_xyz:func") diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py new file mode 100644 index 00000000..926d5313 --- /dev/null +++ b/tests/unit/test_mcp_server.py @@ -0,0 +1,101 @@ +"""Tests for the AgentOps MCP server. + +These tests do not require the ``mcp`` extra to be installed: they +verify that ``cmd_mcp_serve`` is wired into the CLI and that the server +module raises a clear error when the optional dependency is missing. + +When ``mcp`` is installed, we additionally smoke-test that the server +builds successfully and registers the expected tool set. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from agentops.cli.app import app + + +runner = CliRunner() + +_HAS_MCP = importlib.util.find_spec("mcp") is not None + + +def test_cli_exposes_mcp_serve_command() -> None: + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "mcp" in result.stdout + + sub = runner.invoke(app, ["mcp", "--help"]) + assert sub.exit_code == 0 + assert "serve" in sub.stdout + + +def test_mcp_serve_help_runs_without_mcp_extra() -> None: + result = runner.invoke(app, ["mcp", "serve", "--help"]) + assert result.exit_code == 0 + assert "stdio" in result.stdout.lower() or "MCP" in result.stdout + + +@pytest.mark.skipif(_HAS_MCP, reason="mcp extra is installed") +def test_mcp_serve_errors_when_extra_missing() -> None: + result = runner.invoke(app, ["mcp", "serve"]) + assert result.exit_code == 1 + assert "mcp" in result.stdout.lower() or "mcp" in (result.stderr or "").lower() + + +@pytest.mark.skipif(not _HAS_MCP, reason="mcp extra not installed") +def test_build_server_registers_expected_tools() -> None: + from agentops.mcp.server import _build_server + + server = _build_server() + # FastMCP exposes registered tools via _tool_manager._tools (private but + # stable across 1.x); fall back to list_tools() when available. + tools = set() + tm = getattr(server, "_tool_manager", None) + if tm is not None and hasattr(tm, "_tools"): + tools = set(tm._tools.keys()) + expected = { + "agentops_init", + "agentops_eval_run", + "agentops_report_show", + "agentops_results_summary", + "agentops_dataset_add", + "agentops_list_runs", + "agentops_workflow_init", + } + assert expected.issubset(tools), f"missing tools: {expected - tools}" + + +@pytest.mark.skipif(not _HAS_MCP, reason="mcp extra not installed") +def test_dataset_add_tool_appends_rows(tmp_path: Path) -> None: + from agentops.mcp.server import _build_server + + server = _build_server() + tm = server._tool_manager + tool = tm._tools["agentops_dataset_add"] + + target = tmp_path / "rows.jsonl" + rows = [{"input": "hello", "expected": "hi"}, {"input": "ping", "expected": "pong"}] + + fn = getattr(tool, "fn", None) or getattr(tool, "func", None) + assert fn is not None, "could not locate underlying function on Tool" + result = fn(dataset_path=str(target), rows=rows) + + assert result["ok"] is True + assert result["appended"] == 2 + assert target.read_text(encoding="utf-8").count("\n") == 2 + + +@pytest.mark.skipif(not _HAS_MCP, reason="mcp extra not installed") +def test_list_runs_tool_handles_missing_dir(tmp_path: Path) -> None: + from agentops.mcp.server import _build_server + + server = _build_server() + tool = server._tool_manager._tools["agentops_list_runs"] + fn = getattr(tool, "fn", None) or getattr(tool, "func", None) + result = fn(workspace_dir=str(tmp_path)) + assert result == {"ok": True, "runs": []} diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py deleted file mode 100644 index 7a83e792..00000000 --- a/tests/unit/test_models.py +++ /dev/null @@ -1,466 +0,0 @@ -from agentops.core.models import ( - BundleConfig, - BundleRef, - DatasetConfig, - DatasetRef, - ExecutionConfig, - LocalAdapterConfig, - RunConfig, - RowMetricsResult, - TargetConfig, - TargetEndpointConfig, - ThresholdRule, -) - - -def test_bundle_config_parses() -> None: - data = { - "version": 1, - "name": "rag_baseline", - "description": "Baseline eval", - "evaluators": [ - {"name": "GroundednessEvaluator", "source": "foundry", "enabled": True}, - {"name": "exact_match", "source": "local", "enabled": True}, - ], - "thresholds": [ - {"evaluator": "exact_match", "criteria": ">=", "value": 0.8}, - ], - "metadata": {"category": "rag"}, - } - - bundle = BundleConfig.model_validate(data) - assert bundle.name == "rag_baseline" - assert bundle.evaluators[0].source == "foundry" - assert bundle.thresholds[0].criteria == ">=" - - -def test_bundle_config_accepts_foundry_evaluator_config() -> None: - data = { - "version": 1, - "name": "qa_similarity", - "evaluators": [ - { - "name": "SimilarityEvaluator", - "source": "foundry", - "enabled": True, - "config": { - "kind": "builtin", - "class_name": "SimilarityEvaluator", - "input_mapping": { - "query": "$prompt", - "response": "$prediction", - "ground_truth": "$expected", - }, - "score_keys": ["similarity"], - }, - } - ], - "thresholds": [ - {"evaluator": "SimilarityEvaluator", "criteria": ">=", "value": 3}, - ], - "metadata": {}, - } - - bundle = BundleConfig.model_validate(data) - assert bundle.evaluators[0].config["kind"] == "builtin" - - -def test_threshold_legacy_metric_operator_is_supported() -> None: - rule = ThresholdRule.model_validate( - {"metric": "groundedness", "operator": ">=", "value": 0.8} - ) - assert rule.evaluator == "groundedness" - assert rule.criteria == ">=" - - -def test_threshold_operator_validation() -> None: - try: - ThresholdRule.model_validate( - {"evaluator": "groundedness", "criteria": "!=", "value": 0.8} - ) - assert False, "expected validation error" - except Exception as exc: - assert "criteria" in str(exc) - - -def test_threshold_value_must_be_numeric() -> None: - try: - ThresholdRule.model_validate( - {"evaluator": "groundedness", "criteria": ">=", "value": "0.8"} - ) - assert False, "expected validation error" - except Exception as exc: - assert "numeric" in str(exc) - - -def test_boolean_criteria_must_not_have_value() -> None: - try: - ThresholdRule.model_validate( - {"evaluator": "exact_match", "criteria": "true", "value": 1} - ) - assert False, "expected validation error" - except Exception as exc: - assert "must be omitted" in str(exc) - - -def test_row_metrics_requires_positive_row_index() -> None: - try: - RowMetricsResult.model_validate( - { - "row_index": 0, - "metrics": [{"name": "exact_match", "value": 1.0}], - } - ) - assert False, "expected validation error" - except Exception as exc: - assert "row_index" in str(exc) - - -def test_dataset_config_parses() -> None: - data = { - "version": 1, - "name": "smoke", - "description": "Small smoke dataset", - "source": {"type": "file", "path": "./eval/datasets/smoke.jsonl"}, - "format": { - "type": "jsonl", - "input_field": "input", - "expected_field": "expected", - }, - "metadata": {"size_hint": 20}, - } - - dataset = DatasetConfig.model_validate(data) - assert dataset.source.path.name == "smoke.jsonl" - assert dataset.format.context_field is None - - -def test_dataset_config_parses_context_field() -> None: - data = { - "version": 1, - "name": "smoke-rag", - "source": {"type": "file", "path": "./data/smoke-rag.jsonl"}, - "format": { - "type": "jsonl", - "input_field": "input", - "expected_field": "expected", - "context_field": "context", - }, - } - - dataset = DatasetConfig.model_validate(data) - assert dataset.format.context_field == "context" - - -def test_endpoint_rejects_placeholder_model_name() -> None: - try: - TargetEndpointConfig.model_validate( - { - "kind": "foundry_agent", - "model": "", - } - ) - assert False, "expected validation error" - except Exception as exc: - assert "deployment name" in str(exc) - - -def test_target_remote_requires_endpoint() -> None: - try: - TargetConfig.model_validate( - { - "type": "agent", - "hosting": "foundry", - "execution_mode": "remote", - } - ) - assert False, "expected validation error" - except Exception as exc: - assert "endpoint" in str(exc) - - -def test_target_local_requires_local_config() -> None: - try: - TargetConfig.model_validate( - { - "type": "model", - "hosting": "local", - "execution_mode": "local", - } - ) - assert False, "expected validation error" - except Exception as exc: - assert "local" in str(exc) - - -def test_target_agent_mode_only_for_foundry() -> None: - try: - TargetConfig.model_validate( - { - "type": "agent", - "hosting": "local", - "execution_mode": "local", - "agent_mode": "hosted", - "local": {"adapter": "python run.py"}, - } - ) - assert False, "expected validation error" - except Exception as exc: - assert "agent_mode" in str(exc) - - -def test_target_framework_only_for_agent() -> None: - try: - TargetConfig.model_validate( - { - "type": "model", - "hosting": "local", - "execution_mode": "local", - "framework": "langgraph", - "local": {"adapter": "python run.py"}, - } - ) - assert False, "expected validation error" - except Exception as exc: - assert "framework" in str(exc) - - -def test_foundry_agent_endpoint_parses() -> None: - endpoint = TargetEndpointConfig.model_validate( - { - "kind": "foundry_agent", - "agent_id": "my-agent:3", - "model": "gpt-4o", - "project_endpoint_env": "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", - } - ) - assert endpoint.kind == "foundry_agent" - assert endpoint.agent_id == "my-agent:3" - assert endpoint.model == "gpt-4o" - - -def test_http_endpoint_accepts_url() -> None: - endpoint = TargetEndpointConfig.model_validate( - {"kind": "http", "url": "http://localhost:8080/chat"} - ) - assert endpoint.kind == "http" - assert endpoint.url == "http://localhost:8080/chat" - - -def test_http_endpoint_accepts_url_env() -> None: - endpoint = TargetEndpointConfig.model_validate( - {"kind": "http", "url_env": "AGENT_HTTP_URL"} - ) - assert endpoint.kind == "http" - assert endpoint.url_env == "AGENT_HTTP_URL" - - -def test_http_endpoint_requires_url_or_url_env() -> None: - try: - TargetEndpointConfig.model_validate({"kind": "http"}) - assert False, "expected validation error" - except Exception as exc: - assert "url" in str(exc).lower() - - -def test_http_endpoint_accepts_all_optional_fields() -> None: - endpoint = TargetEndpointConfig.model_validate( - { - "kind": "http", - "url": "http://localhost/chat", - "request_field": "query", - "response_field": "output.text", - "headers": {"X-Custom": "value"}, - "auth_header_env": "MY_TOKEN", - "tool_calls_field": "metadata.tool_calls", - "extra_fields": ["session_id", "user_id"], - } - ) - assert endpoint.request_field == "query" - assert endpoint.response_field == "output.text" - assert endpoint.tool_calls_field == "metadata.tool_calls" - assert endpoint.extra_fields == ["session_id", "user_id"] - assert endpoint.headers == {"X-Custom": "value"} - - -def test_target_remote_foundry_agent_parses() -> None: - target = TargetConfig.model_validate( - { - "type": "agent", - "hosting": "foundry", - "execution_mode": "remote", - "agent_mode": "hosted", - "endpoint": { - "kind": "foundry_agent", - "agent_id": "my-agent:3", - "model": "gpt-4o", - }, - } - ) - assert target.type == "agent" - assert target.hosting == "foundry" - assert target.execution_mode == "remote" - assert target.agent_mode == "hosted" - assert target.endpoint is not None - assert target.endpoint.agent_id == "my-agent:3" - - -def test_target_remote_http_parses() -> None: - target = TargetConfig.model_validate( - { - "type": "model", - "hosting": "local", - "execution_mode": "remote", - "endpoint": { - "kind": "http", - "url": "http://localhost:8080/chat", - }, - } - ) - assert target.type == "model" - assert target.endpoint is not None - assert target.endpoint.kind == "http" - - -def test_target_local_adapter_parses() -> None: - target = TargetConfig.model_validate( - { - "type": "model", - "hosting": "local", - "execution_mode": "local", - "local": {"adapter": "python my_adapter.py"}, - } - ) - assert target.type == "model" - assert target.execution_mode == "local" - assert target.local is not None - assert target.local.adapter == "python my_adapter.py" - - -def test_bundle_ref_requires_name_or_path() -> None: - try: - BundleRef.model_validate({}) - assert False, "expected validation error" - except Exception as exc: - assert "name" in str(exc) or "path" in str(exc) - - -def test_bundle_ref_accepts_name() -> None: - ref = BundleRef.model_validate({"name": "model_quality_baseline"}) - assert ref.name == "model_quality_baseline" - assert ref.path is None - - -def test_bundle_ref_accepts_path() -> None: - ref = BundleRef.model_validate({"path": "bundles/custom.yaml"}) - assert ref.path is not None - assert ref.name is None - - -def test_dataset_ref_requires_name_or_path() -> None: - try: - DatasetRef.model_validate({}) - assert False, "expected validation error" - except Exception as exc: - assert "name" in str(exc) or "path" in str(exc) - - -def test_run_config_parses() -> None: - data = { - "version": 1, - "target": { - "type": "model", - "hosting": "foundry", - "execution_mode": "remote", - "endpoint": { - "kind": "foundry_agent", - "model": "gpt-4o", - }, - }, - "bundle": {"name": "model_quality_baseline"}, - "dataset": {"name": "smoke-model-direct"}, - } - run_config = RunConfig.model_validate(data) - assert run_config.version == 1 - assert run_config.target.type == "model" - assert run_config.bundle.name == "model_quality_baseline" - assert run_config.dataset.name == "smoke-model-direct" - assert run_config.execution.timeout_seconds == 300 - assert run_config.output.write_report is True - - -def test_execution_config_defaults() -> None: - cfg = ExecutionConfig.model_validate({}) - assert cfg.concurrency == 1 - assert cfg.timeout_seconds == 300 - - -# ---- LocalAdapterConfig validation ---- - - -def test_local_adapter_config_adapter_only() -> None: - cfg = LocalAdapterConfig.model_validate({"adapter": "python run.py"}) - assert cfg.adapter == "python run.py" - assert cfg.callable is None - - -def test_local_adapter_config_callable_only() -> None: - cfg = LocalAdapterConfig.model_validate({"callable": "my_module:run_eval"}) - assert cfg.callable == "my_module:run_eval" - assert cfg.adapter is None - - -def test_local_adapter_config_both_fails() -> None: - try: - LocalAdapterConfig.model_validate( - {"adapter": "python run.py", "callable": "my_module:run_eval"} - ) - assert False, "expected validation error" - except Exception as exc: - assert "not both" in str(exc) - - -def test_local_adapter_config_neither_fails() -> None: - try: - LocalAdapterConfig.model_validate({}) - assert False, "expected validation error" - except Exception as exc: - assert "adapter" in str(exc) or "callable" in str(exc) - - -def test_local_adapter_config_callable_bad_format() -> None: - try: - LocalAdapterConfig.model_validate({"callable": "no_colon_here"}) - assert False, "expected validation error" - except Exception as exc: - assert "module:function" in str(exc) - - -def test_local_adapter_config_callable_empty_parts() -> None: - try: - LocalAdapterConfig.model_validate({"callable": ":func"}) - assert False, "expected validation error" - except Exception as exc: - assert "module:function" in str(exc) - - -def test_local_adapter_config_callable_empty_string() -> None: - try: - LocalAdapterConfig.model_validate({"callable": " "}) - assert False, "expected validation error" - except Exception as exc: - assert "non-empty" in str(exc) - - -def test_target_local_with_callable_parses() -> None: - target = TargetConfig.model_validate( - { - "type": "model", - "hosting": "local", - "execution_mode": "local", - "local": {"callable": "my_workflow:run_evaluation"}, - } - ) - assert target.local is not None - assert target.local.callable == "my_workflow:run_evaluation" - assert target.local.adapter is None diff --git a/tests/unit/test_pipeline_publisher.py b/tests/unit/test_pipeline_publisher.py new file mode 100644 index 00000000..b063a24b --- /dev/null +++ b/tests/unit/test_pipeline_publisher.py @@ -0,0 +1,354 @@ +"""Unit tests for the optional Foundry publisher.""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path +from typing import Any, Dict, List +from unittest import mock + +import pytest + +from agentops.core.results import ( + RowMetric, + RowResult, + RunResult, + RunSummary, + TargetInfo, +) +from agentops.pipeline import publisher + + +def _build_run_result() -> RunResult: + return RunResult( + started_at="2026-04-27T14:00:00+00:00", + finished_at="2026-04-27T14:00:01+00:00", + duration_seconds=1.0, + target=TargetInfo(kind="foundry_prompt", raw="my-agent:1"), + dataset_path="dataset.jsonl", + evaluators=["F1ScoreEvaluator"], + rows=[ + RowResult( + row_index=0, + input="hi", + expected="hello", + response="hello", + metrics=[RowMetric(name="f1_score", value=1.0)], + ), + RowResult( + row_index=1, + input="bye", + expected="goodbye", + response="goodbye", + metrics=[RowMetric(name="f1_score", value=0.5)], + ), + ], + aggregate_metrics={"f1_score": 0.75}, + summary=RunSummary( + items_total=2, + items_passed_all=2, + items_pass_rate=1.0, + thresholds_total=0, + thresholds_passed=0, + threshold_pass_rate=1.0, + overall_passed=True, + ), + ) + + +def test_build_instance_rows_projects_metrics(): + rows = publisher._build_instance_rows(_build_run_result()) + assert rows == [ + { + "line_number": 0, + "input": "hi", + "response": "hello", + "ground_truth": "hello", + "f1_score": 1.0, + }, + { + "line_number": 1, + "input": "bye", + "response": "goodbye", + "ground_truth": "goodbye", + "f1_score": 0.5, + }, + ] + + +def test_publish_requires_endpoint(monkeypatch): + monkeypatch.delenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", raising=False) + with pytest.raises(ValueError, match="project_endpoint"): + publisher.publish_to_foundry(_build_run_result()) + + +def _install_fake_azure_modules(captured: Dict[str, Any]) -> None: + """Inject lightweight stand-ins for azure-ai-evaluation and pandas.""" + + fake_pandas = types.ModuleType("pandas") + + class _DataFrame: + def __init__(self, rows: List[Dict[str, Any]]): + self.rows = rows + + fake_pandas.DataFrame = _DataFrame # type: ignore[attr-defined] + sys.modules["pandas"] = fake_pandas + + fake_azure = types.ModuleType("azure") + fake_evaluation = types.ModuleType("azure.ai.evaluation") + fake_evaluate = types.ModuleType("azure.ai.evaluation._evaluate") + fake_utils = types.ModuleType("azure.ai.evaluation._evaluate._utils") + + def _log_metrics_and_instance_results_onedp(**kwargs): + captured.update(kwargs) + return "https://ai.azure.com/projects/foo/evaluations/bar" + + fake_utils._log_metrics_and_instance_results_onedp = ( # type: ignore[attr-defined] + _log_metrics_and_instance_results_onedp + ) + fake_ai = types.ModuleType("azure.ai") + + sys.modules["azure"] = fake_azure + sys.modules["azure.ai"] = fake_ai + sys.modules["azure.ai.evaluation"] = fake_evaluation + sys.modules["azure.ai.evaluation._evaluate"] = fake_evaluate + sys.modules["azure.ai.evaluation._evaluate._utils"] = fake_utils + + +def test_publish_calls_onedp_with_expected_payload(monkeypatch): + captured: Dict[str, Any] = {} + _install_fake_azure_modules(captured) + monkeypatch.delenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", raising=False) + + result = publisher.publish_to_foundry( + _build_run_result(), + project_endpoint="https://contoso.services.ai.azure.com/api/projects/p", + evaluation_name="agentops-eval-test", + ) + + assert result.studio_url.startswith("https://ai.azure.com/") + assert result.evaluation_name == "agentops-eval-test" + assert captured["project_url"].endswith("/projects/p") + assert captured["evaluation_name"] == "agentops-eval-test" + assert captured["metrics"] == {"f1_score": 0.75} + assert captured["name_map"] == {"f1_score": "f1_score"} + assert len(captured["instance_results"].rows) == 2 + + +def test_publish_falls_back_to_env_var(monkeypatch): + captured: Dict[str, Any] = {} + _install_fake_azure_modules(captured) + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://contoso.services.ai.azure.com/api/projects/from-env", + ) + + publisher.publish_to_foundry(_build_run_result()) + assert captured["project_url"].endswith("/projects/from-env") + + +def test_orchestrator_skips_publish_when_disabled(tmp_path: Path): + from agentops.core.agentops_config import AgentOpsConfig + from agentops.pipeline import orchestrator + + config = AgentOpsConfig( + version=1, + agent="model:gpt-4o-mini", + dataset=Path("dataset.jsonl"), + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + result = _build_run_result() + + with mock.patch.object(publisher, "publish_to_foundry") as fake: + orchestrator._publish_to_foundry_safely(result, config, output_dir) + + fake.assert_not_called() # never reached because publish is None + # The helper itself only runs when publish == "foundry"; we verify the + # orchestrator branch by emulating that contract. + + +def test_orchestrator_swallows_publish_errors(tmp_path: Path): + from agentops.core.agentops_config import AgentOpsConfig + from agentops.pipeline import orchestrator + + config = AgentOpsConfig( + version=1, + agent="model:gpt-4o-mini", + dataset=Path("dataset.jsonl"), + publish="foundry", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + result = _build_run_result() + + with mock.patch.object( + publisher, "publish_to_foundry", side_effect=ImportError("no SDK") + ): + # Must not raise. + orchestrator._publish_to_foundry_safely(result, config, output_dir) + + assert not (output_dir / "cloud_evaluation.json").exists() + + +def test_orchestrator_writes_cloud_evaluation_metadata(tmp_path: Path): + from agentops.core.agentops_config import AgentOpsConfig + from agentops.pipeline import orchestrator + + config = AgentOpsConfig( + version=1, + agent="model:gpt-4o-mini", + dataset=Path("dataset.jsonl"), + publish="foundry", + project_endpoint="https://contoso.services.ai.azure.com/api/projects/p", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + result = _build_run_result() + + fake_publish = publisher.PublishResult( + studio_url="https://ai.azure.com/projects/p/evaluations/abc", + evaluation_name="agentops-eval-abc", + ) + with mock.patch.object(publisher, "publish_to_foundry", return_value=fake_publish): + orchestrator._publish_to_foundry_safely(result, config, output_dir) + + meta_path = output_dir / "cloud_evaluation.json" + assert meta_path.exists() + import json + payload = json.loads(meta_path.read_text(encoding="utf-8")) + assert payload["report_url"].endswith("/abc") + assert payload["evaluation_name"] == "agentops-eval-abc" + + +def test_orchestrator_dispatches_to_cloud_publisher_when_publish_is_foundry_cloud( + tmp_path: Path, +): + """publish='foundry_cloud' must invoke cloud_publisher (NOT the Classic + publisher) and write a ``mode='cloud'`` cloud_evaluation.json.""" + from agentops.core.agentops_config import AgentOpsConfig + from agentops.pipeline import cloud_publisher as _cp + from agentops.pipeline import orchestrator + + dataset_path = tmp_path / "dataset.jsonl" + dataset_path.write_text('{"input": "hi"}\n', encoding="utf-8") + + config = AgentOpsConfig( + version=1, + agent="support-bot:1", + dataset=dataset_path, + publish="foundry_cloud", + project_endpoint="https://contoso.services.ai.azure.com/api/projects/p", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + + fake_published = _cp.CloudPublishResult( + eval_id="eval-1", + run_id="run-1", + status="completed", + report_url="https://ai.azure.com/foundry/runs/run-1", + evaluation_name="agentops-cloud-abc", + ) + + result = _build_run_result() + result.target = TargetInfo( + kind="foundry_prompt", raw="support-bot:1", + name="support-bot", version="1", + ) + + with mock.patch.object( + _cp, "publish_to_foundry_cloud", return_value=fake_published, + ) as cloud_mock, mock.patch.object( + publisher, "publish_to_foundry", + ) as classic_mock: + orchestrator._publish_to_foundry_cloud_safely( + result, config, output_dir, dataset_path, + ) + + classic_mock.assert_not_called() + cloud_mock.assert_called_once() + + meta_path = output_dir / "cloud_evaluation.json" + assert meta_path.exists() + import json + payload = json.loads(meta_path.read_text(encoding="utf-8")) + assert payload["mode"] == "cloud" + assert payload["eval_id"] == "eval-1" + assert payload["run_id"] == "run-1" + assert payload["report_url"].endswith("/run-1") + + +def test_orchestrator_swallows_cloud_publish_errors(tmp_path: Path): + """A failure in cloud_publisher must not be fatal — local results + remain the source of truth.""" + from agentops.core.agentops_config import AgentOpsConfig + from agentops.pipeline import cloud_publisher as _cp + from agentops.pipeline import orchestrator + + dataset_path = tmp_path / "dataset.jsonl" + dataset_path.write_text('{"input": "hi"}\n', encoding="utf-8") + + config = AgentOpsConfig( + version=1, + agent="support-bot:1", + dataset=dataset_path, + publish="foundry_cloud", + project_endpoint="https://x.example/api/projects/p", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + + result = _build_run_result() + result.target = TargetInfo( + kind="foundry_prompt", raw="support-bot:1", + name="support-bot", version="1", + ) + + notices: list = [] + with mock.patch.object( + _cp, "publish_to_foundry_cloud", side_effect=RuntimeError("boom"), + ): + orchestrator._publish_to_foundry_cloud_safely( + result, config, output_dir, dataset_path, + progress=notices.append, + ) + + # No metadata file written on failure. + assert not (output_dir / "cloud_evaluation.json").exists() + # User saw a clear failure notice. + assert any("foundry_cloud FAILED" in m for m in notices) + + +def test_orchestrator_cloud_publish_requires_project_endpoint(tmp_path: Path, monkeypatch): + """Without project_endpoint or AZURE_AI_FOUNDRY_PROJECT_ENDPOINT, the + cloud branch refuses to call out and emits a clear progress message.""" + from agentops.core.agentops_config import AgentOpsConfig + from agentops.pipeline import cloud_publisher as _cp + from agentops.pipeline import orchestrator + + monkeypatch.delenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", raising=False) + + dataset_path = tmp_path / "dataset.jsonl" + dataset_path.write_text('{"input": "hi"}\n', encoding="utf-8") + + config = AgentOpsConfig( + version=1, + agent="support-bot:1", + dataset=dataset_path, + publish="foundry_cloud", + project_endpoint=None, + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + + result = _build_run_result() + notices: list = [] + with mock.patch.object(_cp, "publish_to_foundry_cloud") as cloud_mock: + orchestrator._publish_to_foundry_cloud_safely( + result, config, output_dir, dataset_path, + progress=notices.append, + ) + cloud_mock.assert_not_called() + assert any("project_endpoint" in m for m in notices) diff --git a/tests/unit/test_reporter.py b/tests/unit/test_reporter.py deleted file mode 100644 index 717ccc12..00000000 --- a/tests/unit/test_reporter.py +++ /dev/null @@ -1,210 +0,0 @@ -from agentops.core.models import RunResult -from agentops.core.reporter import ( - generate_report_markdown, - generate_report_html, - _format_metric_name, - _get_evaluator_description, - _fmt_threshold_value, -) - - -def _sample_result(overall_passed: bool = True, with_row_details: bool = False, with_context: bool = False) -> RunResult: - row_metrics = [] - item_evaluations = [] - if with_row_details: - row_metrics = [ - { - "row_index": 1, - "input": "What is the refund policy?", - "response": "Refunds are available within 30 days.", - "context": "Our company offers a 30-day refund policy for all purchases." if with_context else None, - "metrics": [{"name": "groundedness", "value": 4.0}], - }, - { - "row_index": 2, - "input": "How do I reset my password?", - "response": "Go to Settings > Security > Reset.", - "metrics": [{"name": "groundedness", "value": 2.0}], - }, - ] - item_evaluations = [ - { - "row_index": 1, - "passed_all": True, - "thresholds": [ - {"row_index": 1, "evaluator": "groundedness", "criteria": ">=", "expected": "3.0", "actual": "4.0", "passed": True}, - ], - }, - { - "row_index": 2, - "passed_all": False, - "thresholds": [ - {"row_index": 2, "evaluator": "groundedness", "criteria": ">=", "expected": "3.0", "actual": "2.0", "passed": False}, - ], - }, - ] - return RunResult.model_validate( - { - "version": 1, - "status": "completed", - "bundle": { - "name": "rag_baseline", - "path": ".agentops/bundles/rag_baseline.yaml", - }, - "dataset": {"name": "smoke", "path": ".agentops/datasets/smoke-agent.yaml"}, - "execution": { - "backend": "subprocess", - "command": "python -m fake_eval_runner", - "started_at": "2026-02-23T10:00:00Z", - "finished_at": "2026-02-23T10:00:05Z", - "duration_seconds": 5.0, - "exit_code": 0, - }, - "metrics": [ - {"name": "groundedness", "value": 0.84}, - {"name": "relevance", "value": 0.83}, - ], - "row_metrics": row_metrics, - "item_evaluations": item_evaluations, - "run_metrics": [ - {"name": "run_pass", "value": 0.0 if not overall_passed else 1.0}, - { - "name": "threshold_pass_rate", - "value": 0.5 if not overall_passed else 1.0, - }, - {"name": "accuracy", "value": 0.84}, - ], - "thresholds": [ - { - "evaluator": "groundedness", - "criteria": ">=", - "expected": "0.800000", - "actual": "0.840000", - "passed": True, - }, - { - "evaluator": "relevance", - "criteria": ">=", - "expected": "0.950000", - "actual": "0.830000", - "passed": False, - }, - ], - "summary": { - "metrics_count": 2, - "thresholds_count": 2, - "thresholds_passed": 2 if overall_passed else 1, - "thresholds_failed": 0 if overall_passed else 1, - "overall_passed": overall_passed, - }, - "artifacts": { - "backend_stdout": "backend.stdout.log", - "backend_stderr": "backend.stderr.log", - }, - } - ) - - -def test_report_markdown_contains_required_sections_and_tables() -> None: - markdown = generate_report_markdown(_sample_result(overall_passed=False)) - - assert "# AgentOps Evaluation Report" in markdown - assert "## Overview" in markdown - assert "- Bundle: rag_baseline" in markdown - assert "- Dataset: smoke" in markdown - assert "❌ FAIL" in markdown - - assert "## How Pass/Fail Is Determined" in markdown - - assert "## Execution Summary" in markdown - assert "| Field | Value |" in markdown - assert "| Backend | subprocess |" in markdown - assert "| Duration (s) | 5.000 |" in markdown - - assert "## Metrics" in markdown - assert "| Metric | Value | What It Measures |" in markdown - assert "| Groundedness | 0.84 |" in markdown - assert "Are claims supported by the retrieved context?" in markdown - - assert "## Run Metrics" in markdown - assert "| Run Pass | 0 |" in markdown - - assert "## Threshold Checks" in markdown - assert "| Evaluator | Threshold | Actual | Status |" in markdown - assert "| Relevance | >= 0.95 | 0.83 | ❌ Missed |" in markdown - assert "| Groundedness | >= 0.80 | 0.84 | ✅ Met |" in markdown - - -def test_report_markdown_pass_status() -> None: - markdown = generate_report_markdown(_sample_result(overall_passed=True)) - assert "✅ PASS" in markdown - - -def test_report_markdown_row_details_with_input_response() -> None: - markdown = generate_report_markdown(_sample_result(overall_passed=False, with_row_details=True)) - assert "## Row Details" in markdown - assert "### Row 1" in markdown - assert "**Input:** What is the refund policy?" in markdown - assert "**Response:** Refunds are available within 30 days." in markdown - assert "### Row 2" in markdown - assert "**Input:** How do I reset my password?" in markdown - assert "**Response:** Go to Settings > Security > Reset." in markdown - # Per-row score tables - assert "| Evaluator | Score | Threshold | Status |" in markdown - assert "| Groundedness | 4 | >= 3 | ✅ Met |" in markdown - assert "| Groundedness | 2 | >= 3 | ❌ Missed |" in markdown - # Row status icons - assert "✅ Pass" in markdown - assert "❌ Fail" in markdown - - -def test_report_markdown_context_display() -> None: - markdown = generate_report_markdown(_sample_result(overall_passed=False, with_row_details=True, with_context=True)) - assert "**Retrieved Context:**" in markdown - assert "30-day refund policy" in markdown - - -def test_report_markdown_context_not_shown_when_absent() -> None: - markdown = generate_report_markdown(_sample_result(overall_passed=False, with_row_details=True, with_context=False)) - assert "**Retrieved Context:**" not in markdown - - -def test_format_metric_name() -> None: - assert _format_metric_name("groundedness") == "Groundedness" - assert _format_metric_name("avg_latency_seconds") == "Avg. Latency Seconds" - assert _format_metric_name("SimilarityEvaluator") == "Similarity" - assert _format_metric_name("GroundednessEvaluator_avg") == "Groundedness Avg." - assert _format_metric_name("f1_score") == "F1 Score" - assert _format_metric_name("run_pass") == "Run Pass" - - -def test_get_evaluator_description() -> None: - assert _get_evaluator_description("groundedness") != "" - assert _get_evaluator_description("relevance") != "" - assert _get_evaluator_description("unknown_metric_xyz") == "" - - -def test_fmt_threshold_value() -> None: - assert _fmt_threshold_value("0.800000") == "0.80" - assert _fmt_threshold_value("3.0") == "3" - assert _fmt_threshold_value("0.950000") == "0.95" - assert _fmt_threshold_value("invalid") == "invalid" - - -def test_report_markdown_row_details_without_data() -> None: - markdown = generate_report_markdown(_sample_result(overall_passed=True)) - assert "## Row Details" in markdown - assert "No input/response data captured" in markdown - - -def test_report_html_row_details_with_input_response() -> None: - html = generate_report_html(_sample_result(overall_passed=False, with_row_details=True)) - assert "

Row Details

" in html - assert "What is the refund policy?" in html - assert "Refunds are available within 30 days." in html - assert "How do I reset my password?" in html - assert "Go to Settings > Security > Reset." in html - # Per-row score tables in HTML - assert "Groundedness" in html - assert "How Pass/Fail Is Determined" in html - assert "What It Measures" in html diff --git a/tests/unit/test_runtime_conversation.py b/tests/unit/test_runtime_conversation.py new file mode 100644 index 00000000..605d2b54 --- /dev/null +++ b/tests/unit/test_runtime_conversation.py @@ -0,0 +1,132 @@ +"""Unit tests for the conversation-building helper used by agent evaluators. + +When the dataset row carries ``tool_calls``, AgentOps upgrades the plain +``query`` + ``response`` strings into a conversational message list so +evaluators like ``IntentResolutionEvaluator`` and ``TaskAdherenceEvaluator`` +can see the agent's tool_call + tool_result trace and grade it accurately. +""" + +from __future__ import annotations + +from agentops.pipeline.runtime import _build_conversation_messages + + +def test_builds_text_only_conversation_when_no_tool_calls() -> None: + """Even without tool calls, we build a structured conversation so the + Azure evaluators don't try to parse plain query strings as history and + emit ``WARNING: Conversation history could not be parsed``. + """ + out = _build_conversation_messages( + input_text="Hi", response_text="Hello.", tool_calls=None, + ) + assert out is not None + assert out["query"] == [ + {"role": "user", "content": [{"type": "text", "text": "Hi"}]} + ] + assert out["response"] == [ + {"role": "assistant", "content": [{"type": "text", "text": "Hello."}]} + ] + + out_empty = _build_conversation_messages( + input_text="Hi", response_text="Hello.", tool_calls=[], + ) + assert out_empty == out + + +def test_returns_none_when_no_response_and_no_tool_calls() -> None: + """Nothing to evaluate — caller should fall back to plain kwargs.""" + assert _build_conversation_messages( + input_text="Hi", response_text="", tool_calls=None, + ) is None + + +def test_builds_user_assistant_messages_for_simple_call() -> None: + out = _build_conversation_messages( + input_text="Weather in Paris?", + response_text="It's sunny in Paris.", + tool_calls=[{ + "type": "function_call", + "name": "get_weather", + "arguments": {"location": "Paris"}, + }], + ) + assert out is not None + assert out["query"] == [ + {"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]} + ] + # Two response messages: tool_call + final assistant text. + assert len(out["response"]) == 2 + tool_call_msg = out["response"][0] + assert tool_call_msg["role"] == "assistant" + assert tool_call_msg["content"][0]["type"] == "tool_call" + assert tool_call_msg["content"][0]["name"] == "get_weather" + assert tool_call_msg["content"][0]["arguments"] == {"location": "Paris"} + final = out["response"][-1] + assert final == { + "role": "assistant", + "content": [{"type": "text", "text": "It's sunny in Paris."}], + } + + +def test_includes_tool_result_when_provided() -> None: + out = _build_conversation_messages( + input_text="Weather?", + response_text="It's 20C.", + tool_calls=[{ + "name": "get_weather", + "arguments": {"location": "Tokyo"}, + "result": "20C and clear", + }], + ) + assert out is not None + # tool_call -> tool_result -> assistant final + assert len(out["response"]) == 3 + assert out["response"][1]["role"] == "tool" + assert out["response"][1]["content"][0]["tool_result"] == "20C and clear" + + +def test_parses_json_string_arguments() -> None: + out = _build_conversation_messages( + input_text="?", + response_text="ok", + tool_calls=[{ + "name": "f", + "arguments": '{"x": 1}', + }], + ) + assert out is not None + args = out["response"][0]["content"][0]["arguments"] + assert args == {"x": 1} + + +def test_normalises_nested_function_envelope() -> None: + # Foundry tool calls sometimes nest name/arguments under a ``function`` key. + out = _build_conversation_messages( + input_text="?", + response_text="ok", + tool_calls=[{ + "id": "call_123", + "type": "function", + "function": {"name": "lookup", "arguments": {"q": "x"}}, + }], + ) + assert out is not None + call = out["response"][0]["content"][0] + assert call["name"] == "lookup" + assert call["arguments"] == {"q": "x"} + assert call["tool_call_id"] == "call_123" + + +def test_skips_calls_without_a_name() -> None: + out = _build_conversation_messages( + input_text="?", + response_text="ok", + tool_calls=[ + {"arguments": {"x": 1}}, # no name -> skipped + {"name": "f"}, + ], + ) + assert out is not None + # Only the named call survives, plus the final assistant text. + assert len(out["response"]) == 2 + assert out["response"][0]["content"][0]["name"] == "f" diff --git a/tests/unit/test_skills.py b/tests/unit/test_skills.py index 5b9a4458..1b2fae8e 100644 --- a/tests/unit/test_skills.py +++ b/tests/unit/test_skills.py @@ -28,9 +28,6 @@ ".github/skills/agentops-config/SKILL.md", ".github/skills/agentops-dataset/SKILL.md", ".github/skills/agentops-report/SKILL.md", - ".github/skills/agentops-regression/SKILL.md", - ".github/skills/agentops-trace/SKILL.md", - ".github/skills/agentops-monitor/SKILL.md", ".github/skills/agentops-workflow/SKILL.md", ] @@ -39,9 +36,6 @@ ".claude/commands/agentops-config.md", ".claude/commands/agentops-dataset.md", ".claude/commands/agentops-report.md", - ".claude/commands/agentops-regression.md", - ".claude/commands/agentops-trace.md", - ".claude/commands/agentops-monitor.md", ".claude/commands/agentops-workflow.md", ] @@ -93,7 +87,7 @@ def test_install_creates_copilot_files(tmp_path: Path) -> None: result = install_skills(directory=tmp_path, platforms=["copilot"]) assert result.platforms == ["copilot"] - assert len(result.created_files) == 8 + assert len(result.created_files) == 5 assert len(result.skipped_files) == 0 for rel in _COPILOT_SKILL_PATHS: @@ -120,7 +114,7 @@ def test_install_creates_claude_files(tmp_path: Path) -> None: result = install_skills(directory=tmp_path, platforms=["claude"]) assert result.platforms == ["claude"] - assert len(result.created_files) == 8 + assert len(result.created_files) == 5 for rel in _CLAUDE_SKILL_PATHS: skill_file = tmp_path / rel @@ -143,7 +137,7 @@ def test_claude_files_strip_frontmatter(tmp_path: Path) -> None: def test_install_multi_platform(tmp_path: Path) -> None: result = install_skills(directory=tmp_path, platforms=["copilot", "claude"]) - assert len(result.created_files) == 16 # 8 per platform + assert len(result.created_files) == 10 # 5 per platform assert result.platforms == ["copilot", "claude"] @@ -160,7 +154,7 @@ def test_install_skips_existing(tmp_path: Path) -> None: result = install_skills(directory=tmp_path, platforms=["copilot"], force=False) - assert len(result.skipped_files) == 8 + assert len(result.skipped_files) == 5 assert len(result.created_files) == 0 assert skill.read_text(encoding="utf-8") == "custom content" @@ -173,7 +167,7 @@ def test_install_overwrites_with_force(tmp_path: Path) -> None: result = install_skills(directory=tmp_path, platforms=["copilot"], force=True) - assert len(result.overwritten_files) == 8 + assert len(result.overwritten_files) == 5 content = skill.read_text(encoding="utf-8") assert content != "custom content" assert "AgentOps" in content @@ -242,7 +236,7 @@ def test_cli_init_does_not_install_skills(tmp_path: Path) -> None: result = runner.invoke(app, ["init", "--dir", str(tmp_path)]) assert result.exit_code == 0 - assert "Initialized workspace" in result.stdout + assert "Initialized AgentOps workspace" in result.stdout assert "agentops skills install" in result.stdout # Skills should NOT be created during init @@ -757,3 +751,4 @@ def test_cli_skills_install_from_invalid_ref(tmp_path: Path) -> None: ], ) assert result.exit_code == 1 + diff --git a/tests/unit/test_yaml_loader.py b/tests/unit/test_yaml_loader.py deleted file mode 100644 index e435b82d..00000000 --- a/tests/unit/test_yaml_loader.py +++ /dev/null @@ -1,193 +0,0 @@ -from pathlib import Path - -import pytest - -from agentops.core.config_loader import ( - load_bundle_config, - load_dataset_config, - load_run_config, - load_workspace_config, -) -from agentops.utils.yaml import load_yaml, save_yaml - - -def test_load_yaml_roundtrip(tmp_path: Path) -> None: - path = tmp_path / "config.yaml" - data = {"version": 1, "name": "example"} - - save_yaml(path, data) - loaded = load_yaml(path) - - assert loaded == data - - -def test_load_yaml_missing_file(tmp_path: Path) -> None: - path = tmp_path / "missing.yaml" - with pytest.raises(FileNotFoundError): - load_yaml(path) - - -def test_load_yaml_invalid(tmp_path: Path) -> None: - path = tmp_path / "bad.yaml" - path.write_text("version: [", encoding="utf-8") - - with pytest.raises(ValueError, match="Invalid YAML"): - load_yaml(path) - - -def test_load_workspace_config(tmp_path: Path) -> None: - path = tmp_path / "config.yaml" - path.write_text( - """ -version: 1 -paths: - bundles_dir: ".agentops/bundles" - datasets_dir: ".agentops/datasets" - data_dir: ".agentops/data" - results_dir: ".agentops/results" - -defaults: - backend: "subprocess" - timeout_seconds: 1800 - -report: - generate_markdown: true -""".lstrip(), - encoding="utf-8", - ) - - cfg = load_workspace_config(path) - assert cfg.paths.bundles_dir.as_posix() == ".agentops/bundles" - assert cfg.paths.datasets_dir.as_posix() == ".agentops/datasets" - assert cfg.paths.data_dir.as_posix() == ".agentops/data" - - -def test_load_bundle_config_validation_error(tmp_path: Path) -> None: - path = tmp_path / "bundle.yaml" - path.write_text( - """ -version: 1 -description: "missing name" -""".lstrip(), - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="BundleConfig validation error"): - load_bundle_config(path) - - -def test_load_dataset_config(tmp_path: Path) -> None: - path = tmp_path / "dataset.yaml" - path.write_text( - """ -version: 1 -name: "smoke" -description: "small dataset" -source: - type: "file" - path: "./eval/datasets/smoke.jsonl" -format: - type: "jsonl" - input_field: "input" - expected_field: "expected" -""".lstrip(), - encoding="utf-8", - ) - - cfg = load_dataset_config(path) - assert cfg.name == "smoke" - - -def test_load_run_config_rejects_legacy_format(tmp_path: Path) -> None: - path = tmp_path / "run.yaml" - path.write_text( - """ -version: 1 -bundle: - path: ".agentops/bundles/rag_baseline.yaml" -dataset: - path: ".agentops/datasets/smoke-agent.yaml" -backend: - type: "subprocess" - args: ["-m", "runner"] -output: - write_report: true -""".lstrip(), - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="'backend' key is not supported"): - load_run_config(path) - - -def test_load_run_config_rejects_backend_key(tmp_path: Path) -> None: - path = tmp_path / "run.yaml" - path.write_text( - """ -version: 1 -bundle: - path: ".agentops/bundles/rag_baseline.yaml" -dataset: - path: ".agentops/datasets/smoke-agent.yaml" -backend: - type: "http" - url: "http://localhost/chat" -output: - write_report: true -""".lstrip(), - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="'backend' key is not supported"): - load_run_config(path) - - -def test_load_run_config_backend_error_suggests_target_hosting(tmp_path: Path) -> None: - """Verify the error message includes the migration hint about target.hosting.""" - path = tmp_path / "run.yaml" - path.write_text( - """ -version: 1 -bundle: - path: ".agentops/bundles/rag_baseline.yaml" -dataset: - path: ".agentops/datasets/smoke-agent.yaml" -backend: foundry -output: - write_report: true -""".lstrip(), - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="target.hosting"): - load_run_config(path) - - -def test_load_run_config_parses(tmp_path: Path) -> None: - path = tmp_path / "run.yaml" - path.write_text( - """ -version: 1 -target: - type: model - hosting: local - execution_mode: local - local: - adapter: "python my_adapter.py" -bundle: - name: model_quality_baseline -dataset: - name: smoke-model-direct -execution: - timeout_seconds: 30 -output: - write_report: true -""".lstrip(), - encoding="utf-8", - ) - - cfg = load_run_config(path) - assert cfg.version == 1 - assert cfg.target.type == "model" - assert cfg.target.execution_mode == "local" - assert cfg.bundle.name == "model_quality_baseline"