From a844d9d7ac7f2c3e1c29e5577bd3b17a110d1f18 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 8 May 2026 00:13:19 -0300 Subject: [PATCH] fix: pass evaluator env vars in workflows Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/tutorial-end-to-end.md | 5 +++++ .../templates/workflows/agentops-deploy-dev.yml | 2 ++ .../templates/workflows/agentops-deploy-prod.yml | 2 ++ .../templates/workflows/agentops-deploy-qa.yml | 2 ++ src/agentops/templates/workflows/agentops-pr.yml | 2 ++ tests/unit/test_cicd.py | 15 +++++++++++++++ 6 files changed, 28 insertions(+) diff --git a/docs/tutorial-end-to-end.md b/docs/tutorial-end-to-end.md index f30246b2..ecdb4b9c 100644 --- a/docs/tutorial-end-to-end.md +++ b/docs/tutorial-end-to-end.md @@ -395,6 +395,11 @@ loop you will wire into CI next.** > *shape* of the delta: tool/task metrics down, text-quality metrics > flat. +Before wiring the workflow gate, switch `agentops.yaml` back to the +tool-using version (`support-bot:1` in the example above) if you want +the PR check to pass. Leave it on the degraded version only when you +intentionally want to demonstrate a red quality gate. + ## 7. Generate the GitFlow workflows ```powershell diff --git a/src/agentops/templates/workflows/agentops-deploy-dev.yml b/src/agentops/templates/workflows/agentops-deploy-dev.yml index 10e1a8dc..66249b5e 100644 --- a/src/agentops/templates/workflows/agentops-deploy-dev.yml +++ b/src/agentops/templates/workflows/agentops-deploy-dev.yml @@ -66,6 +66,8 @@ jobs: - name: Run AgentOps eval env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} run: agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" - name: Upload AgentOps results diff --git a/src/agentops/templates/workflows/agentops-deploy-prod.yml b/src/agentops/templates/workflows/agentops-deploy-prod.yml index 6373aac9..8de94689 100644 --- a/src/agentops/templates/workflows/agentops-deploy-prod.yml +++ b/src/agentops/templates/workflows/agentops-deploy-prod.yml @@ -73,6 +73,8 @@ jobs: - name: Run AgentOps eval (production gate) env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} run: agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" - name: Upload AgentOps results diff --git a/src/agentops/templates/workflows/agentops-deploy-qa.yml b/src/agentops/templates/workflows/agentops-deploy-qa.yml index ecec9c61..1e2721e3 100644 --- a/src/agentops/templates/workflows/agentops-deploy-qa.yml +++ b/src/agentops/templates/workflows/agentops-deploy-qa.yml @@ -66,6 +66,8 @@ jobs: - name: Run AgentOps eval env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} run: agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" - name: Upload AgentOps results diff --git a/src/agentops/templates/workflows/agentops-pr.yml b/src/agentops/templates/workflows/agentops-pr.yml index b8bfd595..11b7479f 100644 --- a/src/agentops/templates/workflows/agentops-pr.yml +++ b/src/agentops/templates/workflows/agentops-pr.yml @@ -70,6 +70,8 @@ jobs: id: eval env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} run: | set +e agentops eval run --config "${{ inputs.config || 'agentops.yaml' }}" diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index 2c88fa56..facb234f 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -120,6 +120,21 @@ def test_all_templates_are_valid_yaml(tmp_path: Path) -> None: assert "\non:" in text +def test_all_templates_pass_foundry_and_evaluator_environment(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path) + expected_env = ( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: " + "${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}", + "AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}", + "AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }}", + ) + for rel in ALL_PATHS: + content = (tmp_path / rel).read_text(encoding="utf-8") + + for env_line in expected_env: + assert env_line in content + + def test_pr_template_triggers_and_no_environment(tmp_path: Path) -> None: generate_cicd_workflows(directory=tmp_path, kinds=["pr"]) content = (tmp_path / _PR_PATH).read_text(encoding="utf-8")