python-sdk/.github/workflows/ci.yml at main · eval-protocol/python-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
name: Python CI

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    branches: [main]
    paths-ignore:
      - "docs/**"
      - "*.md"
  pull_request:
    paths-ignore:
      - "docs/**"
      - "*.md"
  workflow_dispatch:

jobs:
  lint-and-type-check:
    name: Lint & Type Check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all tags and branches

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true

      - name: Install the project
        run: uv sync --locked --all-extras --dev

      - name: Install tau2 for testing
        run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main

      - name: Ruff lint
        run: uv run ruff check .

      - name: Type check with pyright
        run: |
          # 'set +e' disables immediate exit on error so we can capture and report errors but exit 0
          # Note: We currently suppress pyright failures to allow CI to pass while we iteratively fix all type issues.
          # Once all type errors are resolved, we will remove this suppression and enforce strict type checking.
          set +e
          uv run basedpyright || true

  test-core:
    name: Core Tests (Python ${{ matrix.python-version }}, Shard ${{ matrix.shard }}/${{ matrix.total-shards }})
    runs-on: ubuntu-latest
    needs: lint-and-type-check
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
        shard: [1, 2, 3, 4]
        total-shards: [4]

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all tags and branches

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true

      - name: Install the project
        run: uv sync --locked --all-extras --dev

      - name: Install tau2 for testing
        run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main

      - name: Run Core Tests with pytest-xdist (Shard ${{ matrix.shard }}/${{ matrix.total-shards }})
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
          FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
          FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
          SUPABASE_PASSWORD: ${{ secrets.SUPABASE_PASSWORD }}
          SUPABASE_HOST: ${{ secrets.SUPABASE_HOST }}
          SUPABASE_PORT: ${{ secrets.SUPABASE_PORT }}
          SUPABASE_DATABASE: ${{ secrets.SUPABASE_DATABASE }}
          SUPABASE_USER: ${{ secrets.SUPABASE_USER }}
          PYTHONWARNINGS: "ignore::DeprecationWarning,ignore::RuntimeWarning"
        run: uv run ./scripts/run_sharded_tests.sh ${{ matrix.shard }} ${{ matrix.total-shards }}

  test-batch-evaluation:
    name: Batch Evaluation Tests
    runs-on: ubuntu-latest
    needs: lint-and-type-check
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all tags and branches

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true

      - name: Install the project
        run: uv sync --locked --all-extras --dev

      - name: Install tau2 for testing
        run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main

      - name: Run Batch Evaluation Tests
        env:
          E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
          FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
          FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
          PYTHONWARNINGS: "ignore::DeprecationWarning,ignore::RuntimeWarning"
        run: |
          # Run only this specific test file, WITHOUT xdist
          uv run pytest tests/test_batch_evaluation.py -v --durations=10

  test-mcp-e2e:
    name: MCP End-to-End Tests
    runs-on: ubuntu-latest
    needs: lint-and-type-check
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all tags and branches
      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true

      - name: Install the project
        run: uv sync --locked --all-extras --dev

      - name: Install tau2 for testing
        run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main