Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 70 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,73 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./coverage.xml

# The `test` job installs .[dev], which pulls openeval-core[fixtura,langchain,openai].
# Every optional dependency is therefore always present there, so it can never
# catch an adapter that has started requiring one. That is exactly how the eager
# `from .langchain import ...` in openeval/adapters/__init__.py shipped: the whole
# suite stayed green while `pip install openeval-core` could not import any adapter
# at all. This job reproduces what a real user gets.
bare-install:
name: Adapter imports must work with no extras
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Install with no extras
run: |
python -m pip install --upgrade pip
pip install .

- name: Confirm the optional deps really are absent
# Without this the job passes vacuously if an adapter dep ever becomes
# a base dependency -- which would "fix" a failure here the wrong way.
run: |
python -c "
import importlib.util, sys
for mod in ('langchain_core', 'openai', 'zstandard'):
if importlib.util.find_spec(mod) is not None:
sys.exit(mod + ' is installed; this job is only meaningful without the adapter extras')
print('confirmed: no optional adapter dependencies present')
"

- name: Adapters that have no third-party dependency must import
run: |
python -c "import openeval.adapters"
python -c "import openeval.adapters.fixtura"
python -c "from openeval.adapters import from_fixtura_trace"
python -c "from openeval.adapters.fixtura import from_fixtura_trace"
python -c "from openeval.adapters import from_openai_messages"

- name: The LangChain adapter must fail only on access, and cleanly
run: |
python -c "
import sys
try:
from openeval.adapters import from_langchain_run
except ModuleNotFoundError as exc:
print('expected on access:', exc)
else:
sys.exit('from_langchain_run imported without langchain-core')
"

- name: Unknown attributes must still raise AttributeError
run: |
python -c "
import sys, openeval.adapters as a
try:
a.does_not_exist
except AttributeError as exc:
print('expected:', exc)
else:
sys.exit('lazy __getattr__ swallowed an unknown attribute')
"

all-tests-passed:
needs: test
needs: [test, bare-install]
runs-on: ubuntu-latest
if: always()
steps:
Expand All @@ -48,3 +113,7 @@ jobs:
echo "One or more matrix jobs failed"
exit 1
fi
if [ "${{ needs.bare-install.result }}" != "success" ]; then
echo "Adapter imports are broken on a bare install"
exit 1
fi
Loading