diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38b6757..01c0189 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,73 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml + # The `test` job installs .[dev], which pulls openeval-core[fixtura,langchain,openai]. + # Every optional dependency is therefore always present there, so it can never + # catch an adapter that has started requiring one. That is exactly how the eager + # `from .langchain import ...` in openeval/adapters/__init__.py shipped: the whole + # suite stayed green while `pip install openeval-core` could not import any adapter + # at all. This job reproduces what a real user gets. + bare-install: + name: Adapter imports must work with no extras + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install with no extras + run: | + python -m pip install --upgrade pip + pip install . + + - name: Confirm the optional deps really are absent + # Without this the job passes vacuously if an adapter dep ever becomes + # a base dependency -- which would "fix" a failure here the wrong way. + run: | + python -c " + import importlib.util, sys + for mod in ('langchain_core', 'openai', 'zstandard'): + if importlib.util.find_spec(mod) is not None: + sys.exit(mod + ' is installed; this job is only meaningful without the adapter extras') + print('confirmed: no optional adapter dependencies present') + " + + - name: Adapters that have no third-party dependency must import + run: | + python -c "import openeval.adapters" + python -c "import openeval.adapters.fixtura" + python -c "from openeval.adapters import from_fixtura_trace" + python -c "from openeval.adapters.fixtura import from_fixtura_trace" + python -c "from openeval.adapters import from_openai_messages" + + - name: The LangChain adapter must fail only on access, and cleanly + run: | + python -c " + import sys + try: + from openeval.adapters import from_langchain_run + except ModuleNotFoundError as exc: + print('expected on access:', exc) + else: + sys.exit('from_langchain_run imported without langchain-core') + " + + - name: Unknown attributes must still raise AttributeError + run: | + python -c " + import sys, openeval.adapters as a + try: + a.does_not_exist + except AttributeError as exc: + print('expected:', exc) + else: + sys.exit('lazy __getattr__ swallowed an unknown attribute') + " + all-tests-passed: - needs: test + needs: [test, bare-install] runs-on: ubuntu-latest if: always() steps: @@ -48,3 +113,7 @@ jobs: echo "One or more matrix jobs failed" exit 1 fi + if [ "${{ needs.bare-install.result }}" != "success" ]; then + echo "Adapter imports are broken on a bare install" + exit 1 + fi