From 283b37b1e8a8c93d62d910196922dfcfadb42b3e Mon Sep 17 00:00:00 2001
From: yash161004 <yash161004@github.com>
Date: Wed, 29 Jul 2026 09:50:37 +0530
Subject: [PATCH] ci: verify adapter imports on a bare install

The `test` job installs .[dev], which pulls
openeval-core[fixtura,langchain,openai], so every optional dependency is always
present and the job cannot detect an adapter that has started requiring one.

That is how the eager `from .langchain import ...` in
openeval/adapters/__init__.py shipped: the suite stayed green the whole time
while `pip install openeval-core` could not import any adapter at all. The fix
in #5 removed the bug but left nothing to stop it coming back.

Add a bare-install job that reproduces what a real user gets and asserts:

- adapters with no third-party dependency import (fixtura, openai -- the openai
  adapter parses dicts and never imports the openai package)
- the LangChain adapter fails only on access, and cleanly
- an unknown attribute still raises AttributeError, so the lazy __getattr__
  cannot silently swallow typos
- the optional deps really are absent, so the job cannot pass vacuously if one
  ever becomes a base dependency

Wired into all-tests-passed, which is the required check, so it actually gates.

Verified against a clean venv: all assertions pass on current main, and
reinstating the eager import makes the import assertion fail as intended.

Closes #6

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 71 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 38b6757..01c0189 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -38,8 +38,73 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml
 
+  # The `test` job installs .[dev], which pulls openeval-core[fixtura,langchain,openai].
+  # Every optional dependency is therefore always present there, so it can never
+  # catch an adapter that has started requiring one. That is exactly how the eager
+  # `from .langchain import ...` in openeval/adapters/__init__.py shipped: the whole
+  # suite stayed green while `pip install openeval-core` could not import any adapter
+  # at all. This job reproduces what a real user gets.
+  bare-install:
+    name: Adapter imports must work with no extras
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install with no extras
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Confirm the optional deps really are absent
+        # Without this the job passes vacuously if an adapter dep ever becomes
+        # a base dependency -- which would "fix" a failure here the wrong way.
+        run: |
+          python -c "
+          import importlib.util, sys
+          for mod in ('langchain_core', 'openai', 'zstandard'):
+              if importlib.util.find_spec(mod) is not None:
+                  sys.exit(mod + ' is installed; this job is only meaningful without the adapter extras')
+          print('confirmed: no optional adapter dependencies present')
+          "
+
+      - name: Adapters that have no third-party dependency must import
+        run: |
+          python -c "import openeval.adapters"
+          python -c "import openeval.adapters.fixtura"
+          python -c "from openeval.adapters import from_fixtura_trace"
+          python -c "from openeval.adapters.fixtura import from_fixtura_trace"
+          python -c "from openeval.adapters import from_openai_messages"
+
+      - name: The LangChain adapter must fail only on access, and cleanly
+        run: |
+          python -c "
+          import sys
+          try:
+              from openeval.adapters import from_langchain_run
+          except ModuleNotFoundError as exc:
+              print('expected on access:', exc)
+          else:
+              sys.exit('from_langchain_run imported without langchain-core')
+          "
+
+      - name: Unknown attributes must still raise AttributeError
+        run: |
+          python -c "
+          import sys, openeval.adapters as a
+          try:
+              a.does_not_exist
+          except AttributeError as exc:
+              print('expected:', exc)
+          else:
+              sys.exit('lazy __getattr__ swallowed an unknown attribute')
+          "
+
   all-tests-passed:
-    needs: test
+    needs: [test, bare-install]
     runs-on: ubuntu-latest
     if: always()
     steps:
@@ -48,3 +113,7 @@ jobs:
             echo "One or more matrix jobs failed"
             exit 1
           fi
+          if [ "${{ needs.bare-install.result }}" != "success" ]; then
+            echo "Adapter imports are broken on a bare install"
+            exit 1
+          fi