From 6c0cb169da8ed67b59bb6eaba7ed8aabb9da5644 Mon Sep 17 00:00:00 2001 From: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:31:42 -0700 Subject: [PATCH] chore: add CI workflow (pytest, ubuntu, py3.11/3.12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This module had no CI at all. Also fixes 3 pre-existing test failures and a stale amplifier-core dependency floor discovered during clean-room verification, plus 2 real-network tests that only surface in a genuinely clean environment (no ambient API keys): 1. tests/test_thinking_cost_and_unpriced_model.py:: test_usage_model_dump_proves_this_was_never_a_serialization_bug failed with 'Decimal is not JSON serializable'. Root cause: the pinned dev-dependency floor (amplifier-core>=1.0.10) resolved to amplifier-core 1.0.10 / pydantic-core 2.41.4, where Usage's field_serializer(when_used="always") for cost_usd does not fire on plain model_dump(). Bumped the floor to amplifier-core>=1.6.0 (matching provider-openai/anthropic), resolving amplifier-core 1.6.1 / pydantic-core 2.46.5, where the serializer fires correctly. 2. tests/test_pricing_and_vision.py::TestFallbackModelsVision (2 tests) asserted on a hardcoded 5-model fallback list that was intentionally removed 2026-04-22 (see list_models()'s own docstring) in favor of hard-failing without live credentials, matching anthropic/openai. These tests could never pass again. Replaced with a single test (TestNoCredentialsHardFails) asserting the current, correct behavior: list_models() raises LLMError without credentials. 3. The inherited amplifier-core behavioral test test_list_models_returns_list, and this repo's own test_image_vision_integration_with_real_api, both make real network calls and cannot pass in CI without genuine Gemini credentials. Marked @pytest.mark.live and deselected via `-m "not live"`. CI sets a placeholder GOOGLE_API_KEY so mount() succeeds for every other behavioral/structural test (mount() only checks presence, not validity). Verified in a genuinely clean environment (all ambient provider keys unset, only the CI placeholder key set): uv run pytest -q -m "not live" -> 204 passed, 2 deselected. 🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> --- .github/workflows/ci.yml | 44 ++++++++++++++++++++++++++++++++ pyproject.toml | 5 +++- tests/test_behavioral.py | 15 +++++++++++ tests/test_image_support.py | 7 +++++ tests/test_pricing_and_vision.py | 34 ++++++++++++------------ uv.lock | 33 +++++++----------------- 6 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ca1729e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + name: pytest (py${{ matrix.python-version }}) + runs-on: ubuntu-latest + # A hang must fail LOUDLY and fast, not sit burning runner time until the + # 6h default -- a job stuck at "in_progress" reads as "not done yet" + # rather than "broken", which is how a false green gets merged. + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # requires-python = ">=3.11" -- cover the floor and a current minor. + python-version: ["3.11", "3.12"] + env: + # Placeholder, non-functional key. mount() only checks for presence + # (not validity) before constructing the provider -- without *some* + # key, mount() returns None and every behavioral/structural test + # that depends on a mounted provider fails/errors. Tests that need + # a real, working key are marked `live` and deselected below. + GOOGLE_API_KEY: test-placeholder-not-a-real-key-ci-only + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: uv sync --all-extras --dev + + - name: Run test suite + # Deselects tests marked `live`: the inherited + # test_list_models_returns_list makes a real network call to + # Google's models.list endpoint. Run locally with a real + # GOOGLE_API_KEY/GEMINI_API_KEY to validate. + run: uv run pytest -q -m "not live" diff --git a/pyproject.toml b/pyproject.toml index 39afc47..3ade5ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,13 @@ allow-direct-references = true testpaths = ["tests"] addopts = "--import-mode=importlib" asyncio_mode = "strict" +markers = [ + "live: requires real network access / real provider credentials; deselected in CI via `-m \"not live\"`", +] [dependency-groups] dev = [ - "amplifier-core>=1.0.10", + "amplifier-core>=1.6.0", "google-api-core>=2.0.0", "pytest>=9.0.3", "pytest-asyncio>=1.2.0", diff --git a/tests/test_behavioral.py b/tests/test_behavioral.py index 89f712a..3370714 100644 --- a/tests/test_behavioral.py +++ b/tests/test_behavioral.py @@ -3,6 +3,8 @@ Inherits authoritative tests from amplifier-core. """ +import pytest + from amplifier_core.validation.behavioral import ProviderBehaviorTests @@ -12,3 +14,16 @@ class TestGeminiProviderBehavior(ProviderBehaviorTests): All tests from ProviderBehaviorTests run automatically. Add module-specific tests below if needed. """ + + @pytest.mark.live + @pytest.mark.asyncio + async def test_list_models_returns_list(self, provider_module): + """Override to mark this inherited test 'live'. + + ProviderBehaviorTests.test_list_models_returns_list calls + provider_module.list_models(), which makes a real call to + Google's models.list endpoint -- it cannot pass in CI without a + genuine GOOGLE_API_KEY/GEMINI_API_KEY. Deselected in CI via + `-m "not live"`; run locally with real credentials to validate. + """ + await super().test_list_models_returns_list(provider_module) diff --git a/tests/test_image_support.py b/tests/test_image_support.py index fc3ac1e..7047823 100644 --- a/tests/test_image_support.py +++ b/tests/test_image_support.py @@ -179,6 +179,7 @@ def test_text_only_message_still_works(gemini_provider): assert parts[0] == {"text": "Hello, how are you?"} +@pytest.mark.live @pytest.mark.asyncio async def test_image_vision_integration_with_real_api(test_image_base64): """Integration test: Verify ImageBlock works with real Gemini API. @@ -190,6 +191,12 @@ async def test_image_vision_integration_with_real_api(test_image_base64): Requires GOOGLE_API_KEY environment variable. Skip if not available (unit tests still validate conversion logic). + + Marked 'live' and deselected in CI via `-m "not live"`. CI sets a + placeholder GOOGLE_API_KEY so other behavioral/structural tests can + mount the provider, which defeats the plain `if not api_key: skip` + check below (a placeholder key is still truthy) -- the marker is the + real CI gate. Run this locally with a real key to validate. """ import os diff --git a/tests/test_pricing_and_vision.py b/tests/test_pricing_and_vision.py index 4a4bd6e..513abb0 100644 --- a/tests/test_pricing_and_vision.py +++ b/tests/test_pricing_and_vision.py @@ -2,31 +2,33 @@ import pytest +from amplifier_core.llm_errors import LLMError from amplifier_module_provider_gemini import GeminiProvider -class TestFallbackModelsVision: - """Verify hardcoded fallback models include vision capability.""" +class TestNoCredentialsHardFails: + """``list_models()`` must hard-fail without credentials, not fall back. + + A hardcoded 5-model fallback list used to be returned here when no + live API call could be made. It was intentionally removed (see the + docstring on ``GeminiProvider.list_models``) because a stale hardcoded + list drifts out of sync with real Google releases and silently masks + API outages -- matching the anthropic/openai providers, which also + hard-fail rather than returning a fallback. + + This replaces the old ``TestFallbackModelsVision`` tests, which + asserted on the removed fallback list's contents (vision/fast + capabilities) and could never pass again now that the list is gone. + """ @pytest.fixture def provider(self): return GeminiProvider(api_key="test-key") @pytest.mark.asyncio - async def test_fallback_all_models_have_vision(self, provider): - provider._client = None - provider._api_key = None - - models = await provider.list_models() - for model in models: - assert "vision" in model.capabilities, f"{model.id} missing vision" - - @pytest.mark.asyncio - async def test_fallback_flash_is_fast(self, provider): + async def test_list_models_without_credentials_raises(self, provider): provider._client = None provider._api_key = None - models = await provider.list_models() - flash_models = [m for m in models if "flash" in m.id] - for model in flash_models: - assert "fast" in model.capabilities, f"{model.id} should be fast" + with pytest.raises(LLMError, match="api_key must be provided"): + await provider.list_models() diff --git a/uv.lock b/uv.lock index ef6f125..c2e50ad 100644 --- a/uv.lock +++ b/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ [[package]] name = "amplifier-core" -version = "1.0.10" +version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -17,28 +17,13 @@ dependencies = [ { name = "tomli" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/3f/c253e35933376df39a1a26e5a5f7555e55d6edb41ec25def76e53e003b0a/amplifier_core-1.0.10.tar.gz", hash = "sha256:3938d446fd8259ea45bd0dff0713d908662731d39a93321e0008c723b4dc2674", size = 235967, upload-time = "2026-03-04T13:37:23.145Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/b6/d470f1b7fc38014c55ef47de8a1e136981c68b9226ba4db7b53b23aedd35/amplifier_core-1.0.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a16b975faca7e28731b56ddc2f64872edd89dab5cf5f18e1cbc28b91c18a65f", size = 765543, upload-time = "2026-03-04T13:36:48.888Z" }, - { url = "https://files.pythonhosted.org/packages/0c/83/f0070db5742693bb2dbaf7b32a9bb00f2381dd39ba2acf10dd61856f731a/amplifier_core-1.0.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfe1fb8a0326e73ccc3a05d534554765b90b0f14fd824427d6b623cedc1f0e11", size = 790667, upload-time = "2026-03-04T13:36:50.553Z" }, - { url = "https://files.pythonhosted.org/packages/b3/61/8c1641ced7d95f0100dd295edfd806177dc22c9c6d41e9f4e84b67df31ed/amplifier_core-1.0.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8ab2ecad8d4ce79b0ef86c81a55201a73a093a55030c748297a9c7539589ab", size = 824412, upload-time = "2026-03-04T13:36:52.358Z" }, - { url = "https://files.pythonhosted.org/packages/05/67/ffc53717d60b9365fb3e7fe4c42f7d1070f31825b8054f5b5bfabdb0eee5/amplifier_core-1.0.10-cp311-cp311-win_amd64.whl", hash = "sha256:b351aaa1bc33c5c5298edfc77bf453c632aa2562a50695a2fdbb7eef191ecf60", size = 832316, upload-time = "2026-03-04T13:36:54.5Z" }, - { url = "https://files.pythonhosted.org/packages/83/dd/6b27bc183ba1e6bf1f226a428f8675e16ad8b11e5a332eae25f5ef482bfa/amplifier_core-1.0.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66e900f685257682b04546c28681fadc771c9e5f138bd8490704c7763f3f86d5", size = 764348, upload-time = "2026-03-04T13:36:56.052Z" }, - { url = "https://files.pythonhosted.org/packages/44/ef/071114e80126e0fd82d9f0511219254f87a0a93cd25f373c14dd6052a1e1/amplifier_core-1.0.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61a5282cbbbee4f59b0f0705e46fdef97f0a05c46859d03599ce990dea2986a2", size = 793215, upload-time = "2026-03-04T13:36:57.811Z" }, - { url = "https://files.pythonhosted.org/packages/a2/60/3cd123a63985782774087975ca6989e4e9e18a31ad71c87d2e25ca0ea816/amplifier_core-1.0.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91d05d76f19d03affbc7d8a140b38826d08c4aa06d9b218bbd6c95065447a7f4", size = 827881, upload-time = "2026-03-04T13:36:59.625Z" }, - { url = "https://files.pythonhosted.org/packages/d4/ff/26ee8571eae0b1cfaadbe8860cb0dde0bccdc4fed20b45f7bdae8f7465a7/amplifier_core-1.0.10-cp312-cp312-win_amd64.whl", hash = "sha256:ce29dc188dab970074a4389d9f74788e02eed7d315f4259fd35f1d8f61b40502", size = 829279, upload-time = "2026-03-04T13:37:01.555Z" }, - { url = "https://files.pythonhosted.org/packages/4a/73/3788548adaa1f854de13a657653d4724e443d2b238f1851522e46ce27fa5/amplifier_core-1.0.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:648f84f1aa5fc7e7c75335c5378f1fb3f0e49e7d7d390133f6e7a9b478ad82e4", size = 763740, upload-time = "2026-03-04T13:37:03.217Z" }, - { url = "https://files.pythonhosted.org/packages/60/f5/c7709e5a680413e8498c051133aaf46f44f78b95f61cf70b79fd22e5eb22/amplifier_core-1.0.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fcb70e36d8d908799761b28a895467a8d1d49b9eb44a20e600bdf16891aba57", size = 792463, upload-time = "2026-03-04T13:37:04.759Z" }, - { url = "https://files.pythonhosted.org/packages/c8/fa/21ada93dabdfb255695c5d86d85ecad959e8b86781fad4f2e94c793f8b84/amplifier_core-1.0.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29878baf51fbdc6c37121a1ba8c2ae20516cadeea600068151b26e55bb84151d", size = 827325, upload-time = "2026-03-04T13:37:06.176Z" }, - { url = "https://files.pythonhosted.org/packages/ed/19/742482e9b93c5dc9329b4fe5211b1b2cb0b33359a9f2e53a5b2676d95f56/amplifier_core-1.0.10-cp313-cp313-win_amd64.whl", hash = "sha256:3487a63b35675e408ad63157fb5c0e76bb19533fc5c34402be07690129c1d092", size = 828745, upload-time = "2026-03-04T13:37:08.161Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0f/46c492805e9f4b9fd9dc8ad49656f140be217a13b0d3455c37f5be8f7f39/amplifier_core-1.0.10-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b30a5cc8fef2778d40781ac5c053b11393bda2a1e8d1b0f160a3049cbe5f999d", size = 786745, upload-time = "2026-03-04T13:37:10.009Z" }, - { url = "https://files.pythonhosted.org/packages/fa/fb/16204db9596c440d993031dffb31b9b09e5402a227bf5862cd8efab5ee4b/amplifier_core-1.0.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5b58c63fef64312831bc10f5ab61321d183810b46a55dbbcf656109d6af2a743", size = 763143, upload-time = "2026-03-04T13:37:11.457Z" }, - { url = "https://files.pythonhosted.org/packages/4d/f2/43d2c60f6e6b432c659d417a0ea484fa9770cfaa57bf7a96c6f9b4ae2ffc/amplifier_core-1.0.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c18f2478d29a96e9e7b74be4919a6a2cd51c39750eddc56fb25354259d177fbb", size = 791646, upload-time = "2026-03-04T13:37:12.903Z" }, - { url = "https://files.pythonhosted.org/packages/72/88/4991af0f71406d00032783f66d8be48e3088dfb6e99eccbda3511b2e261d/amplifier_core-1.0.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:038e1f33d1b49a5e10001f8994aba8e33e923207b710613c255b631450df613b", size = 826816, upload-time = "2026-03-04T13:37:14.626Z" }, - { url = "https://files.pythonhosted.org/packages/f3/51/afdd1c93bdfc9c590778a3395aedb0b7602753d53669d51439edeae1457c/amplifier_core-1.0.10-cp314-cp314-win_amd64.whl", hash = "sha256:916f95b2e0d9be64f8ba2f7f0d32b5bf9bf04bd5583f14a526642461707ed04c", size = 828660, upload-time = "2026-03-04T13:37:16.994Z" }, - { url = "https://files.pythonhosted.org/packages/80/73/7b9095f1f331ec366b95d848bbf906198118ace6def43bb0bd718d005be5/amplifier_core-1.0.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ee5e85432fd9b30f3de456448e4e7668e5c97a5d310ed9581d38142847348c5", size = 786502, upload-time = "2026-03-04T13:37:18.721Z" }, - { url = "https://files.pythonhosted.org/packages/7d/61/a48ee2acfb32f5dc3dbc32390847fb5d833dd6881f52f3435e8eab2c489d/amplifier_core-1.0.10-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b16c88ecc83a83b9f39d4d502aee59bfb0d84f2887522d904b4cf9498a1f7799", size = 791159, upload-time = "2026-03-04T13:37:20.251Z" }, - { url = "https://files.pythonhosted.org/packages/35/e9/216ba241ce3add419091e70a0510f7a00b357d394c574ed681464e260a24/amplifier_core-1.0.10-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcc16ce1d8c7b18b8732df464c61f8504e5ee3d34cff2dc378f28e94cc282c03", size = 825456, upload-time = "2026-03-04T13:37:21.866Z" }, +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/cd/8b0b520bf0de741ea73e069aaf64aca28c9f4ce91a7b8b9239193a6c4c1b/amplifier_core-1.6.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c0f711d8408de78e53e5deddcb38b7240c5c1c497ca51eeaaeff23559b3d3c48", size = 8281633, upload-time = "2026-08-10T02:38:11.98Z" }, + { url = "https://files.pythonhosted.org/packages/14/83/f4fb297d87d35b9d74058da02bb153e12f7891ab62b3aaf7e0857f877798/amplifier_core-1.6.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:b08f37e2c0b1611349a0e25d5bf9bfdfae3afcee35488f8e26bba1cdd400503b", size = 7366930, upload-time = "2026-08-10T02:38:14.105Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ba/5eb9cecf92d8053c5e6d46ad9668c3ed3558d5423845c1dced1f266b2a38/amplifier_core-1.6.1-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ebf7e3993c76ea506e70ac7844b286c3ba2e9127b3bcb350fa4fcd2dcdbd38d", size = 7659512, upload-time = "2026-08-10T02:38:16.314Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/121f054e3d079dc33d83f3d8ba9af50fd9f7694c3e2ba3d7d23d7c157d48/amplifier_core-1.6.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c957cd0671d2a003f2c8f7d6a41bd6e808f97d183c57b97e7700bf4c912621d", size = 8678425, upload-time = "2026-08-10T02:38:18.243Z" }, + { url = "https://files.pythonhosted.org/packages/35/25/bfc217f4a9ed2d033995fc59847f1fee2e1b17130632fcb0e0981a1a311b/amplifier_core-1.6.1-cp311-abi3-win_amd64.whl", hash = "sha256:50c80bcfa1f6efe769b19e7af18c925024c7553d4db08880727241709dd44eae", size = 8976601, upload-time = "2026-08-10T02:38:20.505Z" }, + { url = "https://files.pythonhosted.org/packages/a5/14/5f330452c92c6c5d35c51ad5311301949ce5db4d1a1a901456f3ee43eaac/amplifier_core-1.6.1-cp311-abi3-win_arm64.whl", hash = "sha256:cd8b617f132cf5d1ca3e5187d5f831d1f2a508bb40d07b2ab1085961bcb9e1a9", size = 7744837, upload-time = "2026-08-10T02:38:22.562Z" }, ] [[package]] @@ -63,7 +48,7 @@ requires-dist = [{ name = "google-genai", specifier = ">=1.40.0" }] [package.metadata.requires-dev] dev = [ - { name = "amplifier-core", specifier = ">=1.0.10" }, + { name = "amplifier-core", specifier = ">=1.6.0" }, { name = "google-api-core", specifier = ">=2.0.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.2.0" },