diff --git a/.github/workflows/codecov_aggregator.yml b/.github/workflows/codecov_aggregator.yml index e704e3a0..bacdf548 100644 --- a/.github/workflows/codecov_aggregator.yml +++ b/.github/workflows/codecov_aggregator.yml @@ -32,7 +32,7 @@ jobs: secrets: ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }} - HEADER_EMBEDDING_API_KEY_OPENAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_OPENAI }} + HEADER_EMBEDDING_API_KEY_VOYAGEAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_VOYAGEAI }} local_it: uses: ./.github/workflows/local.yml @@ -46,7 +46,7 @@ jobs: AWS_ECR_ROLE_NAME: ${{ secrets.AWS_ECR_ROLE_NAME }} AWS_ECR_REPOSITORY: ${{ secrets.AWS_ECR_REPOSITORY }} AWS_ECR_REGISTRY: ${{ secrets.AWS_ECR_REGISTRY }} - HEADER_EMBEDDING_API_KEY_OPENAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_OPENAI }} + HEADER_EMBEDDING_API_KEY_VOYAGEAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_VOYAGEAI }} # hardcoding the target DB DOCKER_COMPOSE_LOCAL_DATA_API: "yes" # turn on header-based reranker auth diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml index 73f334a4..7deac31f 100644 --- a/.github/workflows/local.yml +++ b/.github/workflows/local.yml @@ -22,7 +22,7 @@ on: required: true AWS_ECR_REGISTRY: required: true - HEADER_EMBEDDING_API_KEY_OPENAI: + HEADER_EMBEDDING_API_KEY_VOYAGEAI: required: true DOCKER_COMPOSE_LOCAL_DATA_API: required: true @@ -41,7 +41,7 @@ jobs: AWS_ECR_ROLE_NAME: ${{ secrets.AWS_ECR_ROLE_NAME }} AWS_ECR_REPOSITORY: ${{ secrets.AWS_ECR_REPOSITORY }} AWS_ECR_REGISTRY: ${{ secrets.AWS_ECR_REGISTRY }} - HEADER_EMBEDDING_API_KEY_OPENAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_OPENAI }} + HEADER_EMBEDDING_API_KEY_VOYAGEAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_VOYAGEAI }} DOCKER_COMPOSE_LOCAL_DATA_API: ${{ secrets.DOCKER_COMPOSE_LOCAL_DATA_API }} HEADER_RERANKING_API_KEY_NVIDIA: ${{ secrets.HEADER_RERANKING_API_KEY_NVIDIA }} runs-on: ubuntu-latest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6960b409..e2e3c061 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ on: required: true ASTRA_DB_API_ENDPOINT: required: true - HEADER_EMBEDDING_API_KEY_OPENAI: + HEADER_EMBEDDING_API_KEY_VOYAGEAI: required: true jobs: @@ -24,7 +24,7 @@ jobs: env: ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }} - HEADER_EMBEDDING_API_KEY_OPENAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_OPENAI }} + HEADER_EMBEDDING_API_KEY_VOYAGEAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_VOYAGEAI }} LEGACY_INSERTMANY_BEHAVIOUR_PRE2193: ${{ secrets.LEGACY_INSERTMANY_BEHAVIOUR_PRE2193 }} runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index daea2767..5ebe5db0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,7 +124,7 @@ jobs: env: ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }} ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }} - HEADER_EMBEDDING_API_KEY_OPENAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_OPENAI }} + HEADER_EMBEDDING_API_KEY_VOYAGEAI: ${{ secrets.HEADER_EMBEDDING_API_KEY_VOYAGEAI }} LEGACY_INSERTMANY_BEHAVIOUR_PRE2193: ${{ secrets.LEGACY_INSERTMANY_BEHAVIOUR_PRE2193 }} run: make test-integration diff --git a/DEVELOPING.md b/DEVELOPING.md index 7b605e98..527cc164 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -69,10 +69,13 @@ Note that the variables defined in the desired "base" template **must** be set t Additionally, you will need to define the environment variables in `tests/env_templates/env.vectorize-minimal.template`, which are needed by the minimal set of "vectorize" testing belonging to the "base" test group. +These rely on a single embedding provider and model +(the one configured in `embedding_provider_switcher.py`, to match variables in the env template). For Astra DB, you can include "shared secret" vectorize tests (i.e. KMS-based authentication). -To run those tests, you must scope an OpenAI API key -to the target Astra DB with secret name `"SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"` +To run those tests, you must scope an embedding provider API key +to the target Astra DB with secret name matching the name set in +the provider-switcher (e.g. `"SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI"`). and comment the environment flag that suppresses them (see the base Astra env template). For non-Astra, the reranking-related tests run only if one sets diff --git a/tests/base/conftest.py b/tests/base/conftest.py index 1e17ef92..d531f9a1 100644 --- a/tests/base/conftest.py +++ b/tests/base/conftest.py @@ -44,7 +44,11 @@ ADMIN_ENV_LIST, ADMIN_ENV_VARIABLE_MAP, CQL_AVAILABLE, - HEADER_EMBEDDING_API_KEY_OPENAI, + EMBEDDING_PROVIDER_API_KEY, + EMBEDDING_PROVIDER_DIMENSION, + EMBEDDING_PROVIDER_MODEL_NAME, + EMBEDDING_PROVIDER_NAME, + EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME, HEADER_RERANKING_API_KEY_NVIDIA, IS_ASTRA_DB, RUN_SHARED_SECRET_VECTORIZE_TESTS, @@ -227,10 +231,10 @@ def async_empty_collection( @pytest.fixture(scope="session") def service_collection_parameters() -> Iterable[dict[str, Any]]: yield { - "dimension": 1536, - "provider": "openai", - "modelName": "text-embedding-ada-002", - "api_key": HEADER_EMBEDDING_API_KEY_OPENAI, + "dimension": EMBEDDING_PROVIDER_DIMENSION, + "provider": EMBEDDING_PROVIDER_NAME, + "modelName": EMBEDDING_PROVIDER_MODEL_NAME, + "api_key": EMBEDDING_PROVIDER_API_KEY, "reranking_api_key": HEADER_RERANKING_API_KEY_NVIDIA, } @@ -996,7 +1000,11 @@ def async_empty_table_logicalfiltering( "clean_nulls_from_dict", "is_future_version", "sync_fail_if_not_removed", - "HEADER_EMBEDDING_API_KEY_OPENAI", + "EMBEDDING_PROVIDER_NAME", + "EMBEDDING_PROVIDER_MODEL_NAME", + "EMBEDDING_PROVIDER_API_KEY", + "EMBEDDING_PROVIDER_DIMENSION", + "EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME", "IS_ASTRA_DB", "ADMIN_ENV_LIST", "ADMIN_ENV_VARIABLE_MAP", diff --git a/tests/base/integration/collections/test_collection_cursor_async.py b/tests/base/integration/collections/test_collection_cursor_async.py index 985b5eca..23a07949 100644 --- a/tests/base/integration/collections/test_collection_cursor_async.py +++ b/tests/base/integration/collections/test_collection_cursor_async.py @@ -26,8 +26,9 @@ from ..conftest import DefaultAsyncCollection -NUM_DOCS = 25 # keep this between 20 and 39 -NUM_DOCS_PAGINATION = 90 # keep this above 2 * (2 * 20) and below 2 * (3 * 20) +PAGE_SIZE = 20 # TODO: set to 50, default as per Data API config after PR 2461 +NUM_DOCS = 2 * PAGE_SIZE + 5 +NUM_DOCS_PAGINATION = 2 * (2 * PAGE_SIZE) + 5 @pytest.fixture @@ -161,16 +162,16 @@ async def test_collection_cursors_started_properties_async( await cur.__anext__() # now this has 19 items in buffer, one is consumed assert cur.consumed == 1 - assert cur.buffered_count == 19 + assert cur.buffered_count == PAGE_SIZE - 1 assert len(cur.consume_buffer(3)) == 3 assert cur.consumed == 4 - assert cur.buffered_count == 16 + assert cur.buffered_count == PAGE_SIZE - 4 # from time to time the buffer is empty: - for _ in range(16): + for _ in range(PAGE_SIZE - 4): await cur.__anext__() assert cur.buffered_count == 0 assert cur.consume_buffer(3) == [] - assert cur.consumed == 20 + assert cur.consumed == PAGE_SIZE assert cur.buffered_count == 0 with pytest.raises(CursorException): @@ -213,12 +214,12 @@ async def test_collection_cursors_has_next_async( assert curmf.has_next() assert curmf.consumed == 2 assert curmf.state == CursorState.STARTED - for _ in range(18): + for _ in range(PAGE_SIZE - 2): await curmf.__anext__() assert await curmf.has_next() - assert curmf.consumed == 20 + assert curmf.consumed == PAGE_SIZE assert curmf.state == CursorState.STARTED - assert curmf.buffered_count == NUM_DOCS - 20 + assert curmf.buffered_count == PAGE_SIZE cur0 = async_filled_collection.find() cur0.close() @@ -494,11 +495,9 @@ async def test_collection_cursors_initialpagestate_async( self, async_filled_pagination_collection: DefaultAsyncCollection, ) -> None: - page_size = 20 - cur0 = async_filled_pagination_collection.find(filter={"even": True}) ids0: list[int] = [] - for _ in range(page_size): + for _ in range(PAGE_SIZE): doc = await cur0.__anext__() ids0.append(doc["_id"]) nps0 = cur0._next_page_state @@ -509,7 +508,7 @@ async def test_collection_cursors_initialpagestate_async( initial_page_state=nps0, ) ids1: list[int] = [] - for _ in range(page_size): + for _ in range(PAGE_SIZE): doc = await cur1.__anext__() ids1.append(doc["_id"]) nps1 = cur1._next_page_state diff --git a/tests/base/integration/collections/test_collection_cursor_sync.py b/tests/base/integration/collections/test_collection_cursor_sync.py index bee8fcff..16924c75 100644 --- a/tests/base/integration/collections/test_collection_cursor_sync.py +++ b/tests/base/integration/collections/test_collection_cursor_sync.py @@ -26,8 +26,9 @@ from ..conftest import DefaultCollection -NUM_DOCS = 25 # keep this between 20 and 39 -NUM_DOCS_PAGINATION = 90 # keep this above 2 * (2 * 20) and below 2 * (3 * 20) +PAGE_SIZE = 20 # TODO: set to 50, default as per Data API config after PR 2461 +NUM_DOCS = 2 * PAGE_SIZE + 5 +NUM_DOCS_PAGINATION = 2 * (2 * PAGE_SIZE) + 5 @pytest.fixture @@ -159,16 +160,16 @@ def test_collection_cursors_started_properties_sync( next(cur) # now this has 19 items in buffer, one is consumed assert cur.consumed == 1 - assert cur.buffered_count == 19 + assert cur.buffered_count == PAGE_SIZE - 1 assert len(cur.consume_buffer(3)) == 3 assert cur.consumed == 4 - assert cur.buffered_count == 16 + assert cur.buffered_count == PAGE_SIZE - 4 # from time to time the buffer is empty: - for _ in range(16): + for _ in range(PAGE_SIZE - 4): next(cur) assert cur.buffered_count == 0 assert cur.consume_buffer(3) == [] - assert cur.consumed == 20 + assert cur.consumed == PAGE_SIZE assert cur.buffered_count == 0 with pytest.raises(CursorException): @@ -211,12 +212,12 @@ def test_collection_cursors_has_next_sync( assert curmf.has_next() assert curmf.consumed == 2 assert curmf.state == CursorState.STARTED - for _ in range(18): + for _ in range(PAGE_SIZE - 2): next(curmf) assert curmf.has_next() - assert curmf.consumed == 20 + assert curmf.consumed == PAGE_SIZE assert curmf.state == CursorState.STARTED - assert curmf.buffered_count == NUM_DOCS - 20 + assert curmf.buffered_count == PAGE_SIZE cur0 = filled_collection.find() cur0.close() @@ -417,10 +418,8 @@ def test_collection_cursors_initialpagestate_sync( self, filled_pagination_collection: DefaultCollection, ) -> None: - page_size = 20 - cur0 = filled_pagination_collection.find(filter={"even": True}) - ids0 = [doc["_id"] for _, doc in zip(range(page_size), cur0)] + ids0 = [doc["_id"] for _, doc in zip(range(PAGE_SIZE), cur0)] nps0 = cur0._next_page_state assert isinstance(nps0, str) @@ -428,7 +427,7 @@ def test_collection_cursors_initialpagestate_sync( filter={"even": True}, initial_page_state=nps0, ) - ids1 = [doc["_id"] for _, doc in zip(range(page_size), cur1)] + ids1 = [doc["_id"] for _, doc in zip(range(PAGE_SIZE), cur1)] nps1 = cur1._next_page_state assert isinstance(nps1, str) @@ -436,7 +435,7 @@ def test_collection_cursors_initialpagestate_sync( filter={"even": True}, initial_page_state=nps1, ) - ids2 = [doc["_id"] for _, doc in zip(range(page_size), cur2)] + ids2 = [doc["_id"] for doc in cur2] assert cur2._next_page_state is None expected_ids = [i for i in range(NUM_DOCS_PAGINATION) if i % 2 == 0] diff --git a/tests/base/integration/collections/test_collection_vectorize_methods_async.py b/tests/base/integration/collections/test_collection_vectorize_methods_async.py index 22f58aaf..344e8e37 100644 --- a/tests/base/integration/collections/test_collection_vectorize_methods_async.py +++ b/tests/base/integration/collections/test_collection_vectorize_methods_async.py @@ -26,14 +26,14 @@ from astrapy.info import CollectionDefinition from ..conftest import ( - HEADER_EMBEDDING_API_KEY_OPENAI, + EMBEDDING_PROVIDER_API_KEY, DefaultAsyncCollection, ) @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI credential", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) class TestCollectionVectorizeMethodsAsync: @pytest.mark.describe("test of vectorize in collection methods, async") diff --git a/tests/base/integration/collections/test_collection_vectorize_methods_sync.py b/tests/base/integration/collections/test_collection_vectorize_methods_sync.py index 6ca8afad..12207601 100644 --- a/tests/base/integration/collections/test_collection_vectorize_methods_sync.py +++ b/tests/base/integration/collections/test_collection_vectorize_methods_sync.py @@ -24,14 +24,14 @@ from astrapy.info import CollectionDefinition from ..conftest import ( - HEADER_EMBEDDING_API_KEY_OPENAI, + EMBEDDING_PROVIDER_API_KEY, DefaultCollection, ) @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI credential", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) class TestCollectionVectorizeMethodsSync: @pytest.mark.describe("test of vectorize in collection methods, sync") diff --git a/tests/base/integration/conftest.py b/tests/base/integration/conftest.py index e87e7030..c08dbb94 100644 --- a/tests/base/integration/conftest.py +++ b/tests/base/integration/conftest.py @@ -18,7 +18,9 @@ ADMIN_ENV_LIST, ADMIN_ENV_VARIABLE_MAP, CQL_AVAILABLE, - HEADER_EMBEDDING_API_KEY_OPENAI, + EMBEDDING_PROVIDER_API_KEY, + EMBEDDING_PROVIDER_DIMENSION, + EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME, IS_ASTRA_DB, RUN_SHARED_SECRET_VECTORIZE_TESTS, SECONDARY_KEYSPACE, @@ -62,7 +64,9 @@ "async_fail_if_not_removed", "clean_nulls_from_dict", "sync_fail_if_not_removed", - "HEADER_EMBEDDING_API_KEY_OPENAI", + "EMBEDDING_PROVIDER_API_KEY", + "EMBEDDING_PROVIDER_DIMENSION", + "EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME", "IS_ASTRA_DB", "ADMIN_ENV_LIST", "ADMIN_ENV_VARIABLE_MAP", diff --git a/tests/base/integration/tables/test_table_userdefinedtypes_async.py b/tests/base/integration/tables/test_table_userdefinedtypes_async.py index ab1fb00d..d176548e 100644 --- a/tests/base/integration/tables/test_table_userdefinedtypes_async.py +++ b/tests/base/integration/tables/test_table_userdefinedtypes_async.py @@ -567,6 +567,7 @@ async def test_table_udt_weirdcolumns_async( try: for cql_statement in WEIRD_UDT_BASE_INITIALIZE_STATEMENTS: cql_session.execute(cql_statement) + await asyncio.sleep(1.5) # udt propagation requires some time, it seems # test a read and a write for 'base weird' atable_weird_base = async_database.get_table(WEIRD_UDT_BASE_TABLE_NAME) diff --git a/tests/base/integration/tables/test_table_userdefinedtypes_sync.py b/tests/base/integration/tables/test_table_userdefinedtypes_sync.py index 105a06dc..8eac1db0 100644 --- a/tests/base/integration/tables/test_table_userdefinedtypes_sync.py +++ b/tests/base/integration/tables/test_table_userdefinedtypes_sync.py @@ -562,6 +562,7 @@ def test_table_udt_weirdcolumns_sync( try: for cql_statement in WEIRD_UDT_BASE_INITIALIZE_STATEMENTS: cql_session.execute(cql_statement) + time.sleep(1.5) # udt propagation requires some time, it seems # test a read and a write for 'base weird' table_weird_base = sync_database.get_table(WEIRD_UDT_BASE_TABLE_NAME) diff --git a/tests/base/integration/tables/test_table_vectorize_async.py b/tests/base/integration/tables/test_table_vectorize_async.py index 00ddeb8f..10fa3770 100644 --- a/tests/base/integration/tables/test_table_vectorize_async.py +++ b/tests/base/integration/tables/test_table_vectorize_async.py @@ -14,8 +14,6 @@ from __future__ import annotations -import os - import pytest from astrapy.api_options import APIOptions @@ -23,19 +21,19 @@ from astrapy.exceptions import TableInsertManyException from ..conftest import ( + EMBEDDING_PROVIDER_API_KEY, + EMBEDDING_PROVIDER_DIMENSION, IS_ASTRA_DB, RUN_SHARED_SECRET_VECTORIZE_TESTS, VECTORIZE_TEXTS, DefaultAsyncTable, ) -HEADER_EMBEDDING_API_KEY_OPENAI = os.environ.get("HEADER_EMBEDDING_API_KEY_OPENAI") - class TestTableVectorizeAsync: @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI detected", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) @pytest.mark.describe("test of basic table vectorize with key via header, async") async def test_table_vectorize_header_async( @@ -44,10 +42,10 @@ async def test_table_vectorize_header_async( ) -> None: aauthenticated_table = async_empty_table_vectorize.with_options( api_options=APIOptions( - embedding_api_key=HEADER_EMBEDDING_API_KEY_OPENAI or "", + embedding_api_key=EMBEDDING_PROVIDER_API_KEY or "", ), ) - i_vector = DataAPIVector([0.01] * 64) + i_vector = DataAPIVector([0.01] * EMBEDDING_PROVIDER_DIMENSION) await aauthenticated_table.insert_one( {"p_text": "v", "p_vector": i_vector}, ) @@ -104,8 +102,8 @@ async def test_table_vectorize_header_async( # assert isinstance(match0["$similarity"], float) @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI detected", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) @pytest.mark.describe( "test of vectorize-based table insert many grand failures, async" @@ -116,7 +114,7 @@ async def test_table_insertmany_grandfailure_vectorize_async( ) -> None: authenticated_atable = async_empty_table_vectorize.with_options( api_options=APIOptions( - embedding_api_key=HEADER_EMBEDDING_API_KEY_OPENAI or "", + embedding_api_key=EMBEDDING_PROVIDER_API_KEY or "", ), ) await authenticated_atable.insert_many([{"p_text": "0", "p_vector": "Text."}]) @@ -148,7 +146,7 @@ async def test_table_vectorize_shared_secret_async( self, async_empty_table_kms_vectorize: DefaultAsyncTable, ) -> None: - i_vector = DataAPIVector([0.01] * 64) + i_vector = DataAPIVector([0.01] * EMBEDDING_PROVIDER_DIMENSION) await async_empty_table_kms_vectorize.insert_one( {"p_text": "v", "p_vector": i_vector}, ) @@ -206,8 +204,8 @@ async def test_table_vectorize_shared_secret_async( # assert isinstance(match0["$similarity"], float) @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI detected", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) @pytest.mark.describe("test of multiple-vectorize table usage via header, async") async def test_table_multiplevectorize_header_async( diff --git a/tests/base/integration/tables/test_table_vectorize_sync.py b/tests/base/integration/tables/test_table_vectorize_sync.py index 2d2afda4..47244a99 100644 --- a/tests/base/integration/tables/test_table_vectorize_sync.py +++ b/tests/base/integration/tables/test_table_vectorize_sync.py @@ -14,8 +14,6 @@ from __future__ import annotations -import os - import pytest from astrapy.api_options import APIOptions @@ -23,19 +21,19 @@ from astrapy.exceptions import TableInsertManyException from ..conftest import ( + EMBEDDING_PROVIDER_API_KEY, + EMBEDDING_PROVIDER_DIMENSION, IS_ASTRA_DB, RUN_SHARED_SECRET_VECTORIZE_TESTS, VECTORIZE_TEXTS, DefaultTable, ) -HEADER_EMBEDDING_API_KEY_OPENAI = os.environ.get("HEADER_EMBEDDING_API_KEY_OPENAI") - class TestTableVectorizeSync: @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI detected", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) @pytest.mark.describe("test of basic table vectorize with key via header, sync") def test_table_vectorize_header_sync( @@ -44,10 +42,10 @@ def test_table_vectorize_header_sync( ) -> None: authenticated_table = sync_empty_table_vectorize.with_options( api_options=APIOptions( - embedding_api_key=HEADER_EMBEDDING_API_KEY_OPENAI or "", + embedding_api_key=EMBEDDING_PROVIDER_API_KEY or "", ), ) - i_vector = DataAPIVector([0.01] * 64) + i_vector = DataAPIVector([0.01] * EMBEDDING_PROVIDER_DIMENSION) authenticated_table.insert_one( {"p_text": "v", "p_vector": i_vector}, ) @@ -104,8 +102,8 @@ def test_table_vectorize_header_sync( # assert isinstance(match0["$similarity"], float) @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI detected", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) @pytest.mark.describe( "test of vectorize-based table insert many grand failures, sync" @@ -116,7 +114,7 @@ def test_table_insertmany_grandfailure_vectorize_sync( ) -> None: authenticated_table = sync_empty_table_vectorize.with_options( api_options=APIOptions( - embedding_api_key=HEADER_EMBEDDING_API_KEY_OPENAI or "", + embedding_api_key=EMBEDDING_PROVIDER_API_KEY or "", ), ) authenticated_table.insert_many([{"p_text": "0", "p_vector": "Text."}]) @@ -148,7 +146,7 @@ def test_table_vectorize_shared_secret_sync( self, sync_empty_table_kms_vectorize: DefaultTable, ) -> None: - i_vector = DataAPIVector([0.01] * 64) + i_vector = DataAPIVector([0.01] * EMBEDDING_PROVIDER_DIMENSION) sync_empty_table_kms_vectorize.insert_one( {"p_text": "v", "p_vector": i_vector}, ) @@ -206,8 +204,8 @@ def test_table_vectorize_shared_secret_sync( # assert isinstance(match0["$similarity"], float) @pytest.mark.skipif( - HEADER_EMBEDDING_API_KEY_OPENAI is None, - reason="No HEADER_EMBEDDING_API_KEY_OPENAI detected", + EMBEDDING_PROVIDER_API_KEY is None, + reason="No embedding API Key credential", ) @pytest.mark.describe("test of multiple-vectorize table usage via header, sync") def test_table_multiplevectorize_header_sync( diff --git a/tests/base/table_structure_assets.py b/tests/base/table_structure_assets.py index efa513e8..d30820cb 100644 --- a/tests/base/table_structure_assets.py +++ b/tests/base/table_structure_assets.py @@ -30,6 +30,13 @@ VectorServiceOptions, ) +from ..conftest import ( + EMBEDDING_PROVIDER_DIMENSION, + EMBEDDING_PROVIDER_MODEL_NAME, + EMBEDDING_PROVIDER_NAME, + EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME, +) + TEST_ALL_RETURNS_TABLE_NAME = "test_table_all_returns" TEST_ALL_RETURNS_TABLE_DEFINITION = CreateTableDefinition( columns={ @@ -160,10 +167,10 @@ "p_text": TableScalarColumnTypeDescriptor(column_type="text"), "p_vector": TableVectorColumnTypeDescriptor( column_type="vector", - dimension=64, + dimension=EMBEDDING_PROVIDER_DIMENSION, service=VectorServiceOptions( - provider="openai", - model_name="text-embedding-3-small", + provider=EMBEDDING_PROVIDER_NAME, + model_name=EMBEDDING_PROVIDER_MODEL_NAME, ), ), }, @@ -182,12 +189,12 @@ "p_text": TableScalarColumnTypeDescriptor(column_type="text"), "p_vector": TableVectorColumnTypeDescriptor( column_type="vector", - dimension=64, + dimension=EMBEDDING_PROVIDER_DIMENSION, service=VectorServiceOptions( - provider="openai", - model_name="text-embedding-3-small", + provider=EMBEDDING_PROVIDER_NAME, + model_name=EMBEDDING_PROVIDER_MODEL_NAME, authentication={ - "providerKey": "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI" + "providerKey": EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME }, ), ), @@ -210,18 +217,18 @@ "p_text": TableScalarColumnTypeDescriptor(column_type="text"), "p_vector_small": TableVectorColumnTypeDescriptor( column_type="vector", - dimension=1024, + dimension=EMBEDDING_PROVIDER_DIMENSION, service=VectorServiceOptions( - provider="openai", - model_name="text-embedding-3-small", + provider=EMBEDDING_PROVIDER_NAME, + model_name=EMBEDDING_PROVIDER_MODEL_NAME, ), ), "p_vector_large": TableVectorColumnTypeDescriptor( column_type="vector", - dimension=1024, + dimension=EMBEDDING_PROVIDER_DIMENSION, service=VectorServiceOptions( - provider="openai", - model_name="text-embedding-3-large", + provider=EMBEDDING_PROVIDER_NAME, + model_name=EMBEDDING_PROVIDER_MODEL_NAME, ), ), }, diff --git a/tests/conftest.py b/tests/conftest.py index 1a0179c7..650a14f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,13 @@ from astrapy.constants import Environment from astrapy.settings.defaults import DEFAULT_ASTRA_DB_KEYSPACE, DEV_OPS_URL_ENV_MAP +from .embedding_provider_switcher import ( + EMBEDDING_PROVIDER_API_KEY, + EMBEDDING_PROVIDER_DIMENSION, + EMBEDDING_PROVIDER_MODEL_NAME, + EMBEDDING_PROVIDER_NAME, + EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME, +) from .preprocess_env import ( ADMIN_ENV_LIST, ADMIN_ENV_VARIABLE_MAP, @@ -47,7 +54,6 @@ ASTRA_DB_KEYSPACE, ASTRA_DB_TOKEN_PROVIDER, DOCKER_COMPOSE_LOCAL_DATA_API, - HEADER_EMBEDDING_API_KEY_OPENAI, HEADER_RERANKING_API_KEY_NVIDIA, IS_ASTRA_DB, LOCAL_CASSANDRA_CONTACT_POINT, @@ -357,7 +363,11 @@ def cql_session( "ASTRA_DB_KEYSPACE", "CQL_AVAILABLE", "DOCKER_COMPOSE_LOCAL_DATA_API", - "HEADER_EMBEDDING_API_KEY_OPENAI", + "EMBEDDING_PROVIDER_NAME", + "EMBEDDING_PROVIDER_MODEL_NAME", + "EMBEDDING_PROVIDER_API_KEY", + "EMBEDDING_PROVIDER_DIMENSION", + "EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME", "HEADER_RERANKING_API_KEY_NVIDIA", "IS_ASTRA_DB", "LOCAL_DATA_API_ENDPOINT", diff --git a/tests/embedding_provider_switcher.py b/tests/embedding_provider_switcher.py new file mode 100644 index 00000000..c036b5ff --- /dev/null +++ b/tests/embedding_provider_switcher.py @@ -0,0 +1,45 @@ +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Entrypoint for vectorize switch management. + +Env. vars are read and exposed, and the choice of the preferred +vectorize provider is set here for all tests in /base. +""" + +from __future__ import annotations + +import os + +HEADER_EMBEDDING_API_KEY_VOYAGEAI = os.environ.get("HEADER_EMBEDDING_API_KEY_VOYAGEAI") + +# VECTORIZE PROVIDER SWITCHER FOR base TESTS - HARDCODED IN THIS MODULE: +# This controls the actual (auth-requiring) provider being used for the 'base' vectorize tests. +# Provider, model name, dimension and API Key (from env. variable) must match. +EMBEDDING_PROVIDER_NAME = "voyageAI" +EMBEDDING_PROVIDER_MODEL_NAME = "voyage-2" +EMBEDDING_PROVIDER_API_KEY = HEADER_EMBEDDING_API_KEY_VOYAGEAI +EMBEDDING_PROVIDER_DIMENSION = 1024 +# Moreover, for Astra DB KSM testing (if enabled), the database must be scoped +# a matching secret with this name: +EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME = "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI" + +__all__ = [ + "EMBEDDING_PROVIDER_NAME", + "EMBEDDING_PROVIDER_MODEL_NAME", + "EMBEDDING_PROVIDER_API_KEY", + "EMBEDDING_PROVIDER_DIMENSION", + "EMBEDDING_PROVIDER_SHARED_SECRET_KEY_NAME", +] diff --git a/tests/env_templates/env.astra.base.template b/tests/env_templates/env.astra.base.template index a1f1a810..39a7203c 100644 --- a/tests/env_templates/env.astra.base.template +++ b/tests/env_templates/env.astra.base.template @@ -11,5 +11,6 @@ export ASTRA_DB_API_ENDPOINT="https://-.apps.astra.datastax.co # export ASTRA_DB_SECONDARY_KEYSPACE="..." # OPTIONAL. Comment the suppression of "shared secret" (KMS-based) vectorize test: -# (requires a scoped API Key with name 'SHARED_SECRET_EMBEDDING_API_KEY_OPENAI' in the DB) +# (requires a scoped API Key with name 'SHARED_SECRET_EMBEDDING_API_KEY_' +# in the DB, as set in `embedding_provider_switcher.py`) export RUN_SHARED_SECRET_VECTORIZE_TESTS="no" diff --git a/tests/env_templates/env.vectorize-minimal.template b/tests/env_templates/env.vectorize-minimal.template index 800ad128..54551cba 100644 --- a/tests/env_templates/env.vectorize-minimal.template +++ b/tests/env_templates/env.vectorize-minimal.template @@ -2,5 +2,7 @@ # FOR THE VECTORIZE TESTS IN BASE: # #################################### -# Provide a valid OpenAI API Key for embedding computations: -export HEADER_EMBEDDING_API_KEY_OPENAI="..." +# Provide a valid embedding API Key for embedding computations, +# This must match the variable name gathered by `embedding_provider_switcher.py`: +# (you should not need to change this var. name unless you are changing the switcher) +export HEADER_EMBEDDING_API_KEY_VOYAGEAI="..." diff --git a/tests/hcd_compose/docker-compose.yml b/tests/hcd_compose/docker-compose.yml index 10555837..370cb3cb 100644 --- a/tests/hcd_compose/docker-compose.yml +++ b/tests/hcd_compose/docker-compose.yml @@ -1,12 +1,12 @@ services: hcd: - image: 559669398656.dkr.ecr.us-west-2.amazonaws.com/engops-shared/hcd/prod/hcd:1.2.3 + image: 559669398656.dkr.ecr.us-west-2.amazonaws.com/engops-shared/hcd/prod/hcd:2.0.6 networks: - stargate mem_limit: 2G environment: - MAX_HEAP_SIZE=1536M - - CLUSTER_NAME=hcd-1.2.3-cluster + - CLUSTER_NAME=hcd-2.0.6-cluster - DS_LICENSE=accept - HCD_AUTO_CONF_OFF=xcassandra.yaml - JVM_EXTRA_OPTS=-Dcassandra.sai.latest.version=ec @@ -21,7 +21,7 @@ services: retries: 20 data-api: - image: stargateio/data-api:v1.0.39 + image: stargateio/data-api:v1.0.46 depends_on: hcd: condition: service_healthy diff --git a/tests/preprocess_env.py b/tests/preprocess_env.py index 4c96af9b..72974711 100644 --- a/tests/preprocess_env.py +++ b/tests/preprocess_env.py @@ -15,7 +15,7 @@ """ Bottleneck entrypoint for reading os.environ and exposing its contents as (normalized) regular variables. -Except for the vectorize information, which for the time being passes as os.environ. +Except for most vectorize information, handled by the embedding_provider_switcher. """ from __future__ import annotations @@ -204,5 +204,4 @@ def docker_compose_command(self) -> list[str]: } # misc variables -HEADER_EMBEDDING_API_KEY_OPENAI = os.environ.get("HEADER_EMBEDDING_API_KEY_OPENAI") HEADER_RERANKING_API_KEY_NVIDIA = os.environ.get("HEADER_RERANKING_API_KEY_NVIDIA") diff --git a/tests/vectorize/vectorize_models.py b/tests/vectorize/vectorize_models.py index d3889ac4..62555781 100644 --- a/tests/vectorize/vectorize_models.py +++ b/tests/vectorize/vectorize_models.py @@ -115,8 +115,23 @@ # Pending a testable deployment, HuggingFace Dedicated testing is momentarily suspended: ("huggingfaceDedicated", "endpoint-defined-model", "HEADER", "f"), ("huggingfaceDedicated", "endpoint-defined-model", "SHARED_SECRET", "f"), + # Pending a valid OpenAI API Key to use, all OpenAI vectorize testing is suspended: + ("openai", "text-embedding-3-large", "SHARED_SECRET", "0"), + ("openai", "text-embedding-3-large", "SHARED_SECRET", "f"), + ("openai", "text-embedding-3-small", "SHARED_SECRET", "0"), + ("openai", "text-embedding-3-small", "SHARED_SECRET", "f"), + ("openai", "text-embedding-ada-002", "SHARED_SECRET", "0"), + ("openai", "text-embedding-ada-002", "SHARED_SECRET", "f"), + # ... also the header-based testing: + ("openai", "text-embedding-3-large", "HEADER", "0"), + ("openai", "text-embedding-3-large", "HEADER", "f"), + ("openai", "text-embedding-3-small", "HEADER", "0"), + ("openai", "text-embedding-3-small", "HEADER", "f"), + ("openai", "text-embedding-ada-002", "HEADER", "0"), + ("openai", "text-embedding-ada-002", "HEADER", "f"), } + # this is a way to suppress/limit certain combinations of # "full param" testing from the start. If even one of the optional params # in a test model is PARAM_SKIP_MARKER, the combination is not emitted at all. diff --git a/uv.lock b/uv.lock index 5cf15920..95935f5e 100644 --- a/uv.lock +++ b/uv.lock @@ -24,7 +24,7 @@ wheels = [ [[package]] name = "astrapy" -version = "2.2.0" +version = "2.2.1" source = { editable = "." } dependencies = [ { name = "deprecation" },