diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 3d7e9c7..403e8e6 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -1,18 +1,6 @@ name: E2E Tests -on: - pull_request: - branches: - - main - paths: - - 'frontend/**' - - 'deploy/helm/**' - - 'tests/**' - - '.github/workflows/e2e-tests.yaml' - push: - branches: - - main - workflow_dispatch: +on: [pull_request, push, workflow_dispatch] # MaaS configuration - can be overridden with repository secrets for different environments env: @@ -754,3 +742,67 @@ jobs: run: | pkill -f "kubectl port-forward" || true # kind delete cluster --name rag-e2e-ui + + # Single check to mark as required in branch protection (Settings → Branches → + # Require status checks): "PR tests gate". Fails if unit/integration failed, or + # if MaaS-backed jobs ran and failed. Fork PRs may skip secret-dependent jobs. + pr-required-checks: + name: PR tests gate + runs-on: ubuntu-latest + if: always() && github.event_name == 'pull_request' + needs: + - unit-tests + - integration-tests + - llamastack-integration-tests + - ui-e2e-tests + steps: + - name: Enforce test job outcomes + env: + UNIT_RESULT: ${{ needs.unit-tests.result }} + INTEG_RESULT: ${{ needs.integration-tests.result }} + LLAMA_RESULT: ${{ needs.llamastack-integration-tests.result }} + UI_RESULT: ${{ needs.ui-e2e-tests.result }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + THIS_REPO: ${{ github.repository }} + run: | + set -euo pipefail + is_fork=false + if [ "${HEAD_REPO}" != "${THIS_REPO}" ]; then + is_fork=true + fi + + echo "Job results: unit=${UNIT_RESULT} integration=${INTEG_RESULT} llamastack=${LLAMA_RESULT} ui-e2e=${UI_RESULT} fork=${is_fork}" + + if [ "${UNIT_RESULT}" != "success" ]; then + echo "::error::Unit tests did not succeed (result=${UNIT_RESULT})" + exit 1 + fi + if [ "${INTEG_RESULT}" != "success" ]; then + echo "::error::Integration tests (Streamlit) did not succeed (result=${INTEG_RESULT})" + exit 1 + fi + + allow_skipped_secret_jobs() { + local r="$1" + [ "${is_fork}" = true ] && [ "${r}" = "skipped" ] + } + + if [ "${LLAMA_RESULT}" != "success" ]; then + if allow_skipped_secret_jobs "${LLAMA_RESULT}"; then + echo "LlamaStack integration tests were skipped (fork PR; repository secrets are not available)." + else + echo "::error::LlamaStack integration tests did not succeed (result=${LLAMA_RESULT})" + exit 1 + fi + fi + + if [ "${UI_RESULT}" != "success" ]; then + if allow_skipped_secret_jobs "${UI_RESULT}"; then + echo "UI E2E tests were skipped (fork PR; repository secrets are not available)." + else + echo "::error::UI E2E tests did not succeed (result=${UI_RESULT})" + exit 1 + fi + fi + + echo "All required test outcomes for this pull request passed." diff --git a/deploy/helm/rag/Chart.yaml b/deploy/helm/rag/Chart.yaml index 280f3cd..3fd0779 100644 --- a/deploy/helm/rag/Chart.yaml +++ b/deploy/helm/rag/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rag description: A Helm chart for Kubernetes type: application -version: 0.2.42 -appVersion: "0.2.42" +version: 0.2.43 +appVersion: "0.2.43" dependencies: - name: pgvector diff --git a/deploy/helm/rag/values.yaml b/deploy/helm/rag/values.yaml index a1903be..f50da6e 100644 --- a/deploy/helm/rag/values.yaml +++ b/deploy/helm/rag/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: quay.io/rh-ai-quickstart/llamastack-dist-ui pullPolicy: Always - tag: 0.2.42 + tag: 0.2.43 service: type: ClusterIP @@ -164,19 +164,6 @@ pgvector: dbname: rag_blueprint host: pgvector port: "5432" - - # Create a separate vector database for each ingestion pipeline - extraDatabases: - - name: hr_vector_db - vectordb: true - - name: legal_vector_db - vectordb: true - - name: sales_vector_db - vectordb: true - - name: procurement_vector_db - vectordb: true - - name: techsupport_vector_db - vectordb: true # Upload sample files to the minio bucket sampleFileUpload: diff --git a/tests/e2e_ui/requirements.txt b/tests/e2e_ui/requirements.txt index 8e32c60..7aa777d 100644 --- a/tests/e2e_ui/requirements.txt +++ b/tests/e2e_ui/requirements.txt @@ -2,4 +2,6 @@ pytest==8.3.3 pytest-playwright==0.5.2 playwright==1.48.0 requests==2.32.3 +python-docx +openpyxl diff --git a/tests/e2e_ui/test_chat_ui.py b/tests/e2e_ui/test_chat_ui.py index aeec018..8d1e614 100644 --- a/tests/e2e_ui/test_chat_ui.py +++ b/tests/e2e_ui/test_chat_ui.py @@ -183,12 +183,6 @@ def test_initial_greeting_message(self, page: Page): """Test that initial greeting message is displayed""" greeting = page.get_by_text("How can I help you?", exact=False) expect(greeting).to_be_visible(timeout=TEST_TIMEOUT) - - def test_tool_debug_toggle(self, page: Page): - """Test that tool debug toggle is visible""" - debug_toggle = page.get_by_text("Show Tool/Debug Info", exact=False) - expect(debug_toggle).to_be_visible(timeout=TEST_TIMEOUT) - class TestMaaSIntegration: """UI tests for MaaS (Model-as-a-Service) integration through the UI diff --git a/tests/integration/llamastack/test_user_workflow.py b/tests/integration/llamastack/test_user_workflow.py index d67eb98..bf2a448 100644 --- a/tests/integration/llamastack/test_user_workflow.py +++ b/tests/integration/llamastack/test_user_workflow.py @@ -286,18 +286,41 @@ def test_complete_rag_workflow(): print("🤖 Step 4: Checking for available models...") skip_inference = SKIP_MODEL_TESTS == "true" model_available = False - + model_ids = [] + try: - # Llama-stack uses /v1/openai/v1/* paths for OpenAI-compatible API - openai_base_url = f"{LLAMA_STACK_ENDPOINT}/v1/openai/v1" - client = OpenAI( - api_key="not_needed", - base_url=openai_base_url, - timeout=30.0 - ) - - print(f" DEBUG: Calling {openai_base_url}/models endpoint...") - models = client.models.list() + # OpenAI-compatible base URL: current Llama Stack uses /v1 (see conftest.py). + # Older stacks used /v1/openai/v1 (client-examples-python/README.md). + endpoint = LLAMA_STACK_ENDPOINT.rstrip("/") + explicit_openai = os.getenv("LLAMA_STACK_OPENAI_BASE") + openai_base_candidates = [] + if explicit_openai: + openai_base_candidates.append(explicit_openai.rstrip("/")) + openai_base_candidates.append(f"{endpoint}/v1") + openai_base_candidates.append(f"{endpoint}/v1/openai/v1") + + client = None + openai_base_url = None + models = None + last_models_error: Exception | None = None + for base in openai_base_candidates: + candidate = OpenAI( + api_key="not_needed", + base_url=base, + timeout=30.0, + ) + try: + print(f" DEBUG: Calling {base}/models endpoint...") + models = candidate.models.list() + client = candidate + openai_base_url = base + break + except Exception as e: + last_models_error = e + continue + + if client is None or models is None: + raise last_models_error or RuntimeError("Could not list models from any OpenAI base URL") print(f" DEBUG: Raw response type: {type(models)}") print(f" DEBUG: Number of models in response: {len(models.data)}") diff --git a/tests/integration/test_upload_integration.py b/tests/integration/test_upload_integration.py index 378ac1a..6443e29 100644 --- a/tests/integration/test_upload_integration.py +++ b/tests/integration/test_upload_integration.py @@ -64,7 +64,7 @@ def mock_uploaded_file(): class TestDocumentUploadIntegration: """Integration tests for document upload workflow""" - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_single_file_upload_workflow(self, mock_api, mock_uploaded_file): """Test complete workflow for uploading a single file""" from llama_stack_client import RAGDocument @@ -113,7 +113,7 @@ def test_single_file_upload_workflow(self, mock_api, mock_uploaded_file): mock_api.client.vector_dbs.register.assert_called_once() mock_api.client.tool_runtime.rag_tool.insert.assert_called_once() - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_multiple_files_upload_workflow(self, mock_api): """Test uploading multiple files at once""" from llama_stack_client import RAGDocument @@ -182,7 +182,7 @@ def test_file_type_validation(self): class TestVectorDBCreation: """Integration tests for vector database creation""" - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_vector_db_registration_params(self, mock_api): """Test vector DB registration with correct parameters""" mock_api.client.providers.list.return_value = [ @@ -204,7 +204,7 @@ def test_vector_db_registration_params(self, mock_api): assert call_args[1]['embedding_model'] == "all-MiniLM-L6-v2" assert call_args[1]['provider_id'] == "pgvector" - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_vector_db_with_custom_name(self, mock_api): """Test creating vector DB with custom name""" mock_api.client.providers.list.return_value = [ @@ -227,7 +227,7 @@ def test_vector_db_with_custom_name(self, mock_api): class TestProviderDetection: """Integration tests for provider detection""" - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_vector_io_provider_detection(self, mock_api): """Test that vector_io provider is correctly detected""" mock_api.client.providers.list.return_value = [ @@ -244,7 +244,7 @@ def test_vector_io_provider_detection(self, mock_api): assert vector_io_provider == "pgvector" - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_no_vector_io_provider(self, mock_api): """Test handling when no vector_io provider is available""" mock_api.client.providers.list.return_value = [ @@ -264,7 +264,7 @@ def test_no_vector_io_provider(self, mock_api): class TestDocumentInsertion: """Integration tests for document insertion into vector DB""" - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_document_insertion_with_chunks(self, mock_api): """Test document insertion with chunking""" from llama_stack_client import RAGDocument @@ -288,7 +288,7 @@ def test_document_insertion_with_chunks(self, mock_api): call_args = mock_api.client.tool_runtime.rag_tool.insert.call_args assert call_args[1]['chunk_size_in_tokens'] == 512 - @patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api') + @patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api') def test_empty_document_list(self, mock_api): """Test handling of empty document list""" documents = [] diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt index 271ccc3..954a93c 100644 --- a/tests/unit/requirements.txt +++ b/tests/unit/requirements.txt @@ -4,4 +4,5 @@ pytest-cov==5.0.0 llama-stack-client>=0.2.9,<0.2.13 llama-stack streamlit>=1.31.0 - +python-docx +openpyxl