Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 65 additions & 13 deletions .github/workflows/e2e-tests.yaml
Original file line number Diff line number Diff line change
@@ -1,18 +1,6 @@
name: E2E Tests

on:
pull_request:
branches:
- main
paths:
- 'frontend/**'
- 'deploy/helm/**'
- 'tests/**'
- '.github/workflows/e2e-tests.yaml'
push:
branches:
- main
workflow_dispatch:
on: [pull_request, push, workflow_dispatch]

# MaaS configuration - can be overridden with repository secrets for different environments
env:
Expand Down Expand Up @@ -754,3 +742,67 @@ jobs:
run: |
pkill -f "kubectl port-forward" || true
# kind delete cluster --name rag-e2e-ui

# Single check to mark as required in branch protection (Settings → Branches →
# Require status checks): "PR tests gate". Fails if unit/integration failed, or
# if MaaS-backed jobs ran and failed. Fork PRs may skip secret-dependent jobs.
pr-required-checks:
name: PR tests gate
runs-on: ubuntu-latest
if: always() && github.event_name == 'pull_request'
needs:
- unit-tests
- integration-tests
- llamastack-integration-tests
- ui-e2e-tests
steps:
- name: Enforce test job outcomes
env:
UNIT_RESULT: ${{ needs.unit-tests.result }}
INTEG_RESULT: ${{ needs.integration-tests.result }}
LLAMA_RESULT: ${{ needs.llamastack-integration-tests.result }}
UI_RESULT: ${{ needs.ui-e2e-tests.result }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
THIS_REPO: ${{ github.repository }}
run: |
set -euo pipefail
is_fork=false
if [ "${HEAD_REPO}" != "${THIS_REPO}" ]; then
is_fork=true
fi

echo "Job results: unit=${UNIT_RESULT} integration=${INTEG_RESULT} llamastack=${LLAMA_RESULT} ui-e2e=${UI_RESULT} fork=${is_fork}"

if [ "${UNIT_RESULT}" != "success" ]; then
echo "::error::Unit tests did not succeed (result=${UNIT_RESULT})"
exit 1
fi
if [ "${INTEG_RESULT}" != "success" ]; then
echo "::error::Integration tests (Streamlit) did not succeed (result=${INTEG_RESULT})"
exit 1
fi

allow_skipped_secret_jobs() {
local r="$1"
[ "${is_fork}" = true ] && [ "${r}" = "skipped" ]
}

if [ "${LLAMA_RESULT}" != "success" ]; then
if allow_skipped_secret_jobs "${LLAMA_RESULT}"; then
echo "LlamaStack integration tests were skipped (fork PR; repository secrets are not available)."
else
echo "::error::LlamaStack integration tests did not succeed (result=${LLAMA_RESULT})"
exit 1
fi
fi

if [ "${UI_RESULT}" != "success" ]; then
if allow_skipped_secret_jobs "${UI_RESULT}"; then
echo "UI E2E tests were skipped (fork PR; repository secrets are not available)."
else
echo "::error::UI E2E tests did not succeed (result=${UI_RESULT})"
exit 1
fi
fi

echo "All required test outcomes for this pull request passed."
4 changes: 2 additions & 2 deletions deploy/helm/rag/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ apiVersion: v2
name: rag
description: A Helm chart for Kubernetes
type: application
version: 0.2.42
appVersion: "0.2.42"
version: 0.2.43
appVersion: "0.2.43"

dependencies:
- name: pgvector
Expand Down
15 changes: 1 addition & 14 deletions deploy/helm/rag/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ replicaCount: 1
image:
repository: quay.io/rh-ai-quickstart/llamastack-dist-ui
pullPolicy: Always
tag: 0.2.42
tag: 0.2.43

service:
type: ClusterIP
Expand Down Expand Up @@ -164,19 +164,6 @@ pgvector:
dbname: rag_blueprint
host: pgvector
port: "5432"

# Create a separate vector database for each ingestion pipeline
extraDatabases:
- name: hr_vector_db
vectordb: true
- name: legal_vector_db
vectordb: true
- name: sales_vector_db
vectordb: true
- name: procurement_vector_db
vectordb: true
- name: techsupport_vector_db
vectordb: true

# Upload sample files to the minio bucket
sampleFileUpload:
Expand Down
2 changes: 2 additions & 0 deletions tests/e2e_ui/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@ pytest==8.3.3
pytest-playwright==0.5.2
playwright==1.48.0
requests==2.32.3
python-docx
openpyxl

6 changes: 0 additions & 6 deletions tests/e2e_ui/test_chat_ui.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,12 +183,6 @@ def test_initial_greeting_message(self, page: Page):
"""Test that initial greeting message is displayed"""
greeting = page.get_by_text("How can I help you?", exact=False)
expect(greeting).to_be_visible(timeout=TEST_TIMEOUT)

def test_tool_debug_toggle(self, page: Page):
"""Test that tool debug toggle is visible"""
debug_toggle = page.get_by_text("Show Tool/Debug Info", exact=False)
expect(debug_toggle).to_be_visible(timeout=TEST_TIMEOUT)


class TestMaaSIntegration:
"""UI tests for MaaS (Model-as-a-Service) integration through the UI
Expand Down
45 changes: 34 additions & 11 deletions tests/integration/llamastack/test_user_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -286,18 +286,41 @@ def test_complete_rag_workflow():
print("🤖 Step 4: Checking for available models...")
skip_inference = SKIP_MODEL_TESTS == "true"
model_available = False

model_ids = []

try:
# Llama-stack uses /v1/openai/v1/* paths for OpenAI-compatible API
openai_base_url = f"{LLAMA_STACK_ENDPOINT}/v1/openai/v1"
client = OpenAI(
api_key="not_needed",
base_url=openai_base_url,
timeout=30.0
)

print(f" DEBUG: Calling {openai_base_url}/models endpoint...")
models = client.models.list()
# OpenAI-compatible base URL: current Llama Stack uses /v1 (see conftest.py).
# Older stacks used /v1/openai/v1 (client-examples-python/README.md).
endpoint = LLAMA_STACK_ENDPOINT.rstrip("/")
explicit_openai = os.getenv("LLAMA_STACK_OPENAI_BASE")
openai_base_candidates = []
if explicit_openai:
openai_base_candidates.append(explicit_openai.rstrip("/"))
openai_base_candidates.append(f"{endpoint}/v1")
openai_base_candidates.append(f"{endpoint}/v1/openai/v1")

client = None
openai_base_url = None
models = None
last_models_error: Exception | None = None
for base in openai_base_candidates:
candidate = OpenAI(
api_key="not_needed",
base_url=base,
timeout=30.0,
)
try:
print(f" DEBUG: Calling {base}/models endpoint...")
models = candidate.models.list()
client = candidate
openai_base_url = base
break
except Exception as e:
last_models_error = e
continue

if client is None or models is None:
raise last_models_error or RuntimeError("Could not list models from any OpenAI base URL")
print(f" DEBUG: Raw response type: {type(models)}")
print(f" DEBUG: Number of models in response: {len(models.data)}")

Expand Down
16 changes: 8 additions & 8 deletions tests/integration/test_upload_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def mock_uploaded_file():
class TestDocumentUploadIntegration:
"""Integration tests for document upload workflow"""

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_single_file_upload_workflow(self, mock_api, mock_uploaded_file):
"""Test complete workflow for uploading a single file"""
from llama_stack_client import RAGDocument
Expand Down Expand Up @@ -113,7 +113,7 @@ def test_single_file_upload_workflow(self, mock_api, mock_uploaded_file):
mock_api.client.vector_dbs.register.assert_called_once()
mock_api.client.tool_runtime.rag_tool.insert.assert_called_once()

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_multiple_files_upload_workflow(self, mock_api):
"""Test uploading multiple files at once"""
from llama_stack_client import RAGDocument
Expand Down Expand Up @@ -182,7 +182,7 @@ def test_file_type_validation(self):
class TestVectorDBCreation:
"""Integration tests for vector database creation"""

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_vector_db_registration_params(self, mock_api):
"""Test vector DB registration with correct parameters"""
mock_api.client.providers.list.return_value = [
Expand All @@ -204,7 +204,7 @@ def test_vector_db_registration_params(self, mock_api):
assert call_args[1]['embedding_model'] == "all-MiniLM-L6-v2"
assert call_args[1]['provider_id'] == "pgvector"

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_vector_db_with_custom_name(self, mock_api):
"""Test creating vector DB with custom name"""
mock_api.client.providers.list.return_value = [
Expand All @@ -227,7 +227,7 @@ def test_vector_db_with_custom_name(self, mock_api):
class TestProviderDetection:
"""Integration tests for provider detection"""

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_vector_io_provider_detection(self, mock_api):
"""Test that vector_io provider is correctly detected"""
mock_api.client.providers.list.return_value = [
Expand All @@ -244,7 +244,7 @@ def test_vector_io_provider_detection(self, mock_api):

assert vector_io_provider == "pgvector"

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_no_vector_io_provider(self, mock_api):
"""Test handling when no vector_io provider is available"""
mock_api.client.providers.list.return_value = [
Expand All @@ -264,7 +264,7 @@ def test_no_vector_io_provider(self, mock_api):
class TestDocumentInsertion:
"""Integration tests for document insertion into vector DB"""

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_document_insertion_with_chunks(self, mock_api):
"""Test document insertion with chunking"""
from llama_stack_client import RAGDocument
Expand All @@ -288,7 +288,7 @@ def test_document_insertion_with_chunks(self, mock_api):
call_args = mock_api.client.tool_runtime.rag_tool.insert.call_args
assert call_args[1]['chunk_size_in_tokens'] == 512

@patch('llama_stack_ui.distribution.ui.page.upload.upload.llama_stack_api')
@patch('llama_stack_ui.distribution.ui.modules.api.llama_stack_api')
def test_empty_document_list(self, mock_api):
"""Test handling of empty document list"""
documents = []
Expand Down
3 changes: 2 additions & 1 deletion tests/unit/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ pytest-cov==5.0.0
llama-stack-client>=0.2.9,<0.2.13
llama-stack
streamlit>=1.31.0

python-docx
openpyxl
Loading