diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..f1da791a6 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,77 @@ +name: E2E Tests + +on: + pull_request: + push: + branches: [main, alpha, beta, rc] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + +permissions: + contents: read + +jobs: + e2e: + name: E2E Tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + + - name: Setup Tools + uses: TanStack/config/.github/setup@main + + - name: Cache Playwright Browsers + id: playwright-cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ hashFiles('testing/e2e/package.json') }} + restore-keys: | + playwright- + + - name: Build Packages + run: pnpm run build:all + + - name: Install Playwright Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: pnpm --filter @tanstack/ai-e2e exec playwright install --with-deps chromium + + - name: Install Playwright Deps (if cached) + if: steps.playwright-cache.outputs.cache-hit == 'true' + run: pnpm --filter @tanstack/ai-e2e exec playwright install-deps chromium + + - name: Run E2E Tests + run: pnpm --filter @tanstack/ai-e2e test:e2e + + - name: Upload Video Recordings + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-videos + path: testing/e2e/test-results/**/*.webm + retention-days: 14 + + - name: Upload Playwright Report + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-report + path: testing/e2e/playwright-report/ + retention-days: 14 + + - name: Upload Traces (on failure) + uses: actions/upload-artifact@v4 + if: failure() + with: + name: e2e-traces + path: testing/e2e/test-results/**/*.zip + retention-days: 14 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ce140529..70b1b6010 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1646,6 +1646,109 @@ importers: specifier: ^2.11.10 version: 2.11.10(solid-js@1.9.10)(vite@7.2.7(@types/node@25.0.1)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + testing/e2e: + dependencies: + '@copilotkit/llmock': + specifier: latest + version: 1.6.1 + '@tailwindcss/vite': + specifier: ^4.1.18 + version: 4.1.18(vite@7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + '@tanstack/ai': + specifier: workspace:* + version: link:../../packages/typescript/ai + '@tanstack/ai-anthropic': + specifier: workspace:* + version: link:../../packages/typescript/ai-anthropic + '@tanstack/ai-client': + specifier: workspace:* + version: link:../../packages/typescript/ai-client + '@tanstack/ai-gemini': + specifier: workspace:* + version: link:../../packages/typescript/ai-gemini + '@tanstack/ai-grok': + specifier: workspace:* + version: link:../../packages/typescript/ai-grok + '@tanstack/ai-groq': + specifier: workspace:* + version: link:../../packages/typescript/ai-groq + '@tanstack/ai-ollama': + specifier: workspace:* + version: link:../../packages/typescript/ai-ollama + '@tanstack/ai-openai': + specifier: workspace:* + version: link:../../packages/typescript/ai-openai + '@tanstack/ai-openrouter': + specifier: workspace:* + version: link:../../packages/typescript/ai-openrouter + '@tanstack/ai-react': + specifier: workspace:* + version: link:../../packages/typescript/ai-react + '@tanstack/ai-react-ui': + specifier: workspace:* + version: link:../../packages/typescript/ai-react-ui + '@tanstack/nitro-v2-vite-plugin': + specifier: ^1.154.7 + version: 1.154.7(rolldown@1.0.0-beta.53)(vite@7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + '@tanstack/react-router': + specifier: ^1.158.4 + version: 1.159.5(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@tanstack/react-start': + specifier: ^1.159.0 + version: 1.159.5(crossws@0.4.4(srvx@0.11.2))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(vite-plugin-solid@2.11.10(solid-js@1.9.10)(vite@7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)))(vite@7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + react: + specifier: ^19.2.3 + version: 19.2.3 + react-dom: + specifier: ^19.2.3 + version: 19.2.3(react@19.2.3) + react-markdown: + specifier: ^10.1.0 + version: 10.1.0(@types/react@19.2.7)(react@19.2.3) + rehype-highlight: + specifier: ^7.0.2 + version: 7.0.2 + rehype-raw: + specifier: ^7.0.0 + version: 7.0.0 + rehype-sanitize: + specifier: ^6.0.0 + version: 6.0.0 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + tailwindcss: + specifier: ^4.1.18 + version: 4.1.18 + vite-tsconfig-paths: + specifier: ^5.1.4 + version: 5.1.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + zod: + specifier: ^4.2.0 + version: 4.3.6 + devDependencies: + '@playwright/test': + specifier: ^1.57.0 + version: 1.57.0 + '@types/node': + specifier: ^24.10.1 + version: 24.10.3 + '@types/react': + specifier: ^19.2.7 + version: 19.2.7 + '@types/react-dom': + specifier: ^19.2.3 + version: 19.2.3(@types/react@19.2.7) + '@vitejs/plugin-react': + specifier: ^5.1.2 + version: 5.1.2(vite@7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + typescript: + specifier: 5.9.3 + version: 5.9.3 + vite: + specifier: ^7.2.7 + version: 7.3.1(@types/node@24.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + testing/panel: dependencies: '@tailwindcss/vite': @@ -2067,6 +2170,11 @@ packages: '@cloudflare/workers-types@4.20260317.1': resolution: {integrity: sha512-+G4eVwyCpm8Au1ex8vQBCuA9wnwqetz4tPNRoB/53qvktERWBRMQnrtvC1k584yRE3emMThtuY0gWshvSJ++PQ==} + '@copilotkit/llmock@1.6.1': + resolution: {integrity: sha512-5Q8fuoIJ1zN7GYwuUjm6cF2xX4qYeHJOULNcJFPEB7NinaGZviIOfdjQaECVmeRWLjjRLky8DclP8hAoWDYnIQ==} + engines: {node: '>=20.15.0'} + hasBin: true + '@crazydos/vue-markdown@1.1.4': resolution: {integrity: sha512-0I1QMP59LJ3aEjE7bolgvPU4JAFt+pykdDo5674CbsCwFo7OVFos50+MPhGdWflCz1mac5t152lB1qvV/tR/rw==} engines: {node: '>=20.0.0'} @@ -11037,6 +11145,8 @@ snapshots: '@cloudflare/workers-types@4.20260317.1': {} + '@copilotkit/llmock@1.6.1': {} + '@crazydos/vue-markdown@1.1.4(vue@3.5.25(typescript@5.9.3))': dependencies: deepmerge: 4.3.1 diff --git a/testing/e2e/.env b/testing/e2e/.env new file mode 100644 index 000000000..5ad69a16b --- /dev/null +++ b/testing/e2e/.env @@ -0,0 +1 @@ +LLMOCK_URL=http://127.0.0.1:4010 diff --git a/testing/e2e/README.md b/testing/e2e/README.md new file mode 100644 index 000000000..cb83dd330 --- /dev/null +++ b/testing/e2e/README.md @@ -0,0 +1,155 @@ +# TanStack AI E2E Tests + +End-to-end tests for TanStack AI using Playwright and [llmock](https://github.com/CopilotKit/llmock) for deterministic LLM mocking. + +**Architecture:** Playwright drives a TanStack Start app (`testing/e2e/`) which routes requests through provider adapters pointing at llmock. Fixtures define the mock responses. No real API keys needed. + +**Features tested:** chat, one-shot-text, reasoning, multi-turn, tool-calling, parallel-tool-calls, tool-approval, structured-output, agentic-structured, multimodal-image, multimodal-structured, summarize, summarize-stream, image-gen, tts, transcription + +**Providers tested:** openai, anthropic, gemini, ollama, groq, grok, openrouter + +## 1. Quick Start + +```bash +# Install dependencies +pnpm install + +# Run all E2E tests +pnpm --filter @tanstack/ai-e2e test:e2e + +# Run with Playwright UI (useful for debugging) +pnpm --filter @tanstack/ai-e2e test:e2e:ui + +# Run a single spec +pnpm --filter @tanstack/ai-e2e test:e2e -- --grep "openai -- chat" +``` + +## 2. Recording a New Fixture + +```bash +# 1. Set your API keys +export OPENAI_API_KEY=sk-... +export ANTHROPIC_API_KEY=sk-ant-... +# (add whichever providers you need) + +# 2. Start the app in record mode +pnpm --filter @tanstack/ai-e2e record + +# 3. Open the browser and navigate to the feature you want to record +# e.g. http://localhost:3010/openai/tool-calling + +# 4. Interact with the chat - type your message, wait for the response. +# llmock proxies to the real API and saves the response as a fixture. + +# 5. Find your recorded fixture in testing/e2e/fixtures/recorded/ +# Files are named: {provider}-{timestamp}-{uuid}.json + +# 6. Stop the dev server (Ctrl+C) +``` + +## 3. Organizing the Recorded Fixture + +Move from `recorded/` to the appropriate feature directory: + +```bash +mv fixtures/recorded/openai-2026-04-03T*.json fixtures/tool-calling/my-new-scenario.json +``` + +Then edit the fixture to clean it up: + +- **Simplify the `match` field** - use a short, unique `userMessage` that your test will send +- **Verify the `response`** - check that the content, toolCalls, or reasoning fields look correct +- **Remove provider-specific artifacts** - fixtures should be provider-agnostic + +Example - before cleanup: + +```json +{ + "fixtures": [ + { + "match": { + "userMessage": "Hey, I'm looking for a guitar...", + "model": "gpt-4o" + }, + "response": { + "content": "I'd recommend checking out the Fender Stratocaster..." + } + } + ] +} +``` + +After cleanup: + +```json +{ + "fixtures": [ + { + "match": { "userMessage": "recommend a blues guitar" }, + "response": { + "content": "I'd recommend checking out the Fender Stratocaster..." + } + } + ] +} +``` + +## 4. Writing the Test + +```typescript +// In tests/tool-calling.spec.ts (or whichever spec fits) + +test('calls getGuitars with category filter', async ({ page }) => { + await page.goto(`/${provider}/tool-calling`) + if (await isNotSupported(page)) { + test.skip() + return + } + + // Send the exact message that matches your fixture + await sendMessage(page, 'show me acoustic guitars') + await waitForResponse(page) + + // Assert on what should appear in the UI + const toolCalls = await getToolCalls(page) + expect(toolCalls).toHaveLength(1) + expect(toolCalls[0].name).toBe('getGuitars') + + const response = await getLastAssistantMessage(page) + expect(response).toContain('acoustic') +}) +``` + +## 5. Adding a New Feature + +1. **Add the feature to `src/lib/features.ts`** - define tools, modelOptions, outputSchema +2. **Add the feature to `src/lib/feature-support.ts`** - mark which providers support it +3. **Add the feature to `tests/test-matrix.ts`** - so tests iterate over it +4. **Create a fixture directory** - `fixtures/my-new-feature/basic.json` +5. **Create a test spec** - `tests/my-new-feature.spec.ts` +6. **Update the UI if needed** - if the feature needs new UI beyond ChatUI, add a component + +## 6. Adding a New Provider + +1. **Add the adapter factory to `src/lib/providers.ts`** +2. **Add the provider to `src/lib/feature-support.ts`** +3. **Add the provider to `tests/test-matrix.ts`** +4. **No fixture changes needed** - fixtures are provider-agnostic +5. **Verify llmock supports the provider** + +## 7. Fixture Matching Tips + +- **`userMessage`** is the primary match key - use short, unique strings +- **`sequenceIndex`** is essential for multi-turn and tool-call flows +- **`tool`** matches when the model calls a specific tool +- **`model`** matches a specific model name - avoid unless needed (breaks provider-agnosticism) +- **`predicate`** is a custom function for complex matching - last resort +- Fixtures are matched in order - first match wins + +## 8. Troubleshooting + +- **Test times out waiting for response**: Check that `userMessage` in the fixture exactly matches what `sendMessage()` sends +- **Wrong fixture matched**: Make `userMessage` strings more specific or use `sequenceIndex` +- **"Not supported" shows unexpectedly**: Check `src/lib/feature-support.ts` +- **Fixture works for OpenAI but not Anthropic**: Remove `model` from match field +- **Recording doesn't capture the response**: Verify API key env var is set diff --git a/testing/e2e/fixtures/agentic-structured/basic.json b/testing/e2e/fixtures/agentic-structured/basic.json new file mode 100644 index 000000000..9ee487f48 --- /dev/null +++ b/testing/e2e/fixtures/agentic-structured/basic.json @@ -0,0 +1,24 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[agentic] check inventory and recommend", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "getGuitars", + "arguments": "{}" + } + ] + } + }, + { + "match": { "sequenceIndex": 1 }, + "response": { + "content": "{\"name\":\"Fender Stratocaster\",\"price\":1299,\"reason\":\"Most affordable and versatile option in stock\",\"rating\":5}" + } + } + ] +} diff --git a/testing/e2e/fixtures/chat/basic.json b/testing/e2e/fixtures/chat/basic.json new file mode 100644 index 000000000..9161cfbc5 --- /dev/null +++ b/testing/e2e/fixtures/chat/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "[chat] recommend a guitar" }, + "response": { + "content": "I'd recommend the Fender Stratocaster for its versatile tone and comfortable playability. At $1,299, it's great for beginners and pros alike, covering everything from blues to rock." + } + } + ] +} diff --git a/testing/e2e/fixtures/image-gen/basic.json b/testing/e2e/fixtures/image-gen/basic.json new file mode 100644 index 000000000..b63060677 --- /dev/null +++ b/testing/e2e/fixtures/image-gen/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "a guitar in a music store" }, + "response": { + "content": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + } + ] +} diff --git a/testing/e2e/fixtures/multi-turn/conversation.json b/testing/e2e/fixtures/multi-turn/conversation.json new file mode 100644 index 000000000..d2649642e --- /dev/null +++ b/testing/e2e/fixtures/multi-turn/conversation.json @@ -0,0 +1,20 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[multiturn-1] what guitars do you have" + }, + "response": { + "content": "We have four guitars in stock: the Fender Stratocaster ($1,299), Gibson Les Paul ($2,499), Taylor 814ce ($3,299), and Martin D-28 ($2,999). What would you like to know more about?" + } + }, + { + "match": { + "userMessage": "[multiturn-2] tell me about the cheapest one" + }, + "response": { + "content": "The Fender Stratocaster at $1,299 is our most affordable guitar. It's a versatile electric guitar known for its bright, clear tone and comfortable playability. Perfect for blues, rock, and everything in between." + } + } + ] +} diff --git a/testing/e2e/fixtures/multimodal-image/basic.json b/testing/e2e/fixtures/multimodal-image/basic.json new file mode 100644 index 000000000..c5bbb05f4 --- /dev/null +++ b/testing/e2e/fixtures/multimodal-image/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "[mmimage] describe this image" }, + "response": { + "content": "The image shows a guitar in a music store setting. It appears to be a classic electric guitar with a sunburst finish, displayed on a stand with other instruments visible in the background." + } + } + ] +} diff --git a/testing/e2e/fixtures/multimodal-structured/basic.json b/testing/e2e/fixtures/multimodal-structured/basic.json new file mode 100644 index 000000000..91e77874f --- /dev/null +++ b/testing/e2e/fixtures/multimodal-structured/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "[mmstruct] analyze this image" }, + "response": { + "content": "{\"description\":\"A guitar shop with various instruments on display\",\"objects\":[\"guitar\",\"amplifier\",\"music stand\",\"wall hooks\"],\"mood\":\"warm and inviting\"}" + } + } + ] +} diff --git a/testing/e2e/fixtures/one-shot-text/basic.json b/testing/e2e/fixtures/one-shot-text/basic.json new file mode 100644 index 000000000..932e2d7df --- /dev/null +++ b/testing/e2e/fixtures/one-shot-text/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "[oneshot] what is your most popular guitar" }, + "response": { + "content": "Our most popular guitar is the Fender Stratocaster. It's loved for its versatile tone and comfortable neck profile." + } + } + ] +} diff --git a/testing/e2e/fixtures/reasoning/basic.json b/testing/e2e/fixtures/reasoning/basic.json new file mode 100644 index 000000000..9e8dee6b5 --- /dev/null +++ b/testing/e2e/fixtures/reasoning/basic.json @@ -0,0 +1,13 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[reasoning] recommend a guitar for a beginner" + }, + "response": { + "reasoning": "The user is asking for a beginner guitar recommendation. I should consider factors like playability, price point, and versatility. An electric guitar like the Fender Stratocaster has lower action and thinner strings, making it easier to play. It is also the most affordable option in our inventory at $1,299.", + "content": "I'd recommend the Fender Stratocaster for a beginner. It has a comfortable neck profile, versatile tone across genres, and lower string action that's easier on beginner fingers." + } + } + ] +} diff --git a/testing/e2e/fixtures/recorded/.gitkeep b/testing/e2e/fixtures/recorded/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/testing/e2e/fixtures/structured-output/basic.json b/testing/e2e/fixtures/structured-output/basic.json new file mode 100644 index 000000000..438a08605 --- /dev/null +++ b/testing/e2e/fixtures/structured-output/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "[structured] recommend a guitar as json" }, + "response": { + "content": "{\"name\":\"Fender Stratocaster\",\"price\":1299,\"reason\":\"Versatile tone and comfortable playability\",\"rating\":5}" + } + } + ] +} diff --git a/testing/e2e/fixtures/summarize/basic.json b/testing/e2e/fixtures/summarize/basic.json new file mode 100644 index 000000000..e781c965a --- /dev/null +++ b/testing/e2e/fixtures/summarize/basic.json @@ -0,0 +1,12 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[summarize] The Fender Stratocaster is a versatile electric guitar" + }, + "response": { + "content": "The Fender Stratocaster is a popular, versatile electric guitar suitable for multiple genres." + } + } + ] +} diff --git a/testing/e2e/fixtures/tool-approval/approval.json b/testing/e2e/fixtures/tool-approval/approval.json new file mode 100644 index 000000000..2f81d7a2f --- /dev/null +++ b/testing/e2e/fixtures/tool-approval/approval.json @@ -0,0 +1,24 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[approval] add the stratocaster to my cart", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "addToCart", + "arguments": "{\"guitarId\":\"1\",\"quantity\":1}" + } + ] + } + }, + { + "match": { "sequenceIndex": 1 }, + "response": { + "content": "I've added the Fender Stratocaster to your cart." + } + } + ] +} diff --git a/testing/e2e/fixtures/tool-calling/parallel.json b/testing/e2e/fixtures/tool-calling/parallel.json new file mode 100644 index 000000000..00cc033a4 --- /dev/null +++ b/testing/e2e/fixtures/tool-calling/parallel.json @@ -0,0 +1,28 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[parallel] compare the stratocaster and les paul", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "getGuitars", + "arguments": "{}" + }, + { + "name": "compareGuitars", + "arguments": "{\"guitarIds\":[1,2]}" + } + ] + } + }, + { + "match": { "sequenceIndex": 1 }, + "response": { + "content": "Here's a comparison of the two guitars:\n\n| | Fender Stratocaster | Gibson Les Paul |\n|---|---|---|\n| Price | $1,299 | $2,499 |\n\nThe Stratocaster is $1,200 cheaper and more versatile, while the Les Paul offers a warmer, thicker tone." + } + } + ] +} diff --git a/testing/e2e/fixtures/tool-calling/single.json b/testing/e2e/fixtures/tool-calling/single.json new file mode 100644 index 000000000..5170dfbe9 --- /dev/null +++ b/testing/e2e/fixtures/tool-calling/single.json @@ -0,0 +1,24 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[toolcall] what guitars do you have in stock", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "getGuitars", + "arguments": "{}" + } + ] + } + }, + { + "match": { "sequenceIndex": 1 }, + "response": { + "content": "Here's what we have in stock:\n\n1. **Fender Stratocaster** - $1,299\n2. **Gibson Les Paul** - $2,499\n3. **Taylor 814ce** - $3,299\n4. **Martin D-28** - $2,999\n\nWould you like more details on any of these?" + } + } + ] +} diff --git a/testing/e2e/fixtures/transcription/basic.json b/testing/e2e/fixtures/transcription/basic.json new file mode 100644 index 000000000..599f9a865 --- /dev/null +++ b/testing/e2e/fixtures/transcription/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "test-audio-base64" }, + "response": { + "content": "I would like to buy a Fender Stratocaster please." + } + } + ] +} diff --git a/testing/e2e/fixtures/tts/basic.json b/testing/e2e/fixtures/tts/basic.json new file mode 100644 index 000000000..71be9795e --- /dev/null +++ b/testing/e2e/fixtures/tts/basic.json @@ -0,0 +1,10 @@ +{ + "fixtures": [ + { + "match": { "userMessage": "Welcome to the guitar store" }, + "response": { + "content": "data:audio/mp3;base64,AAAA" + } + } + ] +} diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts new file mode 100644 index 000000000..e6f2ae381 --- /dev/null +++ b/testing/e2e/global-setup.ts @@ -0,0 +1,24 @@ +import { LLMock } from '@copilotkit/llmock' +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +export default async function globalSetup() { + const mock = new LLMock({ port: 4010, host: '127.0.0.1', logLevel: 'info' }) + + // loadFixtureDir doesn't recurse into subdirectories, so load each one individually + const fixturesDir = path.resolve(__dirname, 'fixtures') + const entries = fs.readdirSync(fixturesDir, { withFileTypes: true }) + for (const entry of entries) { + if (entry.isDirectory() && entry.name !== 'recorded') { + await mock.loadFixtureDir(path.join(fixturesDir, entry.name)) + } + } + + await mock.start() + console.log(`[global-setup] llmock started at ${mock.url}`) + ;(globalThis as any).__llmock = mock +} diff --git a/testing/e2e/global-teardown.ts b/testing/e2e/global-teardown.ts new file mode 100644 index 000000000..cd9830e28 --- /dev/null +++ b/testing/e2e/global-teardown.ts @@ -0,0 +1,7 @@ +export default async function globalTeardown() { + const mock = (globalThis as any).__llmock + if (mock) { + await mock.stop() + console.log('[global-teardown] llmock stopped') + } +} diff --git a/testing/e2e/package.json b/testing/e2e/package.json new file mode 100644 index 000000000..a84433915 --- /dev/null +++ b/testing/e2e/package.json @@ -0,0 +1,50 @@ +{ + "name": "@tanstack/ai-e2e", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3010", + "build": "vite build", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui", + "record": "LLMOCK_RECORD=true pnpm dev", + "postinstall": "playwright install chromium" + }, + "dependencies": { + "@copilotkit/llmock": "latest", + "@tailwindcss/vite": "^4.1.18", + "@tanstack/ai": "workspace:*", + "@tanstack/ai-anthropic": "workspace:*", + "@tanstack/ai-client": "workspace:*", + "@tanstack/ai-gemini": "workspace:*", + "@tanstack/ai-grok": "workspace:*", + "@tanstack/ai-groq": "workspace:*", + "@tanstack/ai-ollama": "workspace:*", + "@tanstack/ai-openai": "workspace:*", + "@tanstack/ai-openrouter": "workspace:*", + "@tanstack/ai-react": "workspace:*", + "@tanstack/ai-react-ui": "workspace:*", + "@tanstack/nitro-v2-vite-plugin": "^1.154.7", + "@tanstack/react-router": "^1.158.4", + "@tanstack/react-start": "^1.159.0", + "react": "^19.2.3", + "react-dom": "^19.2.3", + "react-markdown": "^10.1.0", + "rehype-highlight": "^7.0.2", + "rehype-raw": "^7.0.0", + "rehype-sanitize": "^6.0.0", + "remark-gfm": "^4.0.1", + "tailwindcss": "^4.1.18", + "vite-tsconfig-paths": "^5.1.4", + "zod": "^4.2.0" + }, + "devDependencies": { + "@playwright/test": "^1.57.0", + "@types/node": "^24.10.1", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.1.2", + "typescript": "5.9.3", + "vite": "^7.2.7" + } +} diff --git a/testing/e2e/playwright.config.ts b/testing/e2e/playwright.config.ts new file mode 100644 index 000000000..cc902bdcd --- /dev/null +++ b/testing/e2e/playwright.config.ts @@ -0,0 +1,34 @@ +import { defineConfig, devices } from '@playwright/test' + +export default defineConfig({ + testDir: './tests', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + workers: process.env.CI ? 4 : undefined, + reporter: [['html', { open: 'never' }], ['list']], + timeout: 30_000, + expect: { + timeout: 15_000, + }, + use: { + baseURL: 'http://localhost:3010', + video: 'on', + screenshot: 'only-on-failure', + trace: 'on-first-retry', + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], + globalSetup: './global-setup.ts', + globalTeardown: './global-teardown.ts', + webServer: { + command: 'pnpm run dev', + url: 'http://localhost:3010', + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, +}) diff --git a/testing/e2e/src/components/ApprovalPrompt.tsx b/testing/e2e/src/components/ApprovalPrompt.tsx new file mode 100644 index 000000000..e1f151e17 --- /dev/null +++ b/testing/e2e/src/components/ApprovalPrompt.tsx @@ -0,0 +1,38 @@ +export function ApprovalPrompt({ + part, + onRespond, +}: { + part: any + onRespond: (response: { id: string; approved: boolean }) => Promise +}) { + return ( +
+
+ Tool {part.name} requires + approval +
+
+ Args: {part.arguments} +
+
+ + +
+
+ ) +} diff --git a/testing/e2e/src/components/AudioPlayer.tsx b/testing/e2e/src/components/AudioPlayer.tsx new file mode 100644 index 000000000..79c827dec --- /dev/null +++ b/testing/e2e/src/components/AudioPlayer.tsx @@ -0,0 +1,12 @@ +export function AudioPlayer({ src }: { src: string }) { + return ( +
+
+ ) +} diff --git a/testing/e2e/src/components/ChatUI.tsx b/testing/e2e/src/components/ChatUI.tsx new file mode 100644 index 000000000..4321f4818 --- /dev/null +++ b/testing/e2e/src/components/ChatUI.tsx @@ -0,0 +1,174 @@ +import { useEffect, useRef, useState } from 'react' +import type { UIMessage } from '@tanstack/ai-react' +import ReactMarkdown from 'react-markdown' +import rehypeRaw from 'rehype-raw' +import rehypeSanitize from 'rehype-sanitize' +import remarkGfm from 'remark-gfm' +import { ToolCallDisplay } from '@/components/ToolCallDisplay' +import { ApprovalPrompt } from '@/components/ApprovalPrompt' + +interface ChatUIProps { + messages: Array + isLoading: boolean + onSendMessage: (text: string) => void + onSendMessageWithImage?: (text: string, file: File) => void + addToolApprovalResponse?: (response: { + id: string + approved: boolean + }) => Promise + showImageInput?: boolean +} + +export function ChatUI({ + messages, + isLoading, + onSendMessage, + onSendMessageWithImage, + addToolApprovalResponse, + showImageInput, +}: ChatUIProps) { + const [input, setInput] = useState('') + const messagesRef = useRef(null) + + useEffect(() => { + if (messagesRef.current) { + messagesRef.current.scrollTop = messagesRef.current.scrollHeight + } + }, [messages]) + + const handleSubmit = () => { + if (!input.trim()) return + onSendMessage(input.trim()) + setInput('') + } + + return ( +
+
+ {messages.map((message) => ( +
+ {message.parts.map((part, i) => { + if (part.type === 'text') { + return ( +
+ + {part.content} + +
+ ) + } + if (part.type === 'thinking') { + return ( +
+ {part.content} +
+ ) + } + if ( + part.type === 'tool-call' && + (part as any).state === 'approval-requested' && + addToolApprovalResponse + ) { + return ( + + ) + } + if (part.type === 'tool-call') { + return + } + if (part.type === 'tool-result') { + return ( +
+ Result: {(part as any).content} +
+ ) + } + return null + })} +
+ ))} +
+ + {isLoading && ( +
+ Generating... +
+ )} + +
+ {showImageInput && ( + { + const file = e.target.files?.[0] + if (file && input.trim() && onSendMessageWithImage) { + onSendMessageWithImage(input.trim(), file) + setInput('') + } + }} + /> + )} + setInput(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault() + handleSubmit() + } + }} + placeholder="Type a message..." + className="flex-1 bg-gray-800 border border-gray-700 rounded px-3 py-2 text-sm focus:outline-none focus:border-orange-500/50" + /> + +
+
+ ) +} diff --git a/testing/e2e/src/components/ImageDisplay.tsx b/testing/e2e/src/components/ImageDisplay.tsx new file mode 100644 index 000000000..f6aa85415 --- /dev/null +++ b/testing/e2e/src/components/ImageDisplay.tsx @@ -0,0 +1,12 @@ +export function ImageDisplay({ src, alt }: { src: string; alt?: string }) { + return ( +
+ {alt +
+ ) +} diff --git a/testing/e2e/src/components/NotSupported.tsx b/testing/e2e/src/components/NotSupported.tsx new file mode 100644 index 000000000..9eeab8c33 --- /dev/null +++ b/testing/e2e/src/components/NotSupported.tsx @@ -0,0 +1,17 @@ +export function NotSupported({ + provider, + feature, +}: { + provider: string + feature: string +}) { + return ( +
+

+ {provider} does + not support{' '} + {feature} +

+
+ ) +} diff --git a/testing/e2e/src/components/SummarizeUI.tsx b/testing/e2e/src/components/SummarizeUI.tsx new file mode 100644 index 000000000..0f1772cf9 --- /dev/null +++ b/testing/e2e/src/components/SummarizeUI.tsx @@ -0,0 +1,39 @@ +import { useState } from 'react' + +interface SummarizeUIProps { + onSubmit: (text: string) => void + result: string | null + isLoading: boolean +} + +export function SummarizeUI({ onSubmit, result, isLoading }: SummarizeUIProps) { + const [input, setInput] = useState('') + + return ( +
+