From 18f75aa8f4380526aede3c16b906191e7ac6cccd Mon Sep 17 00:00:00 2001 From: Steve Robertson Date: Thu, 4 Jun 2026 17:40:35 +0100 Subject: [PATCH 1/3] test(ai-orchestrator): add generic query classification tests Add 6 test cases for mesh router generic query inference: - Status/requirement queries route to po - Architecture/tooling queries route to architect - Accessibility/compliance queries route to a11y --- .../src/__tests__/mesh-router.spec.ts | 62 +++++++++++++++++++ .../src/orchestrator/mesh-router.ts | 18 ++++-- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts b/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts index c274ac7..1f69d29 100644 --- a/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts +++ b/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts @@ -119,6 +119,68 @@ describe('analyzeMeshQuery', () => { expect(result).toEqual({ target: null }); }); }); + + describe('generic query inference (without explicit persona tags)', () => { + it('infers po for status/requirements queries', async () => { + const client = fakeClient('{"target": "po"}'); + const messages = [ + makeMessage('@isolate- What is the status of this issue?'), + ]; + const result = await analyzeMeshQuery(messages, client); + expect(result).toEqual({ target: 'po' }); + }); + + it('infers po for design token queries', async () => { + const client = fakeClient('{"target": "po"}'); + const messages = [ + makeMessage('@isolate- What design tokens are available for colors?'), + ]; + const result = await analyzeMeshQuery(messages, client); + expect(result).toEqual({ target: 'po' }); + }); + + it('infers architect for monorepo structure queries', async () => { + const client = fakeClient('{"target": "architect"}'); + const messages = [ + makeMessage( + '@isolate- How are the presets structured in the monorepo?', + ), + ]; + const result = await analyzeMeshQuery(messages, client); + expect(result).toEqual({ target: 'architect' }); + }); + + it('infers architect for dependency queries', async () => { + const client = fakeClient('{"target": "architect"}'); + const messages = [ + makeMessage('@isolate- What dependencies do we use for testing?'), + ]; + const result = await analyzeMeshQuery(messages, client); + expect(result).toEqual({ target: 'architect' }); + }); + + it('infers a11y for accessibility compliance queries', async () => { + const client = fakeClient('{"target": "a11y"}'); + const messages = [ + makeMessage( + '@isolate- How do we ensure ARIA compliance in components?', + ), + ]; + const result = await analyzeMeshQuery(messages, client); + expect(result).toEqual({ target: 'a11y' }); + }); + + it('infers a11y for keyboard navigation queries', async () => { + const client = fakeClient('{"target": "a11y"}'); + const messages = [ + makeMessage( + '@isolate- What keyboard navigation patterns are supported?', + ), + ]; + const result = await analyzeMeshQuery(messages, client); + expect(result).toEqual({ target: 'a11y' }); + }); + }); }); // ── createMeshRouterNode ────────────────────────────────────────────────────── diff --git a/libs/ai-orchestrator/src/orchestrator/mesh-router.ts b/libs/ai-orchestrator/src/orchestrator/mesh-router.ts index 4b10d1d..9c57c12 100644 --- a/libs/ai-orchestrator/src/orchestrator/mesh-router.ts +++ b/libs/ai-orchestrator/src/orchestrator/mesh-router.ts @@ -49,13 +49,23 @@ export interface MeshQueryResult { const MESH_SYSTEM_PROMPT = `You are a routing classifier for an AI multi-agent orchestration system. -Analyze the provided message and determine whether it contains a directed query or request aimed at a specific agent persona. +Analyze the provided message and determine the best-fit specialist persona for any query, or identify non-query work outputs. -Valid persona IDs: po, architect, dev, a11y, qa, docs +Valid persona IDs and their domains: +- po: Project status, requirements, design tokens, high-level planning +- architect: Monorepo structure, tooling, dependencies, presets +- dev: Code implementation, Panda CSS logic, TypeScript types +- a11y: Accessibility standards, ARIA, keyboard navigation +- qa: Test coverage, edge cases, bug verification +- docs: Documentation, Storybook, usage examples Rules: -- Return a non-null target ONLY when the message explicitly asks a question of, or directs a task to, a specific persona — for example: "I need @isolate-po to clarify the token choice" or "Can qa verify this edge case?". -- Return null for standard work outputs, APPROVED/REJECTED decisions, inline explanations, or any message that does not address a specific persona as the recipient of a query. +- For messages starting with @isolate- that contain a query (?): infer the best-fit persona based on question content. For example: + * "What is the status of this issue?" → po + * "How are the presets structured?" → architect + * "How do we ensure ARIA compliance?" → a11y +- For explicit persona tags (e.g., "@isolate-po", "Can qa verify"): return the explicitly named persona. +- For standard work outputs, APPROVED/REJECTED decisions, or inline explanations with no cross-persona query: return null. Respond with ONLY valid JSON in this exact format, with no additional text: {"target": "persona_id"} From 2d15e97a71f7877739f35644412e9508a72fbf2b Mon Sep 17 00:00:00 2001 From: Steve Robertson Date: Thu, 4 Jun 2026 17:44:28 +0100 Subject: [PATCH 2/3] test(ai-orchestrator): add prompt constant validation tests Validate that MESH_SYSTEM_PROMPT includes all 6 persona domains and contains required routing instruction keywords. This prevents accidental prompt regression during future refactors. --- .../src/__tests__/mesh-router.spec.ts | 17 +++++++++++++++++ .../src/orchestrator/mesh-router.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts b/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts index 1f69d29..f6833fd 100644 --- a/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts +++ b/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts @@ -6,6 +6,7 @@ import { createMeshRouterNode, DEFAULT_MESH_CONFIG, _test_createDefaultMeshClient, + MESH_SYSTEM_PROMPT, type MeshRouterConfig, } from '../orchestrator/mesh-router'; import { createDefaultAgentState } from '../schema'; @@ -401,3 +402,19 @@ describe('_test_createDefaultMeshClient', () => { expect(() => _test_createDefaultMeshClient()).toThrow(/OPENAI_API_KEY/); }); }); + +describe('MESH_SYSTEM_PROMPT', () => { + it('includes all six persona domain descriptions', () => { + const personas = ['po', 'architect', 'dev', 'a11y', 'qa', 'docs']; + + personas.forEach((persona) => { + expect(MESH_SYSTEM_PROMPT).toContain(persona); + }); + }); + + it('contains required routing instruction keywords', () => { + expect(MESH_SYSTEM_PROMPT).toContain('@isolate-'); + expect(MESH_SYSTEM_PROMPT).toContain('best-fit'); + expect(MESH_SYSTEM_PROMPT).toContain('JSON'); + }); +}); diff --git a/libs/ai-orchestrator/src/orchestrator/mesh-router.ts b/libs/ai-orchestrator/src/orchestrator/mesh-router.ts index 9c57c12..7a41095 100644 --- a/libs/ai-orchestrator/src/orchestrator/mesh-router.ts +++ b/libs/ai-orchestrator/src/orchestrator/mesh-router.ts @@ -47,7 +47,7 @@ export interface MeshQueryResult { // ── LLM prompt ──────────────────────────────────────────────────────────────── -const MESH_SYSTEM_PROMPT = `You are a routing classifier for an AI multi-agent orchestration system. +export const MESH_SYSTEM_PROMPT = `You are a routing classifier for an AI multi-agent orchestration system. Analyze the provided message and determine the best-fit specialist persona for any query, or identify non-query work outputs. From 470f23d6ce61a645aa73056bed615fadd2a3565d Mon Sep 17 00:00:00 2001 From: Steve Robertson Date: Thu, 4 Jun 2026 17:49:34 +0100 Subject: [PATCH 3/3] fix(ai-orchestrator): clarify mesh router prompt inference scope Address review feedback: 1. Update MESH_SYSTEM_PROMPT to explicitly state that inference only applies to messages starting with @isolate-, preventing unintended mesh jumps for ordinary in-thread questions without explicit persona tags. 2. Change 'best-fit' to 'ONLY' in the keyword validation test to clarify the restricted scope. 3. Strengthen domain description test to check for actual bullet prefixes ('- po:', etc.) instead of loose substring matching. --- .../src/__tests__/mesh-router.spec.ts | 17 ++++++++++++----- .../src/orchestrator/mesh-router.ts | 7 ++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts b/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts index f6833fd..79214de 100644 --- a/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts +++ b/libs/ai-orchestrator/src/__tests__/mesh-router.spec.ts @@ -404,17 +404,24 @@ describe('_test_createDefaultMeshClient', () => { }); describe('MESH_SYSTEM_PROMPT', () => { - it('includes all six persona domain descriptions', () => { - const personas = ['po', 'architect', 'dev', 'a11y', 'qa', 'docs']; + it('includes all six persona domain descriptions with proper bullet format', () => { + const domainBullets = [ + '- po:', + '- architect:', + '- dev:', + '- a11y:', + '- qa:', + '- docs:', + ]; - personas.forEach((persona) => { - expect(MESH_SYSTEM_PROMPT).toContain(persona); + domainBullets.forEach((bullet) => { + expect(MESH_SYSTEM_PROMPT).toContain(bullet); }); }); it('contains required routing instruction keywords', () => { expect(MESH_SYSTEM_PROMPT).toContain('@isolate-'); - expect(MESH_SYSTEM_PROMPT).toContain('best-fit'); + expect(MESH_SYSTEM_PROMPT).toContain('ONLY'); expect(MESH_SYSTEM_PROMPT).toContain('JSON'); }); }); diff --git a/libs/ai-orchestrator/src/orchestrator/mesh-router.ts b/libs/ai-orchestrator/src/orchestrator/mesh-router.ts index 7a41095..7cc849e 100644 --- a/libs/ai-orchestrator/src/orchestrator/mesh-router.ts +++ b/libs/ai-orchestrator/src/orchestrator/mesh-router.ts @@ -49,7 +49,7 @@ export interface MeshQueryResult { export const MESH_SYSTEM_PROMPT = `You are a routing classifier for an AI multi-agent orchestration system. -Analyze the provided message and determine the best-fit specialist persona for any query, or identify non-query work outputs. +Analyze the provided message and identify the target specialist persona, or return null if no cross-persona routing is needed. Valid persona IDs and their domains: - po: Project status, requirements, design tokens, high-level planning @@ -60,12 +60,13 @@ Valid persona IDs and their domains: - docs: Documentation, Storybook, usage examples Rules: -- For messages starting with @isolate- that contain a query (?): infer the best-fit persona based on question content. For example: +- ONLY for messages starting with @isolate- that contain a query (?): infer the best-fit persona based on question content. For example: * "What is the status of this issue?" → po * "How are the presets structured?" → architect * "How do we ensure ARIA compliance?" → a11y - For explicit persona tags (e.g., "@isolate-po", "Can qa verify"): return the explicitly named persona. -- For standard work outputs, APPROVED/REJECTED decisions, or inline explanations with no cross-persona query: return null. +- For ALL OTHER messages (no @isolate- prefix and no explicit persona tag): return null, even if they contain questions. +- Return null for standard work outputs, APPROVED/REJECTED decisions, and inline explanations. Respond with ONLY valid JSON in this exact format, with no additional text: {"target": "persona_id"}