diff --git a/packages/assertions/src/engine/AssertionEngine.ts b/packages/assertions/src/engine/AssertionEngine.ts index 0d4bef7..b634175 100644 --- a/packages/assertions/src/engine/AssertionEngine.ts +++ b/packages/assertions/src/engine/AssertionEngine.ts @@ -1,6 +1,4 @@ -import { EvaliphyError, EvaliphyErrorCode, logger } from '@evaliphy/core'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { EvaliphyError, EvaliphyErrorCode } from '@evaliphy/core'; import type { BaseMatcher } from '../matchers/base/BaseMatcher.js'; import { PromptLoader } from '../promptManager/promptLoader.js'; import { PromptRenderer } from '../promptManager/promptRenderer.js'; @@ -8,12 +6,15 @@ import { assertionRegistry } from "../registry.js"; import { type AssertionContext, type AssertionResult } from './types.js'; export class AssertionEngine { + /** + * Runs the assertion logic, executing an LLM call if the matcher requires it. + */ static async run( matcher: BaseMatcher, context: AssertionContext ): Promise { const startTime = Date.now(); - const { input, options, llmClient, config } = context; + const { input, options, llmClient } = context; try { matcher.validate(input); @@ -33,7 +34,6 @@ export class AssertionEngine { usage = response.llmUsages; } - const threshold = options.threshold ?? 0.7; const passed = score >= threshold; @@ -58,6 +58,10 @@ export class AssertionEngine { } } + /** + * Prepares the LLM prompt and schema for the assertion. + * Centralizes prompt loading through PromptLoader. + */ private static prepareLLMRequest(matcher: BaseMatcher, context: AssertionContext) { const { input, config } = context; const assertionDef = assertionRegistry[matcher.name]; @@ -69,7 +73,8 @@ export class AssertionEngine { ); } - const loadedPrompt = this.loadPrompt(matcher.name, assertionDef, config); + // Call the centralized PromptLoader to handle path resolution and loading + const loadedPrompt = PromptLoader.resolveAndLoad(matcher.name, assertionDef, config); const variables = this.prepareVariables(input); const finalPrompt = PromptRenderer.render(loadedPrompt.template, variables, assertionDef); const outputSchema = assertionDef.outputSchema.zodSchema as any; @@ -77,67 +82,9 @@ export class AssertionEngine { return { finalPrompt, outputSchema }; } - private static loadPrompt(matcherName: string, assertionDef: any, config: any) { - const configDir = config.configFile ? path.dirname(config.configFile) : process.cwd(); - - // 1. Check for custom prompts directory from config - const customPromptsDir = config.llmAsJudgeConfig?.promptsDir - ? path.resolve(configDir, config.llmAsJudgeConfig.promptsDir) - : null; - - // 2. Check for default prompts directory in consumer's root (convention) - const localPromptsDir = path.join(process.cwd(), 'prompts'); - - // 3. SDK internal prompts directory - const __dirname = path.dirname(fileURLToPath(import.meta.url)); - - // Support both source (../../prompts) and bundled dist (./prompts) - const sdkPromptsDirSource = path.resolve(__dirname, '../../prompts'); - const sdkPromptsDirDist = path.resolve(__dirname, './prompts'); - - const searchPaths = [ - customPromptsDir ? path.join(customPromptsDir, `${matcherName}.md`) : null, - path.join(localPromptsDir, `${matcherName}.md`), - path.join(sdkPromptsDirDist, `${matcherName}.md`), - path.join(sdkPromptsDirSource, `${matcherName}.md`), - ].filter(Boolean) as string[]; - - try { - // 1. Try custom prompts directory from config - if (customPromptsDir) { - const customPath = path.join(customPromptsDir, `${matcherName}.md`); - if (PromptLoader.exists(customPath)) { - return PromptLoader.load(customPath, assertionDef); - } else { - console.warn(`Custom prompt file not found at: ${customPath}. Falling back to defaults.`); - } - } - - // 2. Try other search paths (local convention, SDK dist, SDK source) - const fallbackPaths = [ - path.join(localPromptsDir, `${matcherName}.md`), - path.join(sdkPromptsDirDist, `${matcherName}.md`), - path.join(sdkPromptsDirSource, `${matcherName}.md`), - ]; - - for (const promptPath of fallbackPaths) { - if (PromptLoader.exists(promptPath)) { - return PromptLoader.load(promptPath, assertionDef); - } - } - - // If none found, try the primary SDK path one last time to trigger the standard error - return PromptLoader.load(path.join(sdkPromptsDirDist, `${matcherName}.md`), assertionDef); - } catch (error: any) { - throw new EvaliphyError( - EvaliphyErrorCode.PROMPT_LOAD_ERROR, - `Failed to load prompt for "${matcherName}": ${error.message}`, - 'Check your prompt file formatting and variables.', - error - ); - } - } - + /** + * Prepares the variables for the prompt renderer. + */ private static prepareVariables(input: any) { return { ...Object.fromEntries( @@ -149,6 +96,9 @@ export class AssertionEngine { } as Record; } + /** + * Executes the actual LLM call using the provided client. + */ private static async executeLLMCall(matcher: BaseMatcher, llmClient: any, prompt: string, schema: any) { try { return await llmClient.generateObject(prompt, schema); diff --git a/packages/assertions/src/promptManager/promptLoader.ts b/packages/assertions/src/promptManager/promptLoader.ts index 55a1eb7..e376338 100644 --- a/packages/assertions/src/promptManager/promptLoader.ts +++ b/packages/assertions/src/promptManager/promptLoader.ts @@ -1,5 +1,7 @@ -import { EvaliphyError, EvaliphyErrorCode } from '@evaliphy/core'; +import { EvaliphyError, EvaliphyErrorCode, logger } from '@evaliphy/core'; import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { AssertionDefinition } from "../registry.js"; export interface LoadedPrompt { @@ -11,7 +13,89 @@ export interface LoadedPrompt { }; } +/** + * Service to handle centralized prompt searching and loading. + */ export class PromptLoader { + /** + * Main entry point to resolve and load a prompt. + */ + static resolveAndLoad(matcherName: string, assertion: AssertionDefinition, config: any): LoadedPrompt { + try { + const filePath = this.resolvePromptPath(matcherName, config); + return this.load(filePath, assertion); + } catch (error: any) { + if (error instanceof EvaliphyError) throw error; + throw new EvaliphyError( + EvaliphyErrorCode.PROMPT_LOAD_ERROR, + `Failed to load prompt for "${matcherName}": ${error.message}`, + 'Check your prompt file formatting and variables.', + error + ); + } + } + + /** + * Orchestrates the discovery process using single-responsibility check methods. + */ + private static resolvePromptPath(matcherName: string, config: any): string { + const fileName = `${matcherName}.md`; + + // 1. Check User Config (Priority 1) + const customPath = this.checkUserConfig(config, fileName); + if (customPath) return customPath; + + // 2. Check SDK Dist (Fallback 1) + const distPath = this.checkSdkDist(fileName); + if (distPath) return distPath; + + // 3. Check SDK Source (Fallback 2) + const sourcePath = this.checkSdkSource(fileName); + if (sourcePath) return sourcePath; + + // Default return to Dist path if none found (to trigger FILE_NOT_FOUND error in load()) + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + return path.resolve(__dirname, './prompts', fileName); + } + + /** + * Priority 1: Check for custom prompts directory from user config. + */ + private static checkUserConfig(config: any, fileName: string): string | null { + const configDir = config.configFile ? path.dirname(config.configFile) : process.cwd(); + const promptsDir = config.llmAsJudgeConfig?.promptsDir; + + if (!promptsDir) return null; + + const customPath = path.resolve(configDir, promptsDir, fileName); + if (this.exists(customPath)) { + return customPath; + } + + logger.warn(`Custom prompt file not found at: ${customPath}. Falling back to defaults.`); + return null; + } + + /** + * Fallback 1: Check for default prompts directory in bundled SDK (Dist). + */ + private static checkSdkDist(fileName: string): string | null { + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const distPath = path.resolve(__dirname, './prompts', fileName); + + return this.exists(distPath) ? distPath : null; + } + + /** + * Fallback 2: Check for default prompts directory in SDK Source (for development). + */ + private static checkSdkSource(fileName: string): string | null { + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const sourcePath = path.resolve(__dirname, '../../prompts', fileName); + + return this.exists(sourcePath) ? sourcePath : null; + } + static exists(filePath: string): boolean { return fs.existsSync(filePath); } @@ -52,7 +136,6 @@ export class PromptLoader { } const frontmatter: any = {}; - // Simple YAML-like parser for basic frontmatter yamlStr.split('\n').forEach(line => { const [key, ...rest] = line.split(':'); if (key && rest.length > 0) { @@ -60,14 +143,12 @@ export class PromptLoader { if (value === '') { frontmatter[key.trim()] = []; } else if (value.startsWith('-')) { - // Handle simple lists const listKey = key.trim(); if (!frontmatter[listKey]) frontmatter[listKey] = []; } else { frontmatter[key.trim()] = value; } } else if (line.trim().startsWith('-')) { - // Continue list const lastKey = Object.keys(frontmatter).pop(); if (lastKey && Array.isArray(frontmatter[lastKey])) { frontmatter[lastKey].push(line.trim().substring(1).trim()); @@ -88,7 +169,6 @@ export class PromptLoader { const declared = frontmatter.input_variables ?? []; const usedInTemplate = this.extractTemplateVariables(template); - // check declared variables match what the assertion requires const missingDeclared = required.filter(v => !declared.includes(v)); if (missingDeclared.length > 0) { throw new EvaliphyError( @@ -98,7 +178,6 @@ export class PromptLoader { ); } - // check the template actually uses the variables it declares const missingInTemplate = declared.filter((v: string) => !usedInTemplate.includes(v)); if (missingInTemplate.length > 0) { throw new EvaliphyError( diff --git a/packages/assertions/tests/judge/promptLoader.test.ts b/packages/assertions/tests/judge/promptLoader.test.ts index e70c564..4f7dba2 100644 --- a/packages/assertions/tests/judge/promptLoader.test.ts +++ b/packages/assertions/tests/judge/promptLoader.test.ts @@ -1,101 +1,121 @@ -import { EvaliphyError } from '@evaliphy/core'; +import { EvaliphyErrorCode } from '@evaliphy/core'; import fs from 'node:fs'; -import { describe, expect, it, vi } from 'vitest'; +import path from 'node:path'; +import { describe, expect, it, vi, beforeEach } from 'vitest'; import { PromptLoader } from '../../src/promptManager/promptLoader.js'; -import { assertionRegistry } from "../../src/registry.js"; +// Mock fs to control file existence and content vi.mock('node:fs'); describe('PromptLoader', () => { - const mockAssertion = assertionRegistry.toBeFaithful; + const mockAssertion: any = { + name: 'testMatcher', + inputVariables: ['query', 'response'], + outputSchema: { zodSchema: {} } + }; - it('should load and validate a correct prompt', () => { - const mockContent = `--- -name: toBeFaithful + const mockPromptContent = `--- +name: testMatcher input_variables: - - question - - context + - query - response --- -Question: {{question}} -Context: {{context}} -Response: {{response}} -`; - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockReturnValue(mockContent); - - const result = PromptLoader.load('test.md', mockAssertion); - expect(result.template).toContain('Question: {{question}}'); - expect(result.frontmatter.name).toBe('toBeFaithful'); - }); +Query: {{query}} +Response: {{response}}`; - it('should throw error if file not found', () => { - vi.mocked(fs.existsSync).mockReturnValue(false); - expect(() => PromptLoader.load('missing.md', mockAssertion)).toThrow(EvaliphyError); + beforeEach(() => { + vi.clearAllMocks(); }); - it('should throw error if required variables are missing in frontmatter', () => { - const mockContent = `--- -name: toBeFaithful -input_variables: - - question ---- -{{question}} -`; - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockReturnValue(mockContent); + describe('resolveAndLoad', () => { + it('should load from Priority 1: User Config (promptsDir)', () => { + const config = { + configFile: '/user/project/evaliphy.config.ts', + llmAsJudgeConfig: { + promptsDir: './custom-prompts' + } + }; - expect(() => PromptLoader.load('test.md', mockAssertion)).toThrow(/missing required input_variables/); - }); + // Target path: /user/project/custom-prompts/testMatcher.md + vi.spyOn(fs, 'existsSync').mockImplementation((p: any) => p.includes('custom-prompts')); + vi.spyOn(fs, 'readFileSync').mockReturnValue(mockPromptContent); - it('should throw error if declared variables are not used in template', () => { - const mockContent = `--- -name: toBeFaithful -input_variables: - - question - - context - - response ---- -{{question}} {{context}} {{response}} -`; - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockReturnValue(mockContent); - - // This test was failing because it was missing required variables in input_variables - // but also testing for template usage. Let's fix the template usage test. - const mockContent2 = `--- -name: toBeFaithful -input_variables: - - question - - context - - response ---- -{{question}} {{context}} -`; - vi.mocked(fs.readFileSync).mockReturnValue(mockContent2); - expect(() => PromptLoader.load('test.md', mockAssertion)).toThrow(/never uses them in the template/); - }); + const result = PromptLoader.resolveAndLoad('testMatcher', mockAssertion, config); + + expect(fs.existsSync).toHaveBeenCalled(); + expect(result.template).toContain('Query: {{query}}'); + expect(result.frontmatter.name).toBe('testMatcher'); + }); + + it('should fallback to Fallback 1: SDK Dist if Priority 1 fails', () => { + const config = { + llmAsJudgeConfig: { promptsDir: './non-existent' } + }; - it('should throw error if frontmatter is missing', () => { - const mockContent = `No frontmatter here`; - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockReturnValue(mockContent); + // Mock existsSync to return false for custom but true for dist + vi.spyOn(fs, 'existsSync').mockImplementation((p: any) => { + if (p.includes('non-existent')) return false; + if (p.includes('dist') || p.includes('src/promptManager/prompts')) return true; + return false; + }); + vi.spyOn(fs, 'readFileSync').mockReturnValue(mockPromptContent); - expect(() => PromptLoader.load('test.md', mockAssertion)).toThrow(/Missing frontmatter block/); + const result = PromptLoader.resolveAndLoad('testMatcher', mockAssertion, config); + + expect(result.template).toBeDefined(); + expect(fs.readFileSync).toHaveBeenCalled(); + }); + + it('should fallback to Fallback 2: SDK Source if Dist fails', () => { + const config = { llmAsJudgeConfig: {} }; + + // Mock existsSync to return true only for the source path + vi.spyOn(fs, 'existsSync').mockImplementation((p: any) => { + // Dist path is usually relative to __dirname which is src/promptManager/ + if (p.includes('src/promptManager/prompts')) return false; + if (p.includes('../../prompts')) return true; + return false; + }); + vi.spyOn(fs, 'readFileSync').mockReturnValue(mockPromptContent); + + const result = PromptLoader.resolveAndLoad('testMatcher', mockAssertion, config); + + expect(result.template).toBeDefined(); + expect(fs.readFileSync).toHaveBeenCalled(); + }); + + it('should throw EvaliphyError if prompt is not found anywhere', () => { + const config = {}; + vi.spyOn(fs, 'existsSync').mockReturnValue(false); + + try { + PromptLoader.resolveAndLoad('testMatcher', mockAssertion, config); + } catch (error: any) { + expect(error.code).toBe(EvaliphyErrorCode.FILE_NOT_FOUND); + } + }); }); - it('should throw error if template is empty', () => { - const mockContent = `--- -name: toBeFaithful -input_variables: - - question - - context - - response ---- -`; - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockReturnValue(mockContent); + describe('load', () => { + it('should throw error if file does not exist', () => { + vi.spyOn(fs, 'existsSync').mockReturnValue(false); + expect(() => PromptLoader.load('missing.md', mockAssertion)).toThrow(/Prompt file not found/); + }); + + it('should validate missing input variables in frontmatter', () => { + const invalidContent = `---\nname: test\ninput_variables: [query]\n---\n{{query}}`; + vi.spyOn(fs, 'existsSync').mockReturnValue(true); + vi.spyOn(fs, 'readFileSync').mockReturnValue(invalidContent); + + expect(() => PromptLoader.load('test.md', mockAssertion)).toThrow(/missing required input_variables: response/); + }); + + it('should validate unused variables in template', () => { + const invalidContent = `---\nname: test\ninput_variables: [query, response]\n---\n{{query}}`; + vi.spyOn(fs, 'existsSync').mockReturnValue(true); + vi.spyOn(fs, 'readFileSync').mockReturnValue(invalidContent); - expect(() => PromptLoader.load('test.md', mockAssertion)).toThrow(/template at "test.md" is empty/); + expect(() => PromptLoader.load('test.md', mockAssertion)).toThrow(/declares input_variables \[response\] but never uses them/); + }); }); });