diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 211f571..fcecccd 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -25,7 +25,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt', 'Print full prompt and injected content') .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Print full JSON response from the API') - .option('--output ', 'Output format: line (default), json, or vale-json', 'line') + .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') .option('--config ', 'Path to custom vectorlint.ini config file') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index be3c59a..c909df8 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -4,6 +4,7 @@ import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; +import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; import { locateEvidenceWithMatch } from '../output/location'; import { checkTarget } from '../prompts/target'; @@ -11,6 +12,7 @@ import { isSubjectiveResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; +import { OutputFormat } from './types'; import type { EvaluationOptions, EvaluationResult, ErrorTrackingResult, ReportIssueParams, ExtractMatchTextParams, LocationMatch, ProcessViolationsParams, @@ -56,10 +58,10 @@ async function runWithConcurrency( function reportIssue(params: ReportIssueParams): void { const { file, line, column, severity, summary, ruleName, outputFormat, jsonFormatter, suggestion, scoreText, match } = params; - if (outputFormat === 'line') { + if (outputFormat === OutputFormat.Line) { const locStr = `${line}:${column}`; printIssueRow(locStr, severity, summary, ruleName, suggestion ? { suggestion } : {}); - } else if (outputFormat === 'vale-json') { + } else if (outputFormat === OutputFormat.ValeJson) { const issue: JsonIssue = { file, line, @@ -73,7 +75,8 @@ function reportIssue(params: ReportIssueParams): void { ...(scoreText !== undefined ? { score: scoreText } : {}), }; (jsonFormatter as ValeJsonFormatter).addIssue(issue); - } else if (outputFormat === 'json') { + } else if (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.RdJson) { + const matchLen = match ? match.length : 0; const endColumn = column + matchLen; const issue: Issue = { @@ -82,11 +85,11 @@ function reportIssue(params: ReportIssueParams): void { span: [column, endColumn], severity, message: summary, - eval: ruleName, + rule: ruleName, match: match || '', ...(suggestion ? { suggestion } : {}) }; - (jsonFormatter as JsonFormatter).addIssue(file, issue); + (jsonFormatter as JsonFormatter | RdJsonFormatter).addIssue(file, issue); } } @@ -261,7 +264,7 @@ function extractAndReportCriterion(params: ProcessCriterionParams): ProcessCrite }; } - const got = result.criteria.find(c => c.name === nameKey); + const got = result.criteria.find(c => c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase()); if (!got) { return { errors: 0, @@ -387,15 +390,29 @@ function validateCriteriaCompleteness(params: ValidationParams): boolean { const expectedNames = new Set((meta.criteria || []).map((c) => String(c.name || c.id || ''))); const returnedNames = new Set(result.criteria.map((c: { name: string }) => c.name)); + // Create normalized maps for case-insensitive lookup + const expectedNormalized = new Set(); + const expectedOriginalMap = new Map(); for (const name of expectedNames) { - if (!returnedNames.has(name)) { - console.error(`Missing criterion in model output: ${name}`); + const norm = name.toLowerCase(); + expectedNormalized.add(norm); + expectedOriginalMap.set(norm, name); + } + + const returnedNormalized = new Set(); + for (const name of returnedNames) { + returnedNormalized.add(name.toLowerCase()); + } + + for (const norm of expectedNormalized) { + if (!returnedNormalized.has(norm)) { + console.error(`Missing criterion in model output: ${expectedOriginalMap.get(norm)}`); hadErrors = true; } } for (const name of returnedNames) { - if (!expectedNames.has(name)) { + if (!expectedNormalized.has(name.toLowerCase())) { console.warn(`[vectorlint] Extra criterion returned by model (ignored): ${name}`); } } @@ -412,7 +429,11 @@ function validateScores(params: ValidationParams): boolean { for (const exp of meta.criteria || []) { const nameKey = String(exp.name || exp.id || ''); - const got = result.criteria.find(c => c.name === nameKey); + const got = result.criteria.find( + c => c.name === nameKey + || + c.name.toLowerCase() === nameKey.toLowerCase() + ); if (!got) continue; const score = Number(got.score); @@ -459,7 +480,7 @@ function routePromptResult(params: ProcessPromptResultParams): ErrorTrackingResu jsonFormatter }); hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; - } else if ((outputFormat === 'json' || outputFormat === 'vale-json') && result.message) { + } else if ((outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson) && result.message) { // For JSON, if there's a message but no violations, report it as a general issue reportIssue({ file: relFile, @@ -526,8 +547,8 @@ function routePromptResult(params: ProcessPromptResultParams): ErrorTrackingResu } } - if (outputFormat === 'json' && scoreComponents.length > 0) { - (jsonFormatter as JsonFormatter).addEvaluationScore(relFile, { + if (outputFormat === OutputFormat.Json && scoreComponents.length > 0) { + (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore(relFile, { id: promptId || promptFile.filename.replace(/\.md$/, ''), scores: scoreComponents }); @@ -590,7 +611,7 @@ async function runPromptEvaluation(params: RunPromptEvaluationParams): Promise { const { file, options, jsonFormatter } = params; - const { prompts, provider, searchProvider, concurrency, scanPaths, outputFormat = 'line' } = options; + const { prompts, provider, searchProvider, concurrency, scanPaths, outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -602,7 +623,7 @@ async function evaluateFile(params: EvaluateFileParams): Promise { - const { outputFormat = 'line' } = options; + const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -735,9 +756,11 @@ export async function evaluateFiles( let totalWarnings = 0; let requestFailures = 0; - let jsonFormatter: ValeJsonFormatter | JsonFormatter; - if (outputFormat === 'json') { + let jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + if (outputFormat === OutputFormat.Json) { jsonFormatter = new JsonFormatter(); + } else if (outputFormat === OutputFormat.RdJson) { + jsonFormatter = new RdJsonFormatter(); } else { jsonFormatter = new ValeJsonFormatter(); } @@ -758,9 +781,10 @@ export async function evaluateFiles( } } - // Output results based on format - if (outputFormat === 'json' || outputFormat === 'vale-json') { - console.log(jsonFormatter.toJson()); + // Output results based on format (always to stdout for JSON formats) + if (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson || outputFormat === OutputFormat.RdJson) { + const jsonStr = jsonFormatter.toJson(); + console.log(jsonStr); } return { diff --git a/src/cli/types.ts b/src/cli/types.ts index 5f3818b..ace62ce 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -6,9 +6,17 @@ import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; +import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { EvaluationResult as PromptEvaluationResult, SubjectiveResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; +export enum OutputFormat { + Line = 'line', + Json = 'json', + ValeJson = 'vale-json', + RdJson = 'rdjson' +} + export interface EvaluationOptions { prompts: PromptFile[]; rulesPath: string; @@ -17,7 +25,7 @@ export interface EvaluationOptions { concurrency: number; verbose: boolean; scanPaths: FilePatternConfig[]; - outputFormat?: 'line' | 'json' | 'vale-json'; + outputFormat?: OutputFormat; } export interface EvaluationResult { @@ -40,8 +48,8 @@ export interface ErrorTrackingResult { export interface EvaluationContext { content: string; relFile: string; - outputFormat: 'line' | 'json' | 'vale-json'; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + outputFormat: OutputFormat; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } export interface ReportIssueParams { @@ -51,8 +59,8 @@ export interface ReportIssueParams { severity: Severity summary: string; ruleName: string; - outputFormat: 'line' | 'json' | 'vale-json'; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + outputFormat: OutputFormat; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; suggestion?: string; scoreText?: string; match?: string; @@ -124,7 +132,7 @@ export type RunPromptEvaluationResult = export interface EvaluateFileParams { file: string; options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } export interface EvaluateFileResult extends ErrorTrackingResult { diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index c2311b6..3165a52 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -1,87 +1,98 @@ +import { createRequire } from 'node:module'; +import { z } from 'zod'; import { Severity } from '../evaluators/types'; + +const REQUIRE = createRequire(import.meta.url); + +const PACKAGE_JSON_SCHEMA = z.object({ + version: z.string(), +}); + +// Using require to load JSON in ESM +const RAW_PACKAGE_JSON: unknown = REQUIRE('../../package.json'); +const PKG = PACKAGE_JSON_SCHEMA.parse(RAW_PACKAGE_JSON); export interface ScoreComponent { - criterion?: string; - rawScore: number; - maxScore: number; - weightedScore: number; - weightedMaxScore: number; - normalizedScore: number; - normalizedMaxScore: number; + criterion?: string; + rawScore: number; + maxScore: number; + weightedScore: number; + weightedMaxScore: number; + normalizedScore: number; + normalizedMaxScore: number; } export interface EvaluationScore { - id: string; - scores: ScoreComponent[]; + id: string; + scores: ScoreComponent[]; } export interface Issue { - line: number; - column: number; - span: [number, number]; - severity: Severity; - message: string; - eval: string; // Renamed from rule - match: string; - suggestion?: string; + line: number; + column: number; + span: [number, number]; + severity: Severity; + message: string; + rule: string; + match: string; + suggestion?: string; } export interface FileResult { - issues: Issue[]; - evaluationScores: EvaluationScore[]; + issues: Issue[]; + evaluationScores: EvaluationScore[]; } export interface Result { - files: Record; - summary: { - files: number; - errors: number; - warnings: number; - }; - metadata: { - version: string; - timestamp: string; - }; + files: Record; + summary: { + files: number; + errors: number; + warnings: number; + }; + metadata: { + version: string; + timestamp: string; + }; } export class JsonFormatter { - private files: Record = {}; - private errorCount = 0; - private warningCount = 0; + private files: Record = {}; + private errorCount = 0; + private warningCount = 0; - addIssue(file: string, issue: Issue): void { - if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; - } - this.files[file].issues.push(issue); - - if (issue.severity === Severity.ERROR) { - this.errorCount++; - } else if (issue.severity === Severity.WARNING) { - this.warningCount++; - } + addIssue(file: string, issue: Issue): void { + if (!this.files[file]) { + this.files[file] = { issues: [], evaluationScores: [] }; } + this.files[file].issues.push(issue); - addEvaluationScore(file: string, score: EvaluationScore): void { - if (!this.files[file]) { - this.files[file] = { issues: [], evaluationScores: [] }; - } - this.files[file].evaluationScores.push(score); + if (issue.severity === Severity.ERROR) { + this.errorCount++; + } else if (issue.severity === Severity.WARNING) { + this.warningCount++; } + } - - toJson(): string { - const result: Result = { - files: this.files, - summary: { - files: Object.keys(this.files).length, - errors: this.errorCount, - warnings: this.warningCount, - }, - metadata: { - version: '1.0.0', // TODO: Get from package.json - timestamp: new Date().toISOString(), - }, - }; - return JSON.stringify(result, null, 2); + addEvaluationScore(file: string, score: EvaluationScore): void { + if (!this.files[file]) { + this.files[file] = { issues: [], evaluationScores: [] }; } + this.files[file].evaluationScores.push(score); + } + + toJson(): string { + const result: Result = { + files: this.files, + summary: { + files: Object.keys(this.files).length, + errors: this.errorCount, + warnings: this.warningCount, + }, + metadata: { + version: PKG.version, + timestamp: new Date().toISOString(), + }, + }; + return JSON.stringify(result, null, 2); + } } diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts new file mode 100644 index 0000000..d1daed7 --- /dev/null +++ b/src/output/rdjson-formatter.ts @@ -0,0 +1,134 @@ +import type { Issue, EvaluationScore } from './json-formatter'; +import { Severity } from '../evaluators/types'; + +export interface RdJsonResult { + source: { + name: string; + url: string; + }; + diagnostics: RdJsonDiagnostic[]; +} + +export interface RdJsonDiagnostic { + message: string; + location: { + path: string; + range: { + start: { + line: number; + column: number; + }; + end?: { + line: number; + column: number; + }; + }; + }; + severity: Severity; + code?: { + value: string; + url?: string; + }; + suggestions?: RdJsonSuggestion[]; +} + +export interface RdJsonSuggestion { + range: { + start: { + line: number; + column: number; + }; + end: { + line: number; + column: number; + }; + }; + text: string; +} + +interface FileResult { + issues: Issue[]; + evaluationScores: EvaluationScore[]; +} + +export class RdJsonFormatter { + private files: Record = {}; + + addIssue(file: string, issue: Issue): void { + if (!this.files[file]) { + this.files[file] = { issues: [], evaluationScores: [] }; + } + this.files[file].issues.push(issue); + } + + addEvaluationScore(file: string, score: EvaluationScore): void { + if (!this.files[file]) { + this.files[file] = { issues: [], evaluationScores: [] }; + } + this.files[file].evaluationScores.push(score); + } + + toRdJsonFormat(): RdJsonResult { + const diagnostics: RdJsonDiagnostic[] = []; + + // Iterate over all files and their issues + for (const [filePath, fileResult] of Object.entries(this.files)) { + for (const issue of fileResult.issues) { + const matchLen = issue.match.length; + + const diagnostic: RdJsonDiagnostic = { + message: issue.message, + location: { + path: filePath, + range: { + start: { + line: issue.line, + column: issue.column, + }, + end: { + line: issue.line, + column: issue.column + matchLen, + }, + }, + }, + severity: issue.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, + code: { + value: issue.rule, + }, + }; + + if (issue.suggestion) { + diagnostic.suggestions = [ + { + range: { + start: { + line: issue.line, + column: issue.column, + }, + end: { + line: issue.line, + column: issue.column + matchLen, + }, + }, + text: issue.suggestion, + }, + ]; + } + + diagnostics.push(diagnostic); + } + } + + return { + source: { + name: 'vectorlint', + url: 'https://github.com/TRocket-Labs/vectorlint', + }, + diagnostics, + }; + } + + toJson(): string { + return JSON.stringify(this.toRdJsonFormat(), null, 2); + } +} diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 753f7bf..f216d16 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -6,7 +6,8 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPrompt: z.boolean().default(false), showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), - output: z.enum(['line', 'json', 'vale-json', 'JSON']).default('line'), + output: z.enum(['line', 'json', 'vale-json', 'rdjson']).default('line'), + prompts: z.string().optional(), evals: z.string().optional(), config: z.string().optional(), }); diff --git a/tests/rdjson-formatter.test.ts b/tests/rdjson-formatter.test.ts new file mode 100644 index 0000000..29a5b83 --- /dev/null +++ b/tests/rdjson-formatter.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from 'vitest'; +import { RdJsonFormatter, type RdJsonResult } from '../src/output/rdjson-formatter'; +import { Severity } from '../src/evaluators/types'; + +describe('RdJsonFormatter', () => { + it('should produce valid RDJSON output', () => { + const formatter = new RdJsonFormatter(); + + formatter.addIssue('test.md', { + line: 1, + column: 1, + span: [1, 6], + severity: Severity.ERROR, + message: 'Test message', + rule: 'TestRule', + match: 'match', + suggestion: 'fix' + }); + + const rdjsonOutput = formatter.toJson(); + + // Should be valid JSON + expect(() => { + JSON.parse(rdjsonOutput); + }).not.toThrow(); + + const parsed = JSON.parse(rdjsonOutput) as RdJsonResult; + expect(parsed).toHaveProperty('source'); + expect(parsed.source.name).toBe('vectorlint'); + expect(parsed).toHaveProperty('diagnostics'); + expect(parsed.diagnostics).toHaveLength(1); + + const diag = parsed.diagnostics[0]!; + expect(diag.message).toBe('Test message'); + expect(diag.severity).toBe(Severity.ERROR); + expect(diag.location.path).toBe('test.md'); + expect(diag.location.range.start.line).toBe(1); + expect(diag.location.range.start.column).toBe(1); + expect(diag.location.range.end!.column).toBe(6); // 1 + length of 'match' (5) + + expect(diag.suggestions).toHaveLength(1); + expect(diag.suggestions![0]!.text).toBe('fix'); + }); +});