From b36a642133cb274fcfb8917f0b9b11fc86ecab20 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 26 Nov 2025 18:31:24 +0100 Subject: [PATCH 01/40] feat: Add support for RdJson output formats. --- README.md | 18 ++++++ src/cli/commands.ts | 14 ++--- src/cli/orchestrator.ts | 10 ++-- src/output/json-formatter.ts | 105 ++++++++++++++++++++++++++++++++++- src/schemas/cli-schemas.ts | 2 +- tests/json-formatter.test.ts | 40 ++++++++++++- 6 files changed, 175 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0b3b8a0..da9d048 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,24 @@ npm run dev -- path/to/article.md vectorlint --verbose --show-prompt --debug-json path/to/article.md ``` +### CI/CD Integration (GitHub Actions & reviewdog) + +VectorLint supports the `rdjson` format, making it easy to integrate with [reviewdog](https://github.com/reviewdog/reviewdog) for automated code review comments in Pull Requests. + +**Example Workflow:** + +```yaml +- name: Run VectorLint with reviewdog + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npx vectorlint --output rdjson . | reviewdog -f=rdjson -name="vectorlint" -reporter=github-pr-review -filter-mode=added -fail-on-error=true +``` + +Supported reporters: +- `github-pr-review`: Posts comments on specific lines in the PR. +- `github-check`: Creates annotations in the "Checks" tab. + ### Creating Prompts Prompts are simple Markdown files with YAML frontmatter. diff --git a/src/cli/commands.ts b/src/cli/commands.ts index cbc90e6..a7a2c59 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -25,10 +25,10 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt', 'Print full prompt and injected content') .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Print full JSON response from the API') - .option('--output ', 'Output format: line (default) or JSON', 'line') + .option('--output ', 'Output format: line (default), JSON, or rdjson', 'line') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { - + // Parse and validate CLI options let cliOptions; try { @@ -59,7 +59,7 @@ export function registerMainCommand(program: Command): void { console.error(`Error: ${err.message}`); process.exit(1); } - + const provider = createProvider( env, { @@ -70,7 +70,7 @@ export function registerMainCommand(program: Command): void { }, new DefaultRequestBuilder(directive) ); - + if (cliOptions.verbose) { const directiveLen = directive ? directive.length : 0; console.log(`[vectorlint] Directive active: ${directiveLen} char(s)`); @@ -85,13 +85,13 @@ export function registerMainCommand(program: Command): void { console.error(`Error: ${err.message}`); process.exit(1); } - + const { promptsPath } = config; if (!existsSync(promptsPath)) { console.error(`Error: prompts path does not exist: ${promptsPath}`); process.exit(1); } - + let prompts: PromptFile[]; try { const loaded = loadPrompts(promptsPath, { verbose: cliOptions.verbose }); @@ -120,7 +120,7 @@ export function registerMainCommand(program: Command): void { console.error(`Error: failed to resolve target files: ${err.message}`); process.exit(1); } - + if (targets.length === 0) { console.error('Error: no target files found to evaluate.'); process.exit(1); diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index e1cc215..63a74cc 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -22,7 +22,7 @@ export interface EvaluationOptions { concurrency: number; verbose: boolean; mapping?: PromptMapping; - outputFormat?: 'line' | 'JSON'; + outputFormat?: 'line' | 'JSON' | 'rdjson'; } export interface EvaluationResult { @@ -44,7 +44,7 @@ interface ErrorTrackingResult { interface EvaluationContext { content: string; relFile: string; - outputFormat: 'line' | 'JSON'; + outputFormat: 'line' | 'JSON' | 'rdjson'; jsonFormatter: JsonFormatter; } @@ -64,7 +64,7 @@ interface ReportIssueParams { status: 'ok' | 'warning' | 'error'; summary: string; ruleName: string; - outputFormat: 'line' | 'JSON'; + outputFormat: 'line' | 'JSON' | 'rdjson'; jsonFormatter: JsonFormatter; suggestion?: string; scoreText?: string; @@ -744,7 +744,9 @@ export async function evaluateFiles( // Output results based on format if (outputFormat === 'JSON') { - console.log(jsonFormatter.toJson()); + console.log(jsonFormatter.toJson('vale')); + } else if (outputFormat === 'rdjson') { + console.log(jsonFormatter.toJson('rdjson')); } return { diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index a3dffe4..9a6e9b1 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -23,6 +23,51 @@ export interface JsonResult { }; } +export interface RdJsonResult { + source: { + name: string; + url: string; + }; + diagnostics: RdJsonDiagnostic[]; +} + +export interface RdJsonDiagnostic { + message: string; + location: { + path: string; + range: { + start: { + line: number; + column: number; + }; + end?: { + line: number; + column: number; + }; + }; + }; + severity: 'ERROR' | 'WARNING' | 'INFO'; + code?: { + value: string; + url?: string; + }; + suggestions?: RdJsonSuggestion[]; +} + +export interface RdJsonSuggestion { + range: { + start: { + line: number; + column: number; + }; + end: { + line: number; + column: number; + }; + }; + text: string; +} + export class JsonFormatter { private issues: JsonIssue[] = []; private files = new Set(); @@ -71,7 +116,65 @@ export class JsonFormatter { return result; } - toJson(): string { + toRdJsonFormat(): RdJsonResult { + const diagnostics: RdJsonDiagnostic[] = this.issues.map((issue) => { + const matchLen = issue.matchLength || issue.match.length; + + const diagnostic: RdJsonDiagnostic = { + message: issue.message, + location: { + path: issue.file, + range: { + start: { + line: issue.line, + column: issue.column, + }, + end: { + line: issue.line, + column: issue.column + matchLen, + }, + }, + }, + severity: issue.severity === 'error' ? 'ERROR' : issue.severity === 'warning' ? 'WARNING' : 'INFO', + code: { + value: issue.rule, + }, + }; + + if (issue.suggestion) { + diagnostic.suggestions = [ + { + range: { + start: { + line: issue.line, + column: issue.column, + }, + end: { + line: issue.line, + column: issue.column + matchLen, + }, + }, + text: issue.suggestion, + }, + ]; + } + + return diagnostic; + }); + + return { + source: { + name: 'vectorlint', + url: 'https://github.com/TRocket-Labs/vectorlint', + }, + diagnostics, + }; + } + + toJson(format: 'vale' | 'rdjson' = 'vale'): string { + if (format === 'rdjson') { + return JSON.stringify(this.toRdJsonFormat(), null, 2); + } return JSON.stringify(this.toValeFormat(), null, 2); } diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index aa09b92..c6ef1f0 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -6,7 +6,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPrompt: z.boolean().default(false), showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), - output: z.enum(['line', 'JSON']).default('line'), + output: z.enum(['line', 'JSON', 'rdjson']).default('line'), prompts: z.string().optional(), }); diff --git a/tests/json-formatter.test.ts b/tests/json-formatter.test.ts index f14d9b8..80011df 100644 --- a/tests/json-formatter.test.ts +++ b/tests/json-formatter.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { JsonFormatter } from '../src/output/json-formatter'; +import { JsonFormatter, type RdJsonResult } from '../src/output/json-formatter'; describe('JsonFormatter', () => { it('should format issues into Vale-compatible JSON structure', () => { @@ -128,4 +128,42 @@ describe('JsonFormatter', () => { expect(parsed).toHaveProperty('test.md'); expect(Array.isArray(parsed['test.md'])).toBe(true); }); + it('should produce valid RDJSON output', () => { + const formatter = new JsonFormatter(); + + formatter.addIssue({ + file: 'test.md', + line: 1, + column: 1, + severity: 'error', + message: 'Test message', + rule: 'TestRule', + match: 'match', + suggestion: 'fix' + }); + + const rdjsonOutput = formatter.toJson('rdjson'); + + // Should be valid JSON + expect(() => { + JSON.parse(rdjsonOutput); + }).not.toThrow(); + + const parsed = JSON.parse(rdjsonOutput) as RdJsonResult; + expect(parsed).toHaveProperty('source'); + expect(parsed.source.name).toBe('vectorlint'); + expect(parsed).toHaveProperty('diagnostics'); + expect(parsed.diagnostics).toHaveLength(1); + + const diag = parsed.diagnostics[0]!; + expect(diag.message).toBe('Test message'); + expect(diag.severity).toBe('ERROR'); + expect(diag.location.path).toBe('test.md'); + expect(diag.location.range.start.line).toBe(1); + expect(diag.location.range.start.column).toBe(1); + expect(diag.location.range.end!.column).toBe(6); // 1 + length of 'match' (5) + + expect(diag.suggestions).toHaveLength(1); + expect(diag.suggestions![0]!.text).toBe('fix'); + }); }); From c2bc53183f8a9e52f007a812e306fe5e7b4c40c9 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 1 Dec 2025 13:14:26 +0100 Subject: [PATCH 02/40] feat: Add core evaluation orchestrator, CLI commands, schemas, and GitHub Actions workflows. --- .github/vectorlint-action/action.yml | 124 +++++++++++++++++++++++++++ .github/workflows/vectorlint.yml | 35 ++++++++ src/cli/commands.ts | 3 + src/cli/orchestrator.ts | 50 ++++++++--- src/schemas/cli-schemas.ts | 1 + 5 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 .github/vectorlint-action/action.yml create mode 100644 .github/workflows/vectorlint.yml diff --git a/.github/vectorlint-action/action.yml b/.github/vectorlint-action/action.yml new file mode 100644 index 0000000..ed29c29 --- /dev/null +++ b/.github/vectorlint-action/action.yml @@ -0,0 +1,124 @@ +name: 'VectorLint' +description: 'Run VectorLint with reviewdog on pull requests' +author: 'TRocket Labs' + +inputs: + github_token: + description: 'GitHub token for reviewdog' + required: false + default: ${{ github.token }} + + openai_api_key: + description: 'OpenAI API key' + required: false + + anthropic_api_key: + description: 'Anthropic API key' + required: false + + gemini_api_key: + description: 'Google Gemini API key' + required: false + + reporter: + description: 'Reporter type (github-pr-check, github-pr-review, github-check)' + required: false + default: 'github-pr-check' + + filter_mode: + description: 'Filter mode (added, diff_context, file, nofilter)' + required: false + default: 'added' + + fail_on_error: + description: 'Fail if errors are found' + required: false + default: 'true' + + vectorlint_flags: + description: 'Additional flags to pass to vectorlint' + required: false + default: '' + + working_directory: + description: 'Working directory' + required: false + default: '.' + +runs: + using: 'composite' + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Setup reviewdog + uses: reviewdog/action-setup@v1 + with: + reviewdog_version: latest + + - name: Run VectorLint with reviewdog + shell: bash + working-directory: ${{ inputs.working_directory }} + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ inputs.github_token }} + OPENAI_API_KEY: ${{ inputs.openai_api_key }} + ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }} + GEMINI_API_KEY: ${{ inputs.gemini_api_key }} + run: | + set -e + + # Check if API key is provided and set provider + if [ -n "$OPENAI_API_KEY" ]; then + export LLM_PROVIDER="openai" + elif [ -n "$ANTHROPIC_API_KEY" ]; then + export LLM_PROVIDER="anthropic" + elif [ -n "$GEMINI_API_KEY" ]; then + export LLM_PROVIDER="gemini" + else + echo "Error: An API key (OpenAI, Anthropic, or Gemini) must be provided" + exit 1 + fi + + # Clone, build, and run vectorlint from branch + git clone -b ft/reviewdog-test https://github.com/TRocket-Labs/vectorlint.git .vectorlint-temp + cd .vectorlint-temp + npm ci + npm run build + + # Run using the built artifact from the repo root + cd .. + + # Determine target files (only changed files for PRs) + TARGET_FILES="." + if [ -n "${{ github.base_ref }}" ]; then + echo "Detected PR, calculating changed files against ${{ github.base_ref }}..." + git fetch origin ${{ github.base_ref }} --depth=1 + # Find changed .md files + CHANGED=$(git diff --name-only origin/${{ github.base_ref }} HEAD | grep '\.md$' || true) + + if [ -z "$CHANGED" ]; then + echo "No markdown files changed in this PR. Skipping analysis." + echo '{"source": {"name": "vectorlint", "url": "https://github.com/TRocket-Labs/vectorlint"}, "diagnostics": []}' > vectorlint_result.json + exit 0 + fi + + # Use changed files as targets + TARGET_FILES=$(echo "$CHANGED" | tr '\n' ' ') + echo "Scanning changed files: $TARGET_FILES" + fi + node .vectorlint-temp/dist/index.js --output rdjson --output-file vectorlint_result.json --prompts .vectorlint-temp/prompts ${{ inputs.vectorlint_flags }} $TARGET_FILES || true + + cat vectorlint_result.json | reviewdog -f=rdjson \ + -name="vectorlint" \ + -reporter=${{ inputs.reporter }} \ + -filter-mode=${{ inputs.filter_mode }} \ + -fail-on-error=${{ inputs.fail_on_error }} + + # Cleanup + rm -rf .vectorlint-temp +branding: + icon: 'check-circle' + color: 'blue' + \ No newline at end of file diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml new file mode 100644 index 0000000..1510379 --- /dev/null +++ b/.github/workflows/vectorlint.yml @@ -0,0 +1,35 @@ +name: VectorLint + +on: + pull_request: + paths: + - '**/*.md' + - '**/*.mdx' + - 'vectorlint.ini' + - 'prompts/**' + +permissions: + contents: read + pull-requests: write + checks: write + +concurrency: + group: vectorlint-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run VectorLint + uses: ./.github/vectorlint-action + with: + gemini_api_key: ${{ secrets.GEMINI_API_KEY }} + reporter: github-pr-check + filter_mode: nofilter + fail_on_error: true + vectorlint_flags: '--verbose' + + \ No newline at end of file diff --git a/src/cli/commands.ts b/src/cli/commands.ts index a7a2c59..3852dea 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -26,6 +26,8 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Print full JSON response from the API') .option('--output ', 'Output format: line (default), JSON, or rdjson', 'line') + .option('--output-file ', 'Write output to a file instead of stdout') + .option('--prompts ', 'Path to prompts directory') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { @@ -154,6 +156,7 @@ export function registerMainCommand(program: Command): void { concurrency: config.concurrency, verbose: cliOptions.verbose, outputFormat: cliOptions.output, + ...(cliOptions.outputFile ? { outputFile: cliOptions.outputFile } : {}), ...(mapping ? { mapping } : {}), }); diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 63a74cc..cb479a3 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,4 +1,4 @@ -import { readFileSync } from 'fs'; +import { readFileSync, writeFileSync } from 'fs'; import * as path from 'path'; import type { PromptFile } from '../prompts/prompt-loader'; import type { LLMProvider } from '../providers/llm-provider'; @@ -23,6 +23,7 @@ export interface EvaluationOptions { verbose: boolean; mapping?: PromptMapping; outputFormat?: 'line' | 'JSON' | 'rdjson'; + outputFile?: string; } export interface EvaluationResult { @@ -370,7 +371,7 @@ function processCriterion(params: ProcessCriterionParams): ProcessCriterionResul }; } - const got = result.criteria.find(c => c.name === nameKey); + const got = result.criteria.find(c => c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase()); if (!got) { return { errors: 0, @@ -455,15 +456,29 @@ function validateCriteriaCompleteness(params: ValidationParams): boolean { const expectedNames = new Set((meta.criteria || []).map((c) => String(c.name || c.id || ''))); const returnedNames = new Set(result.criteria.map((c: { name: string }) => c.name)); + // Create normalized maps for case-insensitive lookup + const expectedNormalized = new Set(); + const expectedOriginalMap = new Map(); for (const name of expectedNames) { - if (!returnedNames.has(name)) { - console.error(`Missing criterion in model output: ${name}`); + const norm = name.toLowerCase(); + expectedNormalized.add(norm); + expectedOriginalMap.set(norm, name); + } + + const returnedNormalized = new Set(); + for (const name of returnedNames) { + returnedNormalized.add(name.toLowerCase()); + } + + for (const norm of expectedNormalized) { + if (!returnedNormalized.has(norm)) { + console.error(`Missing criterion in model output: ${expectedOriginalMap.get(norm)}`); hadErrors = true; } } for (const name of returnedNames) { - if (!expectedNames.has(name)) { + if (!expectedNormalized.has(name.toLowerCase())) { console.warn(`[vectorlint] Extra criterion returned by model (ignored): ${name}`); } } @@ -480,7 +495,11 @@ function validateScores(params: ValidationParams): boolean { for (const exp of meta.criteria || []) { const nameKey = String(exp.name || exp.id || ''); - const got = result.criteria.find(c => c.name === nameKey); + const got = result.criteria.find( + c => c.name === nameKey + || + c.name.toLowerCase() === nameKey.toLowerCase() + ); if (!got) continue; const score = Number(got.score); @@ -743,10 +762,21 @@ export async function evaluateFiles( } // Output results based on format - if (outputFormat === 'JSON') { - console.log(jsonFormatter.toJson('vale')); - } else if (outputFormat === 'rdjson') { - console.log(jsonFormatter.toJson('rdjson')); + if (outputFormat === 'JSON' || outputFormat === 'rdjson') { + const jsonStr = jsonFormatter.toJson(outputFormat === 'JSON' ? 'vale' : 'rdjson'); + + if (options.outputFile) { + writeFileSync(options.outputFile, jsonStr, 'utf-8'); + if (options.verbose) { + console.error(`[vectorlint] Wrote output to ${options.outputFile}`); + } + } else { + if (outputFormat === 'rdjson' && options.verbose) { + console.error('[vectorlint] Generated RDJSON:'); + console.error(jsonStr); + } + console.log(jsonStr); + } } return { diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index c6ef1f0..014be5f 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -7,6 +7,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.enum(['line', 'JSON', 'rdjson']).default('line'), + outputFile: z.string().optional(), prompts: z.string().optional(), }); From bd10dcb9d1aa7e5ddaac800f89ac5e4ad9758798 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 1 Dec 2025 13:18:38 +0100 Subject: [PATCH 03/40] Create vectorlint.ini --- .gitignore | 2 +- vectorlint.ini | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 vectorlint.ini diff --git a/.gitignore b/.gitignore index c80090b..9d701b3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ .idea/ coverage/ *.tsbuildinfo -vectorlint.ini +# vectorlint.ini .kiro/ /.idea /npm diff --git a/vectorlint.ini b/vectorlint.ini new file mode 100644 index 0000000..0d2fdbd --- /dev/null +++ b/vectorlint.ini @@ -0,0 +1,7 @@ +PromptsPath=prompts +ScanPaths=["**/*.md"] +Concurrency=4 + +[Defaults] +include=["**/*.md"] +exclude=["node_modules/**", "dist/**", ".github/**"] From 23de209b85bdda86900a16d4535ec591f20c6da0 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 1 Dec 2025 15:46:45 +0100 Subject: [PATCH 04/40] Clean up github action integration with vectorlint --- .github/vectorlint-action/action.yml | 124 --------------------------- .github/workflows/vectorlint.yml | 10 +-- 2 files changed, 4 insertions(+), 130 deletions(-) delete mode 100644 .github/vectorlint-action/action.yml diff --git a/.github/vectorlint-action/action.yml b/.github/vectorlint-action/action.yml deleted file mode 100644 index ed29c29..0000000 --- a/.github/vectorlint-action/action.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: 'VectorLint' -description: 'Run VectorLint with reviewdog on pull requests' -author: 'TRocket Labs' - -inputs: - github_token: - description: 'GitHub token for reviewdog' - required: false - default: ${{ github.token }} - - openai_api_key: - description: 'OpenAI API key' - required: false - - anthropic_api_key: - description: 'Anthropic API key' - required: false - - gemini_api_key: - description: 'Google Gemini API key' - required: false - - reporter: - description: 'Reporter type (github-pr-check, github-pr-review, github-check)' - required: false - default: 'github-pr-check' - - filter_mode: - description: 'Filter mode (added, diff_context, file, nofilter)' - required: false - default: 'added' - - fail_on_error: - description: 'Fail if errors are found' - required: false - default: 'true' - - vectorlint_flags: - description: 'Additional flags to pass to vectorlint' - required: false - default: '' - - working_directory: - description: 'Working directory' - required: false - default: '.' - -runs: - using: 'composite' - steps: - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Setup reviewdog - uses: reviewdog/action-setup@v1 - with: - reviewdog_version: latest - - - name: Run VectorLint with reviewdog - shell: bash - working-directory: ${{ inputs.working_directory }} - env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ inputs.github_token }} - OPENAI_API_KEY: ${{ inputs.openai_api_key }} - ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }} - GEMINI_API_KEY: ${{ inputs.gemini_api_key }} - run: | - set -e - - # Check if API key is provided and set provider - if [ -n "$OPENAI_API_KEY" ]; then - export LLM_PROVIDER="openai" - elif [ -n "$ANTHROPIC_API_KEY" ]; then - export LLM_PROVIDER="anthropic" - elif [ -n "$GEMINI_API_KEY" ]; then - export LLM_PROVIDER="gemini" - else - echo "Error: An API key (OpenAI, Anthropic, or Gemini) must be provided" - exit 1 - fi - - # Clone, build, and run vectorlint from branch - git clone -b ft/reviewdog-test https://github.com/TRocket-Labs/vectorlint.git .vectorlint-temp - cd .vectorlint-temp - npm ci - npm run build - - # Run using the built artifact from the repo root - cd .. - - # Determine target files (only changed files for PRs) - TARGET_FILES="." - if [ -n "${{ github.base_ref }}" ]; then - echo "Detected PR, calculating changed files against ${{ github.base_ref }}..." - git fetch origin ${{ github.base_ref }} --depth=1 - # Find changed .md files - CHANGED=$(git diff --name-only origin/${{ github.base_ref }} HEAD | grep '\.md$' || true) - - if [ -z "$CHANGED" ]; then - echo "No markdown files changed in this PR. Skipping analysis." - echo '{"source": {"name": "vectorlint", "url": "https://github.com/TRocket-Labs/vectorlint"}, "diagnostics": []}' > vectorlint_result.json - exit 0 - fi - - # Use changed files as targets - TARGET_FILES=$(echo "$CHANGED" | tr '\n' ' ') - echo "Scanning changed files: $TARGET_FILES" - fi - node .vectorlint-temp/dist/index.js --output rdjson --output-file vectorlint_result.json --prompts .vectorlint-temp/prompts ${{ inputs.vectorlint_flags }} $TARGET_FILES || true - - cat vectorlint_result.json | reviewdog -f=rdjson \ - -name="vectorlint" \ - -reporter=${{ inputs.reporter }} \ - -filter-mode=${{ inputs.filter_mode }} \ - -fail-on-error=${{ inputs.fail_on_error }} - - # Cleanup - rm -rf .vectorlint-temp -branding: - icon: 'check-circle' - color: 'blue' - \ No newline at end of file diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 1510379..42f41c7 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -24,12 +24,10 @@ jobs: - uses: actions/checkout@v4 - name: Run VectorLint - uses: ./.github/vectorlint-action + uses: ayo6706/vectorlint-test-action@v1 with: gemini_api_key: ${{ secrets.GEMINI_API_KEY }} reporter: github-pr-check - filter_mode: nofilter - fail_on_error: true - vectorlint_flags: '--verbose' - - \ No newline at end of file + filter_mode: added + fail_on_error: false + vectorlint_flags: '--verbose' \ No newline at end of file From 1fed80140c3302b121281a5c45381174c4d4c199 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 02:31:29 +0100 Subject: [PATCH 05/40] update key to anthropic --- .github/workflows/vectorlint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 42f41c7..1bad011 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -26,7 +26,7 @@ jobs: - name: Run VectorLint uses: ayo6706/vectorlint-test-action@v1 with: - gemini_api_key: ${{ secrets.GEMINI_API_KEY }} + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} reporter: github-pr-check filter_mode: added fail_on_error: false From f2f780e7c169a351a8cf0eeaa70ac5cd8d7ce4c6 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 02:54:31 +0100 Subject: [PATCH 06/40] chore: pass LLM_PROVIDER to vectorlint-action - Update vectorlint.yml to pass LLM_PROVIDER secret to action - Allows explicit provider selection instead of defaulting to azure-openai - Fixes 'Missing required Azure OpenAI environment variables' error --- .github/workflows/vectorlint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 1bad011..5536765 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -26,6 +26,7 @@ jobs: - name: Run VectorLint uses: ayo6706/vectorlint-test-action@v1 with: + llm_provider: ${{ secrets.LLM_PROVIDER }} anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} reporter: github-pr-check filter_mode: added From cfcf6a8c6f36d9543c05120d7f5d6d3aeef35e47 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 03:02:07 +0100 Subject: [PATCH 07/40] fix: prioritize CLI --prompts option over config file - Update commands.ts to use cliOptions.prompts if provided - Fixes 'prompts path does not exist' error in GitHub Actions - Ensures action's explicit prompts path takes precedence --- src/cli/commands.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 4bc360a..fed084f 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -88,7 +88,9 @@ export function registerMainCommand(program: Command): void { process.exit(1); } - const { promptsPath } = config; + + // Prioritize CLI prompts path over config + const promptsPath = cliOptions.prompts || config.promptsPath; if (!existsSync(promptsPath)) { console.error(`Error: prompts path does not exist: ${promptsPath}`); process.exit(1); From fa2429089080cd55e4702daee54af9be9d703e61 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 08:15:23 +0100 Subject: [PATCH 08/40] update vectorlint.ini --- vectorlint.ini | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/vectorlint.ini b/vectorlint.ini index 0d2fdbd..4e6d6d5 100644 --- a/vectorlint.ini +++ b/vectorlint.ini @@ -1,7 +1,20 @@ -PromptsPath=prompts -ScanPaths=["**/*.md"] +PromptsPath=evals +ScanPaths=[*.md] Concurrency=4 +DefaultSeverity=warning + +# Optional: map prompts to files +[Prompts] +paths=["Default:prompts", "Blog:prompts/blog"] [Defaults] include=["**/*.md"] -exclude=["node_modules/**", "dist/**", ".github/**"] +exclude=["archived/**"] + +[Directory:Blog] +include=["content/blog/**/*.md"] +exclude=["content/blog/drafts/**"] + +[Prompt:Headline] +include=["content/blog/**/*.md"] +exclude=["content/blog/drafts/**"] \ No newline at end of file From 8b9d99eeb152687a0b9fdeebb024c944636dddf3 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 09:14:51 +0100 Subject: [PATCH 09/40] Update env to include openai --- .github/workflows/vectorlint.yml | 2 ++ src/cli/commands.ts | 3 +-- src/schemas/cli-schemas.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 5536765..433bc53 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -28,6 +28,8 @@ jobs: with: llm_provider: ${{ secrets.LLM_PROVIDER }} anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + openai_api_key: ${{ secrets.OPENAI_API_KEY }} + perplexy_api_key: ${{ secrets.PERPLEXITY_API_KEY }} reporter: github-pr-check filter_mode: added fail_on_error: false diff --git a/src/cli/commands.ts b/src/cli/commands.ts index fed084f..f249c3c 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -27,7 +27,6 @@ export function registerMainCommand(program: Command): void { .option('--debug-json', 'Print full JSON response from the API') .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') .option('--output-file ', 'Write output to a file instead of stdout') - .option('--prompts ', 'Path to prompts directory') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { @@ -149,7 +148,7 @@ export function registerMainCommand(program: Command): void { ? new PerplexitySearchProvider({ debug: false }) : undefined; - const outputFormat = cliOptions.output === 'json' ? 'json' : cliOptions.output; + const outputFormat = cliOptions.output === 'JSON' ? 'json' : cliOptions.output; // Run evaluations via orchestrator const result = await evaluateFiles(targets, { diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index c548fd2..fc50361 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -6,7 +6,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPrompt: z.boolean().default(false), showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), - output: z.enum(['line', 'json', 'vale-json', 'rdjson']).default('line'), + output: z.enum(['line', 'json', 'vale-json', 'JSON', 'rdjson']).default('line'), outputFile: z.string().optional(), prompts: z.string().optional(), }); From 61dbc41ee932b2929a0ed96cf6327b83fe9d9fd1 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 09:27:41 +0100 Subject: [PATCH 10/40] Update vectorlint action to include perplexity --- .github/workflows/vectorlint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 433bc53..7ddffc3 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -6,7 +6,7 @@ on: - '**/*.md' - '**/*.mdx' - 'vectorlint.ini' - - 'prompts/**' + - 'evals/**' permissions: contents: read @@ -29,7 +29,7 @@ jobs: llm_provider: ${{ secrets.LLM_PROVIDER }} anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} openai_api_key: ${{ secrets.OPENAI_API_KEY }} - perplexy_api_key: ${{ secrets.PERPLEXITY_API_KEY }} + perplexity_api_key: ${{ secrets.PERPLEXITY_API_KEY }} reporter: github-pr-check filter_mode: added fail_on_error: false From 85c2c97454abf7388eaad978f6eb6a42619731fd Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 09:35:48 +0100 Subject: [PATCH 11/40] Add openai model and temperature to vectorlint action --- .github/workflows/vectorlint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 7ddffc3..cf32efb 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -29,6 +29,8 @@ jobs: llm_provider: ${{ secrets.LLM_PROVIDER }} anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} openai_api_key: ${{ secrets.OPENAI_API_KEY }} + openai_model: ${{ secrets.OPENAI_MODEL }} + openai_temperature: ${{ secrets.OPENAI_TEMPERATURE }} perplexity_api_key: ${{ secrets.PERPLEXITY_API_KEY }} reporter: github-pr-check filter_mode: added From aaa05479e0ff88c5df60f3c339a1a2ef86e68e32 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 10:12:21 +0100 Subject: [PATCH 12/40] Create test bad article to test vectorlint with reviewdog --- content/test-bad-article.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 content/test-bad-article.md diff --git a/content/test-bad-article.md b/content/test-bad-article.md new file mode 100644 index 0000000..c7097fd --- /dev/null +++ b/content/test-bad-article.md @@ -0,0 +1,13 @@ +# The Future of AI in 2025 + +AI is gonna be huge. It will literally solve everything. + +## New Tools + +Have you heard of **ReactGPT-Pro**? It's a new framework that writes all your code for you. Also, **NodeJS-Turbo** is 100x faster than Rust. + +## Why it matters + +The synergy of the AI paradigm shift is leveraging the blockchain for maximum scalability. We need to utilize the generative capabilities to optimize our workflows. + +This sentence has bad grammer and spelling mistaks. It is very poorly written. \ No newline at end of file From 15ef073f936a48374840c037b41743ef324bda00 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 10:19:14 +0100 Subject: [PATCH 13/40] Update command for output file --- src/cli/commands.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/commands.ts b/src/cli/commands.ts index f249c3c..81c6628 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -160,6 +160,7 @@ export function registerMainCommand(program: Command): void { verbose: cliOptions.verbose, outputFormat: outputFormat, ...(mapping ? { mapping } : {}), + ...(cliOptions.outputFile ? { outputFile: cliOptions.outputFile } : {}), }); // Print global summary (only for line format) From a60eea7e70842796eff0b952d5050c31b88f6215 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 10:31:06 +0100 Subject: [PATCH 14/40] Update json formatter to support rdjson --- src/cli/orchestrator.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 69b01bc..5485894 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -203,7 +203,9 @@ function reportIssue(params: ReportIssueParams): void { ...(scoreText !== undefined ? { score: scoreText } : {}), }; (jsonFormatter as ValeJsonFormatter).addIssue(issue); - } else if (outputFormat === 'json') { + } else if (outputFormat === 'json' || outputFormat === 'rdjson') { + // Both json and rdjson use the same JsonFormatter + // rdjson will convert to reviewdog format when toJson('rdjson') is called const severity = status === 'error' ? 'error' : status === 'warning' ? 'warning' : 'info'; const matchLen = match ? match.length : 0; const endColumn = column + matchLen; @@ -850,7 +852,7 @@ export async function evaluateFiles( let requestFailures = 0; let jsonFormatter: ValeJsonFormatter | JsonFormatter; - if (outputFormat === 'json') { + if (outputFormat === 'json' || outputFormat === 'rdjson') { jsonFormatter = new JsonFormatter(); } else { jsonFormatter = new ValeJsonFormatter(); From fdf0fd5622eff4bbef4908af067f072b41ea6bf3 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 2 Dec 2025 11:14:46 +0100 Subject: [PATCH 15/40] Eslint clean up --- src/cli/commands.ts | 2 -- src/cli/orchestrator.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 81c6628..f4b4b79 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -87,8 +87,6 @@ export function registerMainCommand(program: Command): void { process.exit(1); } - - // Prioritize CLI prompts path over config const promptsPath = cliOptions.prompts || config.promptsPath; if (!existsSync(promptsPath)) { console.error(`Error: prompts path does not exist: ${promptsPath}`); diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 5485894..137b2d7 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -204,8 +204,6 @@ function reportIssue(params: ReportIssueParams): void { }; (jsonFormatter as ValeJsonFormatter).addIssue(issue); } else if (outputFormat === 'json' || outputFormat === 'rdjson') { - // Both json and rdjson use the same JsonFormatter - // rdjson will convert to reviewdog format when toJson('rdjson') is called const severity = status === 'error' ? 'error' : status === 'warning' ? 'warning' : 'info'; const matchLen = match ? match.length : 0; const endColumn = column + matchLen; From 8f15e8698ce8e95a3cb8f3efe2598561b13909eb Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 3 Dec 2025 09:23:01 +0100 Subject: [PATCH 16/40] Implement rdjson formatter and clean up integrations --- src/cli/orchestrator.ts | 26 +++---- src/output/json-formatter.ts | 110 +-------------------------- src/output/rdjson-formatter.ts | 133 +++++++++++++++++++++++++++++++++ tests/json-formatter.test.ts | 6 +- 4 files changed, 148 insertions(+), 127 deletions(-) create mode 100644 src/output/rdjson-formatter.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 137b2d7..175adac 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -9,6 +9,7 @@ import { printFileHeader, printIssueRow, printEvaluationSummaries, type Evaluati import { locateEvidenceWithMatch } from '../output/location'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; +import { RdJsonFormatter } from '../output/rdjson-formatter'; import { checkTarget } from '../prompts/target'; import { resolvePromptMapping, aliasForPromptPath, isMappingConfigured } from '../prompts/prompt-mapping'; import { handleUnknownError } from '../errors/index'; @@ -48,7 +49,7 @@ interface EvaluationContext { content: string; relFile: string; outputFormat: 'line' | 'json' | 'vale-json' | 'rdjson'; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } import type { EvaluationResult as PromptEvaluationResult, SubjectiveResult } from '../prompts/schema'; @@ -75,7 +76,7 @@ interface ReportIssueParams { summary: string; ruleName: string; outputFormat: 'line' | 'json' | 'vale-json' | 'rdjson'; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; suggestion?: string; scoreText?: string; match?: string; @@ -147,7 +148,7 @@ type RunPromptEvaluationResult = interface EvaluateFileParams { file: string; options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } interface EvaluateFileResult extends ErrorTrackingResult { @@ -217,7 +218,7 @@ function reportIssue(params: ReportIssueParams): void { match: match || '', ...(suggestion ? { suggestion } : {}) }; - (jsonFormatter as JsonFormatter).addIssue(file, issue); + (jsonFormatter as JsonFormatter | RdJsonFormatter).addIssue(file, issue); } } @@ -678,7 +679,7 @@ function processPromptResult(params: ProcessPromptResultParams): ErrorTrackingRe } if (outputFormat === 'json' && scoreComponents.length > 0) { - (jsonFormatter as JsonFormatter).addEvaluationScore(relFile, { + (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore(relFile, { id: promptId || promptFile.filename.replace(/\.md$/, ''), scores: scoreComponents }); @@ -849,9 +850,11 @@ export async function evaluateFiles( let totalWarnings = 0; let requestFailures = 0; - let jsonFormatter: ValeJsonFormatter | JsonFormatter; - if (outputFormat === 'json' || outputFormat === 'rdjson') { + let jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + if (outputFormat === 'json') { jsonFormatter = new JsonFormatter(); + } else if (outputFormat === 'rdjson') { + jsonFormatter = new RdJsonFormatter(); } else { jsonFormatter = new ValeJsonFormatter(); } @@ -874,14 +877,7 @@ export async function evaluateFiles( // Output results based on format if (outputFormat === 'json' || outputFormat === 'vale-json' || outputFormat === 'rdjson') { - let jsonStr: string; - if (outputFormat === 'vale-json') { - jsonStr = jsonFormatter.toJson(); - } else if (outputFormat === 'rdjson') { - jsonStr = (jsonFormatter as JsonFormatter).toJson('rdjson'); - } else { - jsonStr = (jsonFormatter as JsonFormatter).toJson(); - } + const jsonStr = jsonFormatter.toJson(); if (options.outputFile) { writeFileSync(options.outputFile, jsonStr, 'utf-8'); diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index a39c1eb..2d9e347 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -43,51 +43,6 @@ export interface Result { }; } -export interface RdJsonResult { - source: { - name: string; - url: string; - }; - diagnostics: RdJsonDiagnostic[]; -} - -export interface RdJsonDiagnostic { - message: string; - location: { - path: string; - range: { - start: { - line: number; - column: number; - }; - end?: { - line: number; - column: number; - }; - }; - }; - severity: 'ERROR' | 'WARNING' | 'INFO'; - code?: { - value: string; - url?: string; - }; - suggestions?: RdJsonSuggestion[]; -} - -export interface RdJsonSuggestion { - range: { - start: { - line: number; - column: number; - }; - end: { - line: number; - column: number; - }; - }; - text: string; -} - export class JsonFormatter { private files: Record = {}; private errorCount = 0; @@ -113,70 +68,7 @@ export class JsonFormatter { this.files[file].evaluationScores.push(score); } - toRdJsonFormat(): RdJsonResult { - const diagnostics: RdJsonDiagnostic[] = []; - - // Iterate over all files and their issues - for (const [filePath, fileResult] of Object.entries(this.files)) { - for (const issue of fileResult.issues) { - const matchLen = issue.match.length; - - const diagnostic: RdJsonDiagnostic = { - message: issue.message, - location: { - path: filePath, - range: { - start: { - line: issue.line, - column: issue.column, - }, - end: { - line: issue.line, - column: issue.column + matchLen, - }, - }, - }, - severity: issue.severity === 'error' ? 'ERROR' : issue.severity === 'warning' ? 'WARNING' : 'INFO', - code: { - value: issue.eval, - }, - }; - - if (issue.suggestion) { - diagnostic.suggestions = [ - { - range: { - start: { - line: issue.line, - column: issue.column, - }, - end: { - line: issue.line, - column: issue.column + matchLen, - }, - }, - text: issue.suggestion, - }, - ]; - } - - diagnostics.push(diagnostic); - } - } - - return { - source: { - name: 'vectorlint', - url: 'https://github.com/TRocket-Labs/vectorlint', - }, - diagnostics, - }; - } - - toJson(format: 'standard' | 'rdjson' = 'standard'): string { - if (format === 'rdjson') { - return JSON.stringify(this.toRdJsonFormat(), null, 2); - } + toJson(): string { const result: Result = { files: this.files, summary: { diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts new file mode 100644 index 0000000..6a84ede --- /dev/null +++ b/src/output/rdjson-formatter.ts @@ -0,0 +1,133 @@ +import type { Issue, EvaluationScore } from './json-formatter'; + +export interface RdJsonResult { + source: { + name: string; + url: string; + }; + diagnostics: RdJsonDiagnostic[]; +} + +export interface RdJsonDiagnostic { + message: string; + location: { + path: string; + range: { + start: { + line: number; + column: number; + }; + end?: { + line: number; + column: number; + }; + }; + }; + severity: 'ERROR' | 'WARNING' | 'INFO'; + code?: { + value: string; + url?: string; + }; + suggestions?: RdJsonSuggestion[]; +} + +export interface RdJsonSuggestion { + range: { + start: { + line: number; + column: number; + }; + end: { + line: number; + column: number; + }; + }; + text: string; +} + +interface FileResult { + issues: Issue[]; + evaluationScores: EvaluationScore[]; +} + +export class RdJsonFormatter { + private files: Record = {}; + + addIssue(file: string, issue: Issue): void { + if (!this.files[file]) { + this.files[file] = { issues: [], evaluationScores: [] }; + } + this.files[file].issues.push(issue); + } + + addEvaluationScore(file: string, score: EvaluationScore): void { + if (!this.files[file]) { + this.files[file] = { issues: [], evaluationScores: [] }; + } + this.files[file].evaluationScores.push(score); + } + + toRdJsonFormat(): RdJsonResult { + const diagnostics: RdJsonDiagnostic[] = []; + + // Iterate over all files and their issues + for (const [filePath, fileResult] of Object.entries(this.files)) { + for (const issue of fileResult.issues) { + const matchLen = issue.match.length; + + const diagnostic: RdJsonDiagnostic = { + message: issue.message, + location: { + path: filePath, + range: { + start: { + line: issue.line, + column: issue.column, + }, + end: { + line: issue.line, + column: issue.column + matchLen, + }, + }, + }, + severity: issue.severity === 'error' ? 'ERROR' : issue.severity === 'warning' ? 'WARNING' : 'INFO', + code: { + value: issue.eval, + }, + }; + + if (issue.suggestion) { + diagnostic.suggestions = [ + { + range: { + start: { + line: issue.line, + column: issue.column, + }, + end: { + line: issue.line, + column: issue.column + matchLen, + }, + }, + text: issue.suggestion, + }, + ]; + } + + diagnostics.push(diagnostic); + } + } + + return { + source: { + name: 'vectorlint', + url: 'https://github.com/TRocket-Labs/vectorlint', + }, + diagnostics, + }; + } + + toJson(): string { + return JSON.stringify(this.toRdJsonFormat(), null, 2); + } +} diff --git a/tests/json-formatter.test.ts b/tests/json-formatter.test.ts index 9080088..6fabbaf 100644 --- a/tests/json-formatter.test.ts +++ b/tests/json-formatter.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { ValeJsonFormatter } from '../src/output/vale-json-formatter'; -import { JsonFormatter, type RdJsonResult } from '../src/output/json-formatter'; +import { RdJsonFormatter, type RdJsonResult } from '../src/output/rdjson-formatter'; describe('ValeJsonFormatter', () => { it('should format issues into Vale-compatible JSON structure', () => { @@ -132,7 +132,7 @@ describe('ValeJsonFormatter', () => { }); it('should produce valid RDJSON output', () => { - const formatter = new JsonFormatter(); + const formatter = new RdJsonFormatter(); formatter.addIssue('test.md', { line: 1, @@ -145,7 +145,7 @@ describe('ValeJsonFormatter', () => { suggestion: 'fix' }); - const rdjsonOutput = formatter.toJson('rdjson'); + const rdjsonOutput = formatter.toJson(); // Should be valid JSON expect(() => { From fa746a777bd43dc24ee9d47eebcb5bad2062d5e6 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 3 Dec 2025 10:07:30 +0100 Subject: [PATCH 17/40] fix micromatch ESM import --- src/boundaries/file-section-resolver.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/boundaries/file-section-resolver.ts b/src/boundaries/file-section-resolver.ts index 1ec7835..681837c 100644 --- a/src/boundaries/file-section-resolver.ts +++ b/src/boundaries/file-section-resolver.ts @@ -1,4 +1,4 @@ -import { isMatch } from 'micromatch'; +import micromatch from 'micromatch'; import { FilePatternConfig } from './file-section-parser'; export interface FileResolution { @@ -23,7 +23,7 @@ export class FileSectionResolver { let mergedOverrides: Record = {}; for (const section of sections) { - if ((isMatch as (filePath: string, pattern: string) => boolean)(filePath, section.pattern)) { + if (micromatch.isMatch(filePath, section.pattern)) { // Handle RunEvals if (section.runEvals.length === 0) { // Explicit exclusion: clear active packs From 31f742e640f821f3500bc435a37b74b2941542ce Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 10:21:04 +0100 Subject: [PATCH 18/40] feat: add support for 'rdjson' output format and formatter in evaluation options --- src/cli/types.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cli/types.ts b/src/cli/types.ts index 5f3818b..7582c92 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -6,6 +6,7 @@ import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; +import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { EvaluationResult as PromptEvaluationResult, SubjectiveResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; @@ -17,7 +18,8 @@ export interface EvaluationOptions { concurrency: number; verbose: boolean; scanPaths: FilePatternConfig[]; - outputFormat?: 'line' | 'json' | 'vale-json'; + outputFormat?: 'line' | 'json' | 'vale-json' | 'rdjson'; + outputFile?: string; } export interface EvaluationResult { @@ -40,8 +42,8 @@ export interface ErrorTrackingResult { export interface EvaluationContext { content: string; relFile: string; - outputFormat: 'line' | 'json' | 'vale-json'; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + outputFormat: 'line' | 'json' | 'vale-json' | 'rdjson'; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } export interface ReportIssueParams { @@ -51,8 +53,8 @@ export interface ReportIssueParams { severity: Severity summary: string; ruleName: string; - outputFormat: 'line' | 'json' | 'vale-json'; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + outputFormat: 'line' | 'json' | 'vale-json' | 'rdjson'; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; suggestion?: string; scoreText?: string; match?: string; @@ -124,7 +126,7 @@ export type RunPromptEvaluationResult = export interface EvaluateFileParams { file: string; options: EvaluationOptions; - jsonFormatter: ValeJsonFormatter | JsonFormatter; + jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } export interface EvaluateFileResult extends ErrorTrackingResult { From 2f905530617ba5953ea89b09428ec218645fea7c Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 10:31:52 +0100 Subject: [PATCH 19/40] eslint fix --- src/cli/orchestrator.ts | 2 +- src/output/rdjson-formatter.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 28676b5..0775398 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -10,7 +10,7 @@ import { locateEvidenceWithMatch } from '../output/location'; import { checkTarget } from '../prompts/target'; import { isSubjectiveResult } from '../prompts/schema'; import { Type, Severity } from '../evaluators/types'; -import { EvaluateFileParams, EvaluateFileResult, ExtractMatchTextParams, LocationMatch, ProcessCriterionParams, ProcessCriterionResult, ProcessPromptResultParams, ProcessViolationsParams, ReportIssueParams, RunPromptEvaluationParams, RunPromptEvaluationResult, ValidationParams, EvaluationOptions, EvaluationResult, ErrorTrackingResult, EvaluationContext } from './types'; +import { EvaluateFileParams, EvaluateFileResult, ExtractMatchTextParams, LocationMatch, ProcessCriterionParams, ProcessCriterionResult, ProcessPromptResultParams, ProcessViolationsParams, ReportIssueParams, RunPromptEvaluationParams, RunPromptEvaluationResult, ValidationParams, EvaluationOptions, EvaluationResult, ErrorTrackingResult } from './types'; import { handleUnknownError, MissingDependencyError } from '../errors'; import { createEvaluator } from '../evaluators'; diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts index 6a84ede..4ec4b1a 100644 --- a/src/output/rdjson-formatter.ts +++ b/src/output/rdjson-formatter.ts @@ -1,4 +1,5 @@ import type { Issue, EvaluationScore } from './json-formatter'; +import { Severity } from '../evaluators/types'; export interface RdJsonResult { source: { @@ -90,7 +91,7 @@ export class RdJsonFormatter { }, }, }, - severity: issue.severity === 'error' ? 'ERROR' : issue.severity === 'warning' ? 'WARNING' : 'INFO', + severity: issue.severity === Severity.ERROR ? 'ERROR' : issue.severity === Severity.WARNING ? 'WARNING' : 'INFO', code: { value: issue.eval, }, From aaf0fe7d9b01e498d4982ac697e338ad7267bdfe Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 10:57:38 +0100 Subject: [PATCH 20/40] update vectorlint action --- .github/workflows/vectorlint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index cf32efb..ceba8d8 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 - name: Run VectorLint - uses: ayo6706/vectorlint-test-action@v1 + uses: TRocket-Labs/vectorlint-action@v1 with: llm_provider: ${{ secrets.LLM_PROVIDER }} anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} From cdf1ea5c5d194239d3c6a311eae24a9fb9c1930f Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 12:08:45 +0100 Subject: [PATCH 21/40] feat: implement silent mode for machine-readable output formats --- .github/workflows/vectorlint.yml | 3 +- src/cli/commands.ts | 9 ++++-- src/cli/orchestrator.ts | 18 +++--------- src/cli/types.ts | 1 - src/output/logger.ts | 50 ++++++++++++++++++++++++++++++++ src/schemas/cli-schemas.ts | 1 - 6 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 src/output/logger.ts diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index ceba8d8..547e5b5 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -35,4 +35,5 @@ jobs: reporter: github-pr-check filter_mode: added fail_on_error: false - vectorlint_flags: '--verbose' \ No newline at end of file + vectorlint_flags: '--verbose' + \ No newline at end of file diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 0299bab..1cadcbb 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -8,6 +8,7 @@ import { loadConfig } from '../boundaries/config-loader'; import { loadPromptFile, type PromptFile } from '../prompts/prompt-loader'; import { EvalPackLoader } from '../boundaries/eval-pack-loader'; import { printGlobalSummary } from '../output/reporter'; +import { setSilentMode } from '../output/logger'; import { DefaultRequestBuilder } from '../providers/request-builder'; import { loadDirective } from '../prompts/directive-loader'; import { resolveTargets } from '../scan/file-resolver'; @@ -26,7 +27,6 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Print full JSON response from the API') .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') - .option('--output-file ', 'Write output to a file instead of stdout') .option('--config ', 'Path to custom vectorlint.ini config file') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { @@ -159,6 +159,12 @@ export function registerMainCommand(program: Command): void { const outputFormat = cliOptions.output === 'JSON' ? 'json' : cliOptions.output; + // Enable silent mode for machine-readable formats (rdjson, json) + // This ensures only the structured output goes to stdout, following Vale's approach + if (outputFormat === 'rdjson' || outputFormat === 'json') { + setSilentMode(true); + } + // Run evaluations via orchestrator const result = await evaluateFiles(targets, { prompts, @@ -168,7 +174,6 @@ export function registerMainCommand(program: Command): void { concurrency: config.concurrency, verbose: cliOptions.verbose, outputFormat: outputFormat, - ...(cliOptions.outputFile ? { outputFile: cliOptions.outputFile } : {}), scanPaths: config.scanPaths, }); diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 0775398..ba81709 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,4 +1,4 @@ -import { readFileSync, writeFileSync } from 'fs'; +import { readFileSync } from 'fs'; import * as path from 'path'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; @@ -779,21 +779,11 @@ export async function evaluateFiles( } // Output results based on format + // For machine-readable formats, output ONLY the JSON to stdout (no logs) + // Silent mode is set in commands.ts for rdjson/json formats if (outputFormat === 'json' || outputFormat === 'vale-json' || outputFormat === 'rdjson') { const jsonStr = jsonFormatter.toJson(); - - if (options.outputFile) { - writeFileSync(options.outputFile, jsonStr, 'utf-8'); - if (options.verbose) { - console.error(`[vectorlint] Wrote output to ${options.outputFile}`); - } - } else { - if (outputFormat === 'rdjson' && options.verbose) { - console.error('[vectorlint] Generated RDJSON:'); - console.error(jsonStr); - } - console.log(jsonStr); - } + console.log(jsonStr); } return { diff --git a/src/cli/types.ts b/src/cli/types.ts index 7582c92..0329de4 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -19,7 +19,6 @@ export interface EvaluationOptions { verbose: boolean; scanPaths: FilePatternConfig[]; outputFormat?: 'line' | 'json' | 'vale-json' | 'rdjson'; - outputFile?: string; } export interface EvaluationResult { diff --git a/src/output/logger.ts b/src/output/logger.ts new file mode 100644 index 0000000..15bae6a --- /dev/null +++ b/src/output/logger.ts @@ -0,0 +1,50 @@ +/** + * Logger utility for VectorLint + * + * Provides console output functions that can be silenced for machine-readable + * output formats (rdjson, json). This follows Vale's approach where machine-readable + * formats output ONLY the structured data to stdout with no other logs. + */ + +let silentMode = false; + +/** + * Enable or disable silent mode. + * When enabled, log() and warn() output nothing. + * error() always outputs to stderr. + */ +export function setSilentMode(silent: boolean): void { + silentMode = silent; +} + +/** + * Get current silent mode status. + */ +export function isSilentMode(): boolean { + return silentMode; +} + +/** + * Log to stdout. Silenced in silent mode. + */ +export function log(...args: unknown[]): void { + if (!silentMode) { + console.log(...args); + } +} + +/** + * Log warning to stderr. Silenced in silent mode. + */ +export function warn(...args: unknown[]): void { + if (!silentMode) { + console.warn(...args); + } +} + +/** + * Log error to stderr. ALWAYS outputs (never silenced). + */ +export function error(...args: unknown[]): void { + console.error(...args); +} diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 6aaa489..4db1b52 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -7,7 +7,6 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.enum(['line', 'json', 'vale-json', 'JSON', 'rdjson']).default('line'), - outputFile: z.string().optional(), prompts: z.string().optional(), evals: z.string().optional(), config: z.string().optional(), From 0d49ab68f022a95a05433ca5f78ab81487447cf7 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 12:58:13 +0100 Subject: [PATCH 22/40] Update vectorlint.ini to new version --- vectorlint.ini | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/vectorlint.ini b/vectorlint.ini index 4e6d6d5..2397594 100644 --- a/vectorlint.ini +++ b/vectorlint.ini @@ -1,20 +1,6 @@ -PromptsPath=evals -ScanPaths=[*.md] +RulesPath=.github/rules Concurrency=4 DefaultSeverity=warning -# Optional: map prompts to files -[Prompts] -paths=["Default:prompts", "Blog:prompts/blog"] - -[Defaults] -include=["**/*.md"] -exclude=["archived/**"] - -[Directory:Blog] -include=["content/blog/**/*.md"] -exclude=["content/blog/drafts/**"] - -[Prompt:Headline] -include=["content/blog/**/*.md"] -exclude=["content/blog/drafts/**"] \ No newline at end of file +[**/*.md] +RunRules=TinyRocket From ac48b8c5b8b2888078fa75baae65141dae68f113 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 15:27:55 +0100 Subject: [PATCH 23/40] Revert to old implementation --- .github/workflows/vectorlint.yml | 3 +- .gitignore | 3 +- src/cli/commands.ts | 9 ++---- src/cli/orchestrator.ts | 18 +++++++++--- src/cli/types.ts | 1 + src/output/logger.ts | 50 -------------------------------- src/schemas/cli-schemas.ts | 1 + 7 files changed, 21 insertions(+), 64 deletions(-) delete mode 100644 src/output/logger.ts diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 547e5b5..b1881cc 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -35,5 +35,4 @@ jobs: reporter: github-pr-check filter_mode: added fail_on_error: false - vectorlint_flags: '--verbose' - \ No newline at end of file + vectorlint_flags: '--verbose' \ No newline at end of file diff --git a/.gitignore b/.gitignore index a481c29..3dbae5d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,10 @@ dist/ .idea/ coverage/ *.tsbuildinfo -# vectorlint.ini +vectorlint.ini .kiro/ # .agent/ /.idea /npm + diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 1cadcbb..0299bab 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -8,7 +8,6 @@ import { loadConfig } from '../boundaries/config-loader'; import { loadPromptFile, type PromptFile } from '../prompts/prompt-loader'; import { EvalPackLoader } from '../boundaries/eval-pack-loader'; import { printGlobalSummary } from '../output/reporter'; -import { setSilentMode } from '../output/logger'; import { DefaultRequestBuilder } from '../providers/request-builder'; import { loadDirective } from '../prompts/directive-loader'; import { resolveTargets } from '../scan/file-resolver'; @@ -27,6 +26,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Print full JSON response from the API') .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') + .option('--output-file ', 'Write output to a file instead of stdout') .option('--config ', 'Path to custom vectorlint.ini config file') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { @@ -159,12 +159,6 @@ export function registerMainCommand(program: Command): void { const outputFormat = cliOptions.output === 'JSON' ? 'json' : cliOptions.output; - // Enable silent mode for machine-readable formats (rdjson, json) - // This ensures only the structured output goes to stdout, following Vale's approach - if (outputFormat === 'rdjson' || outputFormat === 'json') { - setSilentMode(true); - } - // Run evaluations via orchestrator const result = await evaluateFiles(targets, { prompts, @@ -174,6 +168,7 @@ export function registerMainCommand(program: Command): void { concurrency: config.concurrency, verbose: cliOptions.verbose, outputFormat: outputFormat, + ...(cliOptions.outputFile ? { outputFile: cliOptions.outputFile } : {}), scanPaths: config.scanPaths, }); diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index ba81709..0775398 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,4 +1,4 @@ -import { readFileSync } from 'fs'; +import { readFileSync, writeFileSync } from 'fs'; import * as path from 'path'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; @@ -779,11 +779,21 @@ export async function evaluateFiles( } // Output results based on format - // For machine-readable formats, output ONLY the JSON to stdout (no logs) - // Silent mode is set in commands.ts for rdjson/json formats if (outputFormat === 'json' || outputFormat === 'vale-json' || outputFormat === 'rdjson') { const jsonStr = jsonFormatter.toJson(); - console.log(jsonStr); + + if (options.outputFile) { + writeFileSync(options.outputFile, jsonStr, 'utf-8'); + if (options.verbose) { + console.error(`[vectorlint] Wrote output to ${options.outputFile}`); + } + } else { + if (outputFormat === 'rdjson' && options.verbose) { + console.error('[vectorlint] Generated RDJSON:'); + console.error(jsonStr); + } + console.log(jsonStr); + } } return { diff --git a/src/cli/types.ts b/src/cli/types.ts index 0329de4..7582c92 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -19,6 +19,7 @@ export interface EvaluationOptions { verbose: boolean; scanPaths: FilePatternConfig[]; outputFormat?: 'line' | 'json' | 'vale-json' | 'rdjson'; + outputFile?: string; } export interface EvaluationResult { diff --git a/src/output/logger.ts b/src/output/logger.ts deleted file mode 100644 index 15bae6a..0000000 --- a/src/output/logger.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Logger utility for VectorLint - * - * Provides console output functions that can be silenced for machine-readable - * output formats (rdjson, json). This follows Vale's approach where machine-readable - * formats output ONLY the structured data to stdout with no other logs. - */ - -let silentMode = false; - -/** - * Enable or disable silent mode. - * When enabled, log() and warn() output nothing. - * error() always outputs to stderr. - */ -export function setSilentMode(silent: boolean): void { - silentMode = silent; -} - -/** - * Get current silent mode status. - */ -export function isSilentMode(): boolean { - return silentMode; -} - -/** - * Log to stdout. Silenced in silent mode. - */ -export function log(...args: unknown[]): void { - if (!silentMode) { - console.log(...args); - } -} - -/** - * Log warning to stderr. Silenced in silent mode. - */ -export function warn(...args: unknown[]): void { - if (!silentMode) { - console.warn(...args); - } -} - -/** - * Log error to stderr. ALWAYS outputs (never silenced). - */ -export function error(...args: unknown[]): void { - console.error(...args); -} diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 4db1b52..6aaa489 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -7,6 +7,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.enum(['line', 'json', 'vale-json', 'JSON', 'rdjson']).default('line'), + outputFile: z.string().optional(), prompts: z.string().optional(), evals: z.string().optional(), config: z.string().optional(), From 12dd4ccea751f35abc3b76c171ea358c8f2fe33c Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 15:56:32 +0100 Subject: [PATCH 24/40] refactor: remove output file option and update types for consistency --- src/boundaries/file-section-parser.ts | 11 +++++++---- src/cli/commands.ts | 2 -- src/cli/orchestrator.ts | 18 +++--------------- src/cli/types.ts | 1 - src/schemas/cli-schemas.ts | 1 - 5 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/boundaries/file-section-parser.ts b/src/boundaries/file-section-parser.ts index 8cf7f29..6e6c094 100644 --- a/src/boundaries/file-section-parser.ts +++ b/src/boundaries/file-section-parser.ts @@ -1,8 +1,8 @@ export interface FilePatternConfig { pattern: string; - runRules?: string[]; // List of pack names to run (optional) - overrides: Record; + runRules?: string[] | undefined; // List of pack names to run (optional) + overrides: Record; } export class FileSectionParser { @@ -32,10 +32,13 @@ export class FileSectionParser { } } - const overrides: Record = {}; + const overrides: Record = {}; for (const [propKey, propValue] of Object.entries(sectionConfig)) { if (propKey !== 'RunRules') { - overrides[propKey] = propValue; + // INI values are strings, but may be parsed as numbers/booleans + if (typeof propValue === 'string' || typeof propValue === 'number' || typeof propValue === 'boolean') { + overrides[propKey] = propValue; + } } } diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 0299bab..fcecccd 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -26,7 +26,6 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Print full JSON response from the API') .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') - .option('--output-file ', 'Write output to a file instead of stdout') .option('--config ', 'Path to custom vectorlint.ini config file') .argument('[paths...]', 'files or directories to check (optional)') .action(async (paths: string[] = []) => { @@ -168,7 +167,6 @@ export function registerMainCommand(program: Command): void { concurrency: config.concurrency, verbose: cliOptions.verbose, outputFormat: outputFormat, - ...(cliOptions.outputFile ? { outputFile: cliOptions.outputFile } : {}), scanPaths: config.scanPaths, }); diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 0775398..620b06f 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1,4 +1,4 @@ -import { readFileSync, writeFileSync } from 'fs'; +import { readFileSync } from 'fs'; import * as path from 'path'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; @@ -778,22 +778,10 @@ export async function evaluateFiles( } } - // Output results based on format + // Output results based on format (always to stdout for JSON formats) if (outputFormat === 'json' || outputFormat === 'vale-json' || outputFormat === 'rdjson') { const jsonStr = jsonFormatter.toJson(); - - if (options.outputFile) { - writeFileSync(options.outputFile, jsonStr, 'utf-8'); - if (options.verbose) { - console.error(`[vectorlint] Wrote output to ${options.outputFile}`); - } - } else { - if (outputFormat === 'rdjson' && options.verbose) { - console.error('[vectorlint] Generated RDJSON:'); - console.error(jsonStr); - } - console.log(jsonStr); - } + console.log(jsonStr); } return { diff --git a/src/cli/types.ts b/src/cli/types.ts index 7582c92..0329de4 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -19,7 +19,6 @@ export interface EvaluationOptions { verbose: boolean; scanPaths: FilePatternConfig[]; outputFormat?: 'line' | 'json' | 'vale-json' | 'rdjson'; - outputFile?: string; } export interface EvaluationResult { diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 6aaa489..4db1b52 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -7,7 +7,6 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.enum(['line', 'json', 'vale-json', 'JSON', 'rdjson']).default('line'), - outputFile: z.string().optional(), prompts: z.string().optional(), evals: z.string().optional(), config: z.string().optional(), From fa18b23a4b776a7007ebfad32133b694890667bb Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 16:13:21 +0100 Subject: [PATCH 25/40] fix: change logging from console.log to console.error for directive length --- src/cli/commands.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/commands.ts b/src/cli/commands.ts index fcecccd..ff70f57 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -74,7 +74,7 @@ export function registerMainCommand(program: Command): void { if (cliOptions.verbose) { const directiveLen = directive ? directive.length : 0; - console.log(`[vectorlint] Directive active: ${directiveLen} char(s)`); + console.error(`[vectorlint] Directive active: ${directiveLen} char(s)`); } // Load config and prompts From cf0acf6307c3782a86e0d1addf1d723412617574 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 20:08:10 +0100 Subject: [PATCH 26/40] fix: change logging from console.log to console.error to resolve std issues --- src/providers/anthropic-provider.ts | 28 +++++++++++++------------- src/providers/azure-openai-provider.ts | 28 +++++++++++++------------- src/providers/openai-provider.ts | 28 +++++++++++++------------- src/providers/perplexity-provider.ts | 7 +++---- 4 files changed, 45 insertions(+), 46 deletions(-) diff --git a/src/providers/anthropic-provider.ts b/src/providers/anthropic-provider.ts index c489b50..05dfa90 100644 --- a/src/providers/anthropic-provider.ts +++ b/src/providers/anthropic-provider.ts @@ -103,24 +103,24 @@ export class AnthropicProvider implements LLMProvider { } if (this.config.debug) { - console.log('[vectorlint] Sending request to Anthropic:', { + console.error('[vectorlint] Sending request to Anthropic:', { model: this.config.model, maxTokens: this.config.maxTokens, temperature: this.config.temperature, }); if (this.config.showPrompt) { - console.log('[vectorlint] System prompt (full):'); - console.log(systemPrompt); - console.log('[vectorlint] User content (full):'); - console.log(content); + console.error('[vectorlint] System prompt (full):'); + console.error(systemPrompt); + console.error('[vectorlint] User content (full):'); + console.error(content); } else if (this.config.showPromptTrunc) { - console.log('[vectorlint] System prompt (first 500 chars):'); - console.log(systemPrompt.slice(0, 500)); - if (systemPrompt.length > 500) console.log('... [truncated]'); + console.error('[vectorlint] System prompt (first 500 chars):'); + console.error(systemPrompt.slice(0, 500)); + if (systemPrompt.length > 500) console.error('... [truncated]'); const preview = content.slice(0, 500); - console.log('[vectorlint] User content preview (first 500 chars):'); - console.log(preview); - if (content.length > 500) console.log('... [truncated]'); + console.error('[vectorlint] User content preview (first 500 chars):'); + console.error(preview); + if (content.length > 500) console.error('... [truncated]'); } } @@ -180,7 +180,7 @@ export class AnthropicProvider implements LLMProvider { const usage = response.usage; const stopReason = response.stop_reason; if (usage || stopReason) { - console.log('[vectorlint] LLM response meta:', { + console.error('[vectorlint] LLM response meta:', { usage: { input_tokens: usage.input_tokens, output_tokens: usage.output_tokens, @@ -190,8 +190,8 @@ export class AnthropicProvider implements LLMProvider { } if (this.config.debugJson) { try { - console.log('[vectorlint] Full JSON response:'); - console.log(JSON.stringify(response, null, 2)); + console.error('[vectorlint] Full JSON response:'); + console.error(JSON.stringify(response, null, 2)); } catch (e: unknown) { const err = handleUnknownError(e, 'JSON stringify for debug'); console.warn(`[vectorlint] Warning: ${err.message}`); diff --git a/src/providers/azure-openai-provider.ts b/src/providers/azure-openai-provider.ts index e10ecd7..59cc0d0 100644 --- a/src/providers/azure-openai-provider.ts +++ b/src/providers/azure-openai-provider.ts @@ -67,24 +67,24 @@ export class AzureOpenAIProvider implements LLMProvider { } if (this.debug) { - console.log('[vectorlint] Sending request to Azure OpenAI:', { + console.error('[vectorlint] Sending request to Azure OpenAI:', { model: this.deploymentName, apiVersion: this.apiVersion || AZURE_OPENAI_DEFAULT_CONFIG.apiVersion, temperature: this.temperature, }); if (this.showPrompt) { - console.log('[vectorlint] Prompt (full):'); - console.log(prompt); - console.log('[vectorlint] Injected content (full):'); - console.log(content); + console.error('[vectorlint] Prompt (full):'); + console.error(prompt); + console.error('[vectorlint] Injected content (full):'); + console.error(content); } else if (this.showPromptTrunc) { - console.log('[vectorlint] Prompt (first 500 chars):'); - console.log(prompt.slice(0, 500)); - if (prompt.length > 500) console.log('... [truncated]'); + console.error('[vectorlint] Prompt (first 500 chars):'); + console.error(prompt.slice(0, 500)); + if (prompt.length > 500) console.error('... [truncated]'); const preview = content.slice(0, 500); - console.log('[vectorlint] Injected content preview (first 500 chars):'); - console.log(preview); - if (content.length > 500) console.log('... [truncated]'); + console.error('[vectorlint] Injected content preview (first 500 chars):'); + console.error(preview); + if (content.length > 500) console.error('... [truncated]'); } } @@ -116,12 +116,12 @@ export class AzureOpenAIProvider implements LLMProvider { const usage = validatedResponse.usage; const finish = validatedResponse.choices[0]?.finish_reason; if (usage || finish) { - console.log('[vectorlint] LLM response meta:', { usage, finish_reason: finish }); + console.error('[vectorlint] LLM response meta:', { usage, finish_reason: finish }); } if (this.debugJson) { try { - console.log('[vectorlint] Full JSON response:'); - console.log(JSON.stringify(validatedResponse, null, 2)); + console.error('[vectorlint] Full JSON response:'); + console.error(JSON.stringify(validatedResponse, null, 2)); } catch (e: unknown) { const err = handleUnknownError(e, 'JSON stringify for debug'); console.warn(`[vectorlint] Warning: ${err.message}`); diff --git a/src/providers/openai-provider.ts b/src/providers/openai-provider.ts index a61eb3f..a5b5256 100644 --- a/src/providers/openai-provider.ts +++ b/src/providers/openai-provider.ts @@ -86,24 +86,24 @@ export class OpenAIProvider implements LLMProvider { } if (this.config.debug) { - console.log('[vectorlint] Sending request to OpenAI:', { + console.error('[vectorlint] Sending request to OpenAI:', { model: this.config.model, temperature: this.config.temperature, }); if (this.config.showPrompt) { - console.log('[vectorlint] System prompt (full):'); - console.log(systemPrompt); - console.log('[vectorlint] User content (full):'); - console.log(content); + console.error('[vectorlint] System prompt (full):'); + console.error(systemPrompt); + console.error('[vectorlint] User content (full):'); + console.error(content); } else if (this.config.showPromptTrunc) { - console.log('[vectorlint] System prompt (first 500 chars):'); - console.log(systemPrompt.slice(0, 500)); - if (systemPrompt.length > 500) console.log('... [truncated]'); + console.error('[vectorlint] System prompt (first 500 chars):'); + console.error(systemPrompt.slice(0, 500)); + if (systemPrompt.length > 500) console.error('... [truncated]'); const preview = content.slice(0, 500); - console.log('[vectorlint] User content preview (first 500 chars):'); - console.log(preview); - if (content.length > 500) console.log('... [truncated]'); + console.error('[vectorlint] User content preview (first 500 chars):'); + console.error(preview); + if (content.length > 500) console.error('... [truncated]'); } } @@ -134,7 +134,7 @@ export class OpenAIProvider implements LLMProvider { const usage = validatedResponse.usage; const firstChoice = validatedResponse.choices[0]; if (usage || firstChoice) { - console.log('[vectorlint] LLM response meta:', { + console.error('[vectorlint] LLM response meta:', { usage: usage ? { prompt_tokens: usage.prompt_tokens, completion_tokens: usage.completion_tokens, @@ -145,8 +145,8 @@ export class OpenAIProvider implements LLMProvider { } if (this.config.debugJson) { try { - console.log('[vectorlint] Full JSON response:'); - console.log(JSON.stringify(rawResponse, null, 2)); + console.error('[vectorlint] Full JSON response:'); + console.error(JSON.stringify(rawResponse, null, 2)); } catch (e: unknown) { const err = handleUnknownError(e, 'JSON stringify for debug'); console.warn(`[vectorlint] Warning: ${err.message}`); diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts index 0f237eb..45ec592 100644 --- a/src/providers/perplexity-provider.ts +++ b/src/providers/perplexity-provider.ts @@ -24,8 +24,7 @@ export class PerplexitySearchProvider implements SearchProvider { async search(query: string): Promise { if (!query?.trim()) throw new Error('Search query cannot be empty.'); - if (this.debug) console.log(`[Perplexity] Searching: "${query}"`); - + if (this.debug) console.error(`[Perplexity] Searching: "${query}"`); try { const rawResponse: unknown = await this.client.search.create({ query, @@ -41,8 +40,8 @@ export class PerplexitySearchProvider implements SearchProvider { const results = validated.results; if (this.debug) { - console.log(`[Perplexity] Found ${results.length} results`); - console.log(results.slice(0, 2)); + console.error(`[Perplexity] Found ${results.length} results`); + console.error(results.slice(0, 2)); } return results; From 6131be0af75b98828a741d4ce1a09d481cdf0059 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 20:27:59 +0100 Subject: [PATCH 27/40] Revert llm provider logs and remove verbose parameter in vectorlint.yml --- .github/workflows/vectorlint.yml | 2 +- src/providers/anthropic-provider.ts | 28 +++++++++++++------------- src/providers/azure-openai-provider.ts | 28 +++++++++++++------------- src/providers/openai-provider.ts | 28 +++++++++++++------------- src/providers/perplexity-provider.ts | 7 ++++--- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index b1881cc..4c9bbf1 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -35,4 +35,4 @@ jobs: reporter: github-pr-check filter_mode: added fail_on_error: false - vectorlint_flags: '--verbose' \ No newline at end of file + \ No newline at end of file diff --git a/src/providers/anthropic-provider.ts b/src/providers/anthropic-provider.ts index 05dfa90..c489b50 100644 --- a/src/providers/anthropic-provider.ts +++ b/src/providers/anthropic-provider.ts @@ -103,24 +103,24 @@ export class AnthropicProvider implements LLMProvider { } if (this.config.debug) { - console.error('[vectorlint] Sending request to Anthropic:', { + console.log('[vectorlint] Sending request to Anthropic:', { model: this.config.model, maxTokens: this.config.maxTokens, temperature: this.config.temperature, }); if (this.config.showPrompt) { - console.error('[vectorlint] System prompt (full):'); - console.error(systemPrompt); - console.error('[vectorlint] User content (full):'); - console.error(content); + console.log('[vectorlint] System prompt (full):'); + console.log(systemPrompt); + console.log('[vectorlint] User content (full):'); + console.log(content); } else if (this.config.showPromptTrunc) { - console.error('[vectorlint] System prompt (first 500 chars):'); - console.error(systemPrompt.slice(0, 500)); - if (systemPrompt.length > 500) console.error('... [truncated]'); + console.log('[vectorlint] System prompt (first 500 chars):'); + console.log(systemPrompt.slice(0, 500)); + if (systemPrompt.length > 500) console.log('... [truncated]'); const preview = content.slice(0, 500); - console.error('[vectorlint] User content preview (first 500 chars):'); - console.error(preview); - if (content.length > 500) console.error('... [truncated]'); + console.log('[vectorlint] User content preview (first 500 chars):'); + console.log(preview); + if (content.length > 500) console.log('... [truncated]'); } } @@ -180,7 +180,7 @@ export class AnthropicProvider implements LLMProvider { const usage = response.usage; const stopReason = response.stop_reason; if (usage || stopReason) { - console.error('[vectorlint] LLM response meta:', { + console.log('[vectorlint] LLM response meta:', { usage: { input_tokens: usage.input_tokens, output_tokens: usage.output_tokens, @@ -190,8 +190,8 @@ export class AnthropicProvider implements LLMProvider { } if (this.config.debugJson) { try { - console.error('[vectorlint] Full JSON response:'); - console.error(JSON.stringify(response, null, 2)); + console.log('[vectorlint] Full JSON response:'); + console.log(JSON.stringify(response, null, 2)); } catch (e: unknown) { const err = handleUnknownError(e, 'JSON stringify for debug'); console.warn(`[vectorlint] Warning: ${err.message}`); diff --git a/src/providers/azure-openai-provider.ts b/src/providers/azure-openai-provider.ts index 59cc0d0..e10ecd7 100644 --- a/src/providers/azure-openai-provider.ts +++ b/src/providers/azure-openai-provider.ts @@ -67,24 +67,24 @@ export class AzureOpenAIProvider implements LLMProvider { } if (this.debug) { - console.error('[vectorlint] Sending request to Azure OpenAI:', { + console.log('[vectorlint] Sending request to Azure OpenAI:', { model: this.deploymentName, apiVersion: this.apiVersion || AZURE_OPENAI_DEFAULT_CONFIG.apiVersion, temperature: this.temperature, }); if (this.showPrompt) { - console.error('[vectorlint] Prompt (full):'); - console.error(prompt); - console.error('[vectorlint] Injected content (full):'); - console.error(content); + console.log('[vectorlint] Prompt (full):'); + console.log(prompt); + console.log('[vectorlint] Injected content (full):'); + console.log(content); } else if (this.showPromptTrunc) { - console.error('[vectorlint] Prompt (first 500 chars):'); - console.error(prompt.slice(0, 500)); - if (prompt.length > 500) console.error('... [truncated]'); + console.log('[vectorlint] Prompt (first 500 chars):'); + console.log(prompt.slice(0, 500)); + if (prompt.length > 500) console.log('... [truncated]'); const preview = content.slice(0, 500); - console.error('[vectorlint] Injected content preview (first 500 chars):'); - console.error(preview); - if (content.length > 500) console.error('... [truncated]'); + console.log('[vectorlint] Injected content preview (first 500 chars):'); + console.log(preview); + if (content.length > 500) console.log('... [truncated]'); } } @@ -116,12 +116,12 @@ export class AzureOpenAIProvider implements LLMProvider { const usage = validatedResponse.usage; const finish = validatedResponse.choices[0]?.finish_reason; if (usage || finish) { - console.error('[vectorlint] LLM response meta:', { usage, finish_reason: finish }); + console.log('[vectorlint] LLM response meta:', { usage, finish_reason: finish }); } if (this.debugJson) { try { - console.error('[vectorlint] Full JSON response:'); - console.error(JSON.stringify(validatedResponse, null, 2)); + console.log('[vectorlint] Full JSON response:'); + console.log(JSON.stringify(validatedResponse, null, 2)); } catch (e: unknown) { const err = handleUnknownError(e, 'JSON stringify for debug'); console.warn(`[vectorlint] Warning: ${err.message}`); diff --git a/src/providers/openai-provider.ts b/src/providers/openai-provider.ts index a5b5256..a61eb3f 100644 --- a/src/providers/openai-provider.ts +++ b/src/providers/openai-provider.ts @@ -86,24 +86,24 @@ export class OpenAIProvider implements LLMProvider { } if (this.config.debug) { - console.error('[vectorlint] Sending request to OpenAI:', { + console.log('[vectorlint] Sending request to OpenAI:', { model: this.config.model, temperature: this.config.temperature, }); if (this.config.showPrompt) { - console.error('[vectorlint] System prompt (full):'); - console.error(systemPrompt); - console.error('[vectorlint] User content (full):'); - console.error(content); + console.log('[vectorlint] System prompt (full):'); + console.log(systemPrompt); + console.log('[vectorlint] User content (full):'); + console.log(content); } else if (this.config.showPromptTrunc) { - console.error('[vectorlint] System prompt (first 500 chars):'); - console.error(systemPrompt.slice(0, 500)); - if (systemPrompt.length > 500) console.error('... [truncated]'); + console.log('[vectorlint] System prompt (first 500 chars):'); + console.log(systemPrompt.slice(0, 500)); + if (systemPrompt.length > 500) console.log('... [truncated]'); const preview = content.slice(0, 500); - console.error('[vectorlint] User content preview (first 500 chars):'); - console.error(preview); - if (content.length > 500) console.error('... [truncated]'); + console.log('[vectorlint] User content preview (first 500 chars):'); + console.log(preview); + if (content.length > 500) console.log('... [truncated]'); } } @@ -134,7 +134,7 @@ export class OpenAIProvider implements LLMProvider { const usage = validatedResponse.usage; const firstChoice = validatedResponse.choices[0]; if (usage || firstChoice) { - console.error('[vectorlint] LLM response meta:', { + console.log('[vectorlint] LLM response meta:', { usage: usage ? { prompt_tokens: usage.prompt_tokens, completion_tokens: usage.completion_tokens, @@ -145,8 +145,8 @@ export class OpenAIProvider implements LLMProvider { } if (this.config.debugJson) { try { - console.error('[vectorlint] Full JSON response:'); - console.error(JSON.stringify(rawResponse, null, 2)); + console.log('[vectorlint] Full JSON response:'); + console.log(JSON.stringify(rawResponse, null, 2)); } catch (e: unknown) { const err = handleUnknownError(e, 'JSON stringify for debug'); console.warn(`[vectorlint] Warning: ${err.message}`); diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts index 45ec592..0f237eb 100644 --- a/src/providers/perplexity-provider.ts +++ b/src/providers/perplexity-provider.ts @@ -24,7 +24,8 @@ export class PerplexitySearchProvider implements SearchProvider { async search(query: string): Promise { if (!query?.trim()) throw new Error('Search query cannot be empty.'); - if (this.debug) console.error(`[Perplexity] Searching: "${query}"`); + if (this.debug) console.log(`[Perplexity] Searching: "${query}"`); + try { const rawResponse: unknown = await this.client.search.create({ query, @@ -40,8 +41,8 @@ export class PerplexitySearchProvider implements SearchProvider { const results = validated.results; if (this.debug) { - console.error(`[Perplexity] Found ${results.length} results`); - console.error(results.slice(0, 2)); + console.log(`[Perplexity] Found ${results.length} results`); + console.log(results.slice(0, 2)); } return results; From c3b9149285f2a3ca6fc95da879053b42f345cba7 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 20:59:12 +0100 Subject: [PATCH 28/40] eslint fix --- src/boundaries/file-section-parser.ts | 11 ++++------- src/cli/commands.ts | 2 +- src/cli/orchestrator.ts | 14 ++++++++------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/boundaries/file-section-parser.ts b/src/boundaries/file-section-parser.ts index 6e6c094..8cf7f29 100644 --- a/src/boundaries/file-section-parser.ts +++ b/src/boundaries/file-section-parser.ts @@ -1,8 +1,8 @@ export interface FilePatternConfig { pattern: string; - runRules?: string[] | undefined; // List of pack names to run (optional) - overrides: Record; + runRules?: string[]; // List of pack names to run (optional) + overrides: Record; } export class FileSectionParser { @@ -32,13 +32,10 @@ export class FileSectionParser { } } - const overrides: Record = {}; + const overrides: Record = {}; for (const [propKey, propValue] of Object.entries(sectionConfig)) { if (propKey !== 'RunRules') { - // INI values are strings, but may be parsed as numbers/booleans - if (typeof propValue === 'string' || typeof propValue === 'number' || typeof propValue === 'boolean') { - overrides[propKey] = propValue; - } + overrides[propKey] = propValue; } } diff --git a/src/cli/commands.ts b/src/cli/commands.ts index ff70f57..fcecccd 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -74,7 +74,7 @@ export function registerMainCommand(program: Command): void { if (cliOptions.verbose) { const directiveLen = directive ? directive.length : 0; - console.error(`[vectorlint] Directive active: ${directiveLen} char(s)`); + console.log(`[vectorlint] Directive active: ${directiveLen} char(s)`); } // Load config and prompts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 620b06f..0e59827 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -9,13 +9,15 @@ import { printFileHeader, printIssueRow, printEvaluationSummaries, type Evaluati import { locateEvidenceWithMatch } from '../output/location'; import { checkTarget } from '../prompts/target'; import { isSubjectiveResult } from '../prompts/schema'; +import { handleUnknownError, MissingDependencyError } from '../errors/index'; +import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; -import { EvaluateFileParams, EvaluateFileResult, ExtractMatchTextParams, LocationMatch, ProcessCriterionParams, ProcessCriterionResult, ProcessPromptResultParams, ProcessViolationsParams, ReportIssueParams, RunPromptEvaluationParams, RunPromptEvaluationResult, ValidationParams, EvaluationOptions, EvaluationResult, ErrorTrackingResult } from './types'; -import { handleUnknownError, MissingDependencyError } from '../errors'; -import { createEvaluator } from '../evaluators'; - - - +import type { + EvaluationOptions, EvaluationResult, ErrorTrackingResult, + ReportIssueParams, ExtractMatchTextParams, LocationMatch, ProcessViolationsParams, + ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, + RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult +} from './types'; /* * Returns the evaluator type, defaulting to 'base' if not specified. From c9c39c4bfea8658886b32c2885f1723459a69779 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 21:10:20 +0100 Subject: [PATCH 29/40] test: add RDJSON formatter tests for valid output --- tests/json-formatter.test.ts | 40 ------------------------------- tests/rdjson-formatter.test.ts | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 40 deletions(-) create mode 100644 tests/rdjson-formatter.test.ts diff --git a/tests/json-formatter.test.ts b/tests/json-formatter.test.ts index 6fabbaf..dfc85bd 100644 --- a/tests/json-formatter.test.ts +++ b/tests/json-formatter.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect } from 'vitest'; import { ValeJsonFormatter } from '../src/output/vale-json-formatter'; -import { RdJsonFormatter, type RdJsonResult } from '../src/output/rdjson-formatter'; describe('ValeJsonFormatter', () => { it('should format issues into Vale-compatible JSON structure', () => { @@ -130,43 +129,4 @@ describe('ValeJsonFormatter', () => { const parsed: unknown = JSON.parse(jsonOutput); expect(parsed).toHaveProperty('test.md'); }); - - it('should produce valid RDJSON output', () => { - const formatter = new RdJsonFormatter(); - - formatter.addIssue('test.md', { - line: 1, - column: 1, - span: [1, 6], - severity: 'error', - message: 'Test message', - eval: 'TestRule', - match: 'match', - suggestion: 'fix' - }); - - const rdjsonOutput = formatter.toJson(); - - // Should be valid JSON - expect(() => { - JSON.parse(rdjsonOutput); - }).not.toThrow(); - - const parsed = JSON.parse(rdjsonOutput) as RdJsonResult; - expect(parsed).toHaveProperty('source'); - expect(parsed.source.name).toBe('vectorlint'); - expect(parsed).toHaveProperty('diagnostics'); - expect(parsed.diagnostics).toHaveLength(1); - - const diag = parsed.diagnostics[0]!; - expect(diag.message).toBe('Test message'); - expect(diag.severity).toBe('ERROR'); - expect(diag.location.path).toBe('test.md'); - expect(diag.location.range.start.line).toBe(1); - expect(diag.location.range.start.column).toBe(1); - expect(diag.location.range.end!.column).toBe(6); // 1 + length of 'match' (5) - - expect(diag.suggestions).toHaveLength(1); - expect(diag.suggestions![0]!.text).toBe('fix'); - }); }); diff --git a/tests/rdjson-formatter.test.ts b/tests/rdjson-formatter.test.ts new file mode 100644 index 0000000..2a394b8 --- /dev/null +++ b/tests/rdjson-formatter.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from 'vitest'; +import { RdJsonFormatter, type RdJsonResult } from '../src/output/rdjson-formatter'; +import { Severity } from '../src/evaluators/types'; + +describe('RdJsonFormatter', () => { + it('should produce valid RDJSON output', () => { + const formatter = new RdJsonFormatter(); + + formatter.addIssue('test.md', { + line: 1, + column: 1, + span: [1, 6], + severity: Severity.ERROR, + message: 'Test message', + eval: 'TestRule', + match: 'match', + suggestion: 'fix' + }); + + const rdjsonOutput = formatter.toJson(); + + // Should be valid JSON + expect(() => { + JSON.parse(rdjsonOutput); + }).not.toThrow(); + + const parsed = JSON.parse(rdjsonOutput) as RdJsonResult; + expect(parsed).toHaveProperty('source'); + expect(parsed.source.name).toBe('vectorlint'); + expect(parsed).toHaveProperty('diagnostics'); + expect(parsed.diagnostics).toHaveLength(1); + + const diag = parsed.diagnostics[0]!; + expect(diag.message).toBe('Test message'); + expect(diag.severity).toBe('ERROR'); + expect(diag.location.path).toBe('test.md'); + expect(diag.location.range.start.line).toBe(1); + expect(diag.location.range.start.column).toBe(1); + expect(diag.location.range.end!.column).toBe(6); // 1 + length of 'match' (5) + + expect(diag.suggestions).toHaveLength(1); + expect(diag.suggestions![0]!.text).toBe('fix'); + }); +}); From 912333f286c20f499e5f476131ecd27750cfa0e3 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 21:40:26 +0100 Subject: [PATCH 30/40] fix: update vectorlint.yml to remove evals path and add missing parameters for LLM providers --- .github/workflows/vectorlint.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml index 4c9bbf1..f6b7ac5 100644 --- a/.github/workflows/vectorlint.yml +++ b/.github/workflows/vectorlint.yml @@ -6,7 +6,6 @@ on: - '**/*.md' - '**/*.mdx' - 'vectorlint.ini' - - 'evals/**' permissions: contents: read @@ -27,11 +26,29 @@ jobs: uses: TRocket-Labs/vectorlint-action@v1 with: llm_provider: ${{ secrets.LLM_PROVIDER }} + # Anthropic anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + anthropic_model: ${{ secrets.ANTHROPIC_MODEL }} + anthropic_temperature: ${{ secrets.ANTHROPIC_TEMPERATURE }} + anthropic_max_tokens: ${{ secrets.ANTHROPIC_MAX_TOKENS }} + # OpenAI openai_api_key: ${{ secrets.OPENAI_API_KEY }} openai_model: ${{ secrets.OPENAI_MODEL }} openai_temperature: ${{ secrets.OPENAI_TEMPERATURE }} + # Perplexity perplexity_api_key: ${{ secrets.PERPLEXITY_API_KEY }} + # Gemini + gemini_api_key: ${{ secrets.GEMINI_API_KEY }} + gemini_model: ${{ secrets.GEMINI_MODEL }} + gemini_temperature: ${{ secrets.GEMINI_TEMPERATURE }} + # Azure OpenAI + azure_openai_api_key: ${{ secrets.AZURE_OPENAI_API_KEY }} + azure_openai_endpoint: ${{ secrets.AZURE_OPENAI_ENDPOINT }} + azure_openai_deployment_name: ${{ secrets.AZURE_OPENAI_DEPLOYMENT_NAME }} + azure_openai_api_version: ${{ secrets.AZURE_OPENAI_API_VERSION }} + azure_openai_model: ${{ secrets.AZURE_OPENAI_MODEL }} + azure_openai_temperature: ${{ secrets.AZURE_OPENAI_TEMPERATURE }} + # Reviewdog options reporter: github-pr-check filter_mode: added fail_on_error: false From f52a1609eb7cfb6d6aadbdc7eb6306e1e0311c16 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 21:53:34 +0100 Subject: [PATCH 31/40] fix: update README.md for reviewdog integration --- .gitignore | 2 - README.md | 46 +++++++++++++++---- .../technical-accuracy/test-bad-article.md | 2 +- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 3dbae5d..e412ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,3 @@ vectorlint.ini # .agent/ /.idea /npm - - diff --git a/README.md b/README.md index 7919f26..4caeb09 100644 --- a/README.md +++ b/README.md @@ -168,21 +168,47 @@ vectorlint --verbose --show-prompt --debug-json path/to/article.md ### CI/CD Integration (GitHub Actions & reviewdog) -VectorLint supports the `rdjson` format, making it easy to integrate with [reviewdog](https://github.com/reviewdog/reviewdog) for automated code review comments in Pull Requests. +VectorLint integrates with [reviewdog](https://github.com/reviewdog/reviewdog) via the official GitHub Action for automated PR feedback. -**Example Workflow:** +**Example Workflow (`.github/workflows/vectorlint.yml`):** ```yaml -- name: Run VectorLint with reviewdog - env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - npx vectorlint --output rdjson . | reviewdog -f=rdjson -name="vectorlint" -reporter=github-pr-review -filter-mode=added -fail-on-error=true +name: VectorLint + +on: + pull_request: + paths: + - '**/*.md' + - '**/*.mdx' + - 'vectorlint.ini' + +permissions: + contents: read + pull-requests: write + checks: write + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run VectorLint + uses: TRocket-Labs/vectorlint-action@v1 + with: + llm_provider: ${{ secrets.LLM_PROVIDER }} + # Add your provider credentials via GitHub Secrets + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + openai_api_key: ${{ secrets.OPENAI_API_KEY }} + # Reviewdog options + reporter: github-pr-review + filter_mode: added + fail_on_error: false ``` -Supported reporters: -- `github-pr-review`: Posts comments on specific lines in the PR. -- `github-check`: Creates annotations in the "Checks" tab. +**Reviewdog Reporter Options:** +- `github-pr-review`: Posts inline comments on specific lines in the PR diff. +- `github-pr-check`: Creates annotations in the "Checks" tab. ### Creating Prompts diff --git a/tests/fixtures/technical-accuracy/test-bad-article.md b/tests/fixtures/technical-accuracy/test-bad-article.md index c7097fd..eb0bd4f 100644 --- a/tests/fixtures/technical-accuracy/test-bad-article.md +++ b/tests/fixtures/technical-accuracy/test-bad-article.md @@ -10,4 +10,4 @@ Have you heard of **ReactGPT-Pro**? It's a new framework that writes all your co The synergy of the AI paradigm shift is leveraging the blockchain for maximum scalability. We need to utilize the generative capabilities to optimize our workflows. -This sentence has bad grammer and spelling mistaks. It is very poorly written. \ No newline at end of file +This sentence has bad grammer and spelling mistaks. It is very poorly written. From 2f33f4b68efdd3f4bc26f6b1a57f1be4824acab5 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 22:00:26 +0100 Subject: [PATCH 32/40] Update readme file --- README.md | 93 +++---------------------------------------------------- 1 file changed, 4 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 4caeb09..e8e5817 100644 --- a/README.md +++ b/README.md @@ -99,74 +99,7 @@ VectorLint scores your content using error density and a rubric based system, en vectorlint path/to/article.md ``` -## Contributing - -We welcome your contributions! Whether it's adding new rules, fixing bugs, or improving documentation, please check out our [Contributing Guidelines](.github/CONTRIBUTING.md) to get started. - -## Resources - - ```bash - # Inside the vectorlint directory - npm run build - npm link - ``` - -2. **Verify Installation:** - - ```bash - vectorlint --help - ``` - -3. **Usage:** - - Now you can run `vectorlint` in any project: - - ```bash - vectorlint my-article.md - ``` - -## Configuration - -### LLM Provider - -VectorLint supports OpenAI, Azure OpenAI, Anthropic, and Perplexity. - -**Minimal Setup (OpenAI):** - -1. Copy `.env.example` to `.env`. -2. Set `LLM_PROVIDER=openai`. -3. Set `OPENAI_API_KEY=your-key`. - -For other providers (Azure, Anthropic), see the comments in `.env.example`. - -### Project Config (vectorlint.ini) - -To customize which prompts run on which files, use a `vectorlint.ini` file in your project root. - -```bash -cp vectorlint.example.ini vectorlint.ini -``` - -**Key Settings:** -- `PromptsPath`: Directory containing your `.md` prompts. -- `ScanPaths`: Glob patterns for files to scan (e.g., `[content/**/*.md]`). - -## Usage Guide - -### Running Locally - -```bash -# Basic usage (if linked globally) -vectorlint path/to/article.md - -# Using npm script (if not linked) -npm run dev -- path/to/article.md - -# Debug mode (shows prompts and full JSON response) -vectorlint --verbose --show-prompt --debug-json path/to/article.md -``` - -### CI/CD Integration (GitHub Actions & reviewdog) +## CI/CD Integration (GitHub Actions & reviewdog) VectorLint integrates with [reviewdog](https://github.com/reviewdog/reviewdog) via the official GitHub Action for automated PR feedback. @@ -205,34 +138,16 @@ jobs: filter_mode: added fail_on_error: false ``` - **Reviewdog Reporter Options:** - `github-pr-review`: Posts inline comments on specific lines in the PR diff. - `github-pr-check`: Creates annotations in the "Checks" tab. -### Creating Prompts - -Prompts are simple Markdown files with YAML frontmatter. -**Example (`prompts/grammar.md`):** - -```markdown ---- -evaluator: base -type: subjective -id: tone-check -name: Tone and Style Check -severity: error -criteria: - - id: friendlinessure professional writing quality. -``` +## Contributing -## Testing +We welcome your contributions! Whether it's adding new rules, fixing bugs, or improving documentation, please check out our [Contributing Guidelines](.github/CONTRIBUTING.md) to get started. -- `npm test`: Run tests in watch mode -- `npm run test:run`: Single run -- `npm run test:ci`: CI run with coverage +## Resources -Tests live under `tests/` and use Vitest. They validate config parsing (PromptsPath, ScanPaths), file discovery (including prompts exclusion), prompt/file mapping, and prompt aggregation with a mocked provider. - **[Creating Custom Rules](./CREATING_RULES.md)** - Write your own quality checks in Markdown - **[Configuration Guide](./CONFIGURATION.md)** - Complete reference for `vectorlint.ini` From d056593e4e2438a14a5105bb646071cbf8b11ad9 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 12:09:20 +0100 Subject: [PATCH 33/40] Delete vectorlint github action workflow --- .github/workflows/vectorlint.yml | 55 -------------------------------- .gitignore | 2 +- vectorlint.ini | 6 ---- 3 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 .github/workflows/vectorlint.yml delete mode 100644 vectorlint.ini diff --git a/.github/workflows/vectorlint.yml b/.github/workflows/vectorlint.yml deleted file mode 100644 index f6b7ac5..0000000 --- a/.github/workflows/vectorlint.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: VectorLint - -on: - pull_request: - paths: - - '**/*.md' - - '**/*.mdx' - - 'vectorlint.ini' - -permissions: - contents: read - pull-requests: write - checks: write - -concurrency: - group: vectorlint-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run VectorLint - uses: TRocket-Labs/vectorlint-action@v1 - with: - llm_provider: ${{ secrets.LLM_PROVIDER }} - # Anthropic - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - anthropic_model: ${{ secrets.ANTHROPIC_MODEL }} - anthropic_temperature: ${{ secrets.ANTHROPIC_TEMPERATURE }} - anthropic_max_tokens: ${{ secrets.ANTHROPIC_MAX_TOKENS }} - # OpenAI - openai_api_key: ${{ secrets.OPENAI_API_KEY }} - openai_model: ${{ secrets.OPENAI_MODEL }} - openai_temperature: ${{ secrets.OPENAI_TEMPERATURE }} - # Perplexity - perplexity_api_key: ${{ secrets.PERPLEXITY_API_KEY }} - # Gemini - gemini_api_key: ${{ secrets.GEMINI_API_KEY }} - gemini_model: ${{ secrets.GEMINI_MODEL }} - gemini_temperature: ${{ secrets.GEMINI_TEMPERATURE }} - # Azure OpenAI - azure_openai_api_key: ${{ secrets.AZURE_OPENAI_API_KEY }} - azure_openai_endpoint: ${{ secrets.AZURE_OPENAI_ENDPOINT }} - azure_openai_deployment_name: ${{ secrets.AZURE_OPENAI_DEPLOYMENT_NAME }} - azure_openai_api_version: ${{ secrets.AZURE_OPENAI_API_VERSION }} - azure_openai_model: ${{ secrets.AZURE_OPENAI_MODEL }} - azure_openai_temperature: ${{ secrets.AZURE_OPENAI_TEMPERATURE }} - # Reviewdog options - reporter: github-pr-check - filter_mode: added - fail_on_error: false - \ No newline at end of file diff --git a/.gitignore b/.gitignore index e412ad5..b2f1b71 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ .idea/ coverage/ *.tsbuildinfo -vectorlint.ini +# vectorlint.ini .kiro/ # .agent/ /.idea diff --git a/vectorlint.ini b/vectorlint.ini deleted file mode 100644 index 2397594..0000000 --- a/vectorlint.ini +++ /dev/null @@ -1,6 +0,0 @@ -RulesPath=.github/rules -Concurrency=4 -DefaultSeverity=warning - -[**/*.md] -RunRules=TinyRocket From 2ca1d359c4d5178b65084a761fb5f25cee3a1c80 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 12:25:05 +0100 Subject: [PATCH 34/40] eslint fix --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b2f1b71..0b30175 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,9 @@ dist/ .idea/ coverage/ *.tsbuildinfo -# vectorlint.ini +vectorlint.ini .kiro/ # .agent/ /.idea /npm + From f968394de9f1f1ea20f3348c54c404b07d8bf927 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 12:26:59 +0100 Subject: [PATCH 35/40] Delete vectorlint github action docs --- README.md | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/README.md b/README.md index e8e5817..1efe13b 100644 --- a/README.md +++ b/README.md @@ -99,50 +99,6 @@ VectorLint scores your content using error density and a rubric based system, en vectorlint path/to/article.md ``` -## CI/CD Integration (GitHub Actions & reviewdog) - -VectorLint integrates with [reviewdog](https://github.com/reviewdog/reviewdog) via the official GitHub Action for automated PR feedback. - -**Example Workflow (`.github/workflows/vectorlint.yml`):** - -```yaml -name: VectorLint - -on: - pull_request: - paths: - - '**/*.md' - - '**/*.mdx' - - 'vectorlint.ini' - -permissions: - contents: read - pull-requests: write - checks: write - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run VectorLint - uses: TRocket-Labs/vectorlint-action@v1 - with: - llm_provider: ${{ secrets.LLM_PROVIDER }} - # Add your provider credentials via GitHub Secrets - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - openai_api_key: ${{ secrets.OPENAI_API_KEY }} - # Reviewdog options - reporter: github-pr-review - filter_mode: added - fail_on_error: false -``` -**Reviewdog Reporter Options:** -- `github-pr-review`: Posts inline comments on specific lines in the PR diff. -- `github-pr-check`: Creates annotations in the "Checks" tab. - - ## Contributing We welcome your contributions! Whether it's adding new rules, fixing bugs, or improving documentation, please check out our [Contributing Guidelines](.github/CONTRIBUTING.md) to get started. From 3dd672f99fab32bdac5e6289c7d8465fcaa3d5df Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 12:29:40 +0100 Subject: [PATCH 36/40] Delete test bad article --- .../fixtures/technical-accuracy/test-bad-article.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tests/fixtures/technical-accuracy/test-bad-article.md diff --git a/tests/fixtures/technical-accuracy/test-bad-article.md b/tests/fixtures/technical-accuracy/test-bad-article.md deleted file mode 100644 index eb0bd4f..0000000 --- a/tests/fixtures/technical-accuracy/test-bad-article.md +++ /dev/null @@ -1,13 +0,0 @@ -# The Future of AI in 2025 - -AI is gonna be huge. It will literally solve everything. - -## New Tools - -Have you heard of **ReactGPT-Pro**? It's a new framework that writes all your code for you. Also, **NodeJS-Turbo** is 100x faster than Rust. - -## Why it matters - -The synergy of the AI paradigm shift is leveraging the blockchain for maximum scalability. We need to utilize the generative capabilities to optimize our workflows. - -This sentence has bad grammer and spelling mistaks. It is very poorly written. From 39fa6fbd7d08f53e185d4d3475ea5bf64480401f Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 17:19:38 +0100 Subject: [PATCH 37/40] Refactor output format handling to use enum and update schemas accordingly --- src/cli/types.ts | 13 ++++++++++--- src/output/rdjson-formatter.ts | 4 ++-- src/schemas/cli-schemas.ts | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/cli/types.ts b/src/cli/types.ts index 0329de4..ace62ce 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -10,6 +10,13 @@ import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { EvaluationResult as PromptEvaluationResult, SubjectiveResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; +export enum OutputFormat { + Line = 'line', + Json = 'json', + ValeJson = 'vale-json', + RdJson = 'rdjson' +} + export interface EvaluationOptions { prompts: PromptFile[]; rulesPath: string; @@ -18,7 +25,7 @@ export interface EvaluationOptions { concurrency: number; verbose: boolean; scanPaths: FilePatternConfig[]; - outputFormat?: 'line' | 'json' | 'vale-json' | 'rdjson'; + outputFormat?: OutputFormat; } export interface EvaluationResult { @@ -41,7 +48,7 @@ export interface ErrorTrackingResult { export interface EvaluationContext { content: string; relFile: string; - outputFormat: 'line' | 'json' | 'vale-json' | 'rdjson'; + outputFormat: OutputFormat; jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; } @@ -52,7 +59,7 @@ export interface ReportIssueParams { severity: Severity summary: string; ruleName: string; - outputFormat: 'line' | 'json' | 'vale-json' | 'rdjson'; + outputFormat: OutputFormat; jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; suggestion?: string; scoreText?: string; diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts index 4ec4b1a..81429ac 100644 --- a/src/output/rdjson-formatter.ts +++ b/src/output/rdjson-formatter.ts @@ -24,7 +24,7 @@ export interface RdJsonDiagnostic { }; }; }; - severity: 'ERROR' | 'WARNING' | 'INFO'; + severity: Severity; code?: { value: string; url?: string; @@ -91,7 +91,7 @@ export class RdJsonFormatter { }, }, }, - severity: issue.severity === Severity.ERROR ? 'ERROR' : issue.severity === Severity.WARNING ? 'WARNING' : 'INFO', + severity: issue.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, code: { value: issue.eval, }, diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 4db1b52..f216d16 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -6,7 +6,7 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPrompt: z.boolean().default(false), showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), - output: z.enum(['line', 'json', 'vale-json', 'JSON', 'rdjson']).default('line'), + output: z.enum(['line', 'json', 'vale-json', 'rdjson']).default('line'), prompts: z.string().optional(), evals: z.string().optional(), config: z.string().optional(), From 95fe6d679f0628011ac0c48dc70aca5c8fad6a16 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 17:38:34 +0100 Subject: [PATCH 38/40] Update JsonFormatter to retrieve version from package.json and fix severity type in tests --- src/output/json-formatter.ts | 14 +++++++++++++- tests/rdjson-formatter.test.ts | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index 19a694a..285faf2 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -1,4 +1,16 @@ +import { createRequire } from 'node:module'; +import { z } from 'zod'; import { Severity } from '../evaluators/types'; + +const require = createRequire(import.meta.url); + +const PACKAGE_JSON_SCHEMA = z.object({ + version: z.string(), +}); + +// Using require to load JSON in ESM +const rawPackageJson: unknown = require('../../package.json'); +const pkg = PACKAGE_JSON_SCHEMA.parse(rawPackageJson); export interface ScoreComponent { criterion?: string; rawScore: number; @@ -77,7 +89,7 @@ export class JsonFormatter { warnings: this.warningCount, }, metadata: { - version: '1.0.0', // TODO: Get from package.json + version: pkg.version, timestamp: new Date().toISOString(), }, }; diff --git a/tests/rdjson-formatter.test.ts b/tests/rdjson-formatter.test.ts index 2a394b8..b853f2c 100644 --- a/tests/rdjson-formatter.test.ts +++ b/tests/rdjson-formatter.test.ts @@ -32,7 +32,7 @@ describe('RdJsonFormatter', () => { const diag = parsed.diagnostics[0]!; expect(diag.message).toBe('Test message'); - expect(diag.severity).toBe('ERROR'); + expect(diag.severity).toBe(Severity.ERROR); expect(diag.location.path).toBe('test.md'); expect(diag.location.range.start.line).toBe(1); expect(diag.location.range.start.column).toBe(1); From 969be4d6a48bd0aa6ba11502a3ef7acae2746426 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 18:08:31 +0100 Subject: [PATCH 39/40] Refactor output format handling to use enum values in reportIssue and evaluateFiles functions; update JSON version retrieval in JsonFormatter --- src/cli/orchestrator.ts | 25 +++++++++++++------------ src/output/json-formatter.ts | 8 ++++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 0e59827..a7c48b6 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -12,6 +12,7 @@ import { isSubjectiveResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; +import { OutputFormat } from './types'; import type { EvaluationOptions, EvaluationResult, ErrorTrackingResult, ReportIssueParams, ExtractMatchTextParams, LocationMatch, ProcessViolationsParams, @@ -57,10 +58,10 @@ async function runWithConcurrency( function reportIssue(params: ReportIssueParams): void { const { file, line, column, severity, summary, ruleName, outputFormat, jsonFormatter, suggestion, scoreText, match } = params; - if (outputFormat === 'line') { + if (outputFormat === OutputFormat.Line) { const locStr = `${line}:${column}`; printIssueRow(locStr, severity, summary, ruleName, suggestion ? { suggestion } : {}); - } else if (outputFormat === 'vale-json') { + } else if (outputFormat === OutputFormat.ValeJson) { const issue: JsonIssue = { file, line, @@ -74,7 +75,7 @@ function reportIssue(params: ReportIssueParams): void { ...(scoreText !== undefined ? { score: scoreText } : {}), }; (jsonFormatter as ValeJsonFormatter).addIssue(issue); - } else if (outputFormat === 'json' || outputFormat === 'rdjson') { + } else if (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.RdJson) { const matchLen = match ? match.length : 0; const endColumn = column + matchLen; @@ -479,7 +480,7 @@ function routePromptResult(params: ProcessPromptResultParams): ErrorTrackingResu jsonFormatter }); hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; - } else if ((outputFormat === 'json' || outputFormat === 'vale-json') && result.message) { + } else if ((outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson) && result.message) { // For JSON, if there's a message but no violations, report it as a general issue reportIssue({ file: relFile, @@ -546,7 +547,7 @@ function routePromptResult(params: ProcessPromptResultParams): ErrorTrackingResu } } - if (outputFormat === 'json' && scoreComponents.length > 0) { + if (outputFormat === OutputFormat.Json && scoreComponents.length > 0) { (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore(relFile, { id: promptId || promptFile.filename.replace(/\.md$/, ''), scores: scoreComponents @@ -610,7 +611,7 @@ async function runPromptEvaluation(params: RunPromptEvaluationParams): Promise { const { file, options, jsonFormatter } = params; - const { prompts, provider, searchProvider, concurrency, scanPaths, outputFormat = 'line' } = options; + const { prompts, provider, searchProvider, concurrency, scanPaths, outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -622,7 +623,7 @@ async function evaluateFile(params: EvaluateFileParams): Promise { - const { outputFormat = 'line' } = options; + const { outputFormat = OutputFormat.Line } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -756,9 +757,9 @@ export async function evaluateFiles( let requestFailures = 0; let jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; - if (outputFormat === 'json') { + if (outputFormat === OutputFormat.Json) { jsonFormatter = new JsonFormatter(); - } else if (outputFormat === 'rdjson') { + } else if (outputFormat === OutputFormat.RdJson) { jsonFormatter = new RdJsonFormatter(); } else { jsonFormatter = new ValeJsonFormatter(); @@ -781,7 +782,7 @@ export async function evaluateFiles( } // Output results based on format (always to stdout for JSON formats) - if (outputFormat === 'json' || outputFormat === 'vale-json' || outputFormat === 'rdjson') { + if (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson || outputFormat === OutputFormat.RdJson) { const jsonStr = jsonFormatter.toJson(); console.log(jsonStr); } diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index 285faf2..458d0d7 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -2,15 +2,15 @@ import { createRequire } from 'node:module'; import { z } from 'zod'; import { Severity } from '../evaluators/types'; -const require = createRequire(import.meta.url); +const REQUIRE = createRequire(import.meta.url); const PACKAGE_JSON_SCHEMA = z.object({ version: z.string(), }); // Using require to load JSON in ESM -const rawPackageJson: unknown = require('../../package.json'); -const pkg = PACKAGE_JSON_SCHEMA.parse(rawPackageJson); +const RAW_PACKAGE_JSON: unknown = REQUIRE('../../package.json'); +const PKG = PACKAGE_JSON_SCHEMA.parse(RAW_PACKAGE_JSON); export interface ScoreComponent { criterion?: string; rawScore: number; @@ -89,7 +89,7 @@ export class JsonFormatter { warnings: this.warningCount, }, metadata: { - version: pkg.version, + version: PKG.version, timestamp: new Date().toISOString(), }, }; From e827f5b5d4c3bdbaf734bbb9b4915356c97a4ecb Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 11 Dec 2025 09:04:28 +0100 Subject: [PATCH 40/40] Rename eval to rule --- src/cli/orchestrator.ts | 2 +- src/output/json-formatter.ts | 2 +- src/output/rdjson-formatter.ts | 2 +- tests/rdjson-formatter.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index a7c48b6..c909df8 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -85,7 +85,7 @@ function reportIssue(params: ReportIssueParams): void { span: [column, endColumn], severity, message: summary, - eval: ruleName, + rule: ruleName, match: match || '', ...(suggestion ? { suggestion } : {}) }; diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index 458d0d7..3165a52 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -32,7 +32,7 @@ export interface Issue { span: [number, number]; severity: Severity; message: string; - eval: string; // Renamed from rule + rule: string; match: string; suggestion?: string; } diff --git a/src/output/rdjson-formatter.ts b/src/output/rdjson-formatter.ts index 81429ac..d1daed7 100644 --- a/src/output/rdjson-formatter.ts +++ b/src/output/rdjson-formatter.ts @@ -93,7 +93,7 @@ export class RdJsonFormatter { }, severity: issue.severity === Severity.ERROR ? Severity.ERROR : Severity.WARNING, code: { - value: issue.eval, + value: issue.rule, }, }; diff --git a/tests/rdjson-formatter.test.ts b/tests/rdjson-formatter.test.ts index b853f2c..29a5b83 100644 --- a/tests/rdjson-formatter.test.ts +++ b/tests/rdjson-formatter.test.ts @@ -12,7 +12,7 @@ describe('RdJsonFormatter', () => { span: [1, 6], severity: Severity.ERROR, message: 'Test message', - eval: 'TestRule', + rule: 'TestRule', match: 'match', suggestion: 'fix' });