Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 6 additions & 2 deletions src/tools/prompt-score.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ interface ScoreResult {
feedback: string[];
}

function scorePrompt(text: string): ScoreResult {
export function scorePrompt(text: string): ScoreResult {
const feedback: string[] = [];
let specificity: number;
let scope: number;
Expand All @@ -59,8 +59,12 @@ function scorePrompt(text: string): ScoreResult {
}

// Scope: bounded task
if (/\b(only|just|single|one|specific|this)\b/i.test(text) || text.length > 100) {
const hasScopeKeyword = /\b(only|just|single|one|specific|this)\b/i.test(text);
if (hasScopeKeyword) {
scope = 25;
} else if (text.length > 100) {
// Long prompts suggest detail but don't guarantee bounded scope
scope = 20;
} else if (/\b(all|every|entire|whole)\b/i.test(text)) {
scope = 10;
feedback.push("🎯 'All/every' is broad — can you narrow the scope?");
Expand Down
75 changes: 75 additions & 0 deletions tests/tools/prompt-score.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { describe, it, expect } from "vitest";
import { scorePrompt } from "../../src/tools/prompt-score.js";

describe("scorePrompt", () => {
it("gives high score for a well-formed prompt", () => {
const result = scorePrompt(
"Rename the `handleSubmit` function in `src/components/Form.tsx` to `onFormSubmit`. Only this one function. It should still pass the existing tests."
);
expect(result.total).toBeGreaterThanOrEqual(80);
expect(result.grade).toMatch(/^[AB]/);
});

it("gives low score for a vague prompt", () => {
const result = scorePrompt("make it better");
expect(result.total).toBeLessThanOrEqual(40);
expect(result.grade).toMatch(/^[DF]/);
expect(result.feedback.length).toBeGreaterThan(0);
});

it("rewards file paths for specificity", () => {
const withPath = scorePrompt("fix src/lib/utils.ts");
const without = scorePrompt("fix the utility code");
expect(withPath.specificity).toBeGreaterThan(without.specificity);
});

it("rewards scope-bounding keywords", () => {
const bounded = scorePrompt("only update the header component");
const unbounded = scorePrompt("update components");
expect(bounded.scope).toBeGreaterThan(unbounded.scope);
});

it("penalizes broad scope words", () => {
const result = scorePrompt("refactor all the files");
expect(result.scope).toBeLessThanOrEqual(10);
expect(result.feedback.some(f => f.includes("broad"))).toBe(true);
});

it("rewards action verbs", () => {
const specific = scorePrompt("extract the validation logic into a helper");
const vague = scorePrompt("clean up the validation stuff");
expect(specific.actionability).toBeGreaterThan(vague.actionability);
});

it("rewards done conditions", () => {
const withDone = scorePrompt("add a test that should return 404 for missing users");
const without = scorePrompt("add a test for missing users");
expect(withDone.doneCondition).toBeGreaterThan(without.doneCondition);
});

it("treats questions as having implicit done condition", () => {
const result = scorePrompt("Why does the login page crash on mobile?");
expect(result.doneCondition).toBe(20);
});

it("long prompts get partial scope credit, not full", () => {
const longPrompt = "do something with " + "a".repeat(200);
const result = scorePrompt(longPrompt);
// Should get 20 (partial) not 25 (full) since no scope keywords
expect(result.scope).toBe(20);
});

it("returns grade F for minimal input", () => {
const result = scorePrompt("hi");
expect(result.grade).toBe("F");
});

it("returns congratulatory feedback for perfect prompts", () => {
const result = scorePrompt(
"In `src/auth/login.ts`, rename only the `validateToken` function to `verifyAuthToken`. The existing test in `tests/auth.test.ts` should still pass."
);
if (result.total >= 90) {
expect(result.feedback.some(f => f.includes("Excellent"))).toBe(true);
}
});
});
Loading