diff --git a/package.json b/package.json index 9df0b8c..b4a6297 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ "dev": "tsx src/index.ts", "start": "node dist/index.js", "typecheck:tests": "tsc --noEmit -p tsconfig.test.json", - "test": "NATEMPLATE_STUB_PLANNER=1 node --import tsx --test tests/smoke.test.ts", - "ci": "npm run build && npm run typecheck:tests && npm test" + "test": "NATEMPLATE_STUB_ALL=1 node --import tsx --test tests/smoke.test.ts", + "ci": "npm run build && npm run typecheck:tests && npm test", + "cost": "bash scripts/cost.sh" }, "keywords": [], "author": "", diff --git a/scripts/cost.sh b/scripts/cost.sh new file mode 100755 index 0000000..fa91996 --- /dev/null +++ b/scripts/cost.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Aggregate token usage from tmp/trace/*.log and estimate Opus 4.7 cost. +# Each sub-agent emits lines like: +# [02:09:29] usage: 1897 in / 656 out +# This script sums those per agent, prints a table, and estimates USD. +# +# Usage: scripts/cost.sh (or: npm run cost) +# Opus 4.7 rates: $15/Mtok input, $75/Mtok output. + +set -euo pipefail + +trace_dir="$(cd "$(dirname "$0")/.." && pwd)/tmp/trace" + +if [[ ! -d "$trace_dir" ]] || ! compgen -G "$trace_dir/*.log" >/dev/null; then + echo "no trace files at $trace_dir — run \`npm run dev -- \"...\"\` first" + exit 0 +fi + +awk ' + BEGIN { + printf "%-12s %6s %12s %12s %12s\n", "Agent", "Calls", "Tokens in", "Tokens out", "Est. USD" + printf "%-12s %6s %12s %12s %12s\n", "------------", "------", "------------", "------------", "------------" + } + /usage:/ { + agent = FILENAME + sub(/.*\//, "", agent) + sub(/\.log$/, "", agent) + in_tok = 0 + out_tok = 0 + for (i = 1; i <= NF; i++) { + if ($i == "usage:") { in_tok = $(i + 1); out_tok = $(i + 4) } + } + if (!(agent in seen)) { order[++n_agents] = agent; seen[agent] = 1 } + calls[agent]++ + sum_in[agent] += in_tok + sum_out[agent] += out_tok + } + END { + if (n_agents == 0) { + print "no usage: lines found in trace logs" + exit 0 + } + total_calls = 0; total_in = 0; total_out = 0; total_cost = 0 + for (i = 1; i <= n_agents; i++) { + a = order[i] + cost = sum_in[a] * 15 / 1000000 + sum_out[a] * 75 / 1000000 + printf "%-12s %6d %12d %12d %12s\n", a, calls[a], sum_in[a], sum_out[a], sprintf("$%.4f", cost) + total_calls += calls[a] + total_in += sum_in[a] + total_out += sum_out[a] + total_cost += cost + } + printf "%-12s %6s %12s %12s %12s\n", "------------", "------", "------------", "------------", "------------" + printf "%-12s %6d %12d %12d %12s\n", "Total", total_calls, total_in, total_out, sprintf("$%.4f", total_cost) + } +' "$trace_dir"/*.log diff --git a/src/agents/planner.ts b/src/agents/planner.ts index 2e7567f..da1e88d 100644 --- a/src/agents/planner.ts +++ b/src/agents/planner.ts @@ -1,5 +1,6 @@ import Anthropic from "@anthropic-ai/sdk"; import { trace } from "../trace.js"; +import { isStub } from "../stub.js"; import type { DomainSpec } from "./types.js"; const MODEL = "claude-opus-4-7"; @@ -83,7 +84,7 @@ const DOMAIN_TOOL: Anthropic.Messages.Tool = { export async function runPlanner(spec: string): Promise { trace("planner", `received spec: "${spec}"`); - if (process.env['NATEMPLATE_STUB_PLANNER'] === "1") { + if (isStub("planner")) { return runStubPlanner(spec); } @@ -117,7 +118,7 @@ export async function runPlanner(spec: string): Promise { const delay = (ms: number): Promise => new Promise((r) => { setTimeout(r, ms); }); async function runStubPlanner(spec: string): Promise { - trace("planner", "(stub mode — NATEMPLATE_STUB_PLANNER=1)"); + trace("planner", "(stub mode — NATEMPLATE_STUB_ALL or NATEMPLATE_STUB_PLANNER set)"); await delay(200); trace("planner", "extracting entities and fields"); await delay(250); diff --git a/src/stub.ts b/src/stub.ts new file mode 100644 index 0000000..2c538eb --- /dev/null +++ b/src/stub.ts @@ -0,0 +1,6 @@ +import type { AgentName } from "./agents/types.js"; + +export function isStub(agent: AgentName): boolean { + if (process.env['NATEMPLATE_STUB_ALL'] === "1") return true; + return process.env[`NATEMPLATE_STUB_${agent.toUpperCase()}`] === "1"; +}