From c6a53ef465899d261da1de9c98eda1a4ea1bfe6f Mon Sep 17 00:00:00 2001 From: Connor McDonald Date: Tue, 9 Jun 2026 08:26:33 +0200 Subject: [PATCH] feat(stats): add `surf stats` adoption metrics from git history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #13. The proposal's success/kill criteria (§9.2) hinge on two metrics, and the rubber-stamp rate is the one that distinguishes a working gate from one being routed around. Nothing computed them. - New `surf stats [--since ] [--until ] [--format human|json]`. Walks each non-merge commit in the window (one commit ≈ one PR), parsing the hub set as it existed at the commit and its parent via git ls-tree/show. - rubber-stamp rate: re-stamp events (a claim's stored hash changed value) where the claim's prose was left untouched. - in-place update rate: claim-touch events (a commit changed an anchored file) where the claim's hash was updated in the same commit. - JSON emits a versioned `{version, since, until, commits, rubber_stamp, in_place}` envelope; `rate` is null when there were no events. - Stats is a history report, not the advisory git used by `check`: it errors (non-zero) when git history is unavailable rather than printing a zero. - New git helpers: log_commits, commit_files, list_files_at (all best-effort). - Heuristic boundaries (one commit per PR, at:-site claim identity, in-place denominator counts any anchored-file edit) documented in docs/guides/stats.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/guides/stats.md | 55 ++++++ docs/reference/commands.md | 6 + hubs/cli-git.md | 15 ++ hubs/cli-reference.md | 9 +- hubs/cli-stats.md | 28 +++ surf-cli/src/git.rs | 55 ++++++ surf-cli/src/main.rs | 18 ++ surf-cli/src/stats.rs | 378 +++++++++++++++++++++++++++++++++++++ 8 files changed, 560 insertions(+), 4 deletions(-) create mode 100644 docs/guides/stats.md create mode 100644 hubs/cli-stats.md create mode 100644 surf-cli/src/stats.rs diff --git a/docs/guides/stats.md b/docs/guides/stats.md new file mode 100644 index 0000000..b2b4299 --- /dev/null +++ b/docs/guides/stats.md @@ -0,0 +1,55 @@ +--- +title: Adoption metrics (surf stats) +description: How surf stats computes the rubber-stamp and in-place update rates, and how to read them. +--- + +# Adoption metrics + +`surf stats` answers the two falsifiable questions from the proposal's success/kill criteria +(§9.2): *is the gate being routed around, and do docs travel with the code?* Both are computed +deterministically from git history — no model, no network. + +```sh +surf stats # all history +surf stats --since 2026-01-01 # a window +surf stats --since 2026-01-01 --until 2026-04-01 --format json +``` + +## The two rates + +**Rubber-stamp rate** — of *re-stamp events* (a commit changed a claim's stored `hash:` to a new +value), the share where the claim's **prose was left untouched**. Re-sealing without re-reading is +the signal that distinguishes a working gate from one being clicked through. A rising rate is a +kill signal. + +**In-place update rate** — of *claim-touch events* (a commit changed a file a claim anchors), the +share where the claim's stored hash was **updated in the same commit**. Docs that move with the +code score high; drift scores low. + +`--format json` emits a versioned envelope: + +```json +{ + "version": 1, + "since": "2026-01-01", + "commits": 42, + "rubber_stamp": { "n": 3, "d": 12, "rate": 0.25 }, + "in_place": { "n": 30, "d": 40, "rate": 0.75 } +} +``` + +`rate` is `null` when there were no events (`d == 0`). + +## What it assumes (and what that costs) + +These are heuristics, surfaced rather than hidden: + +- **One commit = one PR.** Merge commits are excluded, so a squash-merge workflow maps cleanly; + a merge-commit workflow attributes work to the individual commits instead. +- **A claim's identity is its `at:` site(s).** Re-pointing a claim to a different anchor reads as + a new claim, not an update of the old one. +- **The in-place denominator counts any change to an anchored file** — including comment or + formatting edits that wouldn't actually diverge the claim. So the *true* in-place rate is at + least the reported one; the number is a floor, not a point estimate. +- **History must be reachable.** On a shallow clone or outside a repo, `stats` errors (non-zero) + rather than printing a misleading zero. diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 4d3b907..bfe651b 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -36,6 +36,12 @@ description: The surf CLI — init, new, suggest, for, lint, check, and verify, prose still holds; writes the hash into the frontmatter. `` limits to one anchor. `--follow` re-points a single-segment anchor whose **symbol** was renamed, or whose **file** was moved (detected via git), and re-hashes in one step — only when the code is otherwise unchanged. +- **`surf stats [--since ] [--until ] [--format human|json]`** — adoption metrics from + git history (advisory, never a gate): the **rubber-stamp rate** (re-stamps that changed a + claim's stored hash but left its prose untouched) and the **in-place update rate** (commits + touching an anchored file that re-sealed the claim in the same commit). One commit = one PR + (merges excluded). Heuristic by design — see the [stats guide](../guides/stats.md). Errors + (non-zero) if git history is unavailable. ## Per-claim options diff --git a/hubs/cli-git.md b/hubs/cli-git.md index 9e70af3..76db4dd 100644 --- a/hubs/cli-git.md +++ b/hubs/cli-git.md @@ -20,6 +20,21 @@ anchors: None means git couldn't pair the rename — the deterministic verdict never depends on it. at: surf-cli/src/git.rs > renamed_to hash: 9622170a3b9a + - claim: > + log_commits lists commit SHAs (newest first) in the optional since/until window with merges + excluded, so surf stats treats each SHA as one unit of work. None when git can't answer. + at: surf-cli/src/git.rs > log_commits + hash: 748b15a706c2 + - claim: > + commit_files lists the repo-relative paths a single commit changed versus its first parent + (diff-tree --no-commit-id --name-only -r). None when git can't answer. + at: surf-cli/src/git.rs > commit_files + hash: cef27873a3e1 + - claim: > + list_files_at lists every tracked file at a commit (ls-tree -r --name-only), used to find the + hub set as it existed at a past commit. None when git can't answer. + at: surf-cli/src/git.rs > list_files_at + hash: cbe066de9432 refs: [] --- diff --git a/hubs/cli-reference.md b/hubs/cli-reference.md index 65a4538..2b018f1 100644 --- a/hubs/cli-reference.md +++ b/hubs/cli-reference.md @@ -4,11 +4,12 @@ anchors: - claim: > The CLI exposes exactly these subcommands with these flags: init; new ; lint [--format]; check [--format] [--base ] [--files ]; verify [] [--follow] - [--format]; suggest [--format]; for [symbol] [--format]. Adding, removing, or - renaming a command or flag, or changing a default, diverges this anchor — re-read - docs/reference/commands.md before sealing. + [--format]; suggest [--format]; for [symbol] [--format]; stats + [--since ] [--until ] [--format]. Adding, removing, or renaming a command or + flag, or changing a default, diverges this anchor — re-read docs/reference/commands.md + before sealing. at: surf-cli/src/main.rs > Command - hash: 7dcf2e5a1d8b + hash: b52f85f87955 refs: ["../docs/reference/commands.md"] --- diff --git a/hubs/cli-stats.md b/hubs/cli-stats.md new file mode 100644 index 0000000..728115a --- /dev/null +++ b/hubs/cli-stats.md @@ -0,0 +1,28 @@ +--- +summary: surf stats — git-history adoption metrics (rubber-stamp + in-place rates); advisory, never a gate. +anchors: + - claim: > + run computes the two metrics and prints them human-readable or as a versioned envelope; it + always exits 0 on success and surfaces an error (non-zero) only when git history is + unavailable. The metrics are advisory and never gate. + at: surf-cli/src/stats.rs > run + hash: 7f4ab96fac92 + - claim: > + compute walks each non-merge commit in the since/until window. A rubber-stamp event is an + already-sealed claim whose stored hash value changed in a commit; it counts toward the + rubber-stamp numerator only when the claim's prose was unchanged. A claim-touch event is a + commit that changed a file the claim anchors; it counts toward the in-place numerator when + the claim's stored hash was updated in that same commit. Claim identity is its at: site(s), + and missing git history is a hard error rather than a silent zero. + at: surf-cli/src/stats.rs > compute + hash: 55561222d721 +refs: ["../docs/guides/stats.md"] +--- + +# surf stats + +The proposal's adopt/kill signals (§9.2), computed deterministically from git history. `compute` +parses the hub set as it existed at each commit (and its parent) via `git ls-tree`/`git show`, +compares stored hashes and prose across the pair, and tallies the two rates. Heuristics — one +commit per PR, `at:`-site claim identity, an in-place denominator that counts any anchored-file +edit — are documented in [the stats guide](../docs/guides/stats.md). diff --git a/surf-cli/src/git.rs b/surf-cli/src/git.rs index f3d9051..17db564 100644 --- a/surf-cli/src/git.rs +++ b/surf-cli/src/git.rs @@ -34,6 +34,61 @@ pub fn changed_files(root: &Path, base: &str) -> Option> { }) } +/// Commit SHAs (newest first) in the optional `since`/`until` window, merges excluded so each +/// SHA is one unit of work (`surf stats` treats a commit as a PR). `None` if git can't answer. +pub fn log_commits(root: &Path, since: Option<&str>, until: Option<&str>) -> Option> { + let mut args: Vec = vec!["log".into(), "--no-merges".into(), "--format=%H".into()]; + if let Some(s) = since { + args.push(format!("--since={s}")); + } + if let Some(u) = until { + args.push(format!("--until={u}")); + } + let output = Command::new("git") + .current_dir(root) + .args(&args) + .output() + .ok()?; + output.status.success().then(|| { + String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect() + }) +} + +/// Repo-root-relative paths changed by a single commit (vs its first parent). Empty for the root +/// commit's tree-only diff is fine. `None` if git can't answer. +pub fn commit_files(root: &Path, sha: &str) -> Option> { + let output = Command::new("git") + .current_dir(root) + .args(["diff-tree", "--no-commit-id", "--name-only", "-r", sha]) + .output() + .ok()?; + output.status.success().then(|| { + String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect() + }) +} + +/// Every tracked file at `sha` (repo-root-relative). Used to find the hub set as it existed at a +/// past commit. `None` if git can't answer. +pub fn list_files_at(root: &Path, sha: &str) -> Option> { + let output = Command::new("git") + .current_dir(root) + .args(["ls-tree", "-r", "--name-only", sha]) + .output() + .ok()?; + output.status.success().then(|| { + String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect() + }) +} + /// The contents of `rel_file` at `base` (e.g. `git show HEAD:src/x.rs`). `None` if unavailable. pub fn show(root: &Path, base: &str, rel_file: &str) -> Option { let output = Command::new("git") diff --git a/surf-cli/src/main.rs b/surf-cli/src/main.rs index 4623d58..f6814e2 100644 --- a/surf-cli/src/main.rs +++ b/surf-cli/src/main.rs @@ -7,6 +7,7 @@ mod git; mod init; mod lint; mod new; +mod stats; mod suggest; mod verify; mod workspace; @@ -90,6 +91,18 @@ enum Command { #[arg(long, value_enum, default_value_t = Format::Human)] format: Format, }, + /// Adoption metrics from git history: rubber-stamp rate and in-place update rate. + Stats { + /// Only count commits more recent than this date (passed to `git log --since`). + #[arg(long)] + since: Option, + /// Only count commits older than this date (passed to `git log --until`). + #[arg(long)] + until: Option, + /// Output format for the metrics. + #[arg(long, value_enum, default_value_t = Format::Human)] + format: Format, + }, } fn main() -> std::process::ExitCode { @@ -132,5 +145,10 @@ fn run() -> anyhow::Result { symbol, format, } => for_path::run(&ws, &path, symbol.as_deref(), format), + Command::Stats { + since, + until, + format, + } => stats::run(&ws, since.as_deref(), until.as_deref(), format), } } diff --git a/surf-cli/src/stats.rs b/surf-cli/src/stats.rs new file mode 100644 index 0000000..b975bb1 --- /dev/null +++ b/surf-cli/src/stats.rs @@ -0,0 +1,378 @@ +//! `surf stats` — adoption + rubber-stamp metrics from git history (§9.2, #13). +//! +//! Two falsifiable adopt/kill signals, computed by walking commits in a date range: +//! +//! - **rubber-stamp rate** — of re-stamp events (a claim's stored `hash:` changed value in a +//! commit), the share where the claim's *prose* was left untouched. A high rate means people +//! re-seal to clear the gate without re-reading the claim — the gate is being routed around. +//! - **in-place update rate** — of claim-touch events (a commit changed a file a claim anchors), +//! the share where the claim's stored hash was updated in the *same* commit. A high rate means +//! docs travel with the code rather than drifting. +//! +//! Heuristics, deliberately so (the metrics are advisory, not a gate): +//! - One commit = one PR (merges excluded). Squash-merge workflows map cleanly; merge-commit +//! workflows attribute the work to its individual commits. +//! - Claim identity is its `at:` site(s); renaming a `claim:`'s anchor reads as a new claim. +//! - The in-place denominator counts *any* change to an anchored file, including hash-neutral +//! edits (comments, formatting) that wouldn't actually diverge the claim — so the true rate is +//! at least the reported one. Surfaced rather than hidden. + +use crate::format::Format; +use crate::git; +use crate::workspace::Workspace; +use anyhow::{anyhow, Result}; +use serde::Serialize; +use std::collections::{HashMap, HashSet}; +use std::process::ExitCode; +use surf_core::{parse_anchor, parse_hub, REPORT_VERSION}; + +#[derive(Debug, Clone)] +struct ClaimRec { + hub: String, + id: String, + hash: Option, + prose: String, + files: Vec, +} + +#[derive(Debug, Clone, Serialize)] +struct Rate { + n: usize, + d: usize, + rate: Option, +} + +impl Rate { + fn new(n: usize, d: usize) -> Rate { + let rate = (d > 0).then(|| n as f64 / d as f64); + Rate { n, d, rate } + } +} + +#[derive(Debug, Clone, Serialize)] +struct StatsReport { + version: u32, + #[serde(skip_serializing_if = "Option::is_none")] + since: Option, + #[serde(skip_serializing_if = "Option::is_none")] + until: Option, + commits: usize, + rubber_stamp: Rate, + in_place: Rate, +} + +pub fn run( + ws: &Workspace, + since: Option<&str>, + until: Option<&str>, + format: Format, +) -> Result { + let report = compute(ws, since, until)?; + + match format { + Format::Json => println!("{}", serde_json::to_string_pretty(&report)?), + Format::Human => print_human(&report), + } + Ok(ExitCode::SUCCESS) +} + +fn compute(ws: &Workspace, since: Option<&str>, until: Option<&str>) -> Result { + let patterns: Vec = ws + .config + .hubs + .iter() + .filter_map(|p| glob::Pattern::new(p).ok()) + .collect(); + + // Unlike check's advisory git, stats *is* a history report: if git can't answer, fail loudly + // rather than printing a misleading zero. + let commits = git::log_commits(&ws.root, since, until) + .ok_or_else(|| anyhow!("git history unavailable (not a repo, or a shallow clone?)"))?; + + let (mut rs_n, mut rs_d, mut ip_n, mut ip_d) = (0, 0, 0, 0); + + for sha in &commits { + let changed: HashSet = git::commit_files(&ws.root, sha) + .unwrap_or_default() + .into_iter() + .collect(); + if changed.is_empty() { + continue; + } + + let after = claims_at(&ws.root, sha, &patterns); + let before = claims_at(&ws.root, &format!("{sha}^"), &patterns); + let before_by_key: HashMap<(&str, &str), &ClaimRec> = before + .iter() + .map(|c| ((c.hub.as_str(), c.id.as_str()), c)) + .collect(); + + for c in &after { + let prev = before_by_key.get(&(c.hub.as_str(), c.id.as_str())).copied(); + + // Rubber-stamp: an already-sealed claim whose hash value changed this commit. + if let (Some(prev), Some(new_hash)) = (prev, &c.hash) { + if let Some(old_hash) = &prev.hash { + if old_hash != new_hash { + rs_d += 1; + if prev.prose == c.prose { + rs_n += 1; + } + } + } + } + + // In-place: a commit that touched an anchored file (seeded domain). + if c.files.iter().any(|f| changed.contains(f)) { + ip_d += 1; + let updated = match prev { + Some(prev) => prev.hash != c.hash, + None => c.hash.is_some(), + }; + if updated { + ip_n += 1; + } + } + } + } + + Ok(StatsReport { + version: REPORT_VERSION, + since: since.map(str::to_string), + until: until.map(str::to_string), + commits: commits.len(), + rubber_stamp: Rate::new(rs_n, rs_d), + in_place: Rate::new(ip_n, ip_d), + }) +} + +/// Every claim across the hub set as it existed at `rev`, or empty if the rev/hubs don't exist. +fn claims_at(root: &std::path::Path, rev: &str, patterns: &[glob::Pattern]) -> Vec { + let mut out = Vec::new(); + let files = git::list_files_at(root, rev).unwrap_or_default(); + for hub in files + .iter() + .filter(|f| patterns.iter().any(|p| p.matches(f))) + { + let Some(content) = git::show(root, rev, hub) else { + continue; + }; + let Ok(parsed) = parse_hub(&content) else { + continue; + }; + for claim in &parsed.frontmatter.anchors { + let sites = claim.at.sites(); + let files = sites + .iter() + .filter_map(|s| parse_anchor(s).ok().map(|a| a.file)) + .collect(); + out.push(ClaimRec { + hub: hub.clone(), + id: sites.join(" + "), + hash: claim.hash.clone(), + prose: claim.claim.trim().to_string(), + files, + }); + } + } + out +} + +fn pct(rate: &Rate) -> String { + match rate.rate { + Some(r) => format!("{:.0}% ({}/{})", r * 100.0, rate.n, rate.d), + None => "n/a (no events)".to_string(), + } +} + +fn print_human(report: &StatsReport) { + let range = match (&report.since, &report.until) { + (Some(s), Some(u)) => format!("{s}..{u}"), + (Some(s), None) => format!("since {s}"), + (None, Some(u)) => format!("until {u}"), + (None, None) => "all history".to_string(), + }; + println!("surf stats ({range}, {} commits)", report.commits); + println!(" rubber-stamp rate: {}", pct(&report.rubber_stamp)); + println!(" re-stamps that left the claim's prose untouched"); + println!(" in-place update rate: {}", pct(&report.in_place)); + println!(" anchored-file touches that re-sealed the claim in the same commit"); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::path::Path; + use std::process::Command; + use surf_core::{hash_anchor, parse_anchor, Lang}; + + fn git(root: &Path, args: &[&str]) { + let status = Command::new("git") + .current_dir(root) + .args(["-c", "user.email=t@t", "-c", "user.name=t"]) + .args(args) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + } + + fn write(root: &Path, rel: &str, content: &str) { + let p = root.join(rel); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(p, content).unwrap(); + } + + fn rust_hash(src: &str, anchor: &str) -> String { + hash_anchor(src, Lang::Rust, &parse_anchor(anchor).unwrap()).unwrap() + } + + fn hub(claim: &str, at: &str, hash: &str) -> String { + format!( + "---\nsummary: x\nanchors:\n - claim: {claim}\n at: {at}\n hash: {hash}\n---\n" + ) + } + + fn commit(root: &Path, msg: &str) { + git(root, &["add", "."]); + git(root, &["commit", "-q", "-m", msg]); + } + + fn ws(root: &Path) -> Workspace { + Workspace::discover(root).unwrap() + } + + #[test] + fn rubber_stamp_when_hash_changes_but_prose_does_not() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + git(root, &["init", "-q", "-b", "main"]); + write(root, "surf.toml", ""); + + let v1 = "pub fn add(a: i64) -> i64 { a + 1 }\n"; + write(root, "src/m.rs", v1); + write( + root, + "hubs/a.md", + &hub( + "add increments", + "src/m.rs > add", + &rust_hash(v1, "src/m.rs > add"), + ), + ); + commit(root, "seed"); + + // Code changes; the claim is re-sealed with the SAME prose — a rubber stamp. + let v2 = "pub fn add(a: i64) -> i64 { a + 2 }\n"; + write(root, "src/m.rs", v2); + write( + root, + "hubs/a.md", + &hub( + "add increments", + "src/m.rs > add", + &rust_hash(v2, "src/m.rs > add"), + ), + ); + commit(root, "bump and re-stamp"); + + let r = compute(&ws(root), None, None).unwrap(); + assert_eq!((r.rubber_stamp.n, r.rubber_stamp.d), (1, 1)); + // The same commit touched the anchored file and updated the hash → in-place. + assert_eq!((r.in_place.n, r.in_place.d), (1, 1)); + } + + #[test] + fn genuine_update_is_not_a_rubber_stamp() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + git(root, &["init", "-q", "-b", "main"]); + write(root, "surf.toml", ""); + + let v1 = "pub fn add(a: i64) -> i64 { a + 1 }\n"; + write(root, "src/m.rs", v1); + write( + root, + "hubs/a.md", + &hub( + "add adds one", + "src/m.rs > add", + &rust_hash(v1, "src/m.rs > add"), + ), + ); + commit(root, "seed"); + + let v2 = "pub fn add(a: i64) -> i64 { a + 2 }\n"; + write(root, "src/m.rs", v2); + write( + root, + "hubs/a.md", + &hub( + "add adds two", + "src/m.rs > add", + &rust_hash(v2, "src/m.rs > add"), + ), + ); + commit(root, "bump and re-document"); + + let r = compute(&ws(root), None, None).unwrap(); + assert_eq!((r.rubber_stamp.n, r.rubber_stamp.d), (0, 1)); + } + + #[test] + fn touching_anchored_code_without_resealing_is_not_in_place() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + git(root, &["init", "-q", "-b", "main"]); + write(root, "surf.toml", ""); + + let v1 = "pub fn add(a: i64) -> i64 { a + 1 }\n"; + write(root, "src/m.rs", v1); + write( + root, + "hubs/a.md", + &hub( + "add increments", + "src/m.rs > add", + &rust_hash(v1, "src/m.rs > add"), + ), + ); + commit(root, "seed"); + + // Change the anchored file but DON'T touch the hub — drift left in place. + write(root, "src/m.rs", "pub fn add(a: i64) -> i64 { a + 9 }\n"); + commit(root, "bump only"); + + let r = compute(&ws(root), None, None).unwrap(); + assert_eq!((r.in_place.n, r.in_place.d), (0, 1)); + // No hash value change happened, so no rubber-stamp event either. + assert_eq!(r.rubber_stamp.d, 0); + } + + #[test] + fn errors_when_not_a_git_repo() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + write(root, "surf.toml", ""); + assert!(compute(&ws(root), None, None).is_err()); + } + + #[test] + fn json_envelope_is_versioned() { + let report = StatsReport { + version: REPORT_VERSION, + since: Some("2026-01-01".to_string()), + until: None, + commits: 3, + rubber_stamp: Rate::new(1, 2), + in_place: Rate::new(0, 0), + }; + let json = serde_json::to_value(&report).unwrap(); + assert_eq!(json["version"], REPORT_VERSION); + assert_eq!(json["rubber_stamp"]["rate"], 0.5); + assert!(json["in_place"]["rate"].is_null()); + assert!(json.get("until").is_none()); + } +}