From 38c58bda486dc7cea0993a87d78f5f1cfacc36b6 Mon Sep 17 00:00:00 2001 From: Benjamin Kapner Date: Mon, 17 Aug 2026 08:54:02 +0300 Subject: [PATCH] feat(audit): add scheduled full-repo documentation drift audit A PR diff is a delta; documentation describes end state. Per-PR triggering catches incremental drift but misses accumulated staleness. Add an audit mode for schedule triggers that walks the docs tree using folder indexes, assesses each doc's staleness via LLM, and reports findings as a single updated GitHub Issue (never a new issue per run). Budget cap bounds cost on large repos and reports what was skipped. --- action.yml | 10 +++ src/audit.py | 194 ++++++++++++++++++++++++++++++++++++++++++++ src/suggest_docs.py | 17 ++++ tests/test_audit.py | 26 ++++++ 4 files changed, 247 insertions(+) create mode 100644 src/audit.py create mode 100644 tests/test_audit.py diff --git a/action.yml b/action.yml index cf6382d..a2a5200 100644 --- a/action.yml +++ b/action.yml @@ -73,6 +73,14 @@ inputs: description: 'Path to a Markdown style configuration file (.md) containing documentation style guidelines. If not set, auto-detects .code-to-docs/style.md in the repository root.' required: false default: '' + mode: + description: 'Execution mode: "comment" (default), "detect-only", or "audit" (scheduled full-repo drift check)' + required: false + default: 'comment' + audit-budget: + description: 'Maximum number of doc files to audit per run (default: 20). Controls cost on large repos.' + required: false + default: '20' outputs: status: @@ -104,3 +112,5 @@ runs: GOOGLE_SA_KEY: ${{ inputs.google-sa-key }} MAX_CONTEXT_CHARS: ${{ inputs.max-context-chars }} STYLE_CONFIG_PATH: ${{ inputs.style-config-path }} + MODE: ${{ inputs.mode }} + AUDIT_BUDGET: ${{ inputs.audit-budget }} diff --git a/src/audit.py b/src/audit.py new file mode 100644 index 0000000..e56a6da --- /dev/null +++ b/src/audit.py @@ -0,0 +1,194 @@ +"""Scheduled full-repo documentation drift audit. + +Walks the docs tree using folder indexes, asks the LLM to assess each +doc's staleness, and reports findings grouped by severity. +""" + +import json +import os + +from config import get_client, get_max_context_chars, get_model_name, truncate_content +from doc_index import get_doc_folders, get_docs_in_folder, get_docs_root, load_manifest +from security_utils import run_command_safe, sanitize_output + + +def run_audit(max_files=20): + """Audit documentation for drift against the current codebase. + + Returns a list of {file, severity, reason} dicts. + """ + docs_root = get_docs_root().resolve() + findings = [] + files_checked = 0 + files_skipped = 0 + + folders = get_doc_folders(docs_root) + manifest = load_manifest(docs_root) + + for folder in folders: + docs = get_docs_in_folder(folder, docs_root) + for doc in docs: + if files_checked >= max_files: + files_skipped += 1 + continue + + rel_path = str(doc.relative_to(docs_root)) + try: + content = doc.read_text(encoding="utf-8") + except Exception: + continue + + folder_info = manifest.get("folders", {}).get(folder, {}) + index_summary = folder_info.get("index", "") + + severity, reason = _assess_doc(rel_path, content, index_summary) + if severity != "fresh": + findings.append({"file": rel_path, "severity": severity, "reason": reason}) + files_checked += 1 + + if files_skipped > 0: + print( + f"Audit budget reached: checked {files_checked} files, " + f"skipped {files_skipped}. Increase audit-budget to cover more." + ) + + return findings + + +def _assess_doc(file_path, content, index_summary): + """Ask the LLM to assess a single doc's staleness. + + Returns (severity, reason) where severity is "fresh", "stale", or "very-stale". + """ + prompt = f"""Assess whether this documentation file is likely up to date with the current codebase. + +File: {file_path} + +Folder context: {index_summary[:500] if index_summary else "No index available"} + +Documentation content (first 2000 chars): +{content[:2000]} + +Rate the doc as one of: +- FRESH: content appears current, no obvious staleness +- STALE: some sections may be outdated (e.g. refers to old defaults, missing recent features) +- VERY-STALE: significant portions are likely incorrect or missing + +Respond with exactly one line: SEVERITY: brief reason +Example: STALE: refers to v1 API but v2 has been released""" + + budget = get_max_context_chars() + prompt = truncate_content(prompt, budget, label=f"audit prompt for {file_path}") + + try: + client = get_client() + response = client.chat.completions.create( + model=get_model_name(), + messages=[{"role": "user", "content": prompt}], + ) + text = (response.choices[0].message.content or "").strip() + + for severity in ("VERY-STALE", "STALE", "FRESH"): + if severity in text.upper(): + reason = text.split(":", 1)[1].strip() if ":" in text else text + return severity.lower(), reason + + return "stale", text[:200] + except Exception as e: + print(f"Warning: Could not assess {file_path}: {sanitize_output(str(e))}") + return "fresh", "" + + +def format_audit_report(findings): + """Format findings into a Markdown report grouped by severity.""" + if not findings: + return "No documentation drift detected." + + lines = ["# Documentation Drift Audit", ""] + + for severity in ("very-stale", "stale"): + group = [f for f in findings if f["severity"] == severity] + if not group: + continue + label = "Very Stale" if severity == "very-stale" else "Stale" + lines.append(f"## {label} ({len(group)} file{'s' if len(group) != 1 else ''})") + lines.append("") + for f in group: + lines.append(f"- **{f['file']}**: {f['reason']}") + lines.append("") + + return "\n".join(lines) + + +def post_audit_issue(findings, repo=None): + """Create or update a single GitHub Issue with audit findings.""" + gh_token = os.environ.get("GH_TOKEN") + if not gh_token: + print("Warning: GH_TOKEN not set, cannot post audit issue") + return + + if repo is None: + repo = os.environ.get("GITHUB_REPOSITORY", "") + if not repo: + print("Warning: GITHUB_REPOSITORY not set, cannot post audit issue") + return + + label = "docs-drift-audit" + title = "Documentation Drift Audit Report" + body = format_audit_report(findings) + + run_command_safe( + [ + "gh", + "label", + "create", + label, + "--description", + "Automated docs drift audit", + "--color", + "FBCA04", + "--force", + ], + check=False, + env={**os.environ, "GH_TOKEN": gh_token}, + ) + + existing = run_command_safe( + [ + "gh", + "issue", + "list", + "--label", + label, + "--state", + "open", + "--json", + "number", + "--limit", + "1", + ], + check=False, + env={**os.environ, "GH_TOKEN": gh_token}, + ) + + try: + issues = json.loads(existing.stdout.strip()) if existing.returncode == 0 else [] + except (json.JSONDecodeError, ValueError): + issues = [] + + if issues: + issue_num = issues[0]["number"] + run_command_safe( + ["gh", "issue", "edit", str(issue_num), "--body", body], + check=False, + env={**os.environ, "GH_TOKEN": gh_token}, + ) + print(f"Updated existing audit issue #{issue_num}") + else: + result = run_command_safe( + ["gh", "issue", "create", "--title", title, "--body", body, "--label", label], + check=False, + env={**os.environ, "GH_TOKEN": gh_token}, + ) + if result.returncode == 0: + print(f"Created audit issue: {result.stdout.strip()}") diff --git a/src/suggest_docs.py b/src/suggest_docs.py index 805f09d..139a903 100644 --- a/src/suggest_docs.py +++ b/src/suggest_docs.py @@ -258,6 +258,23 @@ def main(): print(f"Index build complete: {result['status']}") return + # Handle audit mode (scheduled full-repo drift check) + mode = os.environ.get("MODE", "comment") + if mode == "audit": + from audit import format_audit_report, post_audit_issue, run_audit + + print("Mode: audit") + if not setup_docs_environment(): + print("Failed to set up docs environment") + return + budget = int(os.environ.get("AUDIT_BUDGET", "20")) + findings = run_audit(max_files=budget) + report = format_audit_report(findings) + print(report) + if findings and not args.dry_run: + post_audit_issue(findings) + return + # Detect which command was used comment_body = os.environ.get("COMMENT_BODY", "") diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..c224c23 --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,26 @@ +"""Tests for audit.py -- scheduled drift audit.""" + +from audit import format_audit_report + + +class TestFormatAuditReport: + def test_no_findings(self): + assert "No documentation drift" in format_audit_report([]) + + def test_groups_by_severity(self): + findings = [ + {"file": "a.md", "severity": "stale", "reason": "outdated"}, + {"file": "b.md", "severity": "very-stale", "reason": "missing feature"}, + {"file": "c.md", "severity": "stale", "reason": "old defaults"}, + ] + report = format_audit_report(findings) + assert "## Very Stale (1 file)" in report + assert "## Stale (2 files)" in report + assert "b.md" in report + assert "a.md" in report + + def test_skips_empty_groups(self): + findings = [{"file": "a.md", "severity": "stale", "reason": "outdated"}] + report = format_audit_report(findings) + assert "Very Stale" not in report + assert "Stale" in report