feat: complete commit-semantic domain pipeline

prompts/classify_units.md

@@ -0,0 +1,13 @@
+Given the following domain list and commit semantic units, classify each unit into the most appropriate domain.
+
+Domain list:
+{domains_json}
+
+Units to classify:
+{units_json}
+
+Requirements:
+1. For each unit output: {"id": "<unit_id>", "domain": "<domain_name>"}
+2. domain must be a value from the domain list, or "uncategorized"
+3. Judge by semantic content (theme, summary, operation type), not just keywords
+4. Output a JSON array only, no explanation

prompts/discover_domains.md

@@ -0,0 +1,13 @@
+Given the following semantic units from a codebase's git history, cluster them into core domains.
+
+Units summary:
+{units_summary}
+
+Architecture document (if available):
+{architecture_content}
+
+Requirements:
+1. Each domain must have: domain (short identifier), description (one sentence), paths (associated directory prefixes), keywords (associated keywords)
+2. Target 5-15 domains, maximum 20. If fewer than 5 natural domains exist, output the actual count. If more than 20, merge similar domains until under 20.
+3. Cluster based on semantic content (themes, operations, summaries), not just directory structure
+4. Output a JSON array only, no explanation

skills/commit-extract/run.py

@@ -14,6 +14,7 @@
 
 import argparse
 import logging
+import os
 import re
 import subprocess
 import sys
@@ -31,6 +32,7 @@
 
 OUTPUT_BASE = Path("data/commit-extract")
 TMP_DIR = OUTPUT_BASE / "tmp"
+USE_TASK_AGENTS_ENV = "COMMIT_EXTRACT_USE_TASK_AGENTS"
 
 # Adaptive batching constants
 WEIGHT_BUDGET = 3000
@@ -318,12 +320,156 @@ def _run_collect(self, state: HarnessState) -> bool:
         manifest_path = str(TMP_DIR / "manifest.json")
         save_json(manifest, manifest_path)
         print(f"\n  Manifest written to {manifest_path}")
-        print(f"  Workers should write to {TMP_DIR}/batch_NNNN.jsonl")
-        print(f"  After all workers complete, run merge to consolidate.")
+
+        if self._use_task_agents():
+            print(f"  Task-agent orchestration enabled via {USE_TASK_AGENTS_ENV}=1")
+            print(f"  Workers should write to {TMP_DIR}/batch_NNNN.jsonl")
+            print("  After all workers complete, run merge to consolidate.")
+        else:
+            print("  Running local worker fallback...")
+            processed = self._run_local_workers(manifest)
+            merged = merge_tmp_files(OUTPUT_BASE, TMP_DIR)
+            print(f"  Local fallback wrote {processed} records")
+            print(f"  Merged {merged} new records into monthly JSONL")
 
         self.add_artifact(state, str(OUTPUT_BASE))
         return True
 
+    def _use_task_agents(self) -> bool:
+        """Return True when external task-agent orchestration is explicitly enabled."""
+        return os.environ.get(USE_TASK_AGENTS_ENV, "").lower() in ("1", "true", "yes")
+
+    def _run_local_workers(self, manifest: dict) -> int:
+        """Process manifest batches locally with deterministic git-derived extraction."""
+        total = 0
+        for batch in manifest.get("batches", []):
+            output_path = batch.get("output_path")
+            if not output_path:
+                continue
+
+            records = []
+            for sha in batch.get("shas", []):
+                record = self._extract_commit_record(sha)
+                if record is not None:
+                    records.append(record)
+
+            if records:
+                append_jsonl(records, output_path)
+                total += len(records)
+
+        return total
+
+    def _extract_commit_record(self, sha: str) -> dict | None:
+        """Build a schema-valid commit-extract record from git metadata."""
+        try:
+            meta_result = subprocess.run(
+                [
+                    "git", "-C", self.repo_path,
+                    "show", "--no-patch",
+                    "--format=%an%x00%aI%x00%B",
+                    sha,
+                ],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            stat_result = subprocess.run(
+                ["git", "-C", self.repo_path, "show", "--stat", "--summary", "--format=", sha],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            logger.warning("Failed to extract commit %s: %s", sha, e)
+            return None
+
+        parts = meta_result.stdout.split("\x00", 2)
+        author = parts[0].strip() if len(parts) > 0 else ""
+        date = parts[1].strip() if len(parts) > 1 else ""
+        message = parts[2].strip() if len(parts) > 2 else ""
+        summary = next((line.strip() for line in message.splitlines() if line.strip()), "")
+        body_lines = [line.strip() for line in message.splitlines()[1:] if line.strip()]
+
+        weight = parse_stat(stat_result.stdout)
+        summary_lower = summary.lower()
+        op = self._classify_op(summary_lower)
+        theme = self._derive_theme(summary)
+        section_name = self._derive_section_name(summary)
+        item_summary = body_lines[0] if body_lines else (summary or f"Update in {theme}")
+
+        rules_invariants = []
+        for line in body_lines[1:]:
+            if any(keyword in line.lower() for keyword in ("must", "should", "ensure", "always", "never")):
+                rules_invariants.append({
+                    "kind": "rule",
+                    "statement": line,
+                    "enforced_by_commit": False,
+                })
+
+        if not rules_invariants:
+            for line in body_lines:
+                if any(keyword in line.lower() for keyword in ("must", "should", "ensure", "always", "never")):
+                    rules_invariants.append({
+                        "kind": "rule",
+                        "statement": line,
+                        "enforced_by_commit": False,
+                    })
+
+        return {
+            "sha": sha,
+            "author": author,
+            "date": date,
+            "is_large_aggregate": weight >= WEIGHT_BUDGET,
+            "is_mixed": False,
+            "sections": [{
+                "name": section_name,
+                "theme": theme,
+                "importance": "primary",
+                "items": [{
+                    "op": op,
+                    "summary": item_summary,
+                }],
+            }],
+            "rules_invariants": rules_invariants,
+        }
+
+    def _classify_op(self, summary_lower: str) -> str:
+        """Map commit summary text to the existing commit-extract op taxonomy."""
+        if any(token in summary_lower for token in ("bugfix", "fix", "hotfix")):
+            return "bugfix"
+        if "refactor" in summary_lower:
+            return "refactor"
+        if any(token in summary_lower for token in ("test", "spec")):
+            return "test"
+        if any(token in summary_lower for token in ("config", "ci", "build", "infra")):
+            return "config"
+        if any(token in summary_lower for token in ("feat", "feature", "add", "implement")):
+            return "feat"
+        return "other"
+
+    def _derive_theme(self, summary: str) -> str:
+        """Derive a stable-ish theme slug from commit summary text."""
+        text = summary.strip()
+        if not text:
+            return "misc"
+        if ":" in text:
+            text = text.split(":", 1)[1].strip() or text
+        slug = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
+        return slug or "misc"
+
+    def _derive_section_name(self, summary: str) -> str:
+        """Create a readable section name from the commit summary."""
+        text = summary.strip()
+        if not text:
+            return "General changes"
+        if ":" in text:
+            prefix, rest = text.split(":", 1)
+            label = rest.strip() or prefix.strip()
+        else:
+            label = text
+        label = label[:1].upper() + label[1:]
+        return label
+
     def handle_merge(self) -> int:
         """Merge tmp files after workers complete."""
         if not TMP_DIR.exists():

skills/commit-semantic/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: commit-semantic
-description: Analyze commit patterns from structured JSONL (4-stage pipeline)
+description: Analyze commit patterns from structured JSONL (5-stage pipeline)
 entrypoint: skills.commit-semantic.run.run_commit_semantic
 triggers:
   - commit-semantic
@@ -10,69 +10,77 @@ triggers:
 
 # Commit Semantic
 
-4-stage pipeline consuming commit-extract JSONL output: ingest → aggregate → distill → export.
+5-stage pipeline consuming commit-extract JSONL output: discover → ingest → aggregate → distill → export.
 
 ## Prerequisites
 
 Requires `data/commit-extract/*.jsonl` files produced by `/commit-extract run`.
 
 ## Pipeline Stages
 
-### 1. ingest
+### 1. discover
 
-Expand sections into semantic units + collect rules_invariants.
+Bottom-up domain discovery from semantic units, cached by `domains.json` fingerprint.
 
-- Each section's items become individual units with `sha`, `date`, `author`, `theme`, `importance`, `op`, `summary`
-- Commit-level `is_large_aggregate` and `is_mixed` flags carried to each unit
-- `rules_invariants` collected separately
-- Skips invalid JSON lines with warning
+- Builds domains from unit-level semantic signals
+- Reuses cached `domains.json` when fingerprint matches current inputs
+- First run may bootstrap by running ingest first to create units
+- Output: `data/commit-semantic/domains.json`
+
+### 2. ingest
+
+Expand sections into semantic units, collect invariants, and assign domains.
+
+- Each section item becomes a unit with commit metadata, semantic fields, and domain assignment when `domains.json` exists
+- Collects invariants separately into `invariants.jsonl`
+- Mixed or no-path commits may require LLM classification
 - Output: `data/commit-semantic/units/all.jsonl`, `data/commit-semantic/invariants.jsonl`
 
-### 2. aggregate
+### 3. aggregate
 
-Group units by theme, compute statistics.
+Group units by domain and compute domain-level statistics.
 
-- Primary key: `theme` (cross-commit semantic theme)
-- Same theme from different `section_name` values merged
-- Statistics: `op` distribution, `importance` ratio (primary/secondary)
-- Threshold: theme must appear in >= 3 distinct commits
-- Output: `data/commit-semantic/patterns.jsonl`
+- Primary key: `domain`, not theme
+- Preserves `sub_themes` within each domain
+- `uncategorized` remains an independent domain bucket
+- Output: `data/commit-semantic/domains-aggregated.jsonl`
 
-### 3. distill
+### 4. distill
 
-Extract canonical demands from patterns, scored and ranked.
+Extract canonical demands from aggregated domains, score them, and rank them.
 
-- Score: `distinct_commits × importance_weight` where `primary=2, secondary=1`
-- Tie-break: `distinct_commits` desc → `theme` alpha
-- Invariants appearing in >= 3 commits get extra weight
+- Uses multi-dimensional scoring with invariant SHA association and caps
+- Emits score breakdown fields for downstream review
+- Produces ranked canonical demands per domain cluster
 - Output: `data/commit-semantic/canonical-demands.jsonl`
 
-### 4. export
+### 5. export
 
-Generate summary statistics.
+Generate summary statistics for the domain-based pipeline.
 
-- Total units, patterns, op distribution, bugfix ratio
-- Top patterns by score
-- Date range
+- `summary.json` includes `top_domains`, `domain_count`, `uncategorized_ratio`, `file_paths_available`
+- Also includes `op_distribution`, `invariant_count`, `date_range`, and `bugfix_ratio`
+- Also reports runtime provenance via `orchestration_mode`, `discover_mode`, and `classify_mode`
 - Output: `data/commit-semantic/summary.json`
 
 ## Output Schema
 
 ```
 data/commit-semantic/
-  units/all.jsonl              # Expanded semantic units
-  invariants.jsonl             # Rules and invariants
-  patterns.jsonl               # Aggregated patterns (threshold >= 3)
+  domains.json                 # Discovered domains + fingerprint cache
+  domains-aggregated.jsonl     # Aggregated domain statistics
   canonical-demands.jsonl      # Scored and ranked demands
-  summary.json                 # Summary statistics
+  summary.json                 # Domain summary statistics
+  units/all.jsonl              # Expanded semantic units
+  invariants.jsonl             # Collected invariants
 ```
 
 ## Usage
 
 ```bash
-/commit-semantic run              # Full pipeline (4 stages)
-/commit-semantic run --stage ingest  # Run specific stage
-/commit-semantic step             # Run next stage only
-/commit-semantic resume           # Continue from breakpoint
-/commit-semantic reset            # Clear state, keep artifacts
+/commit-semantic run                 # Full pipeline (5 stages)
+/commit-semantic run --stage discover  # Run specific stage
+/commit-semantic step                # Run next stage only
+/commit-semantic resume              # Continue from breakpoint
+/commit-semantic reset               # Clear state, keep artifacts
 ```