From e8f98c078addb02a14c999c38e1f68630e3b3fa1 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 01:42:24 +0800 Subject: [PATCH 1/5] feat(topic24): make Spike toolchain paths portable with graceful degradation --- docs/ARCHITECTURE.md | 5 + .../24-Spike\344\273\277\347\234\237.md" | 26 +- scratchv/standalone/run_spike_bench.py | 72 +- scratchv/standalone/spike_sim.py | 681 +++++++++++++++--- tests/test_spike_sim_paths.py | 340 +++++++++ 5 files changed, 999 insertions(+), 125 deletions(-) create mode 100644 tests/test_spike_sim_paths.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0d1af93..f2a52a6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -384,6 +384,11 @@ ScratchV 项目 — 完整模块地图 │ bench_report.py HTML/JSON/MD 报告生成 │ └─────────────────────────────────────────────────────────────┘ +> 工具解析注记：`spike` / `spike-dasm` / `spike-log-parser` 由 +> `spike_sim.resolve_spike_tools()` 统一解析（CLI > 环境变量 > +> `SCRATCHV_SPIKE_HOME` > `PATH` > 常见目录 > legacy 常量），缺失时按 +> skip / 告警分层降级（详见课题 24 设计文档）。 + ┌─────────────────────────────────────────────────────────────┐ │ CI / Dashboard │ ├─────────────────────────────────────────────────────────────┤ diff --git "a/docs/topics/24-Spike\344\273\277\347\234\237.md" "b/docs/topics/24-Spike\344\273\277\347\234\237.md" index 86f504d..bd7ec69 100644 --- "a/docs/topics/24-Spike\344\273\277\347\234\237.md" +++ "b/docs/topics/24-Spike\344\273\277\347\234\237.md" @@ -135,6 +135,30 @@ def run_spike(elf_path, max_instr, ic_config, dc_config): --- +## 运行方式与降级行为 + +```bash +# 1. 显式指定（优先级最高，CI 推荐固定版本） +python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140 \ + --spike-bin /opt/riscv/bin/spike --json + +# 2. 环境变量（整机安装） +export SCRATCHV_SPIKE_HOME=/opt/coralnpu-spike-rv32 # 约定 $HOME/bin/spike +python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140 +``` + +无 Spike 机器上的行为： + +- 未找到 spike：默认打印 `SKIP: spike binary not found.`（含搜索位置与修复提示）， + 退出码 0，不生成 ELF；加 `--require-spike` 则打印 `ERROR:` 并返回退出码 2。 +- CLI 显式路径无效（不存在 / 不可执行 / 是目录）：`ERROR:` + 退出码 2； + 环境变量指向无效路径：`WARNING:` 后继续向后一层解析。 +- `spike-dasm` / `spike-log-parser` 缺失只产生 `WARNING:`，不阻断仿真。 +- `--json` 报告新增 `status` / `skip_reason` / `spike_binary` / `spike_tools` + / `parse_warnings` / `tool_warnings` 字段，既有字段保持不变。 + +--- + ## 动手练习 ### 练习 1: 对比 Spike vs 估算 @@ -155,7 +179,7 @@ def run_spike(elf_path, max_instr, ic_config, dc_config): | 坑 | 说明 | |----|------| -| **Spike 二进制路径** | 需要自己编译 Spike RV32 版本，当前硬编码路径需要确认存在 | +| **Spike 二进制路径** | 不再硬编码：按 `--spike-bin` > `SCRATCHV_SPIKE_BIN` > `SCRATCHV_SPIKE_HOME` > `PATH` > 常见目录 > legacy 常量解析；`spike-dasm` / `spike-log-parser` 同规则 | | **内存限制** | Spike 默认内存模型可能不够大，CNN 模型需要 `-m512`（512MB） | | **执行时间** | 全量 CNN（32 亿指令）在 Spike 上可能跑数小时，用 `--max-instr` 限制 | | **ELF 兼容性** | 最小 ELF32 只包含必要 header，某些 Spike 版本可能要求更完整的 ELF | diff --git a/scratchv/standalone/run_spike_bench.py b/scratchv/standalone/run_spike_bench.py index a43d232..50751ca 100644 --- a/scratchv/standalone/run_spike_bench.py +++ b/scratchv/standalone/run_spike_bench.py @@ -18,6 +18,10 @@ MORE information than Spike would — since we can classify misses, track per-layer stats, and sample at higher resolution. +All numbers come from the built-in emulator, never from the real Spike +binary (use spike_sim.py for real Spike runs). The JSON report marks this +with a "backend" field so simulated data is not mistaken for Spike data. + Usage: python scratchv/standalone/run_spike_bench.py \\ --binary output.bin --code-size 3140 \\ @@ -666,10 +670,18 @@ def generate_report(result: SpikeStyleResult) -> str: return "\n".join(lines) -def generate_json_report(result: SpikeStyleResult) -> dict: - """Generate structured JSON report.""" +def generate_json_report(result: SpikeStyleResult, + spike_tools: SpikeTools | None = None) -> dict: + """Generate structured JSON report. + + Args: + result: Simulated benchmark result. + spike_tools: Optional resolved Spike toolchain (from --probe-spike); + included as "spike_tools" when provided. + """ total = max(result.total_insns, 1) report = { + "backend": {"kind": "emulator", "spike_style": True}, "summary": { "binary_path": result.binary_path, "code_size": result.code_size, @@ -722,9 +734,54 @@ def generate_json_report(result: SpikeStyleResult) -> dict: if count > 0 }, } + if spike_tools is not None: + report["spike_tools"] = spike_tools.as_dict() return report +# ═══════════════════════════════════════════════════════════════════════════ +# Spike availability probe (external toolchain only, never used for stats) +# ═══════════════════════════════════════════════════════════════════════════ + +def probe_spike_tools() -> "SpikeTools | None": + """Probe the external Spike toolchain and report availability to stderr. + + This never changes the simulation backend: all benchmark numbers still + come from the built-in emulator. Returns the resolved tools (or None when + resolution failed) so callers can attach them to JSON reports. + """ + try: + from scratchv.standalone.spike_sim import ( + SpikeConfigError, + resolve_spike_tools, + ) + except ImportError as e: + print(f"Spike tools: probe failed: {e}", file=sys.stderr) + return None + + try: + tools = resolve_spike_tools() + except SpikeConfigError as e: + print(f"ERROR: {e}", file=sys.stderr) + return None + + parts = [] + for name, path in ( + ("spike", tools.spike), + ("spike-dasm", tools.spike_dasm), + ("spike-log-parser", tools.spike_log_parser), + ): + if path: + parts.append(f"{name}={path} ({tools.sources.get(name, 'missing')})") + else: + parts.append(f"{name}=NOT FOUND") + print(f"Spike tools: {', '.join(parts)}", file=sys.stderr) + if tools.spike is None: + print(" hint: real Spike is unavailable; this run uses the " + "built-in emulator backend", file=sys.stderr) + return tools + + # ═══════════════════════════════════════════════════════════════════════════ # Main # ═══════════════════════════════════════════════════════════════════════════ @@ -753,6 +810,9 @@ def main() -> int: help="Save JSON report to file") parser.add_argument("--markdown", type=str, default="", help="Save markdown report to file") + parser.add_argument("--probe-spike", action="store_true", + help="Probe the external Spike toolchain and report " + "availability (does not change the backend)") args = parser.parse_args() @@ -760,6 +820,10 @@ def main() -> int: print(f"ERROR: binary not found: {args.binary}", file=sys.stderr) return 1 + spike_tools = None + if args.probe_spike: + spike_tools = probe_spike_tools() + # ── Build label address map ──────────────────────────────────── # These are the same labels as used by onnx_to_riscv_standalone.py label_addrs: dict[int, str] = {} @@ -793,14 +857,14 @@ def main() -> int: # ── Output ───────────────────────────────────────────────────── if args.json: - report = generate_json_report(result) + report = generate_json_report(result, spike_tools=spike_tools) print(json.dumps(report, indent=2)) else: report = generate_report(result) print(report) if args.json_output: - json_report = generate_json_report(result) + json_report = generate_json_report(result, spike_tools=spike_tools) with open(args.json_output, "w") as f: json.dump(json_report, f, indent=2) print(f"\n JSON report saved to: {args.json_output}", file=sys.stderr) diff --git a/scratchv/standalone/spike_sim.py b/scratchv/standalone/spike_sim.py index 12e972f..da5b0eb 100644 --- a/scratchv/standalone/spike_sim.py +++ b/scratchv/standalone/spike_sim.py @@ -12,19 +12,29 @@ - Instruction trace samples - Spike execution time -Spike binary: /home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike +Spike tool resolution order: + 1. --spike-bin / --spike-dasm / --spike-log-parser + 2. SCRATCHV_SPIKE_BIN / SCRATCHV_SPIKE_DASM / SCRATCHV_SPIKE_LOG_PARSER + 3. $SCRATCHV_SPIKE_HOME/bin/ + 4. PATH, then common install dirs, then the legacy constants below +If spike is missing, the tool exits 0 with "SKIP: ..." unless --require-spike. + ScratchV binary: output.bin Usage: python scratchv/standalone/spike_sim.py \\ --binary output.bin --code-size 3140 \\ - [--max-instr 50000000] [--ic 64:2:32] [--dc 128:4:32] + [--max-instr 50000000] [--ic 64:2:32] [--dc 128:4:32] \\ + [--spike-bin /path/to/spike] [--require-spike] [--json] """ from __future__ import annotations import argparse +import json import os +import re +import shutil import struct import subprocess import sys @@ -34,10 +44,33 @@ from dataclasses import dataclass, field # ── Paths ────────────────────────────────────────────────────────────────── +# Legacy fallback (may not exist on this machine). New code must use +# resolve_spike_tools(); these constants are only the last resolution layer. SPIKE = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike" SPIKE_DASM = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike-dasm" SPIKE_LOG_PARSER = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike-log-parser" +# Environment variable contract (see docs/topics/24-Spike仿真.md) +ENV_SPIKE_BIN = "SCRATCHV_SPIKE_BIN" +ENV_SPIKE_DASM = "SCRATCHV_SPIKE_DASM" +ENV_SPIKE_LOG_PARSER = "SCRATCHV_SPIKE_LOG_PARSER" +ENV_SPIKE_HOME = "SCRATCHV_SPIKE_HOME" + +COMMON_SPIKE_DIRS: tuple[str, ...] = ( + "/opt/riscv/bin", + "/opt/riscv64/bin", + "/usr/local/bin", + "/usr/bin", + "~/riscv/bin", + "~/.local/bin", + "~/spike/bin", +) + +# Exit code contract: 0 = success/skip, 1 = run failure, 2 = config error. +EXIT_OK = 0 +EXIT_RUN_FAIL = 1 +EXIT_CONFIG = 2 + # ── Constants ────────────────────────────────────────────────────────────── ELF_BASE = 0x80000000 # RISC-V DRAM base (Spike default) STARTUP_SIZE = 20 # 5 instructions × 4 bytes @@ -52,6 +85,176 @@ RV_OP_ECALL = 0b1110011 # ECALL is SYSTEM opcode with funct12=0, rd=0, rs1=0 +# ═══════════════════════════════════════════════════════════════════════════ +# Spike toolchain resolution (no filesystem probing at import time) +# ═══════════════════════════════════════════════════════════════════════════ + +def is_executable(path: str) -> bool: + """Return True if path expands to an existing executable file.""" + path = os.path.expanduser(path) + return bool(path) and os.path.isfile(path) and os.access(path, os.X_OK) + + +class SpikeConfigError(ValueError): + """Raised when an explicitly configured (CLI) tool path is invalid.""" + + +@dataclass(frozen=True) +class SpikeTools: + """Resolved paths for the Spike toolchain and how they were found.""" + spike: str | None = None + spike_dasm: str | None = None + spike_log_parser: str | None = None + sources: dict[str, str] = field(default_factory=dict) + candidates: dict[str, tuple[str, ...]] = field(default_factory=dict) + warnings: tuple[str, ...] = () + + @property + def missing(self) -> list[str]: + """Canonical names of tools that could not be resolved.""" + pairs = ( + ("spike", self.spike), + ("spike-dasm", self.spike_dasm), + ("spike-log-parser", self.spike_log_parser), + ) + return [name for name, path in pairs if not path] + + def as_dict(self) -> dict: + """Machine-readable snapshot for reports.""" + def one(name: str, path: str | None) -> dict: + return { + "path": path, + "source": self.sources.get(name, "missing"), + "candidates": list(self.candidates.get(name, ())), + } + + return { + "spike": one("spike", self.spike), + "spike_dasm": one("spike-dasm", self.spike_dasm), + "spike_log_parser": one("spike-log-parser", self.spike_log_parser), + "warnings": list(self.warnings), + } + + +def _resolve_one( + tool: str, + cli_flag: str, + cli_value: str | None, + env_name: str, + legacy_const: str, + env, + which, + common_dirs, +) -> tuple[str | None, str, list[str], list[str]]: + """Resolve one tool, returning (path, source, candidates, warnings).""" + candidates: list[str] = [] + warnings: list[str] = [] + + # 1. CLI (explicit per-run intent: hard failure on invalid value) + if cli_value: + cand = os.path.expanduser(cli_value.strip()) + candidates.append(cand) + if not is_executable(cand): + raise SpikeConfigError( + f"{cli_flag}={cli_value!r} is not an executable file") + return cand, "cli", candidates, warnings + + # 2. Dedicated environment variable (explicit, possibly stale: warn) + env_value = (env.get(env_name) or "").strip() + if env_value: + cand = os.path.expanduser(env_value) + candidates.append(cand) + if is_executable(cand): + return cand, "env", candidates, warnings + warnings.append(f"{env_name}={env_value} is not executable; ignored") + + # 3. $SCRATCHV_SPIKE_HOME/bin/ + home = (env.get(ENV_SPIKE_HOME) or "").strip() + if home: + cand = os.path.join(os.path.expanduser(home), "bin", tool) + candidates.append(cand) + if is_executable(cand): + return cand, "spike_home", candidates, warnings + + # 4. PATH + found = which(tool) if which else None + if found: + candidates.append(found) + return found, "path", candidates, warnings + + # 5. Common install directories + for directory in common_dirs: + cand = os.path.join(os.path.expanduser(directory), tool) + candidates.append(cand) + if is_executable(cand): + return cand, "common", candidates, warnings + + # 6. Legacy hard-coded constant + if legacy_const and is_executable(legacy_const): + candidates.append(legacy_const) + return legacy_const, "legacy", candidates, warnings + + return None, "missing", candidates, warnings + + +def resolve_spike_tools( + cli_spike: str | None = None, + cli_dasm: str | None = None, + cli_log_parser: str | None = None, + env=None, + which=None, + common_dirs=None, +) -> SpikeTools: + """Resolve spike / spike-dasm / spike-log-parser paths by priority. + + Priority: CLI > dedicated env vars > $SCRATCHV_SPIKE_HOME/bin > PATH > + common install dirs > legacy constants. Invalid CLI paths raise + SpikeConfigError; invalid env paths add a warning and fall through. + """ + env = os.environ if env is None else env + which = shutil.which if which is None else which + common_dirs = COMMON_SPIKE_DIRS if common_dirs is None else common_dirs + + # Legacy constants are read here (not captured at import time) so tests + # can monkeypatch them and so each call sees the current values. + spec = ( + ("spike", "--spike-bin", cli_spike, ENV_SPIKE_BIN, SPIKE), + ("spike-dasm", "--spike-dasm", cli_dasm, ENV_SPIKE_DASM, SPIKE_DASM), + ("spike-log-parser", "--spike-log-parser", cli_log_parser, + ENV_SPIKE_LOG_PARSER, SPIKE_LOG_PARSER), + ) + paths: dict[str, str | None] = {} + sources: dict[str, str] = {} + candidates: dict[str, tuple[str, ...]] = {} + warnings: list[str] = [] + for tool, flag, cli_value, env_name, legacy_const in spec: + path, source, cands, warns = _resolve_one( + tool, flag, cli_value, env_name, legacy_const, + env, which, common_dirs) + paths[tool], sources[tool], candidates[tool] = path, source, tuple(cands) + warnings.extend(warns) + + return SpikeTools( + spike=paths["spike"], + spike_dasm=paths["spike-dasm"], + spike_log_parser=paths["spike-log-parser"], + sources=sources, + candidates=candidates, + warnings=tuple(warnings), + ) + + +def _optional_tool_warnings(tools: SpikeTools) -> list[str]: + """Warnings for missing optional tools (never escalated to failure).""" + warnings: list[str] = [] + if tools.spike_dasm is None: + warnings.append("spike-dasm not found; disassembly features unavailable") + if tools.spike_log_parser is None: + warnings.append( + "spike-log-parser not found; commit log parsing unavailable") + return warnings + + def _sext(v: int, bits: int) -> int: mask = (1 << bits) - 1 v &= mask @@ -274,6 +477,130 @@ class SpikeResult: # Spike internal struct data (from stderr) commited_insns_per_sec: float = 0.0 + # Run status / tool resolution (topic 24) + status: str = "ok" # ok | skipped | timeout | failed + skip_reason: str = "" + spike_path: str = "" + tool_warnings: list[str] = field(default_factory=list) + parse_warnings: list[str] = field(default_factory=list) + + +# ── Output parsing patterns (tolerant of whitespace and thousands separators) ── +_RE_COMMIT = re.compile(r"(?:Commited|Committed)\s+([\d,]+)\s+instructions") +_RE_MIPS = re.compile(r"([\d.]+)\s*MIPS") +_RE_CACHE_HITS = re.compile(r"hits:\s*([\d,]+)") +_RE_CACHE_MISSES = re.compile(r"misses:\s*([\d,]+)") +_RE_CACHE_RATE = re.compile(r"miss rate:\s*([\d.]+)%") +_RE_PC_LINE = re.compile(r"^(0x[0-9a-fA-F]+):\s*(\d+)$") +_RE_ICACHE_HEADER = re.compile(r"^\s*I\$:", re.MULTILINE) +_RE_DCACHE_HEADER = re.compile(r"^\s*D\$:", re.MULTILINE) + + +def _parse_int(text: str) -> int | None: + """Parse an integer with optional thousands separators.""" + try: + return int(text.replace(",", "")) + except (TypeError, ValueError): + return None + + +def parse_commit_stats(stderr: str) -> tuple[int, float]: + """Parse committed instruction count and MIPS rate from Spike stderr. + + Tolerates both the legacy "Commited" and corrected "Committed" spellings + as well as thousands separators. Missing sections yield (0, 0.0). + """ + stderr = stderr or "" + + committed = 0 + match = _RE_COMMIT.search(stderr) + if match: + value = _parse_int(match.group(1)) + if value is not None: + committed = value + + mips = 0.0 + match = _RE_MIPS.search(stderr) + if match: + try: + mips = float(match.group(1)) + except ValueError: + mips = 0.0 + + return committed, mips + + +def parse_cache_stats(stderr: str) -> dict[str, int | float]: + """Parse I$/D$ cache statistics from Spike stderr. + + Always returns the same fixed set of keys; missing sections stay at 0. + """ + stats: dict[str, int | float] = { + "icache_hits": 0, + "icache_misses": 0, + "icache_miss_rate": 0.0, + "dcache_hits": 0, + "dcache_misses": 0, + "dcache_miss_rate": 0.0, + } + current: str | None = None + for line in (stderr or "").splitlines(): + stripped = line.strip() + if stripped.startswith("I$:"): + current = "icache" + continue + if stripped.startswith("D$:"): + current = "dcache" + continue + if current is None: + continue + + hits = _RE_CACHE_HITS.search(stripped) + if hits: + value = _parse_int(hits.group(1)) + if value is not None: + stats[f"{current}_hits"] = value + misses = _RE_CACHE_MISSES.search(stripped) + if misses: + value = _parse_int(misses.group(1)) + if value is not None: + stats[f"{current}_misses"] = value + rate = _RE_CACHE_RATE.search(stripped) + if rate: + try: + stats[f"{current}_miss_rate"] = float(rate.group(1)) + except ValueError: + pass + + return stats + + +def parse_pc_histogram(stdout: str) -> dict[int, int]: + """Parse the `PC histogram` section from Spike stdout. + + Data lines look like `0x80000014: 123`; the section ends at a blank line + and malformed lines are ignored. + """ + histogram: dict[int, int] = {} + in_histogram = False + for line in (stdout or "").splitlines(): + if not in_histogram: + if "PC histogram" in line or ( + "histogram" in line.lower() and "pc" in line.lower() + ): + in_histogram = True + continue + if line.strip() == "": + break + match = _RE_PC_LINE.match(line.strip()) + if not match: + continue + try: + histogram[int(match.group(1), 16)] = int(match.group(2)) + except ValueError: + continue + return histogram + def run_spike( elf_path: str, @@ -285,6 +612,8 @@ def run_spike( timeout_s: int = 600, isa: str = "rv32im", mem_mb: int = 512, + *, + tools: SpikeTools | None = None, ) -> SpikeResult: """Run Spike on an ELF binary and parse results. @@ -298,11 +627,23 @@ def run_spike( timeout_s: Wall-clock timeout. isa: RISC-V ISA string. mem_mb: Target memory in MiB. + tools: Pre-resolved Spike toolchain; resolved on demand when omitted. Returns: SpikeResult with parsed data. """ + tools = tools or resolve_spike_tools() + if tools.spike is None: + return SpikeResult( + status="skipped", + skip_reason="spike binary not found", + exit_code=-2, + stderr="SKIP: spike binary not found; " + "set SCRATCHV_SPIKE_BIN or pass --spike-bin", + tool_warnings=list(tools.warnings), + ) + cmd = [ - SPIKE, + tools.spike, f"--isa={isa}", f"-m{mem_mb}", f"--ic={ic_config}", @@ -319,7 +660,10 @@ def run_spike( print(f" Running Spike: {' '.join(cmd)}", file=sys.stderr) print(f" Max instructions: {max_instr:,}", file=sys.stderr) - result = SpikeResult() + result = SpikeResult( + spike_path=tools.spike, + tool_warnings=list(tools.warnings), + ) t_start = time.perf_counter() try: @@ -333,94 +677,41 @@ def run_spike( result.stderr = proc.stderr result.exit_code = proc.returncode except subprocess.TimeoutExpired: + result.status = "timeout" result.stderr = "TIMEOUT: Spike did not finish within time limit" result.exit_code = -1 result.wall_time_s = timeout_s return result except FileNotFoundError: - result.stderr = f"ERROR: Spike not found at {SPIKE}" + result.status = "failed" + result.stderr = f"ERROR: Spike not found at {tools.spike}" result.exit_code = -2 return result result.wall_time_s = time.perf_counter() - t_start result.total_insns = max_instr # We set the limit - - # ── Parse committed instruction count from stderr ── - # Spike stderr format: "Commited 100000000 instructions" - for line in result.stderr.splitlines(): - if "Commited" in line and "instructions" in line: - parts = line.strip().split() - for i, p in enumerate(parts): - if p == "Commited" or p == "Committed": - try: - result.committed_insns = int(parts[i + 1]) - except (IndexError, ValueError): - pass - if "MIPS" in p or "mips" in p.lower(): - try: - # Extract the number before MIPS - result.commited_insns_per_sec = float(parts[i - 1]) - except (IndexError, ValueError): - pass - - # ── Parse cache stats from stderr ── - # Format: - # I$: 16 sets × 4 ways × 32 B = 2048 B - # hits: 98234567 misses: 12345 miss rate: 0.01% - # D$: 32 sets × 4 ways × 32 B = 4096 B - # hits: 87654321 misses: 23456 miss rate: 0.03% - current_cache = None - for line in result.stderr.splitlines(): - stripped = line.strip() - if stripped.startswith("I$:"): - current_cache = "icache" - elif stripped.startswith("D$:"): - current_cache = "dcache" - elif current_cache and "hits:" in stripped: - import re - hits_m = re.search(r'hits:\s+(\d+)', stripped) - misses_m = re.search(r'misses:\s+(\d+)', stripped) - rate_m = re.search(r'miss rate:\s+([\d.]+)%', stripped) - if current_cache == "icache": - if hits_m: - result.icache_hits = int(hits_m.group(1)) - if misses_m: - result.icache_misses = int(misses_m.group(1)) - if rate_m: - result.icache_miss_rate = float(rate_m.group(1)) - elif current_cache == "dcache": - if hits_m: - result.dcache_hits = int(hits_m.group(1)) - if misses_m: - result.dcache_misses = int(misses_m.group(1)) - if rate_m: - result.dcache_miss_rate = float(rate_m.group(1)) - - # ── Parse PC histogram from stdout (-g flag) ── - # Format (from spike source code): - # PC histogram (number of commits per PC): - # 0x80000014: 12345678 - # 0x80000018: 23456789 - # ... - in_histogram = False - for line in result.stdout.splitlines(): - if "PC histogram" in line or ("histogram" in line.lower() and "pc" in line.lower()): - in_histogram = True - continue - if in_histogram and ':' in line: - parts = line.strip().split(':') - if len(parts) >= 2: - try: - pc_str = parts[0].strip() - count_str = parts[1].strip() - if pc_str.startswith('0x'): - pc = int(pc_str, 16) - count = int(count_str) - result.pc_histogram[pc] = count - except (ValueError, IndexError): - pass - elif in_histogram and line.strip() == "": - in_histogram = False + if proc.returncode != 0: + result.status = "failed" + + # ── Parse stats via pure helpers (missing sections never fail the run) ── + result.committed_insns, result.commited_insns_per_sec = parse_commit_stats( + result.stderr) + cache_stats = parse_cache_stats(result.stderr) + result.icache_hits = int(cache_stats["icache_hits"]) + result.icache_misses = int(cache_stats["icache_misses"]) + result.icache_miss_rate = float(cache_stats["icache_miss_rate"]) + result.dcache_hits = int(cache_stats["dcache_hits"]) + result.dcache_misses = int(cache_stats["dcache_misses"]) + result.dcache_miss_rate = float(cache_stats["dcache_miss_rate"]) + result.pc_histogram = parse_pc_histogram(result.stdout) + + stderr_text = result.stderr or "" + if not _RE_COMMIT.search(stderr_text): + result.parse_warnings.append("commit stats not found in Spike stderr") + if not _RE_ICACHE_HEADER.search(stderr_text): + result.parse_warnings.append("I$ cache stats not found in Spike stderr") + if not _RE_DCACHE_HEADER.search(stderr_text): + result.parse_warnings.append("D$ cache stats not found in Spike stderr") return result @@ -436,14 +727,30 @@ def run_spike_with_log( isa: str = "rv32im", mem_mb: int = 512, timeout_s: int = 600, + *, + tools: SpikeTools | None = None, ) -> tuple[SpikeResult, str]: """Run Spike with --log-commits and save the log. WARNING: Logging every committed instruction is very slow (100-1000× slower). Only use for small instruction counts (e.g., 1M-10M). """ + tools = tools or resolve_spike_tools() + if tools.spike is None: + return ( + SpikeResult( + status="skipped", + skip_reason="spike binary not found", + exit_code=-2, + stderr="SKIP: spike binary not found; " + "set SCRATCHV_SPIKE_BIN or pass --spike-bin", + tool_warnings=list(tools.warnings), + ), + "", + ) + cmd = [ - SPIKE, + tools.spike, f"--isa={isa}", f"-m{mem_mb}", f"--instructions={max_instr}", @@ -458,7 +765,10 @@ def run_spike_with_log( print(f" Max instructions: {max_instr:,}", file=sys.stderr) print(f" Log file: {log_path}", file=sys.stderr) - result = SpikeResult() + result = SpikeResult( + spike_path=tools.spike, + tool_warnings=list(tools.warnings), + ) t_start = time.perf_counter() try: @@ -472,12 +782,20 @@ def run_spike_with_log( result.stderr = proc.stderr result.exit_code = proc.returncode except subprocess.TimeoutExpired: + result.status = "timeout" result.stderr = "TIMEOUT" result.exit_code = -1 result.wall_time_s = timeout_s return result, "" + except FileNotFoundError: + result.status = "failed" + result.stderr = f"ERROR: Spike not found at {tools.spike}" + result.exit_code = -2 + return result, "" result.wall_time_s = time.perf_counter() - t_start + if proc.returncode != 0: + result.status = "failed" # If log was written to file, read it log_content = "" @@ -509,10 +827,30 @@ def generate_spike_report( lines.append(sep) lines.append(f" Binary: {binary_path}") lines.append(f" Code size: {code_size:,} B ({code_size // 4} static insns)") - lines.append(f" Spike: {SPIKE}") + lines.append(f" Spike: {result.spike_path or '(not resolved)'}") lines.append(f" ISA: rv32im") lines.append("") + # ── Run Status (only meaningful when not a clean run) ── + if result.status != "ok" or result.skip_reason: + lines.append(" ── Run Status ──") + lines.append(f" Status: {result.status}") + if result.skip_reason: + lines.append(f" Skip reason: {result.skip_reason}") + lines.append("") + + # ── Warnings (tool resolution / output parsing) ── + if result.tool_warnings: + lines.append(" ── Tool warnings ──") + for warning in result.tool_warnings: + lines.append(f" | {warning}") + lines.append("") + if result.parse_warnings: + lines.append(" ── Parse warnings ──") + for warning in result.parse_warnings: + lines.append(f" | {warning}") + lines.append("") + # ── Timing ── lines.append(" ── Simulation Execution ──") lines.append(f" Wall time: {result.wall_time_s:.2f} s") @@ -589,7 +927,54 @@ def generate_spike_report( # Main entry # ═══════════════════════════════════════════════════════════════════════════ -def main() -> int: +def build_json_report( + result: SpikeResult, + binary_path: str, + code_size: int, + ic_config: str, + dc_config: str, + max_instr: int, + tools: SpikeTools | None = None, +) -> dict: + """Build the machine-readable Spike report (existing keys preserved).""" + report = { + "status": result.status, + "skip_reason": result.skip_reason, + "spike_binary": result.spike_path or None, + "binary": binary_path, + "code_size": code_size, + "static_insns": code_size // 4, + "max_instr": max_instr, + "committed_insns": result.committed_insns, + "wall_time_s": result.wall_time_s, + "exit_code": result.exit_code, + "icache": { + "config": ic_config, + "hits": result.icache_hits, + "misses": result.icache_misses, + "miss_rate_pct": result.icache_miss_rate, + }, + "dcache": { + "config": dc_config, + "hits": result.dcache_hits, + "misses": result.dcache_misses, + "miss_rate_pct": result.dcache_miss_rate, + }, + "top_pcs": sorted( + [{"pc": f"0x{pc:08x}", "count": cnt} + for pc, cnt in result.pc_histogram.items()], + key=lambda x: -x["count"] + )[:15], + "stderr_tail": result.stderr[-2000:] if result.stderr else "", + "parse_warnings": list(result.parse_warnings), + "tool_warnings": list(result.tool_warnings), + } + if tools is not None: + report["spike_tools"] = tools.as_dict() + return report + + +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Spike RISC-V Simulator — ScratchV CNN Benchmark" ) @@ -649,21 +1034,96 @@ def main() -> int: "--json", action="store_true", help="Output results as JSON", ) + parser.add_argument( + "--spike-bin", default=None, + help="Path to the spike executable (highest priority)", + ) + parser.add_argument( + "--spike-dasm", default=None, + help="Path to spike-dasm (optional)", + ) + parser.add_argument( + "--spike-log-parser", default=None, + help="Path to spike-log-parser (optional)", + ) + parser.add_argument( + "--require-spike", action="store_true", + help="Exit with code 2 when spike is missing (default: skip)", + ) - args = parser.parse_args() + args = parser.parse_args(argv) # ── Load binary ──────────────────────────────────────────────────── if not os.path.exists(args.binary): print(f"ERROR: binary not found: {args.binary}", file=sys.stderr) return 1 - with open(args.binary, "rb") as f: - raw_binary = f.read() - code_size = args.code_size if code_size % 4 != 0: code_size += 4 - (code_size % 4) + # ── Resolve Spike toolchain (probing happens only here, never at import) ── + try: + tools = resolve_spike_tools( + args.spike_bin, args.spike_dasm, args.spike_log_parser) + except SpikeConfigError as e: + print(f"ERROR: {e}", file=sys.stderr) + return EXIT_CONFIG + + if tools.spike is None: + searched = ", ".join([ + "--spike-bin", + ENV_SPIKE_BIN, + f"{ENV_SPIKE_HOME}/bin/spike", + "PATH", + *COMMON_SPIKE_DIRS, + "legacy", + ]) + if args.require_spike: + print( + f"ERROR: spike binary not found (--require-spike).\n" + f" searched: {searched}", + file=sys.stderr, + ) + return EXIT_CONFIG + print( + f"SKIP: spike binary not found.\n" + f" searched: {searched}\n" + f" hint: export {ENV_SPIKE_BIN}=/path/to/spike", + file=sys.stderr, + ) + if args.json: + skipped_result = SpikeResult( + status="skipped", + skip_reason="spike binary not found", + tool_warnings=list(tools.warnings), + ) + print(json.dumps( + build_json_report( + skipped_result, args.binary, code_size, + args.ic, args.dc, args.max_instr, tools), + indent=2, + )) + return EXIT_OK + + optional_warnings = _optional_tool_warnings(tools) + for warning in optional_warnings: + print(f"WARNING: {warning}", file=sys.stderr) + + tools_summary = ", ".join( + f"{name}={path} ({tools.sources.get(name, 'missing')})" + if path else f"{name}=missing" + for name, path in ( + ("spike", tools.spike), + ("spike-dasm", tools.spike_dasm), + ("spike-log-parser", tools.spike_log_parser), + ) + ) + print(f"Spike tools: {tools_summary}", file=sys.stderr) + + with open(args.binary, "rb") as f: + raw_binary = f.read() + code = raw_binary[:code_size] data = raw_binary[code_size:] @@ -698,6 +1158,7 @@ def main() -> int: isa=args.isa, mem_mb=args.mem, timeout_s=args.timeout, + tools=tools, ) else: result = run_spike( @@ -710,39 +1171,19 @@ def main() -> int: timeout_s=args.timeout, isa=args.isa, mem_mb=args.mem, + tools=tools, ) + result.tool_warnings.extend(optional_warnings) + # ── Generate report ──────────────────────────────────────────────── if args.json: - import json - report = { - "binary": args.binary, - "code_size": code_size, - "static_insns": code_size // 4, - "max_instr": args.max_instr, - "committed_insns": result.committed_insns, - "wall_time_s": result.wall_time_s, - "exit_code": result.exit_code, - "icache": { - "config": args.ic, - "hits": result.icache_hits, - "misses": result.icache_misses, - "miss_rate_pct": result.icache_miss_rate, - }, - "dcache": { - "config": args.dc, - "hits": result.dcache_hits, - "misses": result.dcache_misses, - "miss_rate_pct": result.dcache_miss_rate, - }, - "top_pcs": sorted( - [{"pc": f"0x{pc:08x}", "count": cnt} - for pc, cnt in result.pc_histogram.items()], - key=lambda x: -x["count"] - )[:15], - "stderr_tail": result.stderr[-2000:] if result.stderr else "", - } - print(json.dumps(report, indent=2)) + print(json.dumps( + build_json_report( + result, args.binary, code_size, + args.ic, args.dc, args.max_instr, tools), + indent=2, + )) else: print(generate_spike_report( result, @@ -760,7 +1201,7 @@ def main() -> int: except OSError: pass - return 0 if result.exit_code == 0 else 1 + return EXIT_OK if result.status == "ok" else EXIT_RUN_FAIL if __name__ == "__main__": diff --git a/tests/test_spike_sim_paths.py b/tests/test_spike_sim_paths.py new file mode 100644 index 0000000..a47b5ea --- /dev/null +++ b/tests/test_spike_sim_paths.py @@ -0,0 +1,340 @@ +"""Topic 24: portable Spike toolchain resolution and graceful degradation. + +All tests are hermetic: they never require a real Spike installation. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import types +from pathlib import Path + +import pytest + +from scratchv.standalone import spike_sim + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def make_fake_tool(tmp_path: Path, name: str) -> Path: + p = tmp_path / name + p.write_text("#!/bin/sh\nexit 0\n") + p.chmod(0o755) + return p + + +@pytest.fixture(autouse=True) +def hermetic_env(monkeypatch, tmp_path): + for var in ("SCRATCHV_SPIKE_BIN", "SCRATCHV_SPIKE_DASM", + "SCRATCHV_SPIKE_LOG_PARSER", "SCRATCHV_SPIKE_HOME"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setattr(spike_sim, "COMMON_SPIKE_DIRS", ()) + monkeypatch.setattr(spike_sim, "SPIKE", str(tmp_path / "legacy-spike")) + monkeypatch.setattr(spike_sim, "SPIKE_DASM", str(tmp_path / "legacy-dasm")) + monkeypatch.setattr(spike_sim, "SPIKE_LOG_PARSER", + str(tmp_path / "legacy-parser")) + monkeypatch.setattr(shutil, "which", lambda name: None) + + +CANNED_STDERR = """\ +Commited 1234 instructions +core 0: 0x80000000 (0x00000013) 1.5 MIPS +I$: 64 sets × 2 ways × 32 B + hits: 10,000 misses: 25 miss rate: 0.25% +D$: 128 sets × 4 ways × 32 B + hits: 20,000 misses: 50 miss rate: 0.25% +""" +CANNED_STDOUT = """\ +PC histogram (number of commits per PC): +0x80000014: 123 +0x80000018: 456 + +""" + + +# ── Import / CLI hygiene ──────────────────────────────────────────────────── + +def test_import_works_without_spike(tmp_path): + env = {"PATH": str(tmp_path), "HOME": str(tmp_path), + "PYTHONPATH": str(REPO_ROOT)} + proc = subprocess.run( + [sys.executable, "-c", + "import scratchv.standalone.spike_sim as s; print(s.SPIKE)"], + capture_output=True, text=True, env=env) + assert proc.returncode == 0, proc.stderr + + +def test_help_exits_zero(capsys): + with pytest.raises(SystemExit) as excinfo: + spike_sim.main(["--help"]) + + assert excinfo.value.code == 0 + out = capsys.readouterr().out + for flag in ("--spike-bin", "--spike-dasm", "--spike-log-parser", + "--require-spike"): + assert flag in out + + +# ── Missing-spike degradation ─────────────────────────────────────────────── + +def test_missing_spike_skips_with_reason(tmp_path, capsys): + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64"]) + + assert rc == spike_sim.EXIT_OK + err = capsys.readouterr().err + assert "SKIP:" in err + assert "spike binary not found" in err + assert "SCRATCHV_SPIKE_BIN" in err + assert "SCRATCHV_SPIKE_HOME/bin/spike" in err + assert "legacy" in err + assert not (tmp_path / "output_spike.elf").exists() + + +def test_missing_spike_strict_returns_config_error(tmp_path, capsys): + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--require-spike"]) + + assert rc == spike_sim.EXIT_CONFIG + err = capsys.readouterr().err + assert "ERROR:" in err + assert "--require-spike" in err + assert not (tmp_path / "output_spike.elf").exists() + + +def test_missing_spike_json_report_fields(tmp_path, capsys): + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", "--json"]) + + assert rc == spike_sim.EXIT_OK + report = json.loads(capsys.readouterr().out) + assert report["status"] == "skipped" + assert report["skip_reason"] == "spike binary not found" + assert report["spike_binary"] is None + for key in ("binary", "code_size", "static_insns", "max_instr", + "committed_insns", "wall_time_s", "exit_code", "icache", + "dcache", "top_pcs", "stderr_tail", "parse_warnings", + "tool_warnings", "spike_tools"): + assert key in report + assert report["spike_tools"]["spike"]["source"] == "missing" + assert report["spike_tools"]["spike"]["path"] is None + + +# ── Resolution priority chain ─────────────────────────────────────────────── + +def test_resolution_cli_over_env(tmp_path, monkeypatch): + fake_env = make_fake_tool(tmp_path, "spike-env") + fake_cli = make_fake_tool(tmp_path, "spike-cli") + monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(fake_env)) + + tools_env = spike_sim.resolve_spike_tools() + assert tools_env.spike == str(fake_env) + assert tools_env.sources["spike"] == "env" + + tools_cli = spike_sim.resolve_spike_tools(cli_spike=str(fake_cli)) + assert tools_cli.spike == str(fake_cli) + assert tools_cli.sources["spike"] == "cli" + + +def test_resolution_env_over_path(tmp_path, monkeypatch): + fake_env = make_fake_tool(tmp_path, "spike-env") + monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(fake_env)) + + tools = spike_sim.resolve_spike_tools() + assert tools.spike == str(fake_env) + assert tools.sources["spike"] == "env" + + monkeypatch.delenv("SCRATCHV_SPIKE_BIN") + fake_path = make_fake_tool(tmp_path, "spike-path") + monkeypatch.setattr( + shutil, "which", + lambda name: str(fake_path) if name == "spike" else None) + + tools = spike_sim.resolve_spike_tools() + assert tools.spike == str(fake_path) + assert tools.sources["spike"] == "path" + + +def test_resolution_spike_home_and_common(tmp_path, monkeypatch): + home = tmp_path / "spike-home" + (home / "bin").mkdir(parents=True) + fake_home_spike = make_fake_tool(home / "bin", "spike") + monkeypatch.setenv("SCRATCHV_SPIKE_HOME", str(home)) + + tools = spike_sim.resolve_spike_tools() + assert tools.spike == str(fake_home_spike) + assert tools.sources["spike"] == "spike_home" + + monkeypatch.delenv("SCRATCHV_SPIKE_HOME") + common = tmp_path / "common" + common.mkdir() + fake_common_spike = make_fake_tool(common, "spike") + monkeypatch.setattr(spike_sim, "COMMON_SPIKE_DIRS", (str(common),)) + + tools = spike_sim.resolve_spike_tools() + assert tools.spike == str(fake_common_spike) + assert tools.sources["spike"] == "common" + + +def test_resolution_legacy_constant(tmp_path, monkeypatch): + fake_legacy = make_fake_tool(tmp_path, "legacy-spike") + monkeypatch.setattr(spike_sim, "SPIKE", str(fake_legacy)) + + tools = spike_sim.resolve_spike_tools() + + assert tools.spike == str(fake_legacy) + assert tools.sources["spike"] == "legacy" + + +# ── Invalid explicit paths ────────────────────────────────────────────────── + +def test_cli_invalid_path_raises(tmp_path): + with pytest.raises(spike_sim.SpikeConfigError) as ei: + spike_sim.resolve_spike_tools(cli_spike=str(tmp_path / "nope")) + assert "--spike-bin" in str(ei.value) + + with pytest.raises(spike_sim.SpikeConfigError) as ei: + spike_sim.resolve_spike_tools(cli_dasm=str(tmp_path / "nope")) + assert "--spike-dasm" in str(ei.value) + + +def test_cli_invalid_path_returns_config_error(tmp_path, capsys): + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--spike-bin", str(tmp_path / "nope")]) + + assert rc == spike_sim.EXIT_CONFIG + assert "ERROR:" in capsys.readouterr().err + assert not (tmp_path / "output_spike.elf").exists() + + +def test_env_invalid_path_warns_and_falls_through(tmp_path, monkeypatch): + monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(tmp_path / "nope")) + + tools = spike_sim.resolve_spike_tools() + + assert tools.spike is None + assert tools.sources["spike"] == "missing" + assert any("SCRATCHV_SPIKE_BIN" in w for w in tools.warnings) + + +# ── Pure parsing helpers ──────────────────────────────────────────────────── + +def test_parse_commit_stats(): + committed, mips = spike_sim.parse_commit_stats(CANNED_STDERR) + assert (committed, mips) == (1234, 1.5) + + corrected, corrected_mips = spike_sim.parse_commit_stats( + "Committed 2,000,000 instructions\n42.0 MIPS") + assert (corrected, corrected_mips) == (2_000_000, 42.0) + + assert spike_sim.parse_commit_stats("nothing here") == (0, 0.0) + + +def test_parse_cache_stats(): + stats = spike_sim.parse_cache_stats(CANNED_STDERR) + assert stats["icache_hits"] == 10_000 + assert stats["icache_misses"] == 25 + assert stats["icache_miss_rate"] == 0.25 + assert stats["dcache_hits"] == 20_000 + assert stats["dcache_misses"] == 50 + assert stats["dcache_miss_rate"] == 0.25 + + empty = spike_sim.parse_cache_stats("no stats here") + assert set(empty) == { + "icache_hits", "icache_misses", "icache_miss_rate", + "dcache_hits", "dcache_misses", "dcache_miss_rate", + } + assert all(value == 0 for value in empty.values()) + + +def test_parse_pc_histogram(): + hist = spike_sim.parse_pc_histogram(CANNED_STDOUT) + assert hist == {0x80000014: 123, 0x80000018: 456} + + messy = ( + "PC histogram (number of commits per PC):\n" + "0x80000010: 7\n" + "garbage line\n" + "0x80000020: 9\n" + "\n" + "0x80000030: 11\n" + ) + assert spike_sim.parse_pc_histogram(messy) == { + 0x80000010: 7, 0x80000020: 9} + + assert spike_sim.parse_pc_histogram("no histogram here") == {} + + +def test_run_spike_mock_subprocess_records_parse_warnings(tmp_path, monkeypatch): + tools = spike_sim.SpikeTools(spike="/fake/spike") + captured: dict[str, list[str]] = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = list(cmd) + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(spike_sim.subprocess, "run", fake_run) + + result = spike_sim.run_spike(str(tmp_path / "x.elf"), tools=tools) + + assert captured["cmd"][0] == "/fake/spike" + assert result.status == "ok" + assert result.spike_path == "/fake/spike" + assert result.committed_insns == 0 + assert result.parse_warnings + assert any("commit" in w.lower() for w in result.parse_warnings) + + +def test_run_spike_missing_tool_returns_skipped(tmp_path): + tools = spike_sim.SpikeTools( + warnings=("SCRATCHV_SPIKE_BIN=/old/spike is not executable; ignored",)) + + result = spike_sim.run_spike(str(tmp_path / "x.elf"), tools=tools) + + assert result.status == "skipped" + assert result.skip_reason == "spike binary not found" + assert result.exit_code == -2 + assert list(result.tool_warnings) == list(tools.warnings) + + +# ── Report fields ─────────────────────────────────────────────────────────── + +def test_report_status_fields(): + result = spike_sim.SpikeResult( + status="skipped", skip_reason="spike binary not found") + + text = spike_sim.generate_spike_report( + result, "output.bin", 64, "64:2:32", "128:4:32", 50_000_000) + assert "Status:" in text + assert "skipped" in text + assert "Skip reason:" in text + assert "spike binary not found" in text + + tools = spike_sim.SpikeTools() + report = spike_sim.build_json_report( + result, "output.bin", 64, "64:2:32", "128:4:32", 50_000_000, tools) + assert report["status"] == "skipped" + assert report["skip_reason"] == "spike binary not found" + assert report["spike_binary"] is None + for key in ("binary", "code_size", "static_insns", "max_instr", + "committed_insns", "wall_time_s", "exit_code", "icache", + "dcache", "top_pcs", "stderr_tail"): + assert key in report + assert report["spike_tools"]["spike"]["path"] is None + + plain = spike_sim.build_json_report( + result, "output.bin", 64, "64:2:32", "128:4:32", 50_000_000) + assert "spike_tools" not in plain From 5937ba3b41a4ec994b051d3fe84299d772822918 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 21:21:40 +0800 Subject: [PATCH 2/5] docs(topic24): add design and development documents --- ...00\345\217\221\346\226\207\346\241\243.md" | 670 ++++++++++++++++++ ...76\350\256\241\346\226\207\346\241\243.md" | 400 +++++++++++ 2 files changed, 1070 insertions(+) create mode 100644 "docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" create mode 100644 "docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" diff --git "a/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" new file mode 100644 index 0000000..a96ac07 --- /dev/null +++ "b/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -0,0 +1,670 @@ +# 课题 24 Spike 仿真：路径可移植化与降级 开发文档 + +> 文档版本：v1.0 +> 编写日期：2026-09-14 +> 目标读者：实现者 / 审查者 +> 配套设计文档：`./设计文档.md` +> 范围边界：只做路径可移植与降级；**不改**周期模型与统计口径；**不改** TinyFive 路径；**不引入**外部依赖（只用 Python 标准库） + +--- + +## 一、接口契约 + +本节所有名称即实现时必须使用的精确名称，不得改名。 + +### 1.1 新增/变更函数 + +| 名称 | 签名 | 语义 | +|------|------|------| +| `is_executable` | `is_executable(path: str) -> bool` | `expanduser` 后同时满足 `isfile` 与 `X_OK` | +| `resolve_spike_tools` | `resolve_spike_tools(cli_spike=None, cli_dasm=None, cli_log_parser=None, env=None, which=None, common_dirs=None) -> SpikeTools` | 按优先级解析三工具；CLI 路径无效抛 `SpikeConfigError`；env 无效记 `warnings` 后继续；`env` 默认 `os.environ`、`which` 默认 `shutil.which`（均在调用时取值，便于 monkeypatch） | +| `SpikeTools.as_dict` | `SpikeTools.as_dict() -> dict` | 输出 `{"spike": {"path": str|None, "source": str}, "spike_dasm": {...}, "spike_log_parser": {...}, "warnings": [...]}` | +| `parse_commit_stats` | `parse_commit_stats(stderr: str) -> tuple[int, float]` | 返回 `(committed_insns, mips)`；容忍 `Commited/Committed` 与千分位 | +| `parse_cache_stats` | `parse_cache_stats(stderr: str) -> dict[str, int | float]` | 返回键固定为 `icache_hits, icache_misses, icache_miss_rate, dcache_hits, dcache_misses, dcache_miss_rate` | +| `parse_pc_histogram` | `parse_pc_histogram(stdout: str) -> dict[int, int]` | 解析 `PC histogram` 段（`0x...: count`，空行结束） | +| `run_spike` | `run_spike(elf_path, max_instr=..., ic_config=..., dc_config=..., track_pc=..., log_commits=..., timeout_s=..., isa=..., mem_mb=..., *, tools: SpikeTools | None = None) -> SpikeResult` | 新增仅关键字参数 `tools`；其余参数名/默认值不变 | +| `run_spike_with_log` | `run_spike_with_log(elf_path, max_instr=..., log_path=..., isa=..., mem_mb=..., timeout_s=..., *, tools: SpikeTools | None = None) -> tuple[SpikeResult, str]` | 同上；缺 spike 时返回 `(skipped_result, "")` | +| `generate_spike_report` | 签名不变，读取 `result.status/skip_reason/spike_path/tool_warnings` | 文本报告新增 `Status:` 等行 | +| `build_json_report` | `build_json_report(result: SpikeResult, binary_path: str, code_size: int, ic_config: str, dc_config: str, max_instr: int, tools: SpikeTools | None = None) -> dict` | 替代 `main()` 内联字典；既有键保留，新增契约字段见 1.5 | +| `run_spike_bench.generate_json_report` | `generate_json_report(result, spike_tools: SpikeTools | None = None) -> dict` | 新增可选参数；新增 `"backend"` 字段 | + +### 1.2 新增类型与异常 + +```python +@dataclass(frozen=True) +class SpikeTools: + spike: str | None = None + spike_dasm: str | None = None + spike_log_parser: str | None = None + sources: dict[str, str] = field(default_factory=dict) # tool -> cli|env|spike_home|path|common|legacy|missing + candidates: dict[str, tuple[str, ...]] = field(default_factory=dict) # tool -> 已尝试的候选路径 + warnings: tuple[str, ...] = () + + @property + def missing(self) -> list[str]: ... # 返回缺失工具的规范名列表 + + def as_dict(self) -> dict: ... + +class SpikeConfigError(ValueError): + """显式配置（CLI）路径无效时抛出。""" +``` + +### 1.3 环境变量契约 + +| 变量 | 精确名称 | 空值语义 | 无效语义 | +|------|---------|---------|---------| +| spike | `SCRATCHV_SPIKE_BIN` | 视为未设置 | WARNING + 继续 | +| dasm | `SCRATCHV_SPIKE_DASM` | 视为未设置 | WARNING + 继续 | +| log parser | `SCRATCHV_SPIKE_LOG_PARSER` | 视为未设置 | WARNING + 继续 | +| 安装根目录 | `SCRATCHV_SPIKE_HOME` | 视为未设置 | 静默跳过该层（候选列表留痕） | + +规则：值先 `strip()`，支持 `~` 展开；大小写敏感；三工具互不影响。 + +### 1.4 CLI 契约（`spike_sim.py`） + +| 参数 | 类型/默认 | 说明 | +|------|----------|------| +| `--spike-bin PATH` | str / 无 | spike 可执行文件；无效 → 退出码 2 | +| `--spike-dasm PATH` | str / 无 | spike-dasm；无效 → 退出码 2 | +| `--spike-log-parser PATH` | str / 无 | spike-log-parser；无效 → 退出码 2 | +| `--require-spike` | store_true / 关 | spike 缺失时硬失败（退出码 2） | +| `--probe-spike` | store_true / 关 | **仅 `run_spike_bench.py`**：探测并打印工具可用性 | + +既有参数 `--binary`、`--code-size`、`--max-instr`、`--ic`、`--dc`、`--isa`、`--mem`、`--timeout`、`--no-pc-histogram`、`--log-commits`、`--log-instr-limit`、`--keep-elf`、`--elf-output`、`--json` 全部保持不变。 + +### 1.5 退出码与数据字段契约 + +``` +EXIT_OK = 0 # 成功或 skip +EXIT_RUN_FAIL = 1 # 运行失败（超时/非零退出/binary 缺失，历史兼容） +EXIT_CONFIG = 2 # 配置错误（CLI 路径无效 / --require-spike 且缺 spike） +``` + +`SpikeResult` 新增字段（既有字段与拼写 `commited_insns_per_sec` 一律不改）： + +```python +status: str = "ok" # ok | skipped | timeout | failed +skip_reason: str = "" +spike_path: str = "" +tool_warnings: list[str] = field(default_factory=list) +parse_warnings: list[str] = field(default_factory=list) +``` + +`build_json_report()` 新增键：`status`、`skip_reason`、`spike_binary`、`parse_warnings`、`tool_warnings`、`spike_tools`（`tools` 为空时不输出该键）。既有键：`binary`、`code_size`、`static_insns`、`max_instr`、`committed_insns`、`wall_time_s`、`exit_code`、`icache`、`dcache`、`top_pcs`、`stderr_tail` 全部保留。 + +`exit_code` 哨兵兼容：`-1` 超时、`-2` 工具不可用/启动失败；`status` 为权威判读字段。 + +--- + +## 二、`spike_sim.py` 逐处改动 + +行号基于 2026-09-14 版本；改动按文件从上到下排列。 + +### 2.1 模块 docstring（L15-22） + +把硬编码路径示例改为新用法： + +```text +Spike tool resolution order: + 1. --spike-bin / --spike-dasm / --spike-log-parser + 2. SCRATCHV_SPIKE_BIN / SCRATCHV_SPIKE_DASM / SCRATCHV_SPIKE_LOG_PARSER + 3. $SCRATCHV_SPIKE_HOME/bin/ + 4. PATH, then common install dirs, then the legacy constants below +If spike is missing, the tool exits 0 with "SKIP: ..." unless --require-spike. +``` + +### 2.2 导入区（L24-34） + +新增 `import shutil`（`os/subprocess/sys` 已有）。不引入第三方包。 + +### 2.3 路径常量与 legacy 回退（L36-39） + +保留三个常量名，改写为「legacy 回退」并新增解析所需常量： + +```python +# ── Paths ────────────────────────────────────────────────────────────────── +# Legacy fallback (may not exist on this machine). New code must use +# resolve_spike_tools(); these constants are only the last resolution layer. +SPIKE = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike" +SPIKE_DASM = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike-dasm" +SPIKE_LOG_PARSER = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike-log-parser" + +ENV_SPIKE_BIN = "SCRATCHV_SPIKE_BIN" +ENV_SPIKE_DASM = "SCRATCHV_SPIKE_DASM" +ENV_SPIKE_LOG_PARSER = "SCRATCHV_SPIKE_LOG_PARSER" +ENV_SPIKE_HOME = "SCRATCHV_SPIKE_HOME" + +COMMON_SPIKE_DIRS: tuple[str, ...] = ( + "/opt/riscv/bin", + "/opt/riscv64/bin", + "/usr/local/bin", + "/usr/bin", + "~/riscv/bin", + "~/.local/bin", + "~/spike/bin", +) + +EXIT_OK = 0 +EXIT_RUN_FAIL = 1 +EXIT_CONFIG = 2 +``` + +**要点**：legacy 常量必须由 `resolve_spike_tools()` 在**调用时**读取（不要 import 时固化进元组），否则测试无法 monkeypatch。 + +### 2.4 新增解析器（建议插在常量区之后） + +```python +def is_executable(path: str) -> bool: + path = os.path.expanduser(path) + return bool(path) and os.path.isfile(path) and os.access(path, os.X_OK) + + +class SpikeConfigError(ValueError): + pass + + +@dataclass(frozen=True) +class SpikeTools: + spike: str | None = None + spike_dasm: str | None = None + spike_log_parser: str | None = None + sources: dict[str, str] = field(default_factory=dict) + candidates: dict[str, tuple[str, ...]] = field(default_factory=dict) + warnings: tuple[str, ...] = () + + @property + def missing(self) -> list[str]: + pairs = (("spike", self.spike), ("spike-dasm", self.spike_dasm), + ("spike-log-parser", self.spike_log_parser)) + return [name for name, path in pairs if not path] + + def as_dict(self) -> dict: + def one(name: str, path: str | None) -> dict: + return {"path": path, "source": self.sources.get(name, "missing"), + "candidates": list(self.candidates.get(name, ()))} + return {"spike": one("spike", self.spike), + "spike_dasm": one("spike-dasm", self.spike_dasm), + "spike_log_parser": one("spike-log-parser", self.spike_log_parser), + "warnings": list(self.warnings)} + + +def _resolve_one(tool, cli_flag, cli_value, env_name, legacy_dir, legacy_const, + env, which, common_dirs): + """返回 (path|None, source, candidates, warnings)。""" + candidates: list[str] = [] + warnings: list[str] = [] + + if cli_value: # 1. CLI（显式，硬失败） + cand = os.path.expanduser(cli_value.strip()) + candidates.append(cand) + if not is_executable(cand): + raise SpikeConfigError( + f"{cli_flag}={cli_value!r} is not an executable file") + return cand, "cli", candidates, warnings + + env_value = (env.get(env_name) or "").strip() # 2. env（显式，告警继续） + if env_value: + cand = os.path.expanduser(env_value) + candidates.append(cand) + if is_executable(cand): + return cand, "env", candidates, warnings + warnings.append(f"{env_name}={env_value} is not executable; ignored") + + home = (env.get(ENV_SPIKE_HOME) or "").strip() # 3. home 提示 + if home: + cand = os.path.join(os.path.expanduser(home), "bin", tool) + candidates.append(cand) + if is_executable(cand): + return cand, "spike_home", candidates, warnings + + found = which(tool) if which else None # 4. PATH + if found: + candidates.append(found) + return found, "path", candidates, warnings + + for d in common_dirs: # 5. 常见目录 + cand = os.path.join(os.path.expanduser(d), tool) + candidates.append(cand) + if is_executable(cand): + return cand, "common", candidates, warnings + + if legacy_const and is_executable(legacy_const): # 6. legacy 常量 + candidates.append(legacy_const) + return legacy_const, "legacy", candidates, warnings + + return None, "missing", candidates, warnings + + +def resolve_spike_tools(cli_spike=None, cli_dasm=None, cli_log_parser=None, + env=None, which=None, common_dirs=None) -> SpikeTools: + env = os.environ if env is None else env + which = shutil.which if which is None else which + common_dirs = COMMON_SPIKE_DIRS if common_dirs is None else common_dirs + + spec = ( + ("spike", "--spike-bin", cli_spike, ENV_SPIKE_BIN, SPIKE), + ("spike-dasm", "--spike-dasm", cli_dasm, ENV_SPIKE_DASM, SPIKE_DASM), + ("spike-log-parser", "--spike-log-parser", cli_log_parser, + ENV_SPIKE_LOG_PARSER, SPIKE_LOG_PARSER), + ) + paths, sources, candidates, warnings = {}, {}, {}, [] + for tool, flag, cli_value, env_name, legacy_const in spec: + path, source, cands, warns = _resolve_one( + tool, flag, cli_value, env_name, None, legacy_const, + env, which, common_dirs) + paths[tool], sources[tool], candidates[tool] = path, source, cands + warnings.extend(warns) + return SpikeTools( + spike=paths["spike"], spike_dasm=paths["spike-dasm"], + spike_log_parser=paths["spike-log-parser"], + sources=sources, candidates={k: tuple(v) for k, v in candidates.items()}, + warnings=tuple(warnings)) +``` + +### 2.5 `SpikeResult` 扩展（L250-275） + +在类中追加 1.5 节五个字段（默认值如上）。不动既有字段顺序与拼写。 + +### 2.6 解析函数抽取（替换 L348-423 内联逻辑） + +```python +def parse_commit_stats(stderr: str) -> tuple[int, float]: ... +def parse_cache_stats(stderr: str) -> dict[str, int | float]: ... +def parse_pc_histogram(stdout: str) -> dict[int, int]: ... +``` + +实现约定： + +- 数字统一 `s.replace(",", "")` 后再 `int()`/`float()`；失败记入告警（函数内可选返回 warnings，若要保持返回类型纯净，则由 `run_spike` 对比「段头存在但值为 0」自行告警；推荐方案：内部 `re` 匹配段头标记，缺失段头时 `run_spike` append 一条 warning）。 +- `parse_commit_stats` 正则：`r"(?:Commited|Committed)\s+([\d,]+)\s+instructions"`；MIPS：`r"([\d.]+)\s*MIPS"`。 +- `parse_cache_stats`：先定位 `^I\$:` / `^D\$:` 段头，再取 `hits:\s*([\d,]+)`、`misses:\s*([\d,]+)`、`miss rate:\s*([\d.]+)%`。 +- `parse_pc_histogram`：段头行含 `PC histogram`；数据行 `r"^(0x[0-9a-fA-F]+):\s*(\d+)$"`；空行结束。 + +### 2.7 `run_spike()`（L278-425） + +1. 签名尾部加 `*, tools: SpikeTools | None = None`。 +2. 函数体开头： + + ```python + tools = tools or resolve_spike_tools() + if tools.spike is None: + return SpikeResult(status="skipped", + skip_reason="spike binary not found", + exit_code=-2, + stderr="SKIP: spike binary not found; " + "set SCRATCHV_SPIKE_BIN or pass --spike-bin", + tool_warnings=list(tools.warnings)) + ``` + +3. `cmd` 首元素由 `SPIKE` 改为 `tools.spike`；`result.spike_path = tools.spike`；`result.tool_warnings` 合并 `tools.warnings`。 +4. `except FileNotFoundError`：`status="failed"`，消息带 `tools.spike`。 +5. `except subprocess.TimeoutExpired`：`status="timeout"`。 +6. 正常返回前用 2.6 的解析函数填充字段；`result.exit_code` 保留 `proc.returncode`。 + +### 2.8 `run_spike_with_log()`（L432-488） + +签名加 `*, tools=None`；开头做与 2.7 相同的缺失判断，返回 `(skipped_result, "")`；`cmd` 首元素改 `tools.spike`。 + +### 2.9 `generate_spike_report()`（L495-585） + +- L512 `Spike: {SPIKE}` → `Spike: {result.spike_path or "(not resolved)"}`；若 `status=="skipped"`，在 Timing 段前输出： + + ```text + ── Run Status ── + Status: skipped + Skip reason: spike binary not found + ``` + +- 有 `tool_warnings` / `parse_warnings` 时各打印一节（`Tool warnings`、`Parse warnings`），每条一行、缩进 ` | `。 + +### 2.10 `main()` 与 JSON（L592-763） + +1. argparse（L592-653）追加 1.4 节的四个参数（`--probe-spike` 不加在 `spike_sim.py`）。 +2. binary 校验之后、构建 ELF 之前插入工具解析与分支： + + ```python + try: + tools = resolve_spike_tools(args.spike_bin, args.spike_dasm, + args.spike_log_parser) + except SpikeConfigError as e: + print(f"ERROR: {e}", file=sys.stderr) + return EXIT_CONFIG + + if tools.spike is None: + searched = ", ".join(["--spike-bin", ENV_SPIKE_BIN, + f"{ENV_SPIKE_HOME}/bin/spike", "PATH", + *COMMON_SPIKE_DIRS, "legacy"]) + if args.require_spike: + print(f"ERROR: spike binary not found (--require-spike).\n" + f" searched: {searched}", file=sys.stderr) + return EXIT_CONFIG + print(f"SKIP: spike binary not found.\n" + f" searched: {searched}\n" + f" hint: export {ENV_SPIKE_BIN}=/path/to/spike", file=sys.stderr) + if args.json: + print(json.dumps(build_json_report( + SpikeResult(status="skipped", + skip_reason="spike binary not found", + tool_warnings=list(tools.warnings)), + args.binary, args.code_size, args.ic, args.dc, + args.max_instr, tools), indent=2)) + return EXIT_OK + ``` + +3. 成功路径：向 `run_spike` / `run_spike_with_log` 传 `tools=tools`。 +4. 抽取 `build_json_report(...)`（L716-745 迁移）：在原有键基础上加入 1.5 节新键；`spike_tools` 用 `tools.as_dict()`。 +5. 最终返回： + + ```python + return EXIT_OK if result.status == "ok" else EXIT_RUN_FAIL + ``` + +6. 运行日志增加一行工具来源摘要（stderr）： + + ```text + Spike tools: spike=/opt/riscv/bin/spike (cli), spike-dasm=missing, spike-log-parser=missing + ``` + +--- + +## 三、`run_spike_bench.py` 适配点 + +**定位**：该文件是「无 Spike 环境」的纯 Python 降级后端，不调用外部 Spike 二进制，因此本课题**不改其任何统计逻辑**。 + +### 3.1 改动清单 + +| 位置 | 改动 | 约束 | +|------|------|------| +| `main()` argparse（L732-757） | 新增 `--probe-spike`（`action="store_true"`，默认关） | 其余参数不动 | +| `main()` 输出前 | 若 `--probe-spike`：惰性 `from scratchv.standalone.spike_sim import resolve_spike_tools, SpikeConfigError`，打印 `Spike tools: ...` 或 `Spike tools: not found (...)`；捕获 `SpikeConfigError` 打印 `ERROR:` 但**不**改变退出码 | 默认路径（未传 flag）行为与输出完全不变 | +| `generate_json_report`（L669-725） | 签名 `generate_json_report(result, spike_tools=None)`；顶层新增 `"backend": {"kind": "emulator", "spike_style": true}`；`spike_tools` 非空时新增 `"spike_tools": spike_tools.as_dict()` | 既有键一个不删；不改任何数值 | +| 模块 docstring（L1-30） | 增加一句：本工具输出为模拟器结果，真实 Spike 实测请用 `spike_sim.py`（本课题新增 `backend` 字段标注） | 不改用法示例 | + +### 3.2 明确不改的部分（防止越界） + +- `run_emulator_with_caches()` 主体（L107-514）：包括 `pc_samples` 采样、`cat_counts`、`branch_*` 计数、cache 访问。 +- 周期估算块（L480-512）：`mul_ratio=0.15`、`PROFILES` 循环——关联问题，不交付。 +- label 表（L633-660）与 `label_addrs` 构建（L763-774）——关联问题，不交付。 +- `ci_benchmark.py` 调用点（`scratchv/ci/ci_benchmark.py:354-389`）：只调 `run_emulator_with_caches`，本适配不影响其行为。 + +### 3.3 输出示例（`--probe-spike`） + +```text +$ python scratchv/standalone/run_spike_bench.py --binary output.bin --code-size 3140 \ + --max-instr 1000000 --probe-spike +Spike tools: spike=NOT FOUND, spike-dasm=NOT FOUND, spike-log-parser=NOT FOUND + hint: real Spike is unavailable; this run uses the built-in emulator backend +Spike-Style RISC-V Simulation +... +``` + +--- + +## 四、文档更新点 + +| 文件 | 更新内容 | +|------|---------| +| `docs/topics/24-Spike仿真.md` | 1) 「常见坑」表「Spike 二进制路径」改为：路径按 `--spike-bin` > `SCRATCHV_SPIKE_BIN` > `SCRATCHV_SPIKE_HOME` > `PATH` > 常见目录 > legacy 解析，不再硬编码；2) 新增「无 Spike 机器上的行为」段：默认 `SKIP:` 退出码 0，`--require-spike` 退出码 2；3) 动手练习 1 前补 CLI 示例 | +| `docs/ARCHITECTURE.md` | standalone 工具清单（L378-379 附近）加一句：Spike 三工具经统一 resolver 解析，缺失时按 skip/告警分层降级（详见课题 24 设计文档） | +| `scratchv/standalone/spike_sim.py` docstring | 见 2.1 | +| `scratchv/standalone/run_spike_bench.py` docstring | 见 3.1 | +| `CLAUDE.md`（可选） | 关键命令区补 `export SCRATCHV_SPIKE_BIN=...` 示例；非必需，视团队习惯 | + +不新增独立 md 文档到仓库；课题文档按既有目录结构更新。 + +--- + +## 五、测试文件与用例 + +**新增文件**：`tests/test_spike_sim_paths.py`。运行：`python -m pytest tests/test_spike_sim_paths.py -q`。所有用例不依赖真实 Spike。 + +### 5.1 公共夹具 + +```python +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from scratchv.standalone import spike_sim + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def make_fake_tool(tmp_path: Path, name: str) -> Path: + p = tmp_path / name + p.write_text("#!/bin/sh\nexit 0\n") + p.chmod(0o755) + return p + + +@pytest.fixture(autouse=True) +def hermetic_env(monkeypatch, tmp_path): + for var in ("SCRATCHV_SPIKE_BIN", "SCRATCHV_SPIKE_DASM", + "SCRATCHV_SPIKE_LOG_PARSER", "SCRATCHV_SPIKE_HOME"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setattr(spike_sim, "COMMON_SPIKE_DIRS", ()) + monkeypatch.setattr(spike_sim, "SPIKE", str(tmp_path / "legacy-spike")) + monkeypatch.setattr(spike_sim, "SPIKE_DASM", str(tmp_path / "legacy-dasm")) + monkeypatch.setattr(spike_sim, "SPIKE_LOG_PARSER", str(tmp_path / "legacy-parser")) + monkeypatch.setattr(shutil, "which", lambda name: None) +``` + +### 5.2 用例清单 + +| 用例 | 名称 | 断言要点 | +|------|------|---------| +| 1 | `test_import_works_without_spike` | 子进程干净 `PATH`/`HOME` 下 `import scratchv.standalone.spike_sim` 退出码 0 | +| 2 | `test_help_exits_zero` | `pytest.raises(SystemExit)`，`code == 0`，不探测工具 | +| 3 | `test_missing_spike_skips_with_reason` | `main()` 返回 0；stderr 含 `SKIP:` 与 `SCRATCHV_SPIKE_BIN` | +| 4 | `test_missing_spike_strict_returns_config_error` | `--require-spike` → 返回 2；stderr 含 `ERROR:` | +| 5 | `test_resolution_cli_over_env` | CLI 命中 `source=="cli"` | +| 6 | `test_resolution_env_over_path` | env 命中 `source=="env"`；无 env 时 `which` 假路径 → `source=="path"` | +| 7 | `test_resolution_spike_home_and_common` | home 命中 `source=="spike_home"`；common 目录命中 `source=="common"` | +| 8 | `test_resolution_legacy_constant` | 仅 legacy 常量可执行时 `source=="legacy"`（monkeypatch 常量指向 tmp 可执行文件） | +| 9 | `test_cli_invalid_path_raises` | `pytest.raises(spike_sim.SpikeConfigError)`，消息含 `--spike-bin` | +| 10 | `test_env_invalid_path_warns_and_falls_through` | 不抛异常；`tools.warnings` 含变量名；最终 `None` | +| 11 | `test_parse_commit_stats` | 样本返回 `(1234, 1.5)`；缺段返回 `(0, 0.0)` | +| 12 | `test_parse_cache_stats` | 千分位样本 → `icache_hits == 10000` 等；缺段全 0 | +| 13 | `test_parse_pc_histogram` | 样本 dict 非空；空行终止；坏行忽略 | +| 14 | `test_report_status_fields` | 文本含 `Status:` / `Skip reason:`；JSON 含 `status` / `skip_reason` / 既有键 | + +### 5.3 关键用例示例代码 + +```python +def test_missing_spike_skips_with_reason(tmp_path, capsys): + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64"]) + + assert rc == spike_sim.EXIT_OK + err = capsys.readouterr().err + assert "SKIP:" in err + assert "SCRATCHV_SPIKE_BIN" in err + assert not (tmp_path / "output_spike.elf").exists() + + +def test_missing_spike_strict_returns_config_error(tmp_path, capsys): + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--require-spike"]) + + assert rc == spike_sim.EXIT_CONFIG + assert "ERROR:" in capsys.readouterr().err + + +def test_resolver_priority(tmp_path, monkeypatch): + fake_env = make_fake_tool(tmp_path, "spike-env") + fake_cli = make_fake_tool(tmp_path, "spike-cli") + monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(fake_env)) + + t_env = spike_sim.resolve_spike_tools() + assert t_env.spike == str(fake_env) + assert t_env.sources["spike"] == "env" + + t_cli = spike_sim.resolve_spike_tools(cli_spike=str(fake_cli)) + assert t_cli.spike == str(fake_cli) + assert t_cli.sources["spike"] == "cli" + + +def test_cli_invalid_path_raises(tmp_path): + with pytest.raises(spike_sim.SpikeConfigError) as ei: + spike_sim.resolve_spike_tools(cli_spike=str(tmp_path / "nope")) + assert "--spike-bin" in str(ei.value) + + +def test_env_invalid_path_warns_and_falls_through(tmp_path, monkeypatch): + monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(tmp_path / "nope")) + + tools = spike_sim.resolve_spike_tools() + + assert tools.spike is None + assert any("SCRATCHV_SPIKE_BIN" in w for w in tools.warnings) + + +CANNED_STDERR = """\ +Commited 1234 instructions +core 0: 0x80000000 (0x00000013) 1.5 MIPS +I$: 64 sets × 2 ways × 32 B + hits: 10,000 misses: 25 miss rate: 0.25% +D$: 128 sets × 4 ways × 32 B + hits: 20,000 misses: 50 miss rate: 0.25% +""" +CANNED_STDOUT = """\ +PC histogram (number of commits per PC): +0x80000014: 123 +0x80000018: 456 + +""" + + +def test_parse_helpers_from_canned_output(): + committed, mips = spike_sim.parse_commit_stats(CANNED_STDERR) + assert (committed, mips) == (1234, 1.5) + + stats = spike_sim.parse_cache_stats(CANNED_STDERR) + assert stats["icache_hits"] == 10_000 + assert stats["icache_misses"] == 25 + assert stats["dcache_hits"] == 20_000 + + hist = spike_sim.parse_pc_histogram(CANNED_STDOUT) + assert hist == {0x80000014: 123, 0x80000018: 456} +``` + +`test_import_works_without_spike` 的子进程写法： + +```python +def test_import_works_without_spike(tmp_path): + env = {"PATH": str(tmp_path), "HOME": str(tmp_path), + "PYTHONPATH": str(REPO_ROOT)} + proc = subprocess.run( + [sys.executable, "-c", + "import scratchv.standalone.spike_sim as s; print(s.SPIKE)"], + capture_output=True, text=True, env=env) + assert proc.returncode == 0, proc.stderr +``` + +--- + +## 六、验收标准 + +在无 Spike 机器上（可先 `unset SCRATCHV_SPIKE_*`）： + +| # | 命令 | 预期 | +|---|------|------| +| 1 | `python -c "import scratchv.standalone.spike_sim"` | 退出码 0，无输出/异常 | +| 2 | `python scratchv/standalone/spike_sim.py --help` | 退出码 0，列出 `--spike-bin/--spike-dasm/--spike-log-parser/--require-spike` | +| 3 | `python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140` | stderr `SKIP: spike binary not found...`，退出码 0，不生成 ELF | +| 4 | 同上加 `--require-spike` | stderr `ERROR:`，退出码 2 | +| 5 | 同上加 `--spike-bin /nonexistent` | stderr `ERROR:`，退出码 2 | +| 6 | 同上加 `--json`（有/无 spike 均可） | JSON 含 `status`、`skip_reason`、`spike_binary`，既有键完整 | +| 7 | `python scratchv/standalone/run_spike_bench.py --binary output.bin --code-size 3140 --max-instr 1000000 --probe-spike` | 退出码 0；stderr 有工具可用性摘要；数值与未加 flag 时一致 | +| 8 | `python -m pytest tests/test_spike_sim_paths.py -q` | 全绿 | +| 9 | `make test` | 全量单测通过 | +| 10 | `python .claude/harness/verify/run.py --level L2` | 通过 | + +在有 Spike 的机器上： + +| # | 命令 | 预期 | +|---|------|------| +| 11 | `--spike-bin <真实路径>` 跑基线 binary | `status=ok`；`committed_insns`、I$/D$ 命中数与改动前一致（数值零回归） | +| 12 | `SCRATCHV_SPIKE_BIN=<旧路径>`（不存在） | stderr `WARNING:` + 回退/跳过，不崩溃 | + +--- + +## 七、风险与回退 + +| 风险 | 缓解 | 回退 | +|------|------|------| +| 解析器误选系统里的其他 Spike 版本 | 报告/日志记录 `spike_path` 与 `source`；CI 用 `--spike-bin` 固定 | 显式 `--spike-bin` 永远最高优先级 | +| 残留的 `SCRATCHV_SPIKE_BIN` 指向旧路径 | 只告警并继续后续层级，不阻断 | `unset` 或 `--spike-bin` 覆盖 | +| monkeypatch 失效（常量被 import 时固化） | 2.3/2.4 明确要求调用时读取常量 | 解析器增加 `legacy_const` 显式参数（本次已设计为参数） | +| 新增 CLI/JSON 字段破坏下游解析 | 只增不删；`ci_benchmark.py` 走函数调用不受影响 | 回退单个 commit；JSON 旧键始终保留 | +| 解析函数抽取引入行为差异 | 用固定样本用例锁定正则与容错行为 | 保留旧内联逻辑在 git 历史中可快速恢复 | +| import 变慢/有副作用 | 导入阶段不调用 `resolve`、不探测文件系统 | 解析全部延迟到 `main()` / `run_spike()` | +| legacy 常量机器（原作者）行为回归 | legacy 作为最后一层，仍可命中 | 若出问题，显式 `--spike-bin` 即可 | + +--- + +## 八、关联不交付项（范围边界） + +| 项 | 位置 | 说明 | +|----|------|------| +| label 表失真 | `run_spike_bench.py:633-660`、`765-774` | `layer_descs` 使用旧标签前缀，`label_addrs` 实际只有 `_start@0`，逐层统计不可用；建议独立课题重做标签映射 | +| 周期估算失真 | `run_spike_bench.py:480-512` | `mul_ratio=0.15` 为经验猜测；`PROFILES` CPI 常量与真实微架构未校准；建议独立课题 | +| TinyFive 路径 | `tinyfive_compare.py` | 本课题明确不触碰（含 `TINYFIVE_AVAILABLE` 探测逻辑） | +| 统计口径 | `run_spike_bench.py` 全部计数 | 不改采样间隔、分类、命中率定义 | +| commit log 深度解析（dasm/log-parser 实际调用） | `spike_sim.py` | 本课题只解析工具路径并定义缺失语义；日志内容仍原样透传，指令分类接入留待后续 | + +--- + +## 九、参考资料 + +- 设计文档：`./设计文档.md` +- 课题文档：`docs/topics/24-Spike仿真.md` +- 降级模式先例：`scratchv/standalone/tinyfive_compare.py:29-86` +- 使用方调用点：`scratchv/ci/ci_benchmark.py:341-393` +- 代码风格：Python 3.12+、type hints、argparse、零外部依赖 + +--- + +## 实现结果（2026-09-14 集成） + +> **集成 commit**：`2f85f32`（`feat(topic24): make Spike toolchain paths portable with graceful degradation`） +> **集成位置**：`Seven_big_summary` 上第 6 个 topic commit（顺序 … → 17 → **24** → 27 → …） +> **集成后全量**：`PYTHONPATH=. python3.11 -m pytest tests/ -q` → **1011 passed / 13 xfailed / 20 xpassed / 0 failed** + +### 实现文件与要点 + +| 文件 | 要点 | +|------|------| +| `scratchv/standalone/spike_sim.py` | 六级路径解析（`--spike-bin` > `SCRATCHV_SPIKE_BIN` > … > legacy）+ 降级分层 | +| `scratchv/standalone/run_spike_bench.py` | 新增 `--probe-spike` | +| `tests/test_spike_sim_paths.py` | 18 个新用例 | +| `docs/ARCHITECTURE.md`、`docs/topics/24-Spike仿真.md` | 随实现更新 | + +### 测试数字 + +| 口径 | 结果 | +|------|------| +| 定向（`tests/test_spike_sim_paths.py`） | 18 passed | +| 分支全量（cherry-pick 前） | 583 passed | +| 集成后全量 | 1011 passed / 13 xfailed / 20 xpassed / 0 failed | + +### 与本文档的偏差 / 未完成项 + +- 新增顶层 `json` / `re` import。 +- `main(argv=None)` 签名适配（便于测试注入参数）。 +- dasm / log-parser 缺失仅 WARNING，不阻断。 + +### 已知限制 + +- 本机 PATH 中无真实 `spike` 二进制，§六 验收 11/12（真实 Spike 端到端与数值零回归）需在装有 Spike 的机器上补跑。 +- label 表失真、周期估算失真、日志深度解析等仍属范围外（§八）。 diff --git "a/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000..a6fa7d9 --- /dev/null +++ "b/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,400 @@ +# ScratchV Spike 仿真工具路径可移植化与降级策略 设计文档 + +> 文档版本：v1.0 +> 编写日期：2026-09-14 +> 涉及模块：`scratchv/standalone/spike_sim.py`（Spike 工具解析/调用/输出解析/报告）、`scratchv/standalone/run_spike_bench.py`（纯仿真降级后端，使用方适配） +> 功能范围：spike / spike-dasm / spike-log-parser 三工具的可移植路径解析、环境变量契约、工具缺失时的降级与报错分层、Spike 输出解析约定、报告状态字段；**不含**周期模型、统计口径、TinyFive 路径的修改 + +--- + +## 一、功能介绍 + +### 1.1 功能概述 + +`spike_sim.py` 的职责是：把 ScratchV 生成的 flat binary 包装成最小 ELF32，调用 Spike（RISC-V 黄金参考模型）执行，解析 commit/cache 输出并生成报告。它依赖三个外部工具： + +| 工具 | 用途 | 当前状态 | +|------|------|---------| +| `spike` | 真正执行 ELF | 必需（缺失则无法仿真） | +| `spike-dasm` | 反汇编指令、供指令分类 | 已定义常量但代码路径尚未使用 | +| `spike-log-parser` | 解析 commit log（trace/热点） | 已定义常量但代码路径尚未使用 | + +当前问题（`spike_sim.py:37-39`）： + +```python +SPIKE = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike" +SPIKE_DASM = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike-dasm" +SPIKE_LOG_PARSER = "/home/kinsomwang/workspace/coralnpu-spike-rv32/bin/spike-log-parser" +``` + +- 三个路径是个人机器绝对路径，换机器后必然 `FileNotFoundError`（`run_spike` 捕获后返回 `exit_code=-2`），但： + - 无法区分「本机没有 Spike（应跳过）」与「Spike 跑了但失败（应报错）」； + - 无 CLI/环境变量覆盖入口，CI 或别的开发者无法在不改代码的前提下指定自己的 Spike； + - `spike-dasm` / `spike-log-parser` 缺失时语义未定义（当前代码甚至不使用它们）。 +- 解析层（commit 统计、cache 统计、PC 直方图）与运行层耦合在 `run_spike()` 内联实现，缺失段落只能静默得到 0，没有任何告警信号。 + +本功能要做的事： + +1. **路径解析器**：按固定优先级自动解析三工具路径（CLI > 环境变量 > `SCRATCHV_SPIKE_HOME` > `PATH` > 常见安装目录 > 旧的硬编码常量回退）。 +2. **降级/报错分层**：默认情况下「工具不存在」是 skip（带原因），「显式配置无效」「`--require-spike` 且工具不存在」是硬失败，二者退出码不同；`spike-dasm` / `spike-log-parser` 缺失只降级为告警。 +3. **解析健壮化**：把内联解析抽成纯函数，容忍千分位数字与缺失段落，并把异常记录进解析告警。 +4. **可观测性**：`SpikeResult`、文本报告、JSON 报告增加 `status` / `skip_reason` / `spike_binary` / 告警字段。 +5. **使用方适配**：`run_spike_bench.py`（当前无 Spike 时的纯 Python 降级后端）增加后端标注与工具可用性探测，不改变任何统计数据。 + +### 1.2 设计目标 + +- **任何机器可 import / `--help`**：模块导入与参数解析阶段不探测、不调用、不写任何外部工具。 +- **单点可覆盖**：`--spike-bin` / `SCRATCHV_SPIKE_BIN` 在任意机器上都能强制指定，优先级最高。 +- **失败语义清晰**：配置错误（退出码 2）≠ 运行失败（退出码 1）≠ 环境不具备的合法跳过（退出码 0 且 `status=skipped`）。 +- **零外部依赖**：只用 `os` / `shutil` / `subprocess` 等标准库，不新增 pip 包。 +- **向后兼容**：三个模块常量保留、`run_spike()` 旧调用参数保持不变、JSON 既有字段不删除；`ci_benchmark.py` 对 `run_spike_bench.run_emulator_with_caches` 的直接函数调用不受影响。 +- **降级可观测**：所有跳过与告警都带机器可读字段（`status`、`skip_reason`、`tool_warnings`、`parse_warnings`），并与 `tinyfive_compare.py` 现有 `TINYFIVE_AVAILABLE` / `_fallback` 模式保持风格一致。 + +--- + +## 二、设计规范 + +### 2.1 路径解析优先级 + +对每个工具（`spike`、`spike-dasm`、`spike-log-parser`）独立解析，规则等价 BNF： + +``` +resolve(tool) ::= cli(tool) + | env(tool) + | spike_home(tool) + | path(tool) + | common(tool) + | legacy(tool) + | MISSING + +cli(tool) ::= PATH -- 来自 --spike-bin / --spike-dasm / --spike-log-parser +env(tool) ::= PATH -- 来自 SCRATCHV_SPIKE_BIN / _DASM / _LOG_PARSER +spike_home(tool) ::= $SCRATCHV_SPIKE_HOME "/bin/" tool +path(tool) ::= shutil.which(tool) +common(tool) ::= dir "/" tool for dir in COMMON_SPIKE_DIRS +legacy(tool) ::= SPIKE | SPIKE_DASM | SPIKE_LOG_PARSER -- 旧硬编码常量，仅当可执行时生效 + +accept(candidate) ::= os.path.isfile(candidate) AND os.access(candidate, os.X_OK) +``` + +| 层级 | 来源 | 显式程度 | 候选无效时的行为 | +|------|------|---------|-----------------| +| 1 | CLI 参数 | 显式（单次运行意图） | **硬失败**：抛配置错误，退出码 2 | +| 2 | 专用环境变量 | 显式（可能跨项目残留） | **告警**：记入 `tool_warnings`，继续下一层 | +| 3 | `SCRATCHV_SPIKE_HOME/bin/` | 隐式（目录提示） | 静默继续（仅在候选列表中留痕） | +| 4 | `PATH` 自动探测 | 隐式 | 静默继续 | +| 5 | 常见安装目录 | 隐式 | 静默继续 | +| 6 | 旧常量回退 | 隐式（legacy） | 静默继续 | +| — | 全部失败 | — | 返回 `path=None, source="missing"` | + +**关键规则**： + +- CLI 与 env 的路径值一律先 `strip()`，空串按「未设置」处理，不报错。 +- env 值支持 `~` 展开（`os.path.expanduser`），不做 `$VAR` 二次展开。 +- 每个工具的解析互不影响：设置了 `SCRATCHV_SPIKE_DASM` 不会影响 `spike` 的解析链。 +- 返回的 `source` 取值固定为 `cli | env | spike_home | path | common | legacy | missing`，写入 `SpikeTools.sources`，供测试与报告断言。 +- 解析器必须在**调用时**读取模块常量（而非 import 时固化），以便测试通过 monkeypatch 覆盖 legacy 回退。 + +### 2.2 环境变量命名 + +| 变量名 | 语义 | 示例 | 生效层级 | +|--------|------|------|---------| +| `SCRATCHV_SPIKE_BIN` | spike 可执行文件路径 | `/opt/riscv/bin/spike` | 2 | +| `SCRATCHV_SPIKE_DASM` | spike-dasm 可执行文件路径 | `/opt/riscv/bin/spike-dasm` | 2 | +| `SCRATCHV_SPIKE_LOG_PARSER` | spike-log-parser 路径 | `/opt/riscv/bin/spike-log-parser` | 2 | +| `SCRATCHV_SPIKE_HOME` | 安装根目录（约定 `/bin/`） | `/opt/coralnpu-spike-rv32` | 3 | + +**命名约定**：统一前缀 `SCRATCHV_SPIKE_`；大小写敏感；无缩写；不引入 `SPIKEDASM` 等变体。新增变量不得与既有 `SPIKE`/`SPIKE_DASM`/`SPIKE_LOG_PARSER` 模块常量同名。 + +### 2.3 工具缺失时的降级/报错策略 + +**分层原则**：环境不具备（未显式配置、工具真的不存在）→ 跳过并说明原因；用户显式配置了错误路径 → 立即失败；仿真已启动后的异常（超时/非零退出）→ 运行失败。 + +| # | 场景 | 默认行为 | `--require-spike` | 退出码 | 报告状态 | +|---|------|---------|-------------------|--------|---------| +| 1 | spike 缺失 | `SKIP:` + 搜索位置 + 修复提示 | `ERROR:` + 搜索位置 | 0 / 2 | `skipped` / `error` | +| 2 | spike-dasm 缺失 | `WARNING:`，其相关功能降级 | 同默认（不升级为失败） | 0 | `tool_warnings` | +| 3 | spike-log-parser 缺失 | `WARNING:`，其相关功能降级 | 同默认 | 0 | `tool_warnings` | +| 4 | CLI 显式路径无效（不存在/不可执行/是目录） | `ERROR:` | `ERROR:` | 2 | `error` | +| 5 | env 显式路径无效 | `WARNING:` + 继续解析 | 同默认 | 0 或后续结果 | `tool_warnings` | +| 6 | Spike 超时（`subprocess.TimeoutExpired`） | `ERROR:` | 同默认 | 1 | `timeout` | +| 7 | Spike 非零退出 / 启动失败 | `ERROR:` | 同默认 | 1 | `failed` | +| 8 | 输出缺少 cache/commit 段 | 继续，零值 + 告警 | 同默认 | 0 | `parse_warnings` | +| 9 | import / `--help` | 永不探测外部工具 | 同默认 | 0 | — | + +**退出码契约（模块常量）**： + +``` +EXIT_OK = 0 # 成功，或合法跳过（status=skipped） +EXIT_RUN_FAIL = 1 # 仿真已启动但失败：超时、非零退出、binary 缺失（历史兼容） +EXIT_CONFIG = 2 # 配置错误：CLI 显式路径无效、--require-spike 且 spike 缺失 +``` + +**消息前缀约定**（stderr，供脚本 grep）： + +``` +SKIP: spike binary not found. + searched: --spike-bin, SCRATCHV_SPIKE_BIN, SCRATCHV_SPIKE_HOME/bin/spike, PATH, <常见目录...>, legacy + hint: export SCRATCHV_SPIKE_BIN=/path/to/spike +ERROR: spike binary not found (--require-spike). +WARNING: SCRATCHV_SPIKE_BIN=/old/path/spike is not executable; ignored +``` + +### 2.4 Spike 输出解析约定 + +解析层拆为三个纯函数（输入字符串、输出结构化数据），不依赖子进程，便于用固定样本测试： + +| 数据 | 数据流 | 匹配模式（容忍空白/千分位） | 缺失/异常行为 | +|------|--------|---------------------------|--------------| +| 提交指令数 | stderr | `(Commited|Committed)\s+([\d,]+)\s+instructions` | 0 + `parse_warnings` | +| 仿真速度 | stderr | `([\d.]+)\s*MIPS` | 0.0 | +| I$ / D$ 统计 | stderr | 段头 `^I\$:` / `^D\$:`；行内 `hits:\s*([\d,]+)`、`misses:\s*([\d,]+)`、`miss rate:\s*([\d.]+)%` | 0 / 0.0 + `parse_warnings` | +| PC 直方图 | stdout | 段头含 `PC histogram`；数据行 `0x[0-9a-fA-F]+:\s*\d+`；空行结束 | 空 dict，报告省略该小节 | +| commit log | 日志文件 | 本期原样透传（`run_spike_with_log` 不解析） | `log_content=""` | + +**容错规则**： + +- 数字中的 `,` 一律去除后再 `int()`；非法数字跳过并记 `parse_warnings`，不抛异常。 +- 未知行忽略；段头缺失时按上表「缺失行为」处理，绝不把解析失败升级为进程失败。 +- `spike` 的已知拼写错误 `Commited` 与修正拼写 `Committed` 均接受（正则二选一）。 +- 现有拼写字段 `SpikeResult.commited_insns_per_sec` 不改名，避免破坏已有消费方。 + +### 2.5 配置合法性规则与示例 + +**合法配置示例**（均不产生硬失败）： + +| # | 配置 | 解析结果 | +|---|------|---------| +| 1 | `--spike-bin /opt/riscv/bin/spike`（文件可执行） | `source=cli` | +| 2 | `SCRATCHV_SPIKE_BIN=$HOME/riscv/bin/spike` | `source=env` | +| 3 | `SCRATCHV_SPIKE_HOME=/opt/coralnpu-spike-rv32`，且 `.../bin/spike` 存在 | `source=spike_home` | +| 4 | 什么都不设，但 `spike` 在 `PATH` 上 | `source=path` | +| 5 | 什么都不设，但 `/usr/local/bin/spike` 存在 | `source=common` | +| 6 | 什么都不设且处处不存在 | `source=missing` → 默认 skip（退出码 0） | + +**非法配置示例**（前 4 项必须报错/告警，不得静默成功）： + +| # | 配置 | 处理 | +|---|------|------| +| 1 | `--spike-bin ./spike`（不存在） | `SpikeConfigError` → 退出码 2 | +| 2 | `SCRATCHV_SPIKE_BIN=/tmp`（是目录，非文件） | CLI 报错；env 场景告警 + 继续 | +| 3 | `SCRATCHV_SPIKE_BIN=/path/not-executable`（无 `x` 位） | CLI 报错；env 场景告警 + 继续 | +| 4 | `SCRATCHV_SPIKE_BIN` 指向不存在的旧路径，且机器上也没有 spike | 告警 + 最终 skip（告警里保留非法值） | +| 5 | `--spike-bin ""` | 视为未设置（等价于不传），不报错 | +| 6 | `SCRATCHV_SPIKE_HOME=/nonexistent` | 非致命：该层跳过，继续 `PATH`/常见目录 | + +--- + +## 三、测试设计 + +所有用例都**不要求机器上安装 Spike**，通过 monkeypatch 环境变量、`shutil.which` 与模块常量构造隔离环境。测试文件：`tests/test_spike_sim_paths.py`。 + +### 测试用例 1：无 Spike 环境下默认跳过 + +- **输入**： + + ```python + monkeypatch.delenv("SCRATCHV_SPIKE_BIN", raising=False) # 及 DASM / LOG_PARSER / HOME + monkeypatch.setattr(spike_sim, "COMMON_SPIKE_DIRS", ()) + monkeypatch.setattr(spike_sim, "SPIKE", str(tmp_path / "legacy-spike")) # 不存在 + monkeypatch.setattr(shutil, "which", lambda name: None) + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64"]) + ``` + +- **预期输出**：`rc == 0`；stderr 含 `SKIP:`、`spike binary not found`、`SCRATCHV_SPIKE_BIN`；不生成 ELF 文件；无异常。 +- **验证点**：合法缺失 ≠ 失败；退出码与 `status` 字段一致；提示包含全部搜索层名称。 + +### 测试用例 2：`--require-spike` 硬失败 + +- **输入**：同用例 1，附带 `--require-spike`。 +- **预期输出**：`rc == 2`；stderr 含 `ERROR:` 与修复提示；stdout 无报告。 +- **验证点**：CI 严格模式可把「环境不具备」升级为可感知失败，且与运行期失败（退出码 1）区分。 + +### 测试用例 3：mock 路径解析优先级 + +- **输入**： + + ```python + fake_env = make_fake_tool(tmp_path, "spike-env") # 0755 可执行 + fake_cli = make_fake_tool(tmp_path, "spike-cli") + monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(fake_env)) + tools_env = spike_sim.resolve_spike_tools() + tools_cli = spike_sim.resolve_spike_tools(cli_spike=str(fake_cli)) + ``` + +- **预期输出**：`tools_env.spike == str(fake_env)` 且 `tools_env.sources["spike"] == "env"`；`tools_cli.spike == str(fake_cli)` 且 `sources["spike"] == "cli"`。补充断言：`which` 返回假路径时 `source == "path"`；`SCRATCHV_SPIKE_HOME` 命中时 `source == "spike_home"`；常见目录命中时 `source == "common"`。 +- **验证点**：完整优先级链（含 legacy 回退）逐层可控，来源标签准确。 + +### 测试用例 4：非法显式路径的差异化处理 + +- **输入**： + - `spike_sim.resolve_spike_tools(cli_spike=str(tmp_path / "nope"))`； + - `monkeypatch.setenv("SCRATCHV_SPIKE_BIN", str(tmp_path / "nope"))` 后调用 `resolve_spike_tools()`。 +- **预期输出**：前者抛 `SpikeConfigError`，消息含 `--spike-bin` 与非法值；后者不抛异常，`tools.spike is None`，`tools.warnings` 中含 `SCRATCHV_SPIKE_BIN`；用 `--spike-bin /nonexistent` 调 `main()` 时 `rc == 2`。 +- **验证点**：显式单次意图 vs 可能残留的环境变量，失败强度不同。 + +### 测试用例 5：报告字段与解析容错 + +- **输入**： + - `SpikeResult(status="skipped", skip_reason="spike binary not found")` 分别调用 `generate_spike_report()` 与 `build_json_report()`； + - 固定样本 stderr（含 `Commited 1234 instructions`、`1.5 MIPS`、`I$:`/`D$:` 段、带千分位 `hits: 10,000`）调用 `parse_commit_stats()` / `parse_cache_stats()`； + - 固定样本 stdout（`PC histogram` 段 + 空行）调用 `parse_pc_histogram()`。 +- **预期输出**：文本报告含 `Status:`、`skipped`、`Skip reason:`；JSON 含 `status == "skipped"`、`skip_reason`、`spike_binary is None`，且既有键（`committed_insns`、`icache`、`dcache` 等）不缺失；解析结果 `(1234, 1.5)`、`icache_hits == 10000`、PC dict 非空；缺段样本返回零值并计入 `parse_warnings`。 +- **验证点**：报告字段可被下游解析；解析器对真实 Spike 输出格式与常见变体都健壮。 + +--- + +## 四、修改模块与实现步骤 + +### 4.1 涉及文件 + +| 文件 | 角色 | 改动类型 | +|------|------|---------| +| `scratchv/standalone/spike_sim.py` | 工具解析、调用、解析、报告 | 修改（核心） | +| `scratchv/standalone/run_spike_bench.py` | 无 Spike 时的纯 Python 降级后端 | 小幅适配（后端标注 + `--probe-spike`） | +| `tests/test_spike_sim_paths.py` | 新增单元测试 | 新增 | +| `docs/topics/24-Spike仿真.md` | 课题文档（路径坑与使用方式） | 更新 | +| `docs/ARCHITECTURE.md` | 架构文档（工具解析注记） | 更新一句 | +| `scratchv/standalone/tinyfive_compare.py` | 降级模式参考 | **不改** | + +（实际路径以仓库为准；本设计中的行号基于 2026-09-14 版本。） + +### 4.2 路径解析器（`spike_sim.py` 新增） + +1. 保留 `spike_sim.py:37-39` 三个常量，但改写注释为「legacy 回退，可能不存在」，作为优先级第 6 层的候选。 +2. 新增常量：`COMMON_SPIKE_DIRS`（有序元组）、`LEGACY_SPIKE_DIR`、四个环境变量名常量、`EXIT_OK/EXIT_RUN_FAIL/EXIT_CONFIG`。 +3. 新增 `is_executable(path: str) -> bool`：`expanduser` + `isfile` + `access(X_OK)`。 +4. 新增 `SpikeConfigError(ValueError)`。 +5. 新增 `SpikeTools` 数据类：三工具路径 + `sources` + `candidates` + `warnings`，并提供 `missing` 属性与 `as_dict()`。 +6. 新增 `resolve_spike_tools(...)`：按 2.1 优先级逐层解析；CLI 无效即抛 `SpikeConfigError`，env 无效记告警后继续；返回 `SpikeTools`。 +7. 解析器不在 import 时执行（模块导入零副作用），在 `main()` 与 `run_spike()` 调用时解析。 + +### 4.3 运行层与解析层改造 + +1. `SpikeResult` 扩展字段：`status`（`ok|skipped|timeout|failed`）、`skip_reason`、`spike_path`、`tool_warnings`、`parse_warnings`；既有字段与拼写保持不变。 +2. 抽取三个纯函数：`parse_commit_stats(stderr)`、`parse_cache_stats(stderr)`、`parse_pc_histogram(stdout)`；`run_spike()` 改为调用它们（移除 `run_spike:348-423` 的内联解析）。 +3. `run_spike()` / `run_spike_with_log()` 增加仅关键字参数 `tools: SpikeTools | None = None`；默认在调用时执行 `resolve_spike_tools()`。 +4. `run_spike()` 缺 spike：不启动子进程，直接返回 `status="skipped"`、`exit_code=-2` 的结果；`FileNotFoundError` 与超时分别映射 `failed` / `timeout`（保留 `-2` / `-1` 哨兵值兼容旧判读）。 +5. `generate_spike_report()`：新增 `Status:`、`Skip reason:`（仅 skip 时）、`Tool warnings:`（有则打印）；`Spike:` 行改用 `result.spike_path`；`spike_sim.py:512` 不再直接引用常量。 + +### 4.4 CLI 与主导出流程 + +1. `main()`（`spike_sim.py:592-653`）新增参数：`--spike-bin`、`--spike-dasm`、`--spike-log-parser`、`--require-spike`；既有参数全部不动。 +2. 流程顺序：校验 binary → 解析工具（捕获 `SpikeConfigError` → 退出码 2）→ 缺失且非严格 → `SKIP:` + JSON（若 `--json`）→ 退出码 0 → 缺失且严格 → `ERROR:` → 退出码 2 → 否则构建 ELF、运行、报告。 +3. 抽取 `build_json_report(result, binary_path, code_size, ic_config, dc_config, max_instr, tools=None) -> dict`（替代 `spike_sim.py:716-745` 内联字典），新增 `status`、`skip_reason`、`spike_binary`、`parse_warnings`、`tool_warnings`、`spike_tools` 字段；既有键全部保留。 +4. 成功路径打印一条工具来源摘要到 stderr（例如 `Spike tools: spike=/opt/... (cli), spike-dasm=missing`）。 + +### 4.5 `run_spike_bench.py` 适配 + +1. 新增 `--probe-spike`（默认关）：惰性导入 `resolve_spike_tools`，把解析结果打印到 stderr；`--json` 时在报告中加入 `spike_tools` 字段。 +2. `generate_json_report(result, spike_tools=None)` 增加常量字段 `"backend": {"kind": "emulator", "spike_style": true}`，用于区分「纯 Python 模拟」与「真实 Spike 实测」。 +3. `run_emulator_with_caches()`、`label_counts`、`label_addrs`、`cycle_estimates`、`cat_counts` **零改动**（统计口径不变），`ci_benchmark.py` 的调用不受影响。 +4. 该文件本身的 label 表与周期估算失真问题记录为关联问题（见 5.3），本课题不交付。 + +### 4.6 测试实现 + +新增 `tests/test_spike_sim_paths.py`：覆盖第三节 5 个用例 + `import`/`--help` 子进程测试（干净 `PATH` 下退出码 0）。全部使用 monkeypatch / 临时可执行文件，不依赖真实 Spike。 + +### 4.7 文档更新 + +- `docs/topics/24-Spike仿真.md`：「常见坑」中「Spike 二进制路径」改为解析优先级 + 环境变量；补一段 CLI/env 用法与 skip 行为。 +- `docs/ARCHITECTURE.md`：在 standalone 工具清单附近加一句工具解析与降级说明。 +- `spike_sim.py` 模块 docstring（`spike_sim.py:15-22`）更新为新用法。 + +### 4.8 集成与回归测试 + +- `python -m pytest tests/test_spike_sim_paths.py -q` 全绿。 +- `make test`（全量单测）与 `python .claude/harness/verify/run.py --level L2` 通过。 +- `python scratchv/standalone/spike_sim.py --help`、`python -c "import scratchv.standalone.spike_sim"` 在无 Spike 机器上退出码 0。 +- 在有 Spike 的机器上，用 `--spike-bin` 跑基线 binary，committed insns 与 cache 命中数与改动前一致。 +- `git diff` 确认 `tinyfive_compare.py` 与周期/统计代码零改动。 + +--- + +## 五、附录 + +### 5.1 CLI 示例与输出 + +**示例 1：显式指定（推荐 CI 固定版本）** + +```bash +python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140 \ + --spike-bin /opt/riscv/bin/spike --json +``` + +**示例 2：环境变量（整机安装）** + +```bash +export SCRATCHV_SPIKE_HOME=/opt/coralnpu-spike-rv32 +python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140 +``` + +**示例 3：工具缺失（默认降级，退出码 0）** + +```text +$ python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140 +SKIP: spike binary not found. + searched: --spike-bin, SCRATCHV_SPIKE_BIN, SCRATCHV_SPIKE_HOME/bin/spike, PATH, + /opt/riscv/bin, /opt/riscv64/bin, /usr/local/bin, /usr/bin, + ~/riscv/bin, ~/.local/bin, ~/spike/bin, legacy + hint: export SCRATCHV_SPIKE_BIN=/path/to/spike +$ echo $? +0 +``` + +**示例 4：`--require-spike`（严格模式，退出码 2）** + +```text +$ python scratchv/standalone/spike_sim.py --binary output.bin --code-size 3140 --require-spike +ERROR: spike binary not found (--require-spike). + searched: --spike-bin, SCRATCHV_SPIKE_BIN, ... , legacy +$ echo $? +2 +``` + +**示例 5：skip 时的 JSON 报告（节选）** + +```json +{ + "status": "skipped", + "skip_reason": "spike binary not found", + "spike_binary": null, + "spike_tools": { + "spike": {"path": null, "source": "missing"}, + "spike_dasm": {"path": null, "source": "missing"}, + "spike_log_parser": {"path": null, "source": "missing"}, + "warnings": [] + }, + "binary": "output.bin", + "code_size": 3140, + "max_instr": 50000000, + "committed_insns": 0 +} +``` + +### 5.2 搜索目录与优先级速查 + +| 层 | 候选 | +|----|------| +| CLI | `--spike-bin` / `--spike-dasm` / `--spike-log-parser` | +| env | `SCRATCHV_SPIKE_BIN` / `SCRATCHV_SPIKE_DASM` / `SCRATCHV_SPIKE_LOG_PARSER` | +| home | `$SCRATCHV_SPIKE_HOME/bin/` | +| PATH | `shutil.which()` | +| common | `/opt/riscv/bin`, `/opt/riscv64/bin`, `/usr/local/bin`, `/usr/bin`, `~/riscv/bin`, `~/.local/bin`, `~/spike/bin` | +| legacy | 旧硬编码常量（可执行才生效） | + +### 5.3 关联问题（本课题不交付） + +| 问题 | 位置 | 现象 | 建议归属 | +|------|------|------|---------| +| label 表失真 | `run_spike_bench.py:633-660`、`765-774` | `layer_descs` 硬编码旧标签前缀；`label_addrs` 实际只有 `_start@0`，逐层统计基本为空 | 独立课题（标签映射/统计口径） | +| 周期估算失真 | `run_spike_bench.py:480-512` | `mul_ratio=0.15` 猜测值、`PROFILES` 常量 CPI 与真实流水线不符 | 独立课题（微架构模型） | +| Spike 真实数据与模拟数据混用风险 | `run_spike_bench.py` docstring | 文件名为 spike_bench 但结果来自内置模拟器 | 本课题已加 `backend` 字段缓解，模型修正不在范围 | +| TinyFive 路径 | `tinyfive_compare.py` | 不涉及 | 明确不改 | + +### 5.4 参考资料 + +- 课题文档：`docs/topics/24-Spike仿真.md` +- 降级模式先例：`scratchv/standalone/tinyfive_compare.py:29-86`（`TINYFIVE_AVAILABLE` / `_fallback`） +- Spike 官方仓库： +- ScratchV 代码仓初始化文档：`/root/Lab/GaoMD/ScratchV/ArcDes/init.md` From 22f6af5b09e30f748b1cd178fa96eca23cabf8be Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:51:48 +0800 Subject: [PATCH 3/5] fix(topic24): harden spike failure handling and optional tool degradation - catch OSError from subprocess spawn (ENOEXEC: no shebang, wrong arch, truncated binary) and map it to status=failed/exit_code=-2 instead of letting a traceback escape - reject fake success: a clean exit with no parsable Spike statistics at all now yields status=failed/exit_code=-2 - fill exit_code=-2 in the CLI skip JSON report, matching the library skip path and the documented sentinel - downgrade invalid --spike-dasm/--spike-log-parser CLI paths to WARNING + fall-through; only --spike-bin stays a hard config error - print resolver warnings to stderr so invalid env/optional CLI values are visible, not just recorded in reports - reword optional-tool warnings: this module does not use dasm or log-parser yet (no feature is degraded) - sync design/development docs with these contracts (F1/F3/F4/F5/F6) --- ...00\345\217\221\346\226\207\346\241\243.md" | 49 ++++++----- ...76\350\256\241\346\226\207\346\241\243.md" | 17 ++-- scratchv/standalone/spike_sim.py | 86 +++++++++++++++---- 3 files changed, 108 insertions(+), 44 deletions(-) diff --git "a/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" index a96ac07..f4719d7 100644 --- "a/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/topics/24-Spike\344\273\277\347\234\237-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -64,9 +64,9 @@ class SpikeConfigError(ValueError): | 参数 | 类型/默认 | 说明 | |------|----------|------| -| `--spike-bin PATH` | str / 无 | spike 可执行文件；无效 → 退出码 2 | -| `--spike-dasm PATH` | str / 无 | spike-dasm；无效 → 退出码 2 | -| `--spike-log-parser PATH` | str / 无 | spike-log-parser；无效 → 退出码 2 | +| `--spike-bin PATH` | str / 无 | spike 可执行文件（必需工具）；无效 → 退出码 2 | +| `--spike-dasm PATH` | str / 无 | spike-dasm（可选工具）；无效 → WARNING + 继续解析，不阻断 | +| `--spike-log-parser PATH` | str / 无 | spike-log-parser（可选工具）；无效 → WARNING + 继续解析，不阻断 | | `--require-spike` | store_true / 关 | spike 缺失时硬失败（退出码 2） | | `--probe-spike` | store_true / 关 | **仅 `run_spike_bench.py`**：探测并打印工具可用性 | @@ -92,7 +92,7 @@ parse_warnings: list[str] = field(default_factory=list) `build_json_report()` 新增键：`status`、`skip_reason`、`spike_binary`、`parse_warnings`、`tool_warnings`、`spike_tools`（`tools` 为空时不输出该键）。既有键：`binary`、`code_size`、`static_insns`、`max_instr`、`committed_insns`、`wall_time_s`、`exit_code`、`icache`、`dcache`、`top_pcs`、`stderr_tail` 全部保留。 -`exit_code` 哨兵兼容：`-1` 超时、`-2` 工具不可用/启动失败；`status` 为权威判读字段。 +`exit_code` 哨兵兼容：`-1` 超时、`-2` 工具不可用/启动失败；skip 路径的 JSON 报告同样填 `-2`；`status` 为权威判读字段。此外，spike 退出 0 但输出中完全没有任何可解析统计段时，防呆为 `status=failed`、`exit_code=-2`（疑似非 Spike 可执行文件）。 --- @@ -188,19 +188,27 @@ class SpikeTools: "warnings": list(self.warnings)} -def _resolve_one(tool, cli_flag, cli_value, env_name, legacy_dir, legacy_const, - env, which, common_dirs): - """返回 (path|None, source, candidates, warnings)。""" +def _resolve_one(tool, cli_flag, cli_value, env_name, legacy_const, + env, which, common_dirs, cli_required=True): + """返回 (path|None, source, candidates, warnings)。 + + cli_required=True（spike）时 CLI 无效抛错；可选工具（dasm/log-parser） + 传 False，CLI 无效仅告警并继续后续层级。 + """ candidates: list[str] = [] warnings: list[str] = [] - if cli_value: # 1. CLI（显式，硬失败） + if cli_value: # 1. CLI（显式） cand = os.path.expanduser(cli_value.strip()) candidates.append(cand) if not is_executable(cand): - raise SpikeConfigError( - f"{cli_flag}={cli_value!r} is not an executable file") - return cand, "cli", candidates, warnings + if cli_required: + raise SpikeConfigError( + f"{cli_flag}={cli_value!r} is not an executable file") + warnings.append( + f"{cli_flag}={cli_value!r} is not executable; ignored") + else: + return cand, "cli", candidates, warnings env_value = (env.get(env_name) or "").strip() # 2. env（显式，告警继续） if env_value: @@ -242,16 +250,17 @@ def resolve_spike_tools(cli_spike=None, cli_dasm=None, cli_log_parser=None, common_dirs = COMMON_SPIKE_DIRS if common_dirs is None else common_dirs spec = ( - ("spike", "--spike-bin", cli_spike, ENV_SPIKE_BIN, SPIKE), - ("spike-dasm", "--spike-dasm", cli_dasm, ENV_SPIKE_DASM, SPIKE_DASM), + ("spike", "--spike-bin", cli_spike, ENV_SPIKE_BIN, SPIKE, True), + ("spike-dasm", "--spike-dasm", cli_dasm, ENV_SPIKE_DASM, SPIKE_DASM, + False), ("spike-log-parser", "--spike-log-parser", cli_log_parser, - ENV_SPIKE_LOG_PARSER, SPIKE_LOG_PARSER), + ENV_SPIKE_LOG_PARSER, SPIKE_LOG_PARSER, False), ) paths, sources, candidates, warnings = {}, {}, {}, [] - for tool, flag, cli_value, env_name, legacy_const in spec: + for tool, flag, cli_value, env_name, legacy_const, cli_required in spec: path, source, cands, warns = _resolve_one( - tool, flag, cli_value, env_name, None, legacy_const, - env, which, common_dirs) + tool, flag, cli_value, env_name, legacy_const, + env, which, common_dirs, cli_required=cli_required) paths[tool], sources[tool], candidates[tool] = path, source, cands warnings.extend(warns) return SpikeTools( @@ -297,13 +306,13 @@ def parse_pc_histogram(stdout: str) -> dict[int, int]: ... ``` 3. `cmd` 首元素由 `SPIKE` 改为 `tools.spike`；`result.spike_path = tools.spike`；`result.tool_warnings` 合并 `tools.warnings`。 -4. `except FileNotFoundError`：`status="failed"`，消息带 `tools.spike`。 +4. `except FileNotFoundError` / `except OSError`：均 `status="failed"`、`exit_code=-2`，消息带 `tools.spike`（`OSError` 覆盖「文件存在且可执行但内核拒绝 exec」的 ENOEXEC 等场景）。 5. `except subprocess.TimeoutExpired`：`status="timeout"`。 -6. 正常返回前用 2.6 的解析函数填充字段；`result.exit_code` 保留 `proc.returncode`。 +6. 正常返回前用 2.6 的解析函数填充字段；`result.exit_code` 保留 `proc.returncode`（唯一例外：退出 0 但完全无可解析统计段时置 `-2`，见设计文档 §2.3 第 8b 行）。 ### 2.8 `run_spike_with_log()`（L432-488） -签名加 `*, tools=None`；开头做与 2.7 相同的缺失判断，返回 `(skipped_result, "")`；`cmd` 首元素改 `tools.spike`。 +签名加 `*, tools=None`；开头做与 2.7 相同的缺失判断，返回 `(skipped_result, "")`；`cmd` 首元素改 `tools.spike`；异常映射与 2.7 相同（含 `OSError` → `failed`/`-2`）。 ### 2.9 `generate_spike_report()`（L495-585） diff --git "a/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" index a6fa7d9..d0ead5f 100644 --- "a/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/24-Spike\344\273\277\347\234\237-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -79,7 +79,7 @@ accept(candidate) ::= os.path.isfile(candidate) AND os.access(candidate, os.X_O | 层级 | 来源 | 显式程度 | 候选无效时的行为 | |------|------|---------|-----------------| -| 1 | CLI 参数 | 显式（单次运行意图） | **硬失败**：抛配置错误，退出码 2 | +| 1 | CLI 参数 | 显式（单次运行意图） | `spike`：**硬失败**（抛配置错误，退出码 2）；`spike-dasm` / `spike-log-parser`：**告警**后继续下一层 | | 2 | 专用环境变量 | 显式（可能跨项目残留） | **告警**：记入 `tool_warnings`，继续下一层 | | 3 | `SCRATCHV_SPIKE_HOME/bin/` | 隐式（目录提示） | 静默继续（仅在候选列表中留痕） | | 4 | `PATH` 自动探测 | 隐式 | 静默继续 | @@ -112,14 +112,16 @@ accept(candidate) ::= os.path.isfile(candidate) AND os.access(candidate, os.X_O | # | 场景 | 默认行为 | `--require-spike` | 退出码 | 报告状态 | |---|------|---------|-------------------|--------|---------| -| 1 | spike 缺失 | `SKIP:` + 搜索位置 + 修复提示 | `ERROR:` + 搜索位置 | 0 / 2 | `skipped` / `error` | -| 2 | spike-dasm 缺失 | `WARNING:`，其相关功能降级 | 同默认（不升级为失败） | 0 | `tool_warnings` | -| 3 | spike-log-parser 缺失 | `WARNING:`，其相关功能降级 | 同默认 | 0 | `tool_warnings` | -| 4 | CLI 显式路径无效（不存在/不可执行/是目录） | `ERROR:` | `ERROR:` | 2 | `error` | +| 1 | spike 缺失 | `SKIP:` + 搜索位置 + 修复提示；`--json` 时输出 `status=skipped`、`exit_code=-2` 的 JSON | `ERROR:` + 搜索位置 | 0 / 2 | `skipped` / 无报告（仅 stderr） | +| 2 | spike-dasm 缺失 | `WARNING:`（本模块尚未使用该工具） | 同默认（不升级为失败） | 0 | `tool_warnings` | +| 3 | spike-log-parser 缺失 | `WARNING:`（本模块尚未使用该工具） | 同默认 | 0 | `tool_warnings` | +| 4 | `--spike-bin` 无效（不存在/不可执行/是目录） | `ERROR:` | `ERROR:` | 2 | 无报告（仅 stderr） | +| 4b | `--spike-dasm` / `--spike-log-parser` 无效（可选工具） | `WARNING:` + 继续解析 | 同默认 | 0 或后续结果 | `tool_warnings` | | 5 | env 显式路径无效 | `WARNING:` + 继续解析 | 同默认 | 0 或后续结果 | `tool_warnings` | | 6 | Spike 超时（`subprocess.TimeoutExpired`） | `ERROR:` | 同默认 | 1 | `timeout` | -| 7 | Spike 非零退出 / 启动失败 | `ERROR:` | 同默认 | 1 | `failed` | -| 8 | 输出缺少 cache/commit 段 | 继续，零值 + 告警 | 同默认 | 0 | `parse_warnings` | +| 7 | Spike 非零退出 / 启动失败（含 `OSError`，如无法 exec 的文件） | `ERROR:` | 同默认 | 1 | `failed` | +| 8 | 输出缺少**部分** cache/commit 段 | 继续，零值 + 告警 | 同默认 | 0 | `parse_warnings` | +| 8b | spike 退出 0 但**完全无可解析统计段**（疑似非 Spike 可执行文件） | `status=failed`、`exit_code=-2` + 告警 | 同默认 | 1 | `failed` + `parse_warnings` | | 9 | import / `--help` | 永不探测外部工具 | 同默认 | 0 | — | **退出码契约（模块常量）**： @@ -359,6 +361,7 @@ $ echo $? "status": "skipped", "skip_reason": "spike binary not found", "spike_binary": null, + "exit_code": -2, "spike_tools": { "spike": {"path": null, "source": "missing"}, "spike_dasm": {"path": null, "source": "missing"}, diff --git a/scratchv/standalone/spike_sim.py b/scratchv/standalone/spike_sim.py index da5b0eb..2bc044f 100644 --- a/scratchv/standalone/spike_sim.py +++ b/scratchv/standalone/spike_sim.py @@ -145,19 +145,29 @@ def _resolve_one( env, which, common_dirs, + cli_required: bool = True, ) -> tuple[str | None, str, list[str], list[str]]: - """Resolve one tool, returning (path, source, candidates, warnings).""" + """Resolve one tool, returning (path, source, candidates, warnings). + + An invalid CLI value raises SpikeConfigError when ``cli_required`` is + true (the spike binary); for optional tools it degrades to a warning + and resolution continues with the lower-priority layers. + """ candidates: list[str] = [] warnings: list[str] = [] - # 1. CLI (explicit per-run intent: hard failure on invalid value) + # 1. CLI (explicit per-run intent) if cli_value: cand = os.path.expanduser(cli_value.strip()) candidates.append(cand) if not is_executable(cand): - raise SpikeConfigError( - f"{cli_flag}={cli_value!r} is not an executable file") - return cand, "cli", candidates, warnings + if cli_required: + raise SpikeConfigError( + f"{cli_flag}={cli_value!r} is not an executable file") + warnings.append( + f"{cli_flag}={cli_value!r} is not executable; ignored") + else: + return cand, "cli", candidates, warnings # 2. Dedicated environment variable (explicit, possibly stale: warn) env_value = (env.get(env_name) or "").strip() @@ -209,7 +219,9 @@ def resolve_spike_tools( Priority: CLI > dedicated env vars > $SCRATCHV_SPIKE_HOME/bin > PATH > common install dirs > legacy constants. Invalid CLI paths raise - SpikeConfigError; invalid env paths add a warning and fall through. + SpikeConfigError for the required spike binary; for the optional tools + (spike-dasm / spike-log-parser) they add a warning and fall through. + Invalid env paths add a warning and fall through. """ env = os.environ if env is None else env which = shutil.which if which is None else which @@ -218,19 +230,20 @@ def resolve_spike_tools( # Legacy constants are read here (not captured at import time) so tests # can monkeypatch them and so each call sees the current values. spec = ( - ("spike", "--spike-bin", cli_spike, ENV_SPIKE_BIN, SPIKE), - ("spike-dasm", "--spike-dasm", cli_dasm, ENV_SPIKE_DASM, SPIKE_DASM), + ("spike", "--spike-bin", cli_spike, ENV_SPIKE_BIN, SPIKE, True), + ("spike-dasm", "--spike-dasm", cli_dasm, ENV_SPIKE_DASM, SPIKE_DASM, + False), ("spike-log-parser", "--spike-log-parser", cli_log_parser, - ENV_SPIKE_LOG_PARSER, SPIKE_LOG_PARSER), + ENV_SPIKE_LOG_PARSER, SPIKE_LOG_PARSER, False), ) paths: dict[str, str | None] = {} sources: dict[str, str] = {} candidates: dict[str, tuple[str, ...]] = {} warnings: list[str] = [] - for tool, flag, cli_value, env_name, legacy_const in spec: + for tool, flag, cli_value, env_name, legacy_const, cli_required in spec: path, source, cands, warns = _resolve_one( tool, flag, cli_value, env_name, legacy_const, - env, which, common_dirs) + env, which, common_dirs, cli_required=cli_required) paths[tool], sources[tool], candidates[tool] = path, source, tuple(cands) warnings.extend(warns) @@ -245,13 +258,20 @@ def resolve_spike_tools( def _optional_tool_warnings(tools: SpikeTools) -> list[str]: - """Warnings for missing optional tools (never escalated to failure).""" + """Warnings for missing optional tools (never escalated to failure). + + Neither tool is called by this module yet, so their absence does not + degrade any current feature; the messages must not imply otherwise. + """ warnings: list[str] = [] if tools.spike_dasm is None: - warnings.append("spike-dasm not found; disassembly features unavailable") + warnings.append( + "spike-dasm not found; not used by this module yet " + "(disassembly support is not implemented)") if tools.spike_log_parser is None: warnings.append( - "spike-log-parser not found; commit log parsing unavailable") + "spike-log-parser not found; not used by this module yet " + "(commit log parsing is not implemented)") return warnings @@ -687,6 +707,14 @@ def run_spike( result.stderr = f"ERROR: Spike not found at {tools.spike}" result.exit_code = -2 return result + except OSError as e: + # Exists and is executable, but the kernel refused to exec it + # (ENOEXEC: no shebang, wrong architecture, truncated binary, ...). + result.status = "failed" + result.stderr = ( + f"ERROR: failed to start Spike at {tools.spike}: {e}") + result.exit_code = -2 + return result result.wall_time_s = time.perf_counter() - t_start result.total_insns = max_instr # We set the limit @@ -706,13 +734,27 @@ def run_spike( result.pc_histogram = parse_pc_histogram(result.stdout) stderr_text = result.stderr or "" - if not _RE_COMMIT.search(stderr_text): + has_commit_stats = bool(_RE_COMMIT.search(stderr_text)) + has_icache_stats = bool(_RE_ICACHE_HEADER.search(stderr_text)) + has_dcache_stats = bool(_RE_DCACHE_HEADER.search(stderr_text)) + if not has_commit_stats: result.parse_warnings.append("commit stats not found in Spike stderr") - if not _RE_ICACHE_HEADER.search(stderr_text): + if not has_icache_stats: result.parse_warnings.append("I$ cache stats not found in Spike stderr") - if not _RE_DCACHE_HEADER.search(stderr_text): + if not has_dcache_stats: result.parse_warnings.append("D$ cache stats not found in Spike stderr") + # Anti-fake-success guard: a clean exit with no recognizable Spike + # statistics at all means the executable produced no usable Spike output + # (e.g. /bin/true). Without this, any executable would look "successful". + if (proc.returncode == 0 + and not (has_commit_stats or has_icache_stats or has_dcache_stats)): + result.status = "failed" + result.exit_code = -2 + result.parse_warnings.append( + "no Spike statistics section found; " + "the executable may not be Spike") + return result @@ -792,6 +834,12 @@ def run_spike_with_log( result.stderr = f"ERROR: Spike not found at {tools.spike}" result.exit_code = -2 return result, "" + except OSError as e: + result.status = "failed" + result.stderr = ( + f"ERROR: failed to start Spike at {tools.spike}: {e}") + result.exit_code = -2 + return result, "" result.wall_time_s = time.perf_counter() - t_start if proc.returncode != 0: @@ -1070,6 +1118,9 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: {e}", file=sys.stderr) return EXIT_CONFIG + for warning in tools.warnings: + print(f"WARNING: {warning}", file=sys.stderr) + if tools.spike is None: searched = ", ".join([ "--spike-bin", @@ -1096,6 +1147,7 @@ def main(argv: list[str] | None = None) -> int: skipped_result = SpikeResult( status="skipped", skip_reason="spike binary not found", + exit_code=-2, # same sentinel as the library skip path tool_warnings=list(tools.warnings), ) print(json.dumps( From 780b5ab795ea5ffb83bcf15eab1a0b9c845ea1db Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:51:53 +0800 Subject: [PATCH 4/5] test(topic24): cover spawn failures, fake success, probe and JSON contracts - F1: real ENOEXEC repro (chmod +x plain text as --spike-bin) asserts exit 1 / no traceback / JSON status=failed; plus mock OSError tests for run_spike and run_spike_with_log - F2: probe_spike_tools() (resolved, missing + hint, config-error) and run_spike_bench.generate_json_report backend/spike_tools contracts - F3: skip JSON exit_code == -2; strict/CLI-invalid modes emit no report - F4: invalid optional CLI paths warn and the simulation still runs - F6: empty-output executable is reported as failed, not ok - F7: timeout->status=timeout, rc!=0->failed/EXIT_RUN_FAIL, _optional_tool_warnings wording, main() success path (ELF write, cleanup, tools passthrough) --- tests/test_spike_sim_paths.py | 352 ++++++++++++++++++++++++++++++++-- 1 file changed, 339 insertions(+), 13 deletions(-) diff --git a/tests/test_spike_sim_paths.py b/tests/test_spike_sim_paths.py index a47b5ea..c386dac 100644 --- a/tests/test_spike_sim_paths.py +++ b/tests/test_spike_sim_paths.py @@ -104,9 +104,11 @@ def test_missing_spike_strict_returns_config_error(tmp_path, capsys): "--require-spike"]) assert rc == spike_sim.EXIT_CONFIG - err = capsys.readouterr().err - assert "ERROR:" in err - assert "--require-spike" in err + captured = capsys.readouterr() + assert "ERROR:" in captured.err + assert "--require-spike" in captured.err + # Design doc §2.3 row 1: strict mode emits no report at all. + assert captured.out == "" assert not (tmp_path / "output_spike.elf").exists() @@ -128,6 +130,8 @@ def test_missing_spike_json_report_fields(tmp_path, capsys): assert key in report assert report["spike_tools"]["spike"]["source"] == "missing" assert report["spike_tools"]["spike"]["path"] is None + # F3: the CLI skip report uses the same -2 sentinel as the library path. + assert report["exit_code"] == -2 # ── Resolution priority chain ─────────────────────────────────────────────── @@ -198,14 +202,29 @@ def test_resolution_legacy_constant(tmp_path, monkeypatch): # ── Invalid explicit paths ────────────────────────────────────────────────── -def test_cli_invalid_path_raises(tmp_path): +def test_cli_invalid_required_path_raises(tmp_path): with pytest.raises(spike_sim.SpikeConfigError) as ei: spike_sim.resolve_spike_tools(cli_spike=str(tmp_path / "nope")) assert "--spike-bin" in str(ei.value) - with pytest.raises(spike_sim.SpikeConfigError) as ei: - spike_sim.resolve_spike_tools(cli_dasm=str(tmp_path / "nope")) - assert "--spike-dasm" in str(ei.value) + +def test_optional_cli_invalid_path_warns_and_falls_through( + tmp_path, monkeypatch): + # F4: spike-dasm / spike-log-parser are optional; an invalid explicit + # path must not abort resolution/execution. + monkeypatch.setenv("SCRATCHV_SPIKE_DASM", str(tmp_path / "nope")) + + tools = spike_sim.resolve_spike_tools( + cli_spike=str(make_fake_tool(tmp_path, "spike")), + cli_dasm=str(tmp_path / "nope-dasm"), + cli_log_parser=str(tmp_path / "nope-parser"), + ) + + assert tools.spike_dasm is None + assert tools.spike_log_parser is None + joined = "\n".join(tools.warnings) + assert "--spike-dasm" in joined + assert "--spike-log-parser" in joined def test_cli_invalid_path_returns_config_error(tmp_path, capsys): @@ -213,10 +232,13 @@ def test_cli_invalid_path_returns_config_error(tmp_path, capsys): binary.write_bytes(b"\x00" * 64) rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", - "--spike-bin", str(tmp_path / "nope")]) + "--spike-bin", str(tmp_path / "nope"), "--json"]) assert rc == spike_sim.EXIT_CONFIG - assert "ERROR:" in capsys.readouterr().err + captured = capsys.readouterr() + assert "ERROR:" in captured.err + # Design doc §2.3 row 4: no report is emitted for an invalid --spike-bin. + assert captured.out == "" assert not (tmp_path / "output_spike.elf").exists() @@ -278,13 +300,14 @@ def test_parse_pc_histogram(): assert spike_sim.parse_pc_histogram("no histogram here") == {} -def test_run_spike_mock_subprocess_records_parse_warnings(tmp_path, monkeypatch): +def test_run_spike_canned_output_is_ok(tmp_path, monkeypatch): tools = spike_sim.SpikeTools(spike="/fake/spike") captured: dict[str, list[str]] = {} def fake_run(cmd, **kwargs): captured["cmd"] = list(cmd) - return types.SimpleNamespace(returncode=0, stdout="", stderr="") + return types.SimpleNamespace( + returncode=0, stdout=CANNED_STDOUT, stderr=CANNED_STDERR) monkeypatch.setattr(spike_sim.subprocess, "run", fake_run) @@ -292,10 +315,111 @@ def fake_run(cmd, **kwargs): assert captured["cmd"][0] == "/fake/spike" assert result.status == "ok" + assert result.exit_code == 0 assert result.spike_path == "/fake/spike" + assert result.committed_insns == 1234 + assert result.icache_hits == 10_000 + assert result.dcache_misses == 50 + assert result.pc_histogram == {0x80000014: 123, 0x80000018: 456} + # Every stats section is present: no parse warnings at all. + assert result.parse_warnings == [] + + +def test_run_spike_empty_output_marks_failed(tmp_path, monkeypatch): + # F6: exit 0 without any recognizable Spike stats is not a success. + tools = spike_sim.SpikeTools(spike="/fake/spike") + + monkeypatch.setattr( + spike_sim.subprocess, "run", + lambda cmd, **kwargs: types.SimpleNamespace( + returncode=0, stdout="", stderr="")) + + result = spike_sim.run_spike(str(tmp_path / "x.elf"), tools=tools) + + assert result.status == "failed" + assert result.exit_code == -2 assert result.committed_insns == 0 - assert result.parse_warnings - assert any("commit" in w.lower() for w in result.parse_warnings) + assert any("no Spike statistics" in w for w in result.parse_warnings) + + +def test_run_spike_oserror_maps_to_failed(tmp_path, monkeypatch): + # F1: an existing executable the kernel refuses to exec (ENOEXEC). + tools = spike_sim.SpikeTools(spike="/fake/spike") + + def fake_run(cmd, **kwargs): + raise OSError(8, "Exec format error", cmd[0]) + + monkeypatch.setattr(spike_sim.subprocess, "run", fake_run) + + result = spike_sim.run_spike(str(tmp_path / "x.elf"), tools=tools) + + assert result.status == "failed" + assert result.exit_code == -2 + assert "failed to start Spike" in result.stderr + assert "/fake/spike" in result.stderr + assert "Exec format error" in result.stderr + + +def test_run_spike_with_log_oserror_maps_to_failed(tmp_path, monkeypatch): + tools = spike_sim.SpikeTools(spike="/fake/spike") + + def fake_run(cmd, **kwargs): + raise OSError(8, "Exec format error", cmd[0]) + + monkeypatch.setattr(spike_sim.subprocess, "run", fake_run) + + result, log_content = spike_sim.run_spike_with_log( + str(tmp_path / "x.elf"), tools=tools) + + assert result.status == "failed" + assert result.exit_code == -2 + assert "failed to start Spike" in result.stderr + assert log_content == "" + + +def test_run_spike_timeout_is_reported(tmp_path, monkeypatch): + tools = spike_sim.SpikeTools(spike="/fake/spike") + + def fake_run(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd, 1) + + monkeypatch.setattr(spike_sim.subprocess, "run", fake_run) + + result = spike_sim.run_spike(str(tmp_path / "x.elf"), tools=tools) + + assert result.status == "timeout" + assert result.exit_code == -1 + assert "TIMEOUT" in result.stderr + + +def test_run_spike_nonzero_exit_marks_failed(tmp_path, monkeypatch): + tools = spike_sim.SpikeTools(spike="/fake/spike") + + monkeypatch.setattr( + spike_sim.subprocess, "run", + lambda cmd, **kwargs: types.SimpleNamespace( + returncode=1, stdout="", stderr=CANNED_STDERR)) + + result = spike_sim.run_spike(str(tmp_path / "x.elf"), tools=tools) + + assert result.status == "failed" + assert result.exit_code == 1 + # Stats are still parsed from the partial output. + assert result.committed_insns == 1234 + + +def test_optional_tool_warnings_do_not_imply_missing_features(): + tools = spike_sim.SpikeTools(spike="/fake/spike") + + warnings = spike_sim._optional_tool_warnings(tools) + + assert len(warnings) == 2 + assert all("not used by this module" in w for w in warnings) + assert not any("unavailable" in w for w in warnings) + + complete = spike_sim.SpikeTools( + spike="s", spike_dasm="d", spike_log_parser="p") + assert spike_sim._optional_tool_warnings(complete) == [] def test_run_spike_missing_tool_returns_skipped(tmp_path): @@ -338,3 +462,205 @@ def test_report_status_fields(): plain = spike_sim.build_json_report( result, "output.bin", 64, "64:2:32", "128:4:32", 50_000_000) assert "spike_tools" not in plain + + +# ── main() end-to-end (mock / real subprocess) ────────────────────────────── + +def _write_binary(tmp_path: Path) -> Path: + binary = tmp_path / "output.bin" + binary.write_bytes(b"\x00" * 64) + return binary + + +def test_main_success_path_writes_and_cleans_elf( + tmp_path, monkeypatch, capsys): + fake_spike = make_fake_tool(tmp_path, "spike") + binary = _write_binary(tmp_path) + captured: dict[str, object] = {} + + def fake_run(cmd, **kwargs): + elf = Path(cmd[-1]) + captured["cmd"] = list(cmd) + captured["elf_magic"] = elf.read_bytes()[:4] + return types.SimpleNamespace( + returncode=0, stdout=CANNED_STDOUT, stderr=CANNED_STDERR) + + monkeypatch.setattr(spike_sim.subprocess, "run", fake_run) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--spike-bin", str(fake_spike), "--json"]) + + report = json.loads(capsys.readouterr().out) + assert rc == spike_sim.EXIT_OK + assert report["status"] == "ok" + assert report["exit_code"] == 0 + assert report["committed_insns"] == 1234 + assert report["icache"]["hits"] == 10_000 + assert report["spike_binary"] == str(fake_spike) + assert report["spike_tools"]["spike"]["source"] == "cli" + # The resolved toolchain reached the runner. + assert captured["cmd"][0] == str(fake_spike) + assert "-g" in captured["cmd"] + # A valid ELF32 was written next to the binary, then cleaned up. + assert captured["elf_magic"] == b"\x7fELF" + assert not (tmp_path / "output_spike.elf").exists() + # F5: optional tools are reported as unused, not as degraded features. + assert len(report["tool_warnings"]) == 2 + assert all("not used by this module" in w + for w in report["tool_warnings"]) + + +def test_main_unexecutable_spike_no_traceback(tmp_path, capsys): + # F1 reproduction: chmod +x plain text file (no shebang) -> ENOEXEC. + bad = tmp_path / "badspike" + bad.write_text("plain text, no shebang\n") + bad.chmod(0o755) + binary = _write_binary(tmp_path) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--spike-bin", str(bad), "--json"]) + + captured = capsys.readouterr() + assert rc == spike_sim.EXIT_RUN_FAIL + assert "Traceback" not in captured.err + report = json.loads(captured.out) + assert report["status"] == "failed" + assert report["exit_code"] == -2 + assert "failed to start Spike" in report["stderr_tail"] + assert not (tmp_path / "output_spike.elf").exists() + + +def test_main_fake_zero_output_marks_failed(tmp_path, capsys): + # F6 reproduction: /bin/true-like executable (exit 0, no output). + fake_spike = make_fake_tool(tmp_path, "spike") + binary = _write_binary(tmp_path) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--spike-bin", str(fake_spike), "--json"]) + + report = json.loads(capsys.readouterr().out) + assert rc == spike_sim.EXIT_RUN_FAIL + assert report["status"] == "failed" + assert report["exit_code"] == -2 + assert any("no Spike statistics" in w for w in report["parse_warnings"]) + assert not (tmp_path / "output_spike.elf").exists() + + +def test_main_nonzero_spike_exit_returns_run_fail(tmp_path, monkeypatch, capsys): + fake_spike = make_fake_tool(tmp_path, "spike") + binary = _write_binary(tmp_path) + + monkeypatch.setattr( + spike_sim.subprocess, "run", + lambda cmd, **kwargs: types.SimpleNamespace( + returncode=1, stdout="", stderr=CANNED_STDERR)) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--spike-bin", str(fake_spike), "--json"]) + + report = json.loads(capsys.readouterr().out) + assert rc == spike_sim.EXIT_RUN_FAIL + assert report["status"] == "failed" + assert report["exit_code"] == 1 + + +def test_main_optional_invalid_cli_still_runs(tmp_path, monkeypatch, capsys): + # F4 reproduction: a bad --spike-dasm must not abort the simulation. + fake_spike = make_fake_tool(tmp_path, "spike") + binary = _write_binary(tmp_path) + + monkeypatch.setattr( + spike_sim.subprocess, "run", + lambda cmd, **kwargs: types.SimpleNamespace( + returncode=0, stdout=CANNED_STDOUT, stderr=CANNED_STDERR)) + + rc = spike_sim.main(["--binary", str(binary), "--code-size", "64", + "--spike-bin", str(fake_spike), + "--spike-dasm", str(tmp_path / "nope"), + "--spike-log-parser", str(tmp_path / "nope2")]) + + captured = capsys.readouterr() + assert rc == spike_sim.EXIT_OK + assert "WARNING:" in captured.err + assert "--spike-dasm" in captured.err + assert "Committed insns:" in captured.out + + +# ── run_spike_bench.py --probe-spike / JSON contract (F2) ─────────────────── + +def test_probe_spike_tools_reports_resolution(monkeypatch, capsys): + from scratchv.standalone import run_spike_bench + + fake = spike_sim.SpikeTools( + spike="/opt/riscv/bin/spike", sources={"spike": "cli"}) + monkeypatch.setattr(spike_sim, "resolve_spike_tools", lambda: fake) + + tools = run_spike_bench.probe_spike_tools() + + assert tools is fake + err = capsys.readouterr().err + assert "Spike tools:" in err + assert "spike=/opt/riscv/bin/spike (cli)" in err + assert "spike-dasm=NOT FOUND" in err + assert "spike-log-parser=NOT FOUND" in err + assert "hint:" not in err + + +def test_probe_spike_tools_missing_prints_hint(monkeypatch, capsys): + from scratchv.standalone import run_spike_bench + + monkeypatch.setattr(spike_sim, "resolve_spike_tools", + lambda: spike_sim.SpikeTools()) + + tools = run_spike_bench.probe_spike_tools() + + assert tools is not None + assert tools.spike is None + err = capsys.readouterr().err + assert "spike=NOT FOUND" in err + assert "built-in emulator backend" in err + + +def test_probe_spike_tools_config_error_returns_none(monkeypatch, capsys): + from scratchv.standalone import run_spike_bench + + def boom(): + raise spike_sim.SpikeConfigError( + "--spike-bin='/x' is not an executable file") + + monkeypatch.setattr(spike_sim, "resolve_spike_tools", boom) + + assert run_spike_bench.probe_spike_tools() is None + assert "ERROR:" in capsys.readouterr().err + + +def test_run_spike_bench_json_backend_and_spike_tools(): + from scratchv.standalone import run_spike_bench + + result = run_spike_bench.SpikeStyleResult( + code_size=3140, binary_path="output.bin", wall_time_s=0.5) + + plain = run_spike_bench.generate_json_report(result) + assert plain["backend"] == {"kind": "emulator", "spike_style": True} + assert "spike_tools" not in plain + for key in ("summary", "instruction_mix", "memory", "cache", + "branch_behavior", "cycle_estimates", "top_pcs", + "per_layer"): + assert key in plain + assert plain["summary"]["binary_path"] == "output.bin" + assert plain["summary"]["code_size"] == 3140 + + tools = spike_sim.SpikeTools( + spike="/opt/riscv/bin/spike", + sources={"spike": "env"}, + warnings=("SCRATCHV_SPIKE_DASM=/old/dasm is not executable; ignored",)) + report = run_spike_bench.generate_json_report(result, spike_tools=tools) + + assert report["backend"] == plain["backend"] + assert report["spike_tools"]["spike"] == { + "path": "/opt/riscv/bin/spike", + "source": "env", + "candidates": [], + } + assert report["spike_tools"]["warnings"] == [ + "SCRATCHV_SPIKE_DASM=/old/dasm is not executable; ignored"] From 35880a9f4bc965c14db5c703a5d718816d3e5273 Mon Sep 17 00:00:00 2001 From: opencode Date: Tue, 15 Sep 2026 00:59:17 +0800 Subject: [PATCH 5/5] feat(topic24): add spike tool-resolution case report and CI regressions --- .github/workflows/ci.yml | 17 + benchmarks/run_topic24_spike_case.py | 677 ++++++++++++++++++++++++ tests/test_topic24_spike_case_report.py | 200 +++++++ 3 files changed, 894 insertions(+) create mode 100644 benchmarks/run_topic24_spike_case.py create mode 100644 tests/test_topic24_spike_case_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15aae4b..9c6d5fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,13 @@ jobs: run: | python3.12 -m pytest tests/test_pr37_regression.py -v --tb=short + - name: Run topic24 spike-tool regressions + run: | + python3.12 -m pytest \ + tests/test_spike_sim_paths.py \ + tests/test_topic24_spike_case_report.py \ + -v --tb=short + - name: Generate test visualization page if: github.ref == 'refs/heads/main' run: | @@ -218,6 +225,13 @@ jobs: --json benchmark_reports/const_merge_report.json \ --markdown benchmark_reports/const_merge_report.md + # ── 3.1.3 课题24：Spike 工具解析 case 报告（hermetic 层级矩阵） ── + - name: Topic 24 spike tool-resolution case report + run: | + python3.12 benchmarks/run_topic24_spike_case.py \ + --json benchmark_reports/spike_tools_report.json \ + --markdown benchmark_reports/spike_tools_report.md + # ── 3.2 DSL 用例编译 + 模拟基准 ──────────────────────────────────── - name: DSL case compilation benchmarks run: | @@ -363,6 +377,9 @@ jobs: if [ -f benchmark_reports/const_merge_report.md ]; then cat benchmark_reports/const_merge_report.md >> $GITHUB_STEP_SUMMARY fi + if [ -f benchmark_reports/spike_tools_report.md ]; then + cat benchmark_reports/spike_tools_report.md >> $GITHUB_STEP_SUMMARY + fi echo "" >> $GITHUB_STEP_SUMMARY if [ -f benchmark_reports/github_summary.md ]; then cat benchmark_reports/github_summary.md >> $GITHUB_STEP_SUMMARY diff --git a/benchmarks/run_topic24_spike_case.py b/benchmarks/run_topic24_spike_case.py new file mode 100644 index 0000000..df4037f --- /dev/null +++ b/benchmarks/run_topic24_spike_case.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +"""Run one Topic 24 Spike tool-resolution feature case and emit CI reports. + +The report proves four separate facts about the portable Spike toolchain +resolution added in this topic: + +1. ``resolve_spike_tools()`` reports a complete record per tool + (path/source/candidates/warnings) for every layer of the priority chain + (CLI > dedicated env vars > ``$SCRATCHV_SPIKE_HOME/bin`` > ``PATH`` > + common install dirs > legacy constant > missing); +2. a fake executable found through ``SCRATCHV_SPIKE_BIN`` resolves with + ``source="env"`` while an explicit ``--spike-bin`` wins over the env var, + proven end to end through ``spike_sim.main(argv=[...])``; +3. missing tools degrade gracefully: an invalid env path only warns and + falls through, a fully missing toolchain prints ``SKIP:`` and exits 0, + and ``--require-spike`` instead exits 2; +4. ``run_spike_bench.py --probe-spike --json`` exits 0 with a ``spike_tools`` + snapshot in its JSON payload, and the pure parsers tolerate thousands + separators as well as missing statistics sections (``parse_warnings``). + +Every fake "spike" is a shell script created in a temporary directory, so +the case is hermetic: it never requires a real Spike installation, and it +makes no claim about real Spike availability or simulator performance. +""" + +from __future__ import annotations + +import argparse +import contextlib +import io +import json +import os +import subprocess +import sys +import tempfile +import types +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scratchv.standalone import spike_sim + +SCHEMA_VERSION = "topic24-spike-case/1" +TOPIC = "topic24-spike-tools" +DEFAULT_CASE = "spike_tools_matrix" +CASES = (DEFAULT_CASE,) +DEFAULT_JSON = Path("benchmark_reports/spike_tools_report.json") +DEFAULT_MARKDOWN = Path("benchmark_reports/spike_tools_report.md") + +EXPECTED_COMMITTED = 1234 +REQUIRED_SOURCES = ("path", "env", "spike_home", "common", "legacy", "missing") + +ENV_SPIKE_BIN = spike_sim.ENV_SPIKE_BIN +ENV_SPIKE_DASM = spike_sim.ENV_SPIKE_DASM +ENV_SPIKE_LOG_PARSER = spike_sim.ENV_SPIKE_LOG_PARSER +ENV_SPIKE_HOME = spike_sim.ENV_SPIKE_HOME + +_ENV_NAMES = (ENV_SPIKE_BIN, ENV_SPIKE_DASM, ENV_SPIKE_LOG_PARSER, + ENV_SPIKE_HOME) + +CANNED_STDERR = ( + "Commited 1234 instructions\n" + "core 0: 0x80000000 (0x00000013) 1.5 MIPS\n" + "I$: 64 sets x 2 ways x 32 B\n" + " hits: 10,000 misses: 25 miss rate: 0.25%\n" + "D$: 128 sets x 4 ways x 32 B\n" + " hits: 20,000 misses: 50 miss rate: 0.25%\n" +) +CANNED_STDOUT = ( + "PC histogram (number of commits per PC):\n" + "0x80000014: 123\n" + "0x80000018: 456\n" + "\n" +) +#: Deliberately different counters: if the env-var stub were executed instead +#: of the CLI stub, the report would show 999999 committed instructions. +POISONED_ENV_STDERR = ( + "Committed 999,999 instructions\n" + "core 0: 0x80000000 (0x00000013) 1.5 MIPS\n" +) + + +def make_fake_tool( + directory: Path, + name: str, + *, + stderr: str = "", + stdout: str = "", + exit_code: int = 0, +) -> Path: + """Write a chmod +x POSIX shell stub; no real Spike is ever used.""" + body: list[str] = ["#!/bin/sh"] + if stderr: + body.append("cat >&2 <<'SPIKE_STUB_EOF'") + body.extend(stderr.rstrip("\n").splitlines()) + body.append("SPIKE_STUB_EOF") + if stdout: + body.append("cat <<'SPIKE_STUB_EOF'") + body.extend(stdout.rstrip("\n").splitlines()) + body.append("SPIKE_STUB_EOF") + body.append(f"exit {int(exit_code)}") + path = directory / name + path.write_text("\n".join(body) + "\n") + path.chmod(0o755) + return path + + +def _case_dir(root: Path, name: str) -> Path: + directory = root / name + directory.mkdir(parents=True, exist_ok=True) + return directory + + +@contextlib.contextmanager +def _patched_module( + tmp: Path, + *, + legacy_spike: str | None = None, + legacy_dasm: str | None = None, + legacy_parser: str | None = None, + which=None, +): + """Isolate the module-level resolution layers (constants + which()).""" + saved = ( + spike_sim.SPIKE, + spike_sim.SPIKE_DASM, + spike_sim.SPIKE_LOG_PARSER, + spike_sim.COMMON_SPIKE_DIRS, + spike_sim.shutil, + ) + finder = which if callable(which) else (lambda name: None) + try: + spike_sim.SPIKE = legacy_spike or str(tmp / "no-legacy-spike") + spike_sim.SPIKE_DASM = legacy_dasm or str(tmp / "no-legacy-dasm") + spike_sim.SPIKE_LOG_PARSER = ( + legacy_parser or str(tmp / "no-legacy-parser")) + spike_sim.COMMON_SPIKE_DIRS = () + spike_sim.shutil = types.SimpleNamespace(which=finder) + yield + finally: + ( + spike_sim.SPIKE, + spike_sim.SPIKE_DASM, + spike_sim.SPIKE_LOG_PARSER, + spike_sim.COMMON_SPIKE_DIRS, + spike_sim.shutil, + ) = saved + + +@contextlib.contextmanager +def _temp_environ(values: dict[str, str]): + """Set only the SCRATCHV_SPIKE_* variables, restoring the host values.""" + saved = {name: os.environ.get(name) for name in _ENV_NAMES} + try: + for name in _ENV_NAMES: + os.environ.pop(name, None) + for name, value in values.items(): + os.environ[name] = value + yield + finally: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _structure_complete(tools: spike_sim.SpikeTools) -> bool: + """Every tool entry carries path/source/candidates plus a warnings list.""" + tool_names = ("spike", "spike-dasm", "spike-log-parser") + if set(tools.sources) != set(tool_names): + return False + if set(tools.candidates) != set(tool_names): + return False + data = tools.as_dict() + for name in ("spike", "spike_dasm", "spike_log_parser"): + entry = data.get(name) + if not isinstance(entry, dict): + return False + if set(entry) != {"path", "source", "candidates"}: + return False + if not isinstance(entry["candidates"], list): + return False + return isinstance(data.get("warnings"), list) + + +def _scenario( + tmp: Path, + name: str, + expected_source: str, + *, + expect_path: Path | None = None, + legacy_spike: str | None = None, + legacy_dasm: str | None = None, + legacy_parser: str | None = None, + **resolve_kwargs, +) -> dict[str, Any]: + with _patched_module( + tmp, + legacy_spike=legacy_spike, + legacy_dasm=legacy_dasm, + legacy_parser=legacy_parser, + ): + tools = spike_sim.resolve_spike_tools(**resolve_kwargs) + observed = tools.sources.get("spike", "missing") + expected_path = str(expect_path) if expect_path else None + return { + "scenario": name, + "expected_source": expected_source, + "observed_source": observed, + "path": tools.spike, + "path_exists": bool(tools.spike and os.path.isfile(tools.spike)), + "candidates": list(tools.candidates.get("spike", ())), + "warnings": list(tools.warnings), + "structure_ok": _structure_complete(tools), + "matches": ( + observed == expected_source + and (expected_path is None or tools.spike == expected_path) + ), + } + + +def build_resolution_matrix(root: Path) -> list[dict[str, Any]]: + """Exercise every resolution layer with fake tools only.""" + tmp = _case_dir(root, "resolution") + fake_cli = make_fake_tool(tmp, "spike-cli") + fake_env = make_fake_tool(tmp, "spike-env") + fake_path = make_fake_tool(tmp, "spike-path") + fake_legacy = make_fake_tool(tmp, "spike-legacy") + home = tmp / "spike-home" + (home / "bin").mkdir(parents=True) + fake_home = make_fake_tool(home / "bin", "spike") + common = tmp / "common" + common.mkdir() + fake_common = make_fake_tool(common, "spike") + + def none_which(_name): + return None + + def path_which(name): + return str(fake_path) if name == "spike" else None + + return [ + _scenario( + tmp, "clean_missing", "missing", + env={}, which=none_which, common_dirs=()), + _scenario( + tmp, "cli", "cli", + cli_spike=str(fake_cli), env={}, which=none_which, + common_dirs=(), expect_path=fake_cli), + _scenario( + tmp, "cli_beats_env", "cli", + cli_spike=str(fake_cli), + env={ENV_SPIKE_BIN: str(fake_env)}, + which=none_which, common_dirs=(), expect_path=fake_cli), + _scenario( + tmp, "env", "env", + env={ENV_SPIKE_BIN: str(fake_env)}, + which=none_which, common_dirs=(), expect_path=fake_env), + _scenario( + tmp, "env_beats_path", "env", + env={ENV_SPIKE_BIN: str(fake_env)}, + which=path_which, common_dirs=(), expect_path=fake_env), + _scenario( + tmp, "spike_home", "spike_home", + env={ENV_SPIKE_HOME: str(home)}, + which=none_which, common_dirs=(), expect_path=fake_home), + _scenario( + tmp, "path", "path", + env={}, which=path_which, common_dirs=(), + expect_path=fake_path), + _scenario( + tmp, "common", "common", + env={}, which=none_which, common_dirs=(str(common),), + expect_path=fake_common), + _scenario( + tmp, "legacy", "legacy", + env={}, which=none_which, common_dirs=(), + legacy_spike=str(fake_legacy), expect_path=fake_legacy), + _scenario( + tmp, "env_invalid_falls_through", "missing", + env={ENV_SPIKE_BIN: str(tmp / "no-such-spike")}, + which=none_which, common_dirs=()), + ] + + +def run_cli_probe(root: Path) -> dict[str, Any]: + """Prove --spike-bin beats SCRATCHV_SPIKE_BIN through main(argv=...).""" + tmp = _case_dir(root, "cli_probe") + fake_env = make_fake_tool( + tmp, "spike-env", stderr=POISONED_ENV_STDERR) + fake_cli = make_fake_tool( + tmp, "spike-cli", stderr=CANNED_STDERR, stdout=CANNED_STDOUT) + binary = tmp / "output.bin" + binary.write_bytes(b"\x00" * 64) + + out, err = io.StringIO(), io.StringIO() + with _temp_environ({ENV_SPIKE_BIN: str(fake_env)}), _patched_module(tmp): + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + exit_code = spike_sim.main([ + "--binary", str(binary), "--code-size", "64", + "--spike-bin", str(fake_cli), "--json", + ]) + + try: + report = json.loads(out.getvalue()) + except (json.JSONDecodeError, ValueError): + report = {} + spike_entry = (report.get("spike_tools") or {}).get("spike") or {} + return { + "exit_code": exit_code, + "status": report.get("status"), + "committed_insns": report.get("committed_insns"), + "resolved_path": report.get("spike_binary"), + "resolved_source": spike_entry.get("source"), + "cli_path": str(fake_cli), + "env_path": str(fake_env), + "stdout_is_json": bool(report), + "stderr_tail": err.getvalue()[-300:], + } + + +def run_degradation_probe(root: Path) -> dict[str, Any]: + """Prove missing-tool degradation: SKIP/exit 0 vs --require-spike/exit 2.""" + tmp = _case_dir(root, "degradation") + binary = tmp / "output.bin" + binary.write_bytes(b"\x00" * 64) + + out, err = io.StringIO(), io.StringIO() + with _temp_environ({}), _patched_module(tmp): + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + skip_exit_code = spike_sim.main([ + "--binary", str(binary), "--code-size", "64", "--json", + ]) + skip_stderr = err.getvalue() + try: + skip_json = json.loads(out.getvalue()) + except (json.JSONDecodeError, ValueError): + skip_json = None + + strict_out, strict_err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(strict_out), \ + contextlib.redirect_stderr(strict_err): + strict_exit_code = spike_sim.main([ + "--binary", str(binary), "--code-size", "64", + "--require-spike", + ]) + + return { + "skip_exit_code": skip_exit_code, + "skip_stderr": skip_stderr, + "skip_json": skip_json, + "strict_exit_code": strict_exit_code, + "strict_stdout_empty": strict_out.getvalue() == "", + "strict_stderr_tail": strict_err.getvalue()[-300:], + "elf_created": (tmp / "output_spike.elf").exists(), + } + + +def run_probe_subprocess(root: Path) -> dict[str, Any]: + """Run ``run_spike_bench.py --probe-spike --json`` as a subprocess.""" + tmp = _case_dir(root, "probe") + binary = tmp / "probe.bin" + binary.write_bytes(b"\x00" * 64) + + env = { + name: value for name, value in os.environ.items() + if not name.startswith("SCRATCHV_SPIKE_") + } + python_path = [str(REPO_ROOT)] + if env.get("PYTHONPATH"): + python_path.append(env["PYTHONPATH"]) + env["PYTHONPATH"] = os.pathsep.join(python_path) + + command = [ + sys.executable, + str(REPO_ROOT / "scratchv" / "standalone" / "run_spike_bench.py"), + "--binary", str(binary), "--code-size", "64", + "--max-instr", "1", "--progress", "1000000", + "--probe-spike", "--json", + ] + proc = subprocess.run( + command, capture_output=True, text=True, env=env, + cwd=str(REPO_ROOT), timeout=120) + try: + report = json.loads(proc.stdout) + except (json.JSONDecodeError, ValueError): + report = None + return { + "command": command, + "exit_code": proc.returncode, + "report_ok": isinstance(report, dict), + "backend": (report or {}).get("backend"), + "summary": (report or {}).get("summary"), + "spike_tools": (report or {}).get("spike_tools"), + "stderr_tail": proc.stderr[-400:], + } + + +def _probe_contract_ok(probe: dict[str, Any]) -> bool: + tools = probe.get("spike_tools") + if not isinstance(tools, dict): + return False + for name in ("spike", "spike_dasm", "spike_log_parser"): + entry = tools.get(name) + if not isinstance(entry, dict): + return False + if set(entry) != {"path", "source", "candidates"}: + return False + return isinstance(tools.get("warnings"), list) + + +def run_parser_tolerance() -> dict[str, Any]: + """Prove thousands separators parse and missing sections only warn.""" + committed, mips = spike_sim.parse_commit_stats( + "Committed 2,000,000 instructions\n42.0 MIPS") + cache = spike_sim.parse_cache_stats(CANNED_STDERR) + histogram = spike_sim.parse_pc_histogram(CANNED_STDOUT) + messy_histogram = spike_sim.parse_pc_histogram( + "PC histogram (number of commits per PC):\n" + "0x80000010: 7\n" + "garbage line\n" + "0x80000020: 9\n" + "\n" + "0x80000030: 11\n" + ) + + original_run = spike_sim.subprocess.run + try: + spike_sim.subprocess.run = lambda cmd, **kwargs: types.SimpleNamespace( + returncode=0, stdout="", stderr="Committed 5 instructions") + partial = spike_sim.run_spike( + "unused.elf", tools=spike_sim.SpikeTools(spike="/fake/spike")) + finally: + spike_sim.subprocess.run = original_run + + return { + "committed": committed, + "mips": mips, + "cache_hits": int(cache["icache_hits"]), + "cache_misses": int(cache["icache_misses"]), + "cache_miss_rate": float(cache["icache_miss_rate"]), + "histogram": {f"0x{pc:08x}": cnt for pc, cnt in histogram.items()}, + "messy_histogram": { + f"0x{pc:08x}": cnt for pc, cnt in messy_histogram.items()}, + "empty_cache_is_zeroed": all( + value == 0 for value in spike_sim.parse_cache_stats("").values()), + "partial_status": partial.status, + "partial_committed": partial.committed_insns, + "partial_parse_warnings": list(partial.parse_warnings), + } + + +def evaluate(root: Path) -> dict[str, Any]: + """Build the full report payload and run the hard invariants.""" + matrix = build_resolution_matrix(root) + by_scenario = {row["scenario"]: row for row in matrix} + observed_sources = {row["observed_source"] for row in matrix} + cli_probe = run_cli_probe(root) + degradation = run_degradation_probe(root) + probe = run_probe_subprocess(root) + tolerance = run_parser_tolerance() + + skip_json = degradation.get("skip_json") or {} + skip_spike = (skip_json.get("spike_tools") or {}).get("spike") or {} + + def row(name: str) -> dict[str, Any]: + return by_scenario.get(name, {}) + + hard_checks = { + "resolution_matrix_all_rows_match": all( + entry["matches"] for entry in matrix), + "resolution_structure_complete": all( + entry["structure_ok"] for entry in matrix), + "resolution_covers_path_env_home_common_legacy_missing": ( + set(REQUIRED_SOURCES) <= observed_sources), + "clean_env_falls_back_to_missing": ( + row("clean_missing").get("observed_source") == "missing" + and row("clean_missing").get("path") is None), + "env_fake_spike_resolves_as_env": ( + row("env").get("observed_source") == "env" + and row("env").get("matches") is True), + "cli_beats_env_resolution": ( + row("cli_beats_env").get("observed_source") == "cli" + and row("cli_beats_env").get("matches") is True), + "env_beats_path_resolution": ( + row("env_beats_path").get("matches") is True), + "invalid_env_warns_and_continues": ( + row("env_invalid_falls_through").get("observed_source") + == "missing" + and any(ENV_SPIKE_BIN in warning for warning in + row("env_invalid_falls_through").get("warnings", ()))), + "cli_probe_exit_zero": cli_probe["exit_code"] == 0, + "cli_probe_resolved_cli_over_env": ( + cli_probe["resolved_source"] == "cli" + and cli_probe["resolved_path"] == cli_probe["cli_path"]), + "cli_probe_ran_cli_stub": ( + cli_probe["committed_insns"] == EXPECTED_COMMITTED), + "missing_tool_skips_exit_zero": ( + degradation["skip_exit_code"] == 0 + and "SKIP:" in degradation["skip_stderr"]), + "missing_tool_json_source_missing": ( + skip_json.get("status") == "skipped" + and skip_spike.get("source") == "missing" + and skip_json.get("exit_code") == -2), + "strict_require_spike_exit_two": degradation["strict_exit_code"] == 2, + "no_elf_written_when_skipped": degradation["elf_created"] is False, + "probe_subprocess_exit_zero": ( + probe["exit_code"] == 0 and probe["report_ok"]), + "probe_json_has_spike_tools_contract": _probe_contract_ok(probe), + "probe_backend_marked_emulator": ( + probe["backend"] == {"kind": "emulator", "spike_style": True}), + "parser_reads_thousands_separators": ( + tolerance["committed"] == 2_000_000 + and tolerance["mips"] == 42.0 + and tolerance["cache_hits"] == 10_000 + and tolerance["histogram"] == {"0x80000014": 123, + "0x80000018": 456}), + "parser_missing_sections_become_warnings": ( + tolerance["partial_status"] == "ok" + and len(tolerance["partial_parse_warnings"]) == 2), + } + failed = sorted(name for name, ok in hard_checks.items() if not ok) + + return { + "schema_version": SCHEMA_VERSION, + "topic": TOPIC, + "generated_at": datetime.now(timezone.utc).isoformat(), + "case": DEFAULT_CASE, + "python": ( + f"{sys.version_info.major}.{sys.version_info.minor}." + f"{sys.version_info.micro}"), + "resolution_matrix": matrix, + "resolution_sources_observed": sorted(observed_sources), + "cli_probe": cli_probe, + "degradation": degradation, + "probe": probe, + "parser_tolerance": tolerance, + "hard_checks": hard_checks, + "hard_failures": failed, + "honesty": ( + "Hermetic deterministic feature case: every spike executable " + "used here is a shell stub created under a temporary directory, " + "and the contract is proven through spike_sim.main(argv=[...]) " + "plus one run_spike_bench.py --probe-spike subprocess. Resolution " + "sources prove the documented priority order of the search " + "layers only; they do not prove that a real Spike binary is " + "installed, that a real Spike run succeeds, or anything about " + "simulator performance." + ), + } + + +def render_markdown(report: dict[str, Any]) -> str: + cli_probe = report["cli_probe"] + degradation = report["degradation"] + probe = report["probe"] + tolerance = report["parser_tolerance"] + total = len(report["hard_checks"]) + passed = total - len(report["hard_failures"]) + + lines = [ + "# Topic 24 Spike Tool-Resolution Feature Case", + "", + f"- Schema: `{report['schema_version']}`", + f"- Case: `{report['case']}` (hermetic, program-built matrix)", + f"- Generated: {report['generated_at']}", + f"- Python: {report['python']}", + f"- Hard checks: {'PASS' if not report['hard_failures'] else 'FAIL'} " + f"({passed}/{total})", + "", + "## Resolution matrix (source / candidates / warnings)", + "", + "| Scenario | Expected source | Observed source | Tool found | " + "Candidates | Warnings |", + "|----------|-----------------|-----------------|------------|" + "------------|----------|", + ] + for row in report["resolution_matrix"]: + tool_name = Path(row["path"]).name if row["path"] else "-" + lines.append( + f"| `{row['scenario']}` | `{row['expected_source']}` | " + f"`{row['observed_source']}` | {tool_name} | " + f"{len(row['candidates'])} | {len(row['warnings'])} |") + lines += [ + "", + f"- Observed sources: " + f"{', '.join(f'`{s}`' for s in report['resolution_sources_observed'])}", + "- Priority proven: CLI > dedicated env vars > " + "`$SCRATCHV_SPIKE_HOME/bin` > `PATH` > common install dirs > " + "legacy constant > missing", + "", + "## Degradation path", + "", + f"- Invalid `{ENV_SPIKE_BIN}`: warning emitted, resolution falls " + f"through to `missing` (no exception)", + f"- Missing toolchain: exit `{degradation['skip_exit_code']}`, " + f"stderr contains `SKIP:`, JSON status " + f"`{(degradation['skip_json'] or {}).get('status')}` with source " + f"`{((degradation['skip_json'] or {}).get('spike_tools') or {}).get('spike', {}).get('source')}`", + f"- `--require-spike`: exit `{degradation['strict_exit_code']}`, " + f"no report on stdout", + f"- ELF written while skipped: `{degradation['elf_created']}`", + "", + "## CLI-level priority (`--spike-bin` over env)", + "", + f"- `spike_sim.main(argv=[...])` exit code: {cli_probe['exit_code']}", + f"- Resolved source: `{cli_probe['resolved_source']}` at " + f"`{cli_probe['resolved_path']}`", + f"- Parsed committed instructions: {cli_probe['committed_insns']} " + f"(the env stub is poisoned with 999,999 to prove it was not used)", + "", + "## Probe JSON summary (`run_spike_bench.py --probe-spike --json`)", + "", + f"- Exit code: {probe['exit_code']}", + f"- Backend: `{json.dumps(probe['backend'])}`", + "", + "```json", + json.dumps(probe["spike_tools"], indent=2), + "```", + "", + "## Parser tolerance", + "", + f"- `parse_commit_stats` thousands separator: " + f"{tolerance['committed']:,} instructions, " + f"{tolerance['mips']} MIPS", + f"- `parse_cache_stats` thousands separator: " + f"hits={tolerance['cache_hits']:,}, " + f"misses={tolerance['cache_misses']}", + f"- `parse_pc_histogram`: {tolerance['histogram']}", + f"- Missing sections: `parse_warnings`=" + f"{tolerance['partial_parse_warnings']}", + "", + "## Hard checks", + "", + ] + for name, ok in report["hard_checks"].items(): + lines.append(f"- [{'x' if ok else ' '}] {name}") + lines += [ + "", + "## Honesty", + "", + report["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", default=DEFAULT_CASE, choices=CASES) + parser.add_argument("--json", type=Path, default=DEFAULT_JSON) + parser.add_argument("--markdown", type=Path, default=DEFAULT_MARKDOWN) + args = parser.parse_args(argv) + + with tempfile.TemporaryDirectory(prefix="topic24-spike-case-") as tmp: + report = evaluate(Path(tmp)) + + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(report, indent=2) + "\n") + args.markdown.parent.mkdir(parents=True, exist_ok=True) + markdown = render_markdown(report) + args.markdown.write_text(markdown + "\n") + print(markdown) + if report["hard_failures"]: + print("HARD FAILURES: " + ", ".join(report["hard_failures"])) + return 1 + print(f"reports written: {args.json}, {args.markdown}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_topic24_spike_case_report.py b/tests/test_topic24_spike_case_report.py new file mode 100644 index 0000000..50ef91d --- /dev/null +++ b/tests/test_topic24_spike_case_report.py @@ -0,0 +1,200 @@ +"""Tests for the Topic 24 Spike tool-resolution feature case report. + +The report is the CI artifact that proves the six-layer Spike toolchain +resolution, CLI-over-env priority, missing-tool degradation, the +``run_spike_bench.py --probe-spike`` JSON contract, and parser tolerance. +All tests are hermetic: fakes are shell stubs in temporary directories, so +no real Spike installation is ever required. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from benchmarks.run_topic24_spike_case import ( + SCHEMA_VERSION, + build_resolution_matrix, + evaluate, + main, + make_fake_tool, + run_cli_probe, + run_degradation_probe, + run_parser_tolerance, + run_probe_subprocess, +) +from scratchv.standalone import spike_sim + +CASE = "spike_tools_matrix" +ALL_SOURCES = { + "cli", "env", "spike_home", "path", "common", "legacy", "missing", +} + + +@pytest.fixture(autouse=True) +def hermetic_env(monkeypatch): + for var in ("SCRATCHV_SPIKE_BIN", "SCRATCHV_SPIKE_DASM", + "SCRATCHV_SPIKE_LOG_PARSER", "SCRATCHV_SPIKE_HOME"): + monkeypatch.delenv(var, raising=False) + + +def test_resolution_matrix_covers_all_sources_and_structure(tmp_path): + matrix = build_resolution_matrix(tmp_path) + by_scenario = {row["scenario"]: row for row in matrix} + + assert ALL_SOURCES <= {row["observed_source"] for row in matrix} + assert all(row["matches"] for row in matrix) + assert all(row["structure_ok"] for row in matrix) + + assert by_scenario["clean_missing"]["observed_source"] == "missing" + assert by_scenario["clean_missing"]["path"] is None + assert by_scenario["cli_beats_env"]["path"].endswith("spike-cli") + assert by_scenario["env_beats_path"]["path"].endswith("spike-env") + assert by_scenario["spike_home"]["path"].endswith("bin/spike") + assert by_scenario["path"]["path"].endswith("spike-path") + assert by_scenario["legacy"]["path"].endswith("spike-legacy") + + env_row = by_scenario["env"] + assert env_row["candidates"][0] == env_row["path"] + invalid = by_scenario["env_invalid_falls_through"] + assert invalid["observed_source"] == "missing" + assert any(spike_sim.ENV_SPIKE_BIN in item + for item in invalid["warnings"]) + + +def test_env_fake_spike_resolves_env_and_cli_wins(tmp_path): + fake_env = make_fake_tool(tmp_path, "spike-env") + fake_cli = make_fake_tool(tmp_path, "spike-cli") + + tools_env = spike_sim.resolve_spike_tools( + env={spike_sim.ENV_SPIKE_BIN: str(fake_env)}, + which=lambda name: None, common_dirs=()) + assert tools_env.spike == str(fake_env) + assert tools_env.sources["spike"] == "env" + + tools_cli = spike_sim.resolve_spike_tools( + cli_spike=str(fake_cli), + env={spike_sim.ENV_SPIKE_BIN: str(fake_env)}, + which=lambda name: None, common_dirs=()) + assert tools_cli.spike == str(fake_cli) + assert tools_cli.sources["spike"] == "cli" + + +def test_cli_probe_via_main_executes_cli_tool(tmp_path): + probe = run_cli_probe(tmp_path) + + assert probe["exit_code"] == 0 + assert probe["status"] == "ok" + assert probe["resolved_source"] == "cli" + assert probe["resolved_path"] == probe["cli_path"] + assert probe["resolved_path"].endswith("spike-cli") + assert probe["committed_insns"] == 1234 + assert probe["stdout_is_json"] + + +def test_missing_tool_degrades_to_skip_and_strict_config_error(tmp_path): + probe = run_degradation_probe(tmp_path) + + assert probe["skip_exit_code"] == 0 + assert "SKIP:" in probe["skip_stderr"] + skip_json = probe["skip_json"] + assert skip_json["status"] == "skipped" + assert skip_json["exit_code"] == -2 + assert skip_json["spike_tools"]["spike"]["source"] == "missing" + assert skip_json["spike_tools"]["spike"]["path"] is None + + assert probe["strict_exit_code"] == 2 + assert probe["strict_stdout_empty"] + assert probe["elf_created"] is False + + +def test_probe_spike_subprocess_reports_spike_tools(tmp_path): + probe = run_probe_subprocess(tmp_path) + + assert probe["exit_code"] == 0 + assert probe["report_ok"] + assert probe["backend"] == {"kind": "emulator", "spike_style": True} + tools = probe["spike_tools"] + for name in ("spike", "spike_dasm", "spike_log_parser"): + assert set(tools[name]) == {"path", "source", "candidates"} + assert isinstance(tools["warnings"], list) + + +def test_parse_helpers_tolerate_thousands_and_missing_sections(): + tolerance = run_parser_tolerance() + + assert tolerance["committed"] == 2_000_000 + assert tolerance["mips"] == 42.0 + assert tolerance["cache_hits"] == 10_000 + assert tolerance["cache_misses"] == 25 + assert tolerance["cache_miss_rate"] == 0.25 + assert tolerance["histogram"] == { + "0x80000014": 123, "0x80000018": 456} + assert tolerance["messy_histogram"] == { + "0x80000010": 7, "0x80000020": 9} + assert tolerance["empty_cache_is_zeroed"] + assert tolerance["partial_status"] == "ok" + assert tolerance["partial_committed"] == 5 + assert len(tolerance["partial_parse_warnings"]) == 2 + + +def test_main_writes_json_and_markdown(tmp_path, capsys): + json_path = tmp_path / "report.json" + markdown_path = tmp_path / "report.md" + + exit_code = main([ + "--case", CASE, + "--json", str(json_path), + "--markdown", str(markdown_path), + ]) + + assert exit_code == 0 + data = json.loads(json_path.read_text()) + assert data["schema_version"] == SCHEMA_VERSION + assert data["topic"] == "topic24-spike-tools" + assert data["hard_failures"] == [] + assert all(data["hard_checks"].values()) + markdown = markdown_path.read_text() + assert "Topic 24 Spike Tool-Resolution Feature Case" in markdown + assert "Resolution matrix" in markdown + assert "Honesty" in markdown + assert capsys.readouterr().out + + with pytest.raises(SystemExit) as excinfo: + main(["--case", "no_such_case"]) + assert excinfo.value.code == 2 + + +def test_hard_check_gate_is_not_vacuous(tmp_path, monkeypatch): + """A broken matrix must surface as hard failures and exit code 1.""" + def broken_matrix(_root): + return [{ + "scenario": "clean_missing", + "expected_source": "missing", + "observed_source": "path", + "path": "/usr/bin/spike", + "path_exists": True, + "candidates": [], + "warnings": [], + "structure_ok": True, + "matches": False, + }] + + monkeypatch.setattr( + "benchmarks.run_topic24_spike_case.build_resolution_matrix", + broken_matrix) + + report = evaluate(tmp_path) + assert "resolution_matrix_all_rows_match" in report["hard_failures"] + assert "resolution_covers_path_env_home_common_legacy_missing" in ( + report["hard_failures"]) + + exit_code = main([ + "--case", CASE, + "--json", str(tmp_path / "broken.json"), + "--markdown", str(tmp_path / "broken.md"), + ]) + assert exit_code == 1 + assert json.loads((tmp_path / "broken.json").read_text())["hard_failures"]