From d5b79ac5b04519959a4bc488dd85e330309b6565 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 01:43:24 +0800 Subject: [PATCH 1/6] feat(topic06): add pytest-based DSL benchmark suite with xfail policy --- .github/workflows/ci.yml | 17 + benchmarks/cases/001_simple_add.meta.json | 15 + benchmarks/cases/002_simple_mul.meta.json | 15 + benchmarks/cases/003_sub_div.meta.json | 15 + benchmarks/cases/004_relu.meta.json | 15 + benchmarks/cases/005_gelu.meta.json | 15 + benchmarks/cases/006_softmax.meta.json | 15 + benchmarks/cases/007_matmul.meta.json | 10 + benchmarks/cases/008_dot.meta.json | 10 + benchmarks/cases/009_maxpool.meta.json | 22 + benchmarks/cases/010_exp_neg.meta.json | 15 + benchmarks/cases/011_multi_op_chain.meta.json | 15 + benchmarks/cases/012_nn_pipeline.meta.json | 15 + benchmarks/cases/013_for_sum.meta.json | 27 + benchmarks/cases/014_for_dot.meta.json | 29 + benchmarks/cases/015_for_relu.meta.json | 27 + benchmarks/cases/016_if_simple.meta.json | 27 + benchmarks/cases/017_while_sum.dsl | 1 + benchmarks/cases/017_while_sum.meta.json | 29 + benchmarks/cases/018_nested_if.meta.json | 27 + benchmarks/cases/019_nested_loop.meta.json | 29 + .../cases/020_constant_propagation.meta.json | 10 + benchmarks/cases/021_dsl_if_else.meta.json | 27 + benchmarks/cases/022_dsl_while_sum.dsl | 1 + benchmarks/cases/022_dsl_while_sum.meta.json | 31 + benchmarks/cases/023_large_chain.meta.json | 15 + benchmarks/dsl_suite.py | 1531 +++++++++++++++++ benchmarks/run_suite.py | 162 ++ tests/stress/reg_pressure_32.meta.json | 20 +- tests/stress/reg_pressure_loop.meta.json | 26 +- tests/test_dsl_suite.py | 235 +++ 31 files changed, 2443 insertions(+), 5 deletions(-) create mode 100644 benchmarks/cases/001_simple_add.meta.json create mode 100644 benchmarks/cases/002_simple_mul.meta.json create mode 100644 benchmarks/cases/003_sub_div.meta.json create mode 100644 benchmarks/cases/004_relu.meta.json create mode 100644 benchmarks/cases/005_gelu.meta.json create mode 100644 benchmarks/cases/006_softmax.meta.json create mode 100644 benchmarks/cases/007_matmul.meta.json create mode 100644 benchmarks/cases/008_dot.meta.json create mode 100644 benchmarks/cases/009_maxpool.meta.json create mode 100644 benchmarks/cases/010_exp_neg.meta.json create mode 100644 benchmarks/cases/011_multi_op_chain.meta.json create mode 100644 benchmarks/cases/012_nn_pipeline.meta.json create mode 100644 benchmarks/cases/013_for_sum.meta.json create mode 100644 benchmarks/cases/014_for_dot.meta.json create mode 100644 benchmarks/cases/015_for_relu.meta.json create mode 100644 benchmarks/cases/016_if_simple.meta.json create mode 100644 benchmarks/cases/017_while_sum.meta.json create mode 100644 benchmarks/cases/018_nested_if.meta.json create mode 100644 benchmarks/cases/019_nested_loop.meta.json create mode 100644 benchmarks/cases/020_constant_propagation.meta.json create mode 100644 benchmarks/cases/021_dsl_if_else.meta.json create mode 100644 benchmarks/cases/022_dsl_while_sum.meta.json create mode 100644 benchmarks/cases/023_large_chain.meta.json create mode 100644 benchmarks/dsl_suite.py create mode 100644 benchmarks/run_suite.py create mode 100644 tests/test_dsl_suite.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15aae4b..644db5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,11 @@ jobs: --junit-xml=benchmark_reports/test_results.xml \ --ignore=tests/test_simulator.py + # ── 课题06：DSL 基准套件（硬门禁，xfail 不红灯） ───────────────── + - name: Run DSL benchmark suite + run: | + python3.12 -m pytest tests/test_dsl_suite.py -v --tb=short + - name: Run assembly-beautifier regressions run: | python3.12 -m pytest \ @@ -227,6 +232,13 @@ jobs: --output-html benchmark_reports/dsl_bench.html \ --output-md benchmark_reports/dsl_bench.md + # ── 课题06：DSL 基准套件 JSON/MD 报告（同时作为第二道门禁） ─────── + - name: DSL benchmark suite report + run: | + python3.12 benchmarks/run_suite.py \ + --json benchmark_reports/dsl_suite.json \ + --markdown benchmark_reports/dsl_suite.md + # ── 3.3 CNN RISC-V 编译 + 性能估算 ──────────────────────────────── - name: CNN RISC-V compilation & estimation run: | @@ -381,6 +393,11 @@ jobs: if [ -f benchmark_reports/regalloc_bench.md ]; then cat benchmark_reports/regalloc_bench.md >> $GITHUB_STEP_SUMMARY fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "### DSL Benchmark Suite" >> $GITHUB_STEP_SUMMARY + if [ -f benchmark_reports/dsl_suite.md ]; then + cat benchmark_reports/dsl_suite.md >> $GITHUB_STEP_SUMMARY + fi # ═════════════════════════════════════════════════════════════════════════ # GitHub Pages 部署 (仅 main 分支) diff --git a/benchmarks/cases/001_simple_add.meta.json b/benchmarks/cases/001_simple_add.meta.json new file mode 100644 index 0000000..8ca0672 --- /dev/null +++ b/benchmarks/cases/001_simple_add.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Basic addition of two vectors", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 2.0, + 4.0, + 6.0, + 8.0 + ], + "max_instructions": 3 +} diff --git a/benchmarks/cases/002_simple_mul.meta.json b/benchmarks/cases/002_simple_mul.meta.json new file mode 100644 index 0000000..82a3e2a --- /dev/null +++ b/benchmarks/cases/002_simple_mul.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Element-wise multiplication of two vectors", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 1.0, + 4.0, + 9.0, + 16.0 + ], + "max_instructions": 3 +} diff --git a/benchmarks/cases/003_sub_div.meta.json b/benchmarks/cases/003_sub_div.meta.json new file mode 100644 index 0000000..8ee8727 --- /dev/null +++ b/benchmarks/cases/003_sub_div.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Chained subtraction and division", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "max_instructions": 4 +} diff --git a/benchmarks/cases/004_relu.meta.json b/benchmarks/cases/004_relu.meta.json new file mode 100644 index 0000000..5b961f4 --- /dev/null +++ b/benchmarks/cases/004_relu.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "ReLU activation function", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 1.0, + 2.0, + 3.0, + 4.0 + ], + "max_instructions": 3 +} diff --git a/benchmarks/cases/005_gelu.meta.json b/benchmarks/cases/005_gelu.meta.json new file mode 100644 index 0000000..9f4dda5 --- /dev/null +++ b/benchmarks/cases/005_gelu.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "GELU activation function", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 0.841192, + 1.954598, + 2.996363, + 3.99993 + ], + "max_instructions": 5 +} diff --git a/benchmarks/cases/006_softmax.meta.json b/benchmarks/cases/006_softmax.meta.json new file mode 100644 index 0000000..3ee827f --- /dev/null +++ b/benchmarks/cases/006_softmax.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Softmax activation on a vector", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 0.0320586, + 0.08714432, + 0.23688282, + 0.64391428 + ], + "max_instructions": 3 +} diff --git a/benchmarks/cases/007_matmul.meta.json b/benchmarks/cases/007_matmul.meta.json new file mode 100644 index 0000000..1de29ab --- /dev/null +++ b/benchmarks/cases/007_matmul.meta.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "description": "Matrix multiplication (2x2 matrices)", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": 30.0, + "max_instructions": 3 +} diff --git a/benchmarks/cases/008_dot.meta.json b/benchmarks/cases/008_dot.meta.json new file mode 100644 index 0000000..3126695 --- /dev/null +++ b/benchmarks/cases/008_dot.meta.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "description": "Dot product of two length-4 vectors", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": 30.0, + "max_instructions": 3 +} diff --git a/benchmarks/cases/009_maxpool.meta.json b/benchmarks/cases/009_maxpool.meta.json new file mode 100644 index 0000000..8e8cc4a --- /dev/null +++ b/benchmarks/cases/009_maxpool.meta.json @@ -0,0 +1,22 @@ +{ + "schema_version": 1, + "description": "1D MaxPool with kernel=2, stride=2", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 2.0, + 4.0 + ], + "max_instructions": 9, + "xfail": { + "stages": [ + "assemble", + "budget" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a1 # .Lmp_gt_1", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/010_exp_neg.meta.json b/benchmarks/cases/010_exp_neg.meta.json new file mode 100644 index 0000000..d2f7f0b --- /dev/null +++ b/benchmarks/cases/010_exp_neg.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Chained exponentiation and negation", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + -2.71828183, + -7.3890561, + -20.08553692, + -54.59815003 + ], + "max_instructions": 5 +} diff --git a/benchmarks/cases/011_multi_op_chain.meta.json b/benchmarks/cases/011_multi_op_chain.meta.json new file mode 100644 index 0000000..f0978e7 --- /dev/null +++ b/benchmarks/cases/011_multi_op_chain.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Multi-operation chain with three arithmetic ops", + "category": "complex", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "max_instructions": 5 +} diff --git a/benchmarks/cases/012_nn_pipeline.meta.json b/benchmarks/cases/012_nn_pipeline.meta.json new file mode 100644 index 0000000..c43f139 --- /dev/null +++ b/benchmarks/cases/012_nn_pipeline.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Typical NN layer: matmul + bias + relu", + "category": "nn", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 31.0, + 32.0, + 33.0, + 34.0 + ], + "max_instructions": 5 +} diff --git a/benchmarks/cases/013_for_sum.meta.json b/benchmarks/cases/013_for_sum.meta.json new file mode 100644 index 0000000..6ad84e6 --- /dev/null +++ b/benchmarks/cases/013_for_sum.meta.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "description": "For-loop accumulation (sum)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "x": 1, + "acc": 0 + }, + "input_registers": { + "x": "a0", + "acc": "a1" + }, + "expected_return": 4, + "max_instructions": 9, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/014_for_dot.meta.json b/benchmarks/cases/014_for_dot.meta.json new file mode 100644 index 0000000..b985177 --- /dev/null +++ b/benchmarks/cases/014_for_dot.meta.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "description": "Dot product simulated with for-loop", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "a": 2, + "b": 3, + "acc": 0 + }, + "input_registers": { + "a": "a0", + "b": "a1", + "acc": "a2" + }, + "expected_return": 24, + "max_instructions": 9, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/015_for_relu.meta.json b/benchmarks/cases/015_for_relu.meta.json new file mode 100644 index 0000000..4b3a036 --- /dev/null +++ b/benchmarks/cases/015_for_relu.meta.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "description": "ReLU activation inside a for-loop", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "x": 1, + "y": 0 + }, + "input_registers": { + "x": "a0", + "y": "a1" + }, + "expected_return": 4, + "max_instructions": 10, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/016_if_simple.meta.json b/benchmarks/cases/016_if_simple.meta.json new file mode 100644 index 0000000..78ace3c --- /dev/null +++ b/benchmarks/cases/016_if_simple.meta.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "description": "If-else branching (extended DSL parser required)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "a": 4, + "b": 1 + }, + "input_registers": { + "a": "a0", + "b": "a1" + }, + "expected_return": 5, + "max_instructions": 8, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a1 # if_then1", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/017_while_sum.dsl b/benchmarks/cases/017_while_sum.dsl index 8c417d8..b36eb20 100644 --- a/benchmarks/cases/017_while_sum.dsl +++ b/benchmarks/cases/017_while_sum.dsl @@ -1,5 +1,6 @@ # Sum with while loop (extended parser) while (i < 10): acc = add(acc, x) + i = add(i, 1) endwhile return acc diff --git a/benchmarks/cases/017_while_sum.meta.json b/benchmarks/cases/017_while_sum.meta.json new file mode 100644 index 0000000..47f3831 --- /dev/null +++ b/benchmarks/cases/017_while_sum.meta.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "description": "While-loop accumulation (extended DSL parser required)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "i": 0, + "x": 1, + "acc": 0 + }, + "input_registers": { + "i": "a0", + "x": "a1", + "acc": "a2" + }, + "expected_return": 10, + "max_instructions": 9, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: j # while_hdr1", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/018_nested_if.meta.json b/benchmarks/cases/018_nested_if.meta.json new file mode 100644 index 0000000..0faa34b --- /dev/null +++ b/benchmarks/cases/018_nested_if.meta.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "description": "Nested if-else branches (extended DSL parser required)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "a": 4, + "b": 1 + }, + "input_registers": { + "a": "a0", + "b": "a1" + }, + "expected_return": 5, + "max_instructions": 14, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a1 # if_then1", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/019_nested_loop.meta.json b/benchmarks/cases/019_nested_loop.meta.json new file mode 100644 index 0000000..94afc15 --- /dev/null +++ b/benchmarks/cases/019_nested_loop.meta.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "description": "Double-nested for-loops", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "x": 1, + "y": 1, + "acc": 0 + }, + "input_registers": { + "x": "a0", + "y": "a1", + "acc": "a2" + }, + "expected_return": 8, + "max_instructions": 14, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/020_constant_propagation.meta.json b/benchmarks/cases/020_constant_propagation.meta.json new file mode 100644 index 0000000..d589c0d --- /dev/null +++ b/benchmarks/cases/020_constant_propagation.meta.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "description": "Arithmetic with literal constants for constant folding test", + "category": "const", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": 20.0, + "max_instructions": 3 +} diff --git a/benchmarks/cases/021_dsl_if_else.meta.json b/benchmarks/cases/021_dsl_if_else.meta.json new file mode 100644 index 0000000..f0077a3 --- /dev/null +++ b/benchmarks/cases/021_dsl_if_else.meta.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "description": "If-else with multi-instruction branches and relu (extended parser required)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "a": 4, + "b": 1 + }, + "input_registers": { + "a": "a0", + "b": "a1" + }, + "expected_return": 10, + "max_instructions": 12, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a2 # if_then1", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/022_dsl_while_sum.dsl b/benchmarks/cases/022_dsl_while_sum.dsl index db83f7b..ff437b0 100644 --- a/benchmarks/cases/022_dsl_while_sum.dsl +++ b/benchmarks/cases/022_dsl_while_sum.dsl @@ -2,5 +2,6 @@ while (i < 5): t1 = mul(x, y) acc = add(acc, t1) + i = add(i, 1) endwhile return acc diff --git a/benchmarks/cases/022_dsl_while_sum.meta.json b/benchmarks/cases/022_dsl_while_sum.meta.json new file mode 100644 index 0000000..0a78272 --- /dev/null +++ b/benchmarks/cases/022_dsl_while_sum.meta.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "description": "While-loop with inner operations (extended parser required)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": { + "i": 0, + "x": 2, + "y": 3, + "acc": 0 + }, + "input_registers": { + "i": "a0", + "x": "a1", + "y": "a2", + "acc": "a3" + }, + "expected_return": 30, + "max_instructions": 10, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: j # while_hdr1", + "owner": "backend/asm-encoder", + "strict": false + } +} diff --git a/benchmarks/cases/023_large_chain.meta.json b/benchmarks/cases/023_large_chain.meta.json new file mode 100644 index 0000000..0595b27 --- /dev/null +++ b/benchmarks/cases/023_large_chain.meta.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "description": "Long 6-operation chain with mixed ops (add, sub, relu, mul, gelu, div)", + "category": "complex", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "max_instructions": 10 +} diff --git a/benchmarks/dsl_suite.py b/benchmarks/dsl_suite.py new file mode 100644 index 0000000..2fa119a --- /dev/null +++ b/benchmarks/dsl_suite.py @@ -0,0 +1,1531 @@ +"""Core implementation of the ScratchV DSL benchmark suite (topic 06). + +The suite discovers ``*.dsl`` cases under configurable roots, validates the +``*.meta.json`` contract, and evaluates each case through four stages: +compile, assemble, instruction budget and semantic golden check. It is +consumed by ``tests/test_dsl_suite.py`` (pytest gate) and by +``benchmarks/run_suite.py`` (JSON/Markdown report + exit code CLI). + +Design constraints: + - ``scratchv/**`` is read-only for this suite; compiler defects are + recorded as ``xfail`` declarations in case metadata, never fixed here. + - No silent pass: every stage exception is captured into a case outcome. +""" + +from __future__ import annotations + +import datetime +import json +import re +import shutil +import tempfile +import time +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any, Sequence + +import numpy as np + +from scratchv.backend._asm_parser import parse_asm + + +DEFAULT_ROOTS: tuple[str, ...] = ("benchmarks/cases", "tests/stress") +SCHEMA_VERSION: int = 1 +SUITE_NAME: str = "dsl-benchmark" + +STAGE_COMPILE: str = "compile" +STAGE_ASSEMBLE: str = "assemble" +STAGE_BUDGET: str = "budget" +STAGE_SEMANTIC: str = "semantic" +STAGE_META: str = "meta" + +ALL_STAGES: tuple[str, ...] = ( + STAGE_COMPILE, + STAGE_ASSEMBLE, + STAGE_BUDGET, + STAGE_SEMANTIC, +) + +ASSERT_COMPILE_OK: str = "compile_ok" +ASSERT_ASM_ENCODABLE: str = "asm_encodable" +ASSERT_INST_BUDGET: str = "inst_budget" +ASSERT_SEMANTIC_GOLDEN: str = "semantic_golden" + +DEFAULT_ASSERTIONS: tuple[str, ...] = ( + ASSERT_COMPILE_OK, + ASSERT_ASM_ENCODABLE, + ASSERT_INST_BUDGET, + ASSERT_SEMANTIC_GOLDEN, +) + +ORACLE_INTERPRETER: str = "interpreter" +ORACLE_EXECUTION: str = "execution" +ORACLE_NONE: str = "none" + +FLOW_LINEAR: str = "linear" +FLOW_CONTROL: str = "control" + +STATUS_PASS: str = "pass" +STATUS_FAIL: str = "fail" +STATUS_XFAIL: str = "xfail" +STATUS_XPASS: str = "xpass" +STATUS_SKIP: str = "skip" + +ALLOWED_CATEGORIES: frozenset[str] = frozenset( + {"arith", "nn", "control", "complex", "const", "stress"} +) + +_CONTROL_KEYWORDS: tuple[str, ...] = ( + "if (", "else:", "endif", "while (", "endwhile", "for ", "endfor", +) + +_REGISTER_RE = re.compile(r"^(a[0-7]|s[0-9]|s1[01])$") + +_KNOWN_META_FIELDS: frozenset[str] = frozenset( + { + "schema_version", "description", "category", "flow", "oracle", + "inputs", "input_registers", "expected_output_type", + "expected_return", "rtol", "atol", "assertions", + "max_instructions", "timeout_s", "xfail", + } +) + +_OP_PATTERN = ( + r"\b(add|sub|mul|div|relu|gelu|exp|neg|" + r"matmul|dot|maxpool|softmax)\(([^)]+)" +) + +_INPUT_KEYWORDS: frozenset[str] = frozenset( + { + "add", "sub", "mul", "div", "relu", "gelu", "exp", "neg", + "matmul", "dot", "maxpool", "softmax", "return", "for", + "endfor", "if", "else", "endif", "while", "endwhile", + "m", "n", "k", "rows", "cols", "inner", "len", + "axis", "kernel", "stride", "padding", + "out_channels", "kernel_size", + "transA", "transB", "alpha", "beta", + "i", "j", "t1", "t2", "t3", "t4", + "acc", "sum", "tmp", + } +) + + +class DSLSuiteError(Exception): + """Base error for suite configuration problems.""" + + +class CaseSpecError(DSLSuiteError): + """Raised when a case specification cannot be loaded at all.""" + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class XFailSpec: + """Declared, located and owned expected-failure description.""" + + stages: tuple[str, ...] + reason: str + owner: str + strict: bool = False + + +@dataclass(frozen=True) +class CaseSpec: + """Immutable case specification produced at discovery time.""" + + case_id: str + pytest_id: str + name: str + root: Path + dsl_path: Path + meta_path: Path | None + expected_path: Path | None + description: str + category: str + flow: str + oracle: str + inputs: dict[str, Any] + input_registers: dict[str, str] + expected_return: Any + expected_text: str + rtol: float + atol: float + assertions: tuple[str, ...] + max_instructions: int | None + timeout_s: float + xfail: XFailSpec | None + skip_reason: str | None + meta_errors: tuple[str, ...] + warnings: tuple[str, ...] = () + + +@dataclass +class CompileOutcome: + ok: bool + asm_text: str + output_path: Path | None + ir_instruction_count: int + duration_s: float + error: str | None + + +@dataclass +class AssembleOutcome: + ok: bool + binary_len: int + instruction_count: int | None + duration_s: float + error: str | None + + +@dataclass +class BudgetOutcome: + ok: bool | None + instruction_count: int | None + limit: int | None + skipped: bool + error: str | None + + +@dataclass +class SemanticOutcome: + ok: bool | None + oracle: str + expected: Any + actual: Any + duration_s: float + blocked_reason: str | None + error: str | None + + +@dataclass +class CaseOutcome: + case_id: str + pytest_id: str + status: str + compile: CompileOutcome + assemble: AssembleOutcome | None + budget: BudgetOutcome | None + semantic: SemanticOutcome | None + error_stage: str | None + error: str | None + xfail: XFailSpec | None + + +@dataclass +class SuiteReport: + """Aggregated suite result with JSON/Markdown rendering.""" + + results: list[CaseOutcome] + roots: tuple[str, ...] + compiler: dict[str, str] + generated_at: str + specs: dict[str, CaseSpec] = field(default_factory=dict) + + @property + def pass_count(self) -> int: + return sum(1 for r in self.results if r.status == STATUS_PASS) + + @property + def xfail_count(self) -> int: + return sum(1 for r in self.results if r.status == STATUS_XFAIL) + + @property + def xpass_count(self) -> int: + return sum(1 for r in self.results if r.status == STATUS_XPASS) + + @property + def fail_count(self) -> int: + return sum(1 for r in self.results if r.status == STATUS_FAIL) + + @property + def skip_count(self) -> int: + return sum(1 for r in self.results if r.status == STATUS_SKIP) + + def summary(self) -> dict[str, int]: + return { + "total": len(self.results), + "passed": self.pass_count, + "xfailed": self.xfail_count, + "xpassed": self.xpass_count, + "failed": self.fail_count, + "skipped": self.skip_count, + } + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "suite": SUITE_NAME, + "generated_at": self.generated_at, + "roots": list(self.roots), + "compiler": dict(self.compiler), + "summary": self.summary(), + "results": [self._result_to_dict(r) for r in self.results], + } + + def save_json(self, path: str | Path) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.to_dict(), indent=2) + "\n") + + def to_markdown(self) -> str: + summary = self.summary() + lines = [ + "# ScratchV DSL Benchmark Suite", + "", + f"- generated_at: {self.generated_at}", + f"- roots: {', '.join(self.roots)}", + "- compiler: " + + ", ".join(f"{k}={v}" for k, v in self.compiler.items()), + "", + "## Summary", + "", + "| total | passed | xfailed | xpassed | failed | skipped |", + "|-------|--------|---------|---------|--------|---------|", + "| {total} | {passed} | {xfailed} | {xpassed} | {failed} | " + "{skipped} |".format(**summary), + "", + "## Cases", + "", + "| case_id | status | compile | asm | inst | semantic | oracle | " + "xfail reason |", + "|---------|--------|---------|-----|------|----------|--------|" + "-------------|", + ] + for r in self.results: + spec = self.specs.get(r.case_id) + oracle = spec.oracle if spec else "-" + reason = r.xfail.reason if r.xfail else "" + reason = reason.replace("|", "\\|") + lines.append( + f"| {r.case_id} | {r.status} | {_mark(r.compile.ok)} | " + f"{_mark(r.assemble.ok) if r.assemble else '-'} | " + f"{_mark(r.budget.ok) if r.budget else '-'} | " + f"{_mark(r.semantic.ok) if r.semantic else '-'} | " + f"{oracle} | {reason} |" + ) + + problems = [r for r in self.results if r.status == STATUS_FAIL] + lines += ["", "## Failures", ""] + if problems: + groups: dict[str, list[CaseOutcome]] = {} + for result in problems: + groups.setdefault(result.error_stage or "unknown", []).append( + result + ) + for stage, staged in groups.items(): + lines.append(f"### {stage}") + lines.append("") + for result in staged: + lines.append(f"- **{result.case_id}**: {result.error}") + lines.append("") + else: + lines.append("No hard failures.") + + blocked_or_xfail = [ + r for r in self.results + if r.status in (STATUS_XFAIL, STATUS_XPASS) + ] + lines += ["", "## Expected failures / unexpected passes", ""] + if blocked_or_xfail: + for r in blocked_or_xfail: + which = ( + r.error_stage if r.status == STATUS_XFAIL + else "declared stages now pass" + ) + lines.append(f"- {r.case_id} [{r.status}] ({which})") + else: + lines.append("None.") + + lines += [ + "", + "## 数据缺口与缺陷登记", + "", + "- C1 backend/asm-encoder: symbolic branch targets are emitted in " + "comments; affected assemble semantics stages are xfailed.", + "- C2 backend/asm-emitter: constant materialisation emits invalid " + "`mv rd, imm`.", + "- C3 backend/regalloc: spill/reload stack slots read uninitialised " + "frames (reg_pressure_32 returns 178 instead of 64).", + "- C4 backend/op-lowering: matmul/dot/maxpool/softmax are not " + "lowered; execution oracle would expose it.", + "- C5 backend/numeric semantics: float division/activations are " + "executed as integer instructions.", + "- C6 backend/regalloc: input variable to physical register mapping " + "has no published contract; declared in `input_registers`.", + "- B1 verification: `DSLInterpreter` does not implement control " + "flow; control cases use the execution oracle.", + "- D1/D2: five control cases had no golden and two while loops had " + "no induction step; fixed in this suite's data files.", + "", + ] + return "\n".join(lines) + + def save_markdown(self, path: str | Path) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(self.to_markdown()) + + def _result_to_dict(self, r: CaseOutcome) -> dict[str, Any]: + spec = self.specs.get(r.case_id) + skipped = r.status == STATUS_SKIP + asm_count: int | None = None + if r.assemble is not None: + asm_count = r.assemble.instruction_count + elif r.budget is not None: + asm_count = r.budget.instruction_count + + expected: Any = None + actual: Any = None + if r.semantic is not None: + expected = r.semantic.expected + actual = r.semantic.actual + if expected is None and spec is not None: + expected = ( + spec.expected_return + if spec.expected_return is not None + else (spec.expected_text or None) + ) + + return { + "case_id": r.case_id, + "pytest_id": r.pytest_id, + "category": spec.category if spec else "unknown", + "flow": spec.flow if spec else "unknown", + "oracle": spec.oracle if spec else "unknown", + "status": r.status, + "stages": { + "compile_ok": None if skipped else r.compile.ok, + "asm_encodable": None if skipped or r.assemble is None + else r.assemble.ok, + "inst_budget_ok": None if skipped or r.budget is None + else r.budget.ok, + "semantic_ok": None if skipped or r.semantic is None + else r.semantic.ok, + }, + "metrics": { + "ir_instructions": None if skipped + else r.compile.ir_instruction_count, + "asm_instructions": None if skipped else asm_count, + "compile_time_s": 0.0 if skipped else r.compile.duration_s, + "assemble_time_s": 0.0 if skipped or r.assemble is None + else r.assemble.duration_s, + "semantic_time_s": 0.0 if skipped or r.semantic is None + else r.semantic.duration_s, + }, + "expected": _jsonable(expected), + "actual": None if skipped else _jsonable(actual), + "xfail": _xfail_to_dict(r.xfail), + "error_stage": r.error_stage, + "error": r.error, + "warnings": list(spec.warnings) if spec else [], + } + + +def _mark(value: bool | None) -> str: + if value is None: + return "-" + return "ok" if value else "FAIL" + + +def _xfail_to_dict(spec: XFailSpec | None) -> dict[str, Any] | None: + if spec is None: + return None + return { + "stages": list(spec.stages), + "reason": spec.reason, + "owner": spec.owner, + "strict": spec.strict, + } + + +# --------------------------------------------------------------------------- +# Value helpers +# --------------------------------------------------------------------------- + + +def infer_flow(source: str) -> str: + """Return ``control`` when the source uses extended control flow.""" + return ( + FLOW_CONTROL + if any(keyword in source for keyword in _CONTROL_KEYWORDS) + else FLOW_LINEAR + ) + + +def default_inputs(source: str, *, fill: float = 1.0) -> dict[str, np.ndarray]: + """Default vector inputs for the interpreter oracle (``[1, 2, 3, 4]``).""" + base = np.array([fill, 2.0 * fill, 3.0 * fill, 4.0 * fill], dtype=np.float32) + names: set[str] = set() + for match in re.finditer(_OP_PATTERN, source): + for arg in match.group(2).split(","): + arg = arg.strip().split(":")[0].strip() + if arg and not arg[0].isdigit(): + names.add(arg) + names = { + name for name in names + if name.lower() not in _INPUT_KEYWORDS and not name.startswith("_") + } + return {name: base.copy() for name in names} + + +def _as_vector(value: Any) -> np.ndarray | None: + if value is None or isinstance(value, (bool, np.bool_)): + return None + if isinstance(value, np.ndarray): + try: + return np.asarray(value, dtype=float).ravel() + except (TypeError, ValueError): + return None + if isinstance(value, (int, float, np.integer, np.floating)): + return np.array([float(value)], dtype=float) + if isinstance(value, (list, tuple)): + try: + return np.asarray(value, dtype=float).ravel() + except (TypeError, ValueError): + return None + if isinstance(value, str): + text = value.strip() + if not text: + return None + parts = [p for p in re.split(r"[,;\s]+", text.strip("[]")) if p] + try: + return np.array([float(p) for p in parts], dtype=float) + except ValueError: + return None + return None + + +def compare_values( + actual: Any, + expected: Any, + *, + rtol: float = 1e-3, + atol: float = 1e-6, +) -> bool: + """Numerically compare scalars, vectors or stringified vectors.""" + if isinstance(actual, str) and isinstance(expected, str): + if actual.strip() == expected.strip(): + return True + vector_a = _as_vector(actual) + vector_e = _as_vector(expected) + if vector_a is None or vector_e is None: + return False + if vector_a.shape != vector_e.shape: + return False + if vector_a.size == 0: + return True + return bool(np.allclose( + vector_a, vector_e, rtol=rtol, atol=atol, equal_nan=True, + )) + + +def count_asm_instructions(asm_text: str) -> int: + """Count literal instructions with the shared ``parse_asm`` semantics.""" + return sum( + 1 for line in parse_asm(asm_text) + if line.opcode and not line.is_directive + ) + + +def _stringify(value: Any) -> Any: + if isinstance(value, np.ndarray): + return np.array2string(value, precision=6, suppress_small=True) + if isinstance(value, np.generic): + return value.item() + return value + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, np.ndarray): + return _stringify(value) + if isinstance(value, np.generic): + return value.item() + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, dict): + return {str(k): _jsonable(v) for k, v in value.items()} + return str(value) + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float, np.integer, np.floating)) and not ( + isinstance(value, bool) + ) + + +# --------------------------------------------------------------------------- +# Metadata loading and validation +# --------------------------------------------------------------------------- + + +def infer_category(name: str, source: str, flow: str, root: Path) -> str: + if root.name == "stress" or name.startswith("stress"): + return "stress" + if flow == FLOW_CONTROL: + return "control" + if "constant" in name: + return "const" + lowered = source.lower() + if any( + op in lowered + for op in ("matmul(", "dot(", "relu(", "gelu(", "softmax(", "maxpool(") + ): + return "nn" + if len(re.findall(r"\w+\s*=\s*\w+\(", source)) >= 3: + return "complex" + if any(op in lowered for op in ("add(", "sub(", "mul(", "div(")): + return "arith" + return "unknown" + + +def _parse_xfail(value: Any, errors: list[str]) -> XFailSpec | None: + if value is None: + return None + if not isinstance(value, dict): + errors.append("xfail must be a JSON object") + return None + stages_raw = value.get("stages") + if isinstance(stages_raw, str): + stages: tuple[str, ...] = (stages_raw,) + elif isinstance(stages_raw, list) and all( + isinstance(s, str) for s in stages_raw + ): + stages = tuple(stages_raw) + else: + errors.append( + "xfail.stages must be a string or a list of strings" + ) + stages = () + if not stages: + errors.append("xfail.stages must not be empty") + for stage in stages: + if stage not in ALL_STAGES: + errors.append( + f"xfail.stages contains unknown stage '{stage}' " + f"(expected one of {list(ALL_STAGES)})" + ) + reason = value.get("reason", "") + owner = value.get("owner", "") + if not isinstance(reason, str) or not reason.strip(): + errors.append("xfail.reason must be a non-empty string") + if not isinstance(owner, str) or not owner.strip(): + errors.append("xfail.owner must be a non-empty string") + strict = value.get("strict", False) + if not isinstance(strict, bool): + errors.append("xfail.strict must be a boolean") + strict = False + return XFailSpec( + stages=stages, + reason=reason if isinstance(reason, str) else "", + owner=owner if isinstance(owner, str) else "", + strict=strict, + ) + + +def _validate_inputs(value: Any, errors: list[str]) -> dict[str, Any]: + if not isinstance(value, dict): + errors.append("inputs must be a JSON object") + return {} + for key, item in value.items(): + if _is_number(item): + continue + if ( + isinstance(item, list) + and all(_is_number(element) for element in item) + ): + continue + errors.append( + f"inputs['{key}'] must be a number or a one-dimensional list " + "of numbers" + ) + return dict(value) + + +def _validate_input_registers( + value: Any, + inputs: dict[str, Any], + oracle: str, + errors: list[str], +) -> dict[str, str]: + if not isinstance(value, dict): + errors.append("input_registers must be a JSON object") + return {} + registers: dict[str, str] = {} + for key, item in value.items(): + if not isinstance(item, str): + errors.append( + f"input_registers['{key}'] must be a register name string" + ) + continue + if not _REGISTER_RE.match(item): + errors.append( + f"input_registers['{key}'] = '{item}' does not match " + r"^(a[0-7]|s[0-9]|s1[01])$" + ) + continue + registers[str(key)] = item + if oracle == ORACLE_EXECUTION and inputs and not registers: + errors.append( + "input_registers is required when oracle=execution and inputs " + "is non-empty" + ) + if registers and inputs and set(registers) != set(inputs): + errors.append( + "input_registers keys must match inputs keys " + f"(inputs={sorted(inputs)}, registers={sorted(registers)})" + ) + return registers + + +def _validate_assertions(value: Any, errors: list[str]) -> tuple[str, ...]: + if value is None: + return DEFAULT_ASSERTIONS + if not isinstance(value, list) or not all( + isinstance(item, str) for item in value + ): + errors.append("assertions must be a list of strings") + return DEFAULT_ASSERTIONS + unknown = [item for item in value if item not in DEFAULT_ASSERTIONS] + if unknown: + errors.append(f"assertions contains unknown entries: {unknown}") + valid = tuple(item for item in value if item in DEFAULT_ASSERTIONS) + if not valid: + errors.append("assertions must contain at least one known assertion") + return DEFAULT_ASSERTIONS + return valid + + +def _validate_expected_return( + value: Any, errors: list[str] +) -> Any: + if value is None: + return None + if _is_number(value): + return value + if ( + isinstance(value, list) + and all(_is_number(element) for element in value) + ): + return value + errors.append("expected_return must be a number or a list of numbers") + return None + + +def load_case_spec(dsl_path: Path, root: Path) -> CaseSpec: + """Load and validate one case; contract violations land in ``meta_errors``.""" + dsl_path = Path(dsl_path) + root = Path(root) + relative = dsl_path.relative_to(root) + rel_stem = relative.with_suffix("").as_posix() + case_id = f"{root.name}/{rel_stem}" + pytest_id = case_id.replace("/", "-") + + source = dsl_path.read_text() + inferred_flow = infer_flow(source) + errors: list[str] = [] + warnings: list[str] = [] + + meta_path = dsl_path.with_suffix(".meta.json") + expected_path = dsl_path.with_suffix(".expected") + expected_text = "" + if expected_path.exists(): + expected_text = expected_path.read_text().strip() + + meta: dict[str, Any] = {} + if meta_path.exists(): + try: + loaded = json.loads(meta_path.read_text()) + except json.JSONDecodeError as exc: + errors.append(f"meta.json parse error: {exc}") + loaded = {} + if not isinstance(loaded, dict): + errors.append("meta.json must decode to a JSON object") + loaded = {} + meta = loaded + else: + meta_path = None + errors.append("meta.json not found") + + for key in meta: + if key not in _KNOWN_META_FIELDS: + warnings.append(f"unknown meta field: {key}") + + schema_version = meta.get("schema_version", SCHEMA_VERSION) + if schema_version != SCHEMA_VERSION: + errors.append( + f"schema_version {schema_version!r} is not supported " + f"(expected {SCHEMA_VERSION})" + ) + + description = meta.get("description") + if description is None: + desc_path = dsl_path.with_suffix(".desc") + if desc_path.exists(): + description = desc_path.read_text().strip() + if not isinstance(description, str) or not description.strip(): + errors.append("description must be a non-empty string") + description = description if isinstance(description, str) else "" + + meta_flow = meta.get("flow") + if meta_flow is None: + flow = inferred_flow + elif meta_flow not in (FLOW_LINEAR, FLOW_CONTROL): + errors.append(f"flow must be '{FLOW_LINEAR}' or '{FLOW_CONTROL}'") + flow = inferred_flow + else: + flow = meta_flow + if flow != inferred_flow: + errors.append( + f"flow conflict: meta declares '{flow}' but the source " + f"implies '{inferred_flow}'" + ) + + inferred_category = infer_category( + dsl_path.stem, source, flow, root, + ) + category = meta.get("category", inferred_category) + if not isinstance(category, str) or category not in ALLOWED_CATEGORIES: + errors.append( + f"category {category!r} is not one of " + f"{sorted(ALLOWED_CATEGORIES)}" + ) + category = inferred_category + + oracle = meta.get("oracle") + if oracle is None: + oracle = ( + ORACLE_EXECUTION if flow == FLOW_CONTROL else ORACLE_INTERPRETER + ) + warnings.append(f"oracle not declared; inferred '{oracle}'") + elif oracle not in (ORACLE_INTERPRETER, ORACLE_EXECUTION, ORACLE_NONE): + errors.append( + f"oracle {oracle!r} is not one of " + f"'{ORACLE_INTERPRETER}', '{ORACLE_EXECUTION}', '{ORACLE_NONE}'" + ) + oracle = ORACLE_NONE + if oracle == ORACLE_INTERPRETER and flow != FLOW_LINEAR: + errors.append( + "oracle/flow conflict: oracle=interpreter requires flow=linear" + ) + + inputs = _validate_inputs(meta.get("inputs", {}), errors) + input_registers = _validate_input_registers( + meta.get("input_registers", {}), inputs, oracle, errors, + ) + + expected_output_type = meta.get("expected_output_type", "return_value") + if expected_output_type != "return_value": + errors.append( + "expected_output_type must be 'return_value' for this schema " + f"version (got {expected_output_type!r})" + ) + + expected_return = _validate_expected_return( + meta.get("expected_return"), errors, + ) + if ( + oracle != ORACLE_NONE + and expected_return is None + and not (oracle == ORACLE_INTERPRETER and expected_text) + ): + errors.append( + "expected_return is required when oracle != none " + "(interpreter cases may fall back to a non-empty .expected)" + ) + + rtol = meta.get("rtol", 1e-3) + atol = meta.get("atol", 1e-6) + for name, value in (("rtol", rtol), ("atol", atol)): + if not _is_number(value) or float(value) < 0: + errors.append(f"{name} must be a non-negative number") + if name == "rtol": + rtol = 1e-3 + else: + atol = 1e-6 + + assertions = _validate_assertions(meta.get("assertions"), errors) + + max_instructions = meta.get("max_instructions") + if max_instructions is not None: + if ( + not isinstance(max_instructions, int) + or isinstance(max_instructions, bool) + or max_instructions <= 0 + ): + errors.append("max_instructions must be a positive integer") + max_instructions = None + + timeout_s = meta.get("timeout_s", 30.0) + if not _is_number(timeout_s) or float(timeout_s) <= 0: + errors.append("timeout_s must be a positive number") + timeout_s = 30.0 + + xfail = _parse_xfail(meta.get("xfail"), errors) + if oracle == ORACLE_NONE: + if xfail is None: + errors.append("oracle=none requires an explicit xfail declaration") + elif STAGE_SEMANTIC not in xfail.stages: + errors.append( + "oracle=none requires xfail.stages to include 'semantic'" + ) + + return CaseSpec( + case_id=case_id, + pytest_id=pytest_id, + name=dsl_path.stem, + root=root, + dsl_path=dsl_path, + meta_path=meta_path, + expected_path=expected_path if expected_path.exists() else None, + description=description, + category=category, + flow=flow, + oracle=oracle, + inputs=inputs, + input_registers=input_registers, + expected_return=expected_return, + expected_text=expected_text, + rtol=float(rtol), + atol=float(atol), + assertions=assertions, + max_instructions=max_instructions, + timeout_s=float(timeout_s), + xfail=xfail, + skip_reason=None, + meta_errors=tuple(errors), + warnings=tuple(warnings), + ) + + +def discover_cases( + roots: Sequence[str | Path] = DEFAULT_ROOTS, + *, + verbose: bool = False, +) -> list[CaseSpec]: + """Discover cases under *roots* following the suite's skip rules.""" + cases: list[CaseSpec] = [] + for root in roots: + root_path = Path(root) + if not root_path.is_dir(): + if verbose: + print(f"warning: root not found, skipping: {root_path}") + continue + for dsl_path in root_path.rglob("*.dsl"): + relative = dsl_path.relative_to(root_path) + if dsl_path.name.startswith("_"): + continue + if "fixtures" in relative.parts[:-1]: + continue + try: + spec = load_case_spec(dsl_path, root_path) + except Exception as exc: # pragma: no cover - defensive + spec = _broken_spec(dsl_path, root_path, exc) + skip_path = dsl_path.with_suffix(".skip") + if skip_path.exists(): + reason = skip_path.read_text().strip() + spec = replace( + spec, skip_reason=reason or "skipped by marker file", + ) + cases.append(spec) + cases.sort(key=lambda case: case.case_id) + return cases + + +def _broken_spec(dsl_path: Path, root: Path, exc: Exception) -> CaseSpec: + relative = dsl_path.relative_to(root) + rel_stem = relative.with_suffix("").as_posix() + case_id = f"{root.name}/{rel_stem}" + return CaseSpec( + case_id=case_id, + pytest_id=case_id.replace("/", "-"), + name=dsl_path.stem, + root=root, + dsl_path=dsl_path, + meta_path=None, + expected_path=None, + description="", + category="unknown", + flow=FLOW_LINEAR, + oracle=ORACLE_NONE, + inputs={}, + input_registers={}, + expected_return=None, + expected_text="", + rtol=1e-3, + atol=1e-6, + assertions=DEFAULT_ASSERTIONS, + max_instructions=None, + timeout_s=30.0, + xfail=None, + skip_reason=None, + meta_errors=(f"case loading failed: {type(exc).__name__}: {exc}",), + ) + + +# --------------------------------------------------------------------------- +# Execution oracle +# --------------------------------------------------------------------------- + + +_REG_NUMS: dict[str, int] = { + "a0": 10, "a1": 11, "a2": 12, "a3": 13, + "a4": 14, "a5": 15, "a6": 16, "a7": 17, + "s0": 8, "s1": 9, + "s2": 18, "s3": 19, "s4": 20, "s5": 21, + "s6": 22, "s7": 23, "s8": 24, "s9": 25, + "s10": 26, "s11": 27, +} + + +class ExecutionOracle: + """Execute assembled RISC-V and read the ``a0`` return register.""" + + def __init__(self, *, mem_size: int = 128 * 1024 * 1024) -> None: + self.mem_size = mem_size + + def available(self) -> bool: + try: + from scratchv.simulator.tinyfive import ProfiledMachine + except ImportError: + return False + return bool(ProfiledMachine(mem_size=4096).available) + + def execute( + self, + asm_text: str, + input_registers: dict[str, str], + inputs: dict[str, object], + *, + timeout_s: float = 5.0, + ) -> SemanticOutcome: + started = time.perf_counter() + + def elapsed() -> float: + return time.perf_counter() - started + + if not self.available(): + return SemanticOutcome( + ok=None, oracle=ORACLE_EXECUTION, expected=None, actual=None, + duration_s=0.0, blocked_reason="tinyfive not installed", + error=None, + ) + try: + from scratchv.backend.riscv_encoder import assemble_to_binary + from scratchv.simulator.tinyfive import ProfiledMachine + + binary = assemble_to_binary(asm_text) + if not binary: + return SemanticOutcome( + ok=False, oracle=ORACLE_EXECUTION, expected=None, + actual=None, duration_s=elapsed(), + blocked_reason=None, + error="assembler produced empty binary", + ) + words = [ + int.from_bytes(binary[i:i + 4], "little") + for i in range(0, len(binary), 4) + ] + machine = ProfiledMachine(mem_size=self.mem_size) + machine.load_binary(words, origin=0) + for name, register in input_registers.items(): + value = float(inputs.get(name, 0)) + if value != int(value): + return SemanticOutcome( + ok=None, oracle=ORACLE_EXECUTION, expected=None, + actual=None, duration_s=elapsed(), + blocked_reason=( + "non-integer input not supported by integer " + "register semantics" + ), + error=None, + ) + machine.set_reg(_REG_NUMS[register], int(value)) + machine.run(instructions=len(words), start=0, strict=True) + actual = machine.get_reg(10) + return SemanticOutcome( + ok=None, oracle=ORACLE_EXECUTION, expected=None, + actual=actual, duration_s=elapsed(), + blocked_reason=None, error=None, + ) + except Exception as exc: + return SemanticOutcome( + ok=False, oracle=ORACLE_EXECUTION, expected=None, actual=None, + duration_s=elapsed(), blocked_reason=None, + error=f"{type(exc).__name__}: {exc}", + ) + + +def _first_bad_asm_line( + asm_text: str, +) -> tuple[int, str, str] | None: + from scratchv.backend.riscv_encoder import RISCVAEncoder + + for index, line in enumerate(asm_text.splitlines()): + stripped = line.strip() + if not stripped or stripped.startswith((".", "#")) or ":" in stripped: + continue + try: + RISCVAEncoder().assemble(stripped + "\n") + except Exception as exc: + return index + 1, stripped, f"{type(exc).__name__}: {exc}" + return None + + +# --------------------------------------------------------------------------- +# Suite runner +# --------------------------------------------------------------------------- + + +def _empty_compile_outcome() -> CompileOutcome: + return CompileOutcome( + ok=False, asm_text="", output_path=None, + ir_instruction_count=0, duration_s=0.0, error=None, + ) + + +class DSLSuiteRunner: + """Discover, execute and grade DSL benchmark cases.""" + + def __init__( + self, + roots: tuple[str | Path, ...] = DEFAULT_ROOTS, + *, + workdir: str | Path | None = None, + backend: str = "riscv", + optimize_level: str = "all", + reg_alloc: str = "linear", + timeout_s: float = 30.0, + strict_xfail: bool = False, + verbose: bool = False, + ) -> None: + self._roots = tuple(Path(root) for root in roots) + self.root_strings: tuple[str, ...] = tuple(str(root) for root in roots) + if workdir is None: + self.workdir = Path(tempfile.mkdtemp(prefix="dsl_suite_")) + self._owns_workdir = True + else: + self.workdir = Path(workdir) + self.workdir.mkdir(parents=True, exist_ok=True) + self._owns_workdir = False + self.backend = backend + self.optimize_level = optimize_level + self.reg_alloc = reg_alloc + self.timeout_s = timeout_s + self.strict_xfail = strict_xfail + self.verbose = verbose + self._cases: list[CaseSpec] | None = None + + @property + def compiler_info(self) -> dict[str, str]: + return { + "backend": self.backend, + "optimize_level": self.optimize_level, + "reg_alloc": self.reg_alloc, + } + + def discover(self) -> list[CaseSpec]: + if self._cases is None: + self._cases = discover_cases(self._roots, verbose=self.verbose) + return list(self._cases) + + def cleanup(self) -> None: + if self._owns_workdir: + shutil.rmtree(self.workdir, ignore_errors=True) + + def compile_case(self, spec: CaseSpec) -> CompileOutcome: + out_path = self.workdir / f"{spec.pytest_id}.s" + started = time.perf_counter() + asm_text = "" + ok = False + error: str | None = None + output_path: Path | None = None + try: + from scratchv.compiler import CompilerConfig, CompilerDriver + + driver = CompilerDriver( + CompilerConfig( + backend=self.backend, + optimize_level=self.optimize_level, + reg_alloc=self.reg_alloc, + ) + ) + result = driver.compile(str(spec.dsl_path), str(out_path)) + asm_text = result.output_text or "" + ok = bool(result.success and asm_text) + if ok: + output_path = out_path + else: + messages = list(result.errors) or ["compilation failed"] + error = "; ".join(messages) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + duration = time.perf_counter() - started + return CompileOutcome( + ok=ok, + asm_text=asm_text, + output_path=output_path, + ir_instruction_count=self._count_ir_instructions(spec), + duration_s=duration, + error=error, + ) + + def _count_ir_instructions(self, spec: CaseSpec) -> int: + try: + source = spec.dsl_path.read_text() + except OSError: + return 0 + program = None + try: + from scratchv.frontend.dsl_extended import ExtendedDSLParser + + program = ExtendedDSLParser().parse(source) + except Exception: + try: + from scratchv.frontend.dsl_parser import DSLParser + + program = DSLParser().parse(source) + except Exception: + return 0 + return sum( + 1 + for function in program.functions + for block in function.blocks + for _ in block.instructions + ) + + def assemble_asm(self, asm_text: str) -> AssembleOutcome: + started = time.perf_counter() + binary: bytearray | None = None + error: str | None = None + try: + from scratchv.backend.riscv_encoder import assemble_to_binary + + binary = assemble_to_binary(asm_text) + except Exception as exc: + detail = _first_bad_asm_line(asm_text) + if detail is not None: + line_no, bad_line, _ = detail + error = ( + f"{type(exc).__name__}: {exc} at line {line_no}: " + f"{bad_line!r}" + ) + else: + error = f"{type(exc).__name__}: {exc}" + duration = time.perf_counter() - started + instruction_count: int | None + try: + instruction_count = count_asm_instructions(asm_text) + except Exception: + instruction_count = None + return AssembleOutcome( + ok=bool(binary), + binary_len=len(binary) if binary else 0, + instruction_count=instruction_count, + duration_s=duration, + error=error, + ) + + def check_instruction_budget( + self, spec: CaseSpec, asm_text: str, + ) -> BudgetOutcome: + count = count_asm_instructions(asm_text) + if spec.max_instructions is None: + return BudgetOutcome( + ok=None, instruction_count=count, limit=None, + skipped=True, error=None, + ) + ok = count <= spec.max_instructions + return BudgetOutcome( + ok=ok, + instruction_count=count, + limit=spec.max_instructions, + skipped=False, + error=None if ok else f"{count} > {spec.max_instructions}", + ) + + def evaluate_semantics( + self, spec: CaseSpec, compile_outcome: CompileOutcome, + ) -> SemanticOutcome: + if spec.oracle == ORACLE_NONE: + return SemanticOutcome( + ok=None, oracle=ORACLE_NONE, expected=spec.expected_return, + actual=None, duration_s=0.0, + blocked_reason="oracle=none", error=None, + ) + if spec.oracle == ORACLE_INTERPRETER: + return self._interpreter_semantics(spec) + return self._execution_semantics(spec, compile_outcome) + + def _interpreter_semantics(self, spec: CaseSpec) -> SemanticOutcome: + started = time.perf_counter() + source = spec.dsl_path.read_text() + if spec.inputs: + inputs = { + name: np.asarray(value, dtype=np.float32) + for name, value in spec.inputs.items() + } + else: + inputs = default_inputs(source) + try: + from scratchv.verification.verifier import DSLInterpreter + + result = DSLInterpreter().run(source, inputs) + actual = _stringify(result) + expected = ( + spec.expected_return + if spec.expected_return is not None + else spec.expected_text + ) + ok = compare_values( + result, expected, rtol=spec.rtol, atol=spec.atol, + ) + error = None + if not ok: + error = ( + f"semantic mismatch: expected {expected!r}, " + f"actual {actual!r}" + ) + return SemanticOutcome( + ok=ok, oracle=ORACLE_INTERPRETER, expected=expected, + actual=actual, duration_s=time.perf_counter() - started, + blocked_reason=None, error=error, + ) + except Exception as exc: + return SemanticOutcome( + ok=False, oracle=ORACLE_INTERPRETER, + expected=spec.expected_return, actual=None, + duration_s=time.perf_counter() - started, + blocked_reason=None, error=f"{type(exc).__name__}: {exc}", + ) + + def _execution_semantics( + self, spec: CaseSpec, compile_outcome: CompileOutcome, + ) -> SemanticOutcome: + oracle = ExecutionOracle() + outcome = oracle.execute( + compile_outcome.asm_text, + spec.input_registers, + spec.inputs, + timeout_s=spec.timeout_s, + ) + if outcome.ok is None and ( + outcome.blocked_reason or outcome.error + ): + return replace(outcome, expected=spec.expected_return) + ok = compare_values( + outcome.actual, spec.expected_return, + rtol=spec.rtol, atol=spec.atol, + ) + error = None + if not ok: + error = ( + f"semantic mismatch: expected {spec.expected_return!r}, " + f"actual {outcome.actual!r}" + ) + return replace( + outcome, ok=ok, expected=spec.expected_return, error=error, + ) + + def run_case(self, spec: CaseSpec) -> CaseOutcome: + if spec.meta_errors: + return CaseOutcome( + case_id=spec.case_id, + pytest_id=spec.pytest_id, + status=STATUS_FAIL, + compile=_empty_compile_outcome(), + assemble=None, + budget=None, + semantic=None, + error_stage=STAGE_META, + error="; ".join(spec.meta_errors), + xfail=spec.xfail, + ) + if spec.skip_reason: + return CaseOutcome( + case_id=spec.case_id, + pytest_id=spec.pytest_id, + status=STATUS_SKIP, + compile=_empty_compile_outcome(), + assemble=None, + budget=None, + semantic=None, + error_stage=None, + error=None, + xfail=spec.xfail, + ) + + stage_order = { + STAGE_COMPILE: 0, + STAGE_ASSEMBLE: 1, + STAGE_BUDGET: 2, + STAGE_SEMANTIC: 3, + } + xfail_stages = set(spec.xfail.stages) if spec.xfail else set() + failures: list[tuple[str, str]] = [] + blocked: list[tuple[str, str]] = [] + + compile_outcome = self.compile_case(spec) + assemble_outcome: AssembleOutcome | None = None + budget_outcome: BudgetOutcome | None = None + semantic_outcome: SemanticOutcome | None = None + + if ASSERT_COMPILE_OK in spec.assertions and not compile_outcome.ok: + failures.append( + (STAGE_COMPILE, compile_outcome.error or "compile failed"), + ) + else: + if compile_outcome.asm_text: + assemble_outcome = self.assemble_asm(compile_outcome.asm_text) + else: + assemble_outcome = AssembleOutcome( + ok=False, binary_len=0, instruction_count=None, + duration_s=0.0, error="no assembly text produced", + ) + if ( + ASSERT_ASM_ENCODABLE in spec.assertions + and not assemble_outcome.ok + ): + failures.append( + ( + STAGE_ASSEMBLE, + assemble_outcome.error or "assembly not encodable", + ), + ) + + if ASSERT_INST_BUDGET in spec.assertions: + if spec.max_instructions is None: + budget_outcome = self.check_instruction_budget( + spec, compile_outcome.asm_text, + ) + elif assemble_outcome.ok: + budget_outcome = self.check_instruction_budget( + spec, compile_outcome.asm_text, + ) + if budget_outcome.ok is False: + failures.append( + ( + STAGE_BUDGET, + budget_outcome.error + or "instruction budget exceeded", + ), + ) + else: + budget_outcome = BudgetOutcome( + ok=None, + instruction_count=assemble_outcome.instruction_count, + limit=spec.max_instructions, + skipped=False, + error="blocked by assemble stage", + ) + blocked.append( + (STAGE_BUDGET, "blocked by assemble stage"), + ) + elif assemble_outcome.ok: + budget_outcome = self.check_instruction_budget( + spec, compile_outcome.asm_text, + ) + + if ASSERT_SEMANTIC_GOLDEN in spec.assertions: + if ( + spec.oracle == ORACLE_EXECUTION + and not assemble_outcome.ok + ): + semantic_outcome = SemanticOutcome( + ok=None, oracle=spec.oracle, + expected=spec.expected_return, actual=None, + duration_s=0.0, + blocked_reason="blocked by assemble stage", + error=None, + ) + blocked.append( + (STAGE_SEMANTIC, "blocked by assemble stage"), + ) + else: + semantic_outcome = self.evaluate_semantics( + spec, compile_outcome, + ) + if semantic_outcome.ok is False: + failures.append( + ( + STAGE_SEMANTIC, + semantic_outcome.error + or "semantic golden check failed", + ), + ) + elif semantic_outcome.ok is None: + blocked.append( + ( + STAGE_SEMANTIC, + semantic_outcome.blocked_reason + or "semantic check blocked", + ), + ) + + hard = [ + (stage, message) + for stage, message in failures if stage not in xfail_stages + ] + covered = [ + (stage, message) + for stage, message in failures if stage in xfail_stages + ] + blocked_covered = [ + (stage, message) + for stage, message in blocked if stage in xfail_stages + ] + + status = STATUS_PASS + error_stage: str | None = None + error: str | None = None + if hard: + status = STATUS_FAIL + error_stage, error = min( + hard, key=lambda item: stage_order[item[0]], + ) + elif covered or blocked_covered: + status = STATUS_XFAIL + error_stage, error = min( + covered + blocked_covered, + key=lambda item: stage_order[item[0]], + ) + elif spec.xfail is not None: + status = STATUS_XPASS + if self.strict_xfail: + status = STATUS_FAIL + error = ( + "strict xfail: declared failing stage(s) now pass " + f"({', '.join(spec.xfail.stages)})" + ) + + return CaseOutcome( + case_id=spec.case_id, + pytest_id=spec.pytest_id, + status=status, + compile=compile_outcome, + assemble=assemble_outcome, + budget=budget_outcome, + semantic=semantic_outcome, + error_stage=error_stage, + error=error, + xfail=spec.xfail, + ) + + def run_cases(self, cases: Sequence[CaseSpec]) -> SuiteReport: + results = [self.run_case(case) for case in cases] + return SuiteReport( + results=results, + roots=self.root_strings, + compiler=self.compiler_info, + generated_at=datetime.datetime.now().isoformat(timespec="seconds"), + specs={case.case_id: case for case in cases}, + ) + + def run_all(self) -> SuiteReport: + return self.run_cases(self.discover()) diff --git a/benchmarks/run_suite.py b/benchmarks/run_suite.py new file mode 100644 index 0000000..5a7e8fd --- /dev/null +++ b/benchmarks/run_suite.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""CLI entry point for the ScratchV DSL benchmark suite (topic 06). + +Produces JSON/Markdown reports and returns the suite exit code: + + 0 = no hard failures (pass/xfail/xpass/skip) + 1 = at least one hard failure + 2 = configuration error (invalid roots, no cases, invalid options) +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from benchmarks.dsl_suite import ( # noqa: E402 + DEFAULT_ROOTS, + STATUS_FAIL, + STATUS_SKIP, + STATUS_XFAIL, + STATUS_XPASS, + DSLSuiteRunner, + SuiteReport, +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="ScratchV DSL benchmark suite (topic 06)", + ) + parser.add_argument( + "--roots", action="append", default=None, + help="Case root directory (repeatable; default: " + + ", ".join(DEFAULT_ROOTS) + ")", + ) + parser.add_argument("--json", default=None, help="Write JSON report") + parser.add_argument( + "--markdown", default=None, + help="Write Markdown report", + ) + parser.add_argument( + "--backend", default="riscv", choices=("riscv", "llvm"), + help="Compiler backend (default: riscv)", + ) + parser.add_argument( + "--optimize-level", default="all", + choices=("none", "basic", "all"), + help="Optimization level (default: all)", + ) + parser.add_argument( + "--reg-alloc", default="linear", + choices=("naive", "greedy", "linear"), + help="Register allocator (default: linear)", + ) + parser.add_argument( + "--timeout", type=float, default=30.0, + help="Per-case timeout budget in seconds (default: 30)", + ) + parser.add_argument( + "--filter", default=None, + help="Only run cases whose case_id contains this substring", + ) + parser.add_argument( + "--list", action="store_true", + help="List discovered case ids and exit", + ) + parser.add_argument( + "--strict-xfail", action="store_true", + help="Treat unexpected passes of xfail-declared cases as failures", + ) + parser.add_argument( + "--quiet", action="store_true", + help="Only write reports; suppress the stdout summary", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + roots = tuple(args.roots) if args.roots else DEFAULT_ROOTS + + missing = [root for root in roots if not Path(root).is_dir()] + if missing: + print( + f"error: root director{'y' if len(missing) == 1 else 'ies'} " + f"not found: {', '.join(missing)}", + file=sys.stderr, + ) + return 2 + if args.timeout <= 0: + print("error: --timeout must be positive", file=sys.stderr) + return 2 + + runner = DSLSuiteRunner( + roots, + backend=args.backend, + optimize_level=args.optimize_level, + reg_alloc=args.reg_alloc, + timeout_s=args.timeout, + strict_xfail=args.strict_xfail, + verbose=False, + ) + try: + cases = runner.discover() + if not cases: + print("error: no cases discovered", file=sys.stderr) + return 2 + if args.list: + for case in cases: + print(case.case_id) + return 0 + if args.filter: + cases = [case for case in cases if args.filter in case.case_id] + if not cases: + print( + f"error: no cases match filter {args.filter!r}", + file=sys.stderr, + ) + return 2 + report = runner.run_cases(cases) + finally: + runner.cleanup() + + _print_summary(report, quiet=args.quiet) + + if args.json: + report.save_json(args.json) + if args.markdown: + report.save_markdown(args.markdown) + + return 1 if report.fail_count > 0 else 0 + + +def _print_summary(report: SuiteReport, *, quiet: bool) -> None: + summary = report.summary() + if quiet: + return + print( + "DSL benchmark suite: " + f"{summary['passed']} passed, {summary['xfailed']} xfailed, " + f"{summary['xpassed']} xpassed, {summary['failed']} failed, " + f"{summary['skipped']} skipped (total {summary['total']})" + ) + for result in report.results: + if result.status == STATUS_FAIL: + print( + f" [FAIL] {result.case_id} ({result.error_stage}): " + f"{result.error}" + ) + elif result.status in (STATUS_XFAIL, STATUS_XPASS, STATUS_SKIP): + detail = result.error or "" + suffix = f": {detail}" if detail else "" + print(f" [{result.status}] {result.case_id}{suffix}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/stress/reg_pressure_32.meta.json b/tests/stress/reg_pressure_32.meta.json index 06d4e06..dc161cf 100644 --- a/tests/stress/reg_pressure_32.meta.json +++ b/tests/stress/reg_pressure_32.meta.json @@ -1,8 +1,24 @@ { + "schema_version": 1, "description": "32-vreg stress test using input variables (no constants). Fan-in tree over 32 intermediate add(x,x) values triggers spill/reload path. 32 > 19 (greedy allocator register count).", - "expected_output_type": "return_value", + "category": "stress", + "flow": "linear", + "oracle": "execution", "inputs": { "x": 1 }, - "expected_return": 64 + "input_registers": { + "x": "a0" + }, + "expected_output_type": "return_value", + "expected_return": 64, + "max_instructions": 77, + "xfail": { + "stages": [ + "semantic" + ], + "reason": "C3: spill/reload stack path reads uninitialised frame; executing the fan-in with x=1 returns 178 instead of 64", + "owner": "backend/regalloc", + "strict": false + } } diff --git a/tests/stress/reg_pressure_loop.meta.json b/tests/stress/reg_pressure_loop.meta.json index c5e94ed..57ad01f 100644 --- a/tests/stress/reg_pressure_loop.meta.json +++ b/tests/stress/reg_pressure_loop.meta.json @@ -1,8 +1,28 @@ { + "schema_version": 1, "description": "Loop regression test: accumulator survives across 10 loop back-edges while loop body creates 6-chain of dependent intermediates (Fibonacci-like). Validates that LABEL at loop header does not corrupt vreg-to-physreg mappings across iterations.", - "expected_output_type": "return_value", + "category": "stress", + "flow": "control", + "oracle": "execution", "inputs": { - "x": 1 + "x": 1, + "acc": 0 + }, + "input_registers": { + "x": "a0", + "acc": "a1" }, - "expected_return": 210 + "expected_output_type": "return_value", + "expected_return": 210, + "max_instructions": 16, + "xfail": { + "stages": [ + "assemble", + "budget", + "semantic" + ], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 10 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false + } } diff --git a/tests/test_dsl_suite.py b/tests/test_dsl_suite.py new file mode 100644 index 0000000..0fd01a9 --- /dev/null +++ b/tests/test_dsl_suite.py @@ -0,0 +1,235 @@ +"""Pytest gate for the ScratchV DSL benchmark suite (topic 06). + +Each stage of each discovered case becomes an independently addressable +test node (``case_id + stage``), so failures pinpoint the exact stage. +Failures already located, owned and declared in ``*.meta.json`` are marked +``xfail`` (non-strict by default; xpasses are visible but do not fail CI). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from benchmarks.dsl_suite import ( + ASSERT_INST_BUDGET, + ORACLE_EXECUTION, + ORACLE_NONE, + STAGE_ASSEMBLE, + STAGE_BUDGET, + STAGE_SEMANTIC, + CaseSpec, + DSLSuiteRunner, + SemanticOutcome, + compare_values, + discover_cases, + load_case_spec, +) + +ALL_CASES: tuple[CaseSpec, ...] = tuple(discover_cases()) +if not ALL_CASES: + pytest.fail("dsl suite discovery found 0 cases", pytrace=False) + + +def marks_for(case: CaseSpec, stage: str) -> list[pytest.MarkDecorator]: + """Return the xfail marker for *stage* when the case declares it.""" + if case.xfail is not None and stage in case.xfail.stages: + return [ + pytest.mark.xfail( + reason=case.xfail.reason, + strict=case.xfail.strict, + ) + ] + return [] + + +def params_for(stage: str | None) -> list[pytest.ParameterSet]: + """Build per-case parameters; marks must be injected per parameter.""" + return [ + pytest.param( + case, + id=case.pytest_id, + marks=marks_for(case, stage) if stage else [], + ) + for case in ALL_CASES + ] + + +@pytest.fixture(scope="session") +def suite_runner() -> DSLSuiteRunner: + runner = DSLSuiteRunner(verbose=False) + yield runner + runner.cleanup() + + +@pytest.fixture(scope="session") +def compile_results(suite_runner: DSLSuiteRunner) -> dict: + return { + case.case_id: suite_runner.compile_case(case) for case in ALL_CASES + } + + +@pytest.fixture(scope="session") +def asm_results( + suite_runner: DSLSuiteRunner, compile_results: dict, +) -> dict: + return { + case_id: suite_runner.assemble_asm(outcome.asm_text) + for case_id, outcome in compile_results.items() + } + + +@pytest.fixture(scope="session") +def semantic_results( + suite_runner: DSLSuiteRunner, compile_results: dict, asm_results: dict, +) -> dict: + results: dict = {} + for case in ALL_CASES: + if case.skip_reason or case.oracle == ORACLE_NONE: + results[case.case_id] = None + continue + if case.oracle == ORACLE_EXECUTION and not asm_results[case.case_id].ok: + results[case.case_id] = SemanticOutcome( + ok=None, + oracle=case.oracle, + expected=case.expected_return, + actual=None, + duration_s=0.0, + blocked_reason="blocked by assemble stage", + error=None, + ) + continue + results[case.case_id] = suite_runner.evaluate_semantics( + case, compile_results[case.case_id], + ) + return results + + +@pytest.mark.parametrize("case", params_for(None)) +def test_meta_contract(case: CaseSpec) -> None: + assert case.meta_errors == (), ( + f"meta contract violations: {case.meta_errors}" + ) + if case.xfail is not None: + assert case.xfail.owner.strip(), "xfail.owner must be non-empty" + assert case.xfail.reason.strip(), "xfail.reason must be non-empty" + + +@pytest.mark.parametrize("case", params_for(None)) +def test_compile_ok(case: CaseSpec, compile_results: dict) -> None: + if case.skip_reason: + pytest.skip(case.skip_reason) + outcome = compile_results[case.case_id] + assert outcome.ok, f"compile failed: {outcome.error}" + + +@pytest.mark.parametrize("case", params_for(STAGE_ASSEMBLE)) +def test_asm_encodable(case: CaseSpec, asm_results: dict) -> None: + if case.skip_reason: + pytest.skip(case.skip_reason) + outcome = asm_results[case.case_id] + assert outcome.ok, f"assemble failed: {outcome.error}" + + +@pytest.mark.parametrize("case", params_for(STAGE_BUDGET)) +def test_instruction_budget(case: CaseSpec, asm_results: dict) -> None: + if case.skip_reason: + pytest.skip(case.skip_reason) + if ASSERT_INST_BUDGET not in case.assertions: + pytest.skip("inst_budget assertion disabled") + if case.max_instructions is None: + pytest.skip("no max_instructions declared") + outcome = asm_results[case.case_id] + assert outcome.ok, "blocked by assemble stage" + assert outcome.instruction_count is not None + assert outcome.instruction_count <= case.max_instructions, ( + f"{outcome.instruction_count} > {case.max_instructions}" + ) + + +@pytest.mark.parametrize("case", params_for(STAGE_SEMANTIC)) +def test_semantic_golden( + case: CaseSpec, semantic_results: dict, +) -> None: + if case.skip_reason: + pytest.skip(case.skip_reason) + if case.oracle == ORACLE_NONE: + pytest.skip("oracle=none") + outcome = semantic_results[case.case_id] + assert outcome is not None + assert outcome.ok, outcome.blocked_reason or outcome.error + + +def test_compare_values_handles_scalars_vectors_and_text() -> None: + assert compare_values([2, 4, 6, 8], "[2. 4. 6. 8.]") + assert compare_values(30.0, "30.") + assert compare_values( + [0.032059, 0.087144, 0.236883, 0.643914], + [0.0320586, 0.08714432, 0.23688282, 0.64391428], + ) + assert not compare_values([1.0, 2.0], [1.0, 2.0, 3.0]) + assert not compare_values(1.0, 2.0) + assert not compare_values(None, 1.0) + + +def _write_pseudo_case( + directory: Path, + name: str, + source: str, + meta: dict, +) -> Path: + dsl_path = directory / f"{name}.dsl" + dsl_path.write_text(source) + (directory / f"{name}.meta.json").write_text(json.dumps(meta)) + return dsl_path + + +@pytest.mark.parametrize( + ("name", "source", "meta", "fragment"), + [ + ( + "interpreter_control", + "while (i < 3):\n i = add(i, 1)\nendwhile\nreturn i\n", + { + "description": "pseudo case", + "oracle": "interpreter", + "expected_return": 3, + }, + "oracle/flow conflict", + ), + ( + "execution_missing_registers", + "c = add(a, b)\nreturn c\n", + { + "description": "pseudo case", + "oracle": "execution", + "inputs": {"a": 1, "b": 2}, + "expected_return": 3, + }, + "input_registers is required", + ), + ( + "none_without_xfail", + "c = add(a, b)\nreturn c\n", + { + "description": "pseudo case", + "oracle": "none", + }, + "oracle=none requires", + ), + ], +) +def test_meta_contract_rejects_invalid_metadata( + tmp_path: Path, + name: str, + source: str, + meta: dict, + fragment: str, +) -> None: + dsl_path = _write_pseudo_case(tmp_path, name, source, meta) + spec = load_case_spec(dsl_path, tmp_path) + assert any(fragment in error for error in spec.meta_errors), ( + f"expected {fragment!r} in {spec.meta_errors}" + ) From 6c85290c49e1ab92c104513fb1fe1ceaf8884ace Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 21:22:15 +0800 Subject: [PATCH 2/6] docs(topic06): add design and development documents --- ...00\345\217\221\346\226\207\346\241\243.md" | 892 ++++++++++++++++++ ...76\350\256\241\346\226\207\346\241\243.md" | 617 ++++++++++++ 2 files changed, 1509 insertions(+) create mode 100644 "docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" create mode 100644 "docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" new file mode 100644 index 0000000..33bae9d --- /dev/null +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -0,0 +1,892 @@ +# ScratchV 课题 06 性能基准套件 开发文档 + +> 文档版本：v1.0 +> 编写日期：2026-09-14 +> 对应设计文档：`./设计文档.md`（本目录） +> 交付物：`benchmarks/dsl_suite.py`、`benchmarks/run_suite.py`、`tests/test_dsl_suite.py`、23 份新增 `meta.json` + 2 份修订、`.github/workflows/ci.yml` 两处接线 +> 代码基线：HEAD `d146515`；本文所有实测数据来自 2026-09-14 实跑（Python 3.11 + numpy 2.4.6 + pytest 9.1.1 + tinyfive 已安装，编译配置 `backend=riscv, optimize_level=all, reg_alloc=linear`） + +--- + +## 一、接口契约（先契约后实现） + +### 1.1 文件与入口 + +| 文件 | 类型 | 入口符号 | 用途 | +|------|------|----------|------| +| `benchmarks/dsl_suite.py` | 新增 | `DSLSuiteRunner`、`discover_cases`、`compare_values`、`ExecutionOracle` | 套件核心（无 CLI） | +| `benchmarks/run_suite.py` | 新增 | `main(argv: list[str] \| None = None) -> int` | CLI：报告 + 退出码 | +| `tests/test_dsl_suite.py` | 新增 | `ALL_CASES`、5 个 `test_*`、3 个 session fixture | pytest 门禁 | +| `benchmarks/cases/{name}.meta.json` | 新增 23 | — | 元数据（§1.7） | +| `tests/stress/{name}.meta.json` | 修改 2 | — | 接入统一 schema | +| `.github/workflows/ci.yml` | 修改 | — | 门禁与报告（§五） | + +约束：`scratchv/**` 零改动；`benchmarks/bench_runner.py`、`tests/test_bench_runner.py` 零改动。 + +### 1.2 常量（`benchmarks/dsl_suite.py` 模块级，精确名称） + +```python +DEFAULT_ROOTS: tuple[str, ...] = ("benchmarks/cases", "tests/stress") +SCHEMA_VERSION: int = 1 + +STAGE_COMPILE: str = "compile" +STAGE_ASSEMBLE: str = "assemble" +STAGE_BUDGET: str = "budget" +STAGE_SEMANTIC: str = "semantic" +STAGE_META: str = "meta" + +ASSERT_COMPILE_OK: str = "compile_ok" +ASSERT_ASM_ENCODABLE: str = "asm_encodable" +ASSERT_INST_BUDGET: str = "inst_budget" +ASSERT_SEMANTIC_GOLDEN: str = "semantic_golden" + +DEFAULT_ASSERTIONS: tuple[str, ...] = ( + ASSERT_COMPILE_OK, ASSERT_ASM_ENCODABLE, ASSERT_INST_BUDGET, + ASSERT_SEMANTIC_GOLDEN, +) + +ORACLE_INTERPRETER: str = "interpreter" +ORACLE_EXECUTION: str = "execution" +ORACLE_NONE: str = "none" + +STATUS_PASS: str = "pass" +STATUS_FAIL: str = "fail" +STATUS_XFAIL: str = "xfail" +STATUS_XPASS: str = "xpass" +STATUS_SKIP: str = "skip" +``` + +### 1.3 数据类（字段精确名称） + +```python +@dataclass(frozen=True) +class XFailSpec: + stages: tuple[str, ...] # STAGE_* 的非空子集（compile/assemble/budget/semantic） + reason: str + owner: str + strict: bool = False + +@dataclass(frozen=True) +class CaseSpec: + case_id: str # "cases/001_simple_add" + pytest_id: str # "cases-001_simple_add" + name: str # "001_simple_add" + root: Path + dsl_path: Path + meta_path: Path | None + expected_path: Path | None + description: str + category: str # arith|nn|control|complex|const|stress|unknown + flow: str # linear|control + oracle: str # interpreter|execution|none + inputs: dict[str, object] + input_registers: dict[str, str] + expected_return: object | None + expected_text: str + rtol: float + atol: float + assertions: tuple[str, ...] + max_instructions: int | None + timeout_s: float + xfail: XFailSpec | None + skip_reason: str | None + meta_errors: tuple[str, ...] + +@dataclass +class CompileOutcome: + ok: bool + asm_text: str + output_path: Path | None + ir_instruction_count: int + duration_s: float + error: str | None + +@dataclass +class AssembleOutcome: + ok: bool + binary_len: int + instruction_count: int | None # parse_asm 文本口径 + duration_s: float + error: str | None + +@dataclass +class BudgetOutcome: + ok: bool | None # None = 未声明/跳过 + instruction_count: int | None + limit: int | None + skipped: bool + error: str | None + +@dataclass +class SemanticOutcome: + ok: bool | None # None = blocked/skipped + oracle: str + expected: object | None + actual: object | None + duration_s: float + blocked_reason: str | None + error: str | None + +@dataclass +class CaseOutcome: + case_id: str + pytest_id: str + status: str # STATUS_* + compile: CompileOutcome + assemble: AssembleOutcome | None + budget: BudgetOutcome | None + semantic: SemanticOutcome | None + error_stage: str | None + error: str | None + xfail: XFailSpec | None + +@dataclass +class SuiteReport: + results: list[CaseOutcome] + roots: tuple[str, ...] + compiler: dict[str, str] + generated_at: str + # 属性/方法 + @property + def pass_count(self) -> int: ... + @property + def xfail_count(self) -> int: ... + @property + def xpass_count(self) -> int: ... + @property + def fail_count(self) -> int: ... + @property + def skip_count(self) -> int: ... + def summary(self) -> dict[str, int]: ... + def to_dict(self) -> dict: ... + def save_json(self, path: str | Path) -> None: ... + def to_markdown(self) -> str: ... + def save_markdown(self, path: str | Path) -> None: ... +``` + +### 1.4 `DSLSuiteRunner` 方法签名 + +```python +class DSLSuiteRunner: + def __init__( + self, + roots: tuple[str | Path, ...] = DEFAULT_ROOTS, + *, + workdir: str | Path | None = None, # None -> tempfile.mkdtemp("dsl_suite_") + backend: str = "riscv", + optimize_level: str = "all", + reg_alloc: str = "linear", + timeout_s: float = 30.0, + strict_xfail: bool = False, + verbose: bool = False, + ) -> None: ... + + def discover(self) -> list[CaseSpec]: ... + def compile_case(self, spec: CaseSpec) -> CompileOutcome: ... + def assemble_asm(self, asm_text: str) -> AssembleOutcome: ... + def check_instruction_budget( + self, spec: CaseSpec, asm_text: str) -> BudgetOutcome: ... + def evaluate_semantics( + self, spec: CaseSpec, compile_outcome: CompileOutcome) -> SemanticOutcome: ... + def run_case(self, spec: CaseSpec) -> CaseOutcome: ... + def run_all(self) -> SuiteReport: ... + def cleanup(self) -> None: ... # 删除 workdir（若为自建临时目录） +``` + +模块级函数： + +```python +def discover_cases( + roots: Sequence[str | Path] = DEFAULT_ROOTS, + *, + verbose: bool = False, +) -> list[CaseSpec]: ... + +def load_case_spec(dsl_path: Path, root: Path) -> CaseSpec: ... + # 解析 meta + .expected/.desc 回退 + 契约校验，违规写入 meta_errors + +def infer_flow(source: str) -> str: ... + # 含 ("if (", "else:", "endif", "while (", "endwhile", "for ", "endfor") -> "control" 否则 "linear" + +def default_inputs(source: str, *, fill: float = 1.0) -> dict[str, np.ndarray]: ... + # interpreter oracle 的默认向量输入，规则与 bench_runner 一致（[1,2,3,4]） + +def compare_values(actual, expected, *, rtol: float = 1e-3, atol: float = 1e-6) -> bool: ... + # 支持标量/向量/字符串化向量（"[2. 4. 6. 8.]"） + +def count_asm_instructions(asm_text: str) -> int: ... + # parse_asm 口径：opcode 非空且 not is_directive +``` + +### 1.5 执行 oracle（`ExecutionOracle`） + +```python +class ExecutionOracle: + def __init__(self, *, mem_size: int = 128 * 1024 * 1024) -> None: ... + + def available(self) -> bool: + # ProfiledMachine.available；tinyfive 缺失返回 False + + def execute( + self, + asm_text: str, + input_registers: dict[str, str], # {"x": "a0"} + inputs: dict[str, object], # {"x": 1} + *, + timeout_s: float = 5.0, + ) -> SemanticOutcome: ... +``` + +实现约定（不改任何 `scratchv/` 文件）： + +1. `binary = assemble_to_binary(asm_text)`；空 → `error="assembler produced empty binary"`； +2. `words = [int.from_bytes(binary[i:i+4], "little") for i in range(0, len(binary), 4)]`； +3. `m = ProfiledMachine(mem_size=...)`；`available` 为假 → `blocked_reason="tinyfive not installed"`； +4. `m.load_binary(words, origin=0)`； +5. 对每个输入：`value = int(inputs[name])`（非整数 float → `blocked_reason="non-integer input not supported by integer register semantics"`）；`m.set_reg(register_number(name), value)`，寄存器名→编号用本地 `_REG_NUMS` 表（`a0`=10 起）； +6. `m.run(instructions=len(words), start=0, strict=True)`；`actual = m.get_reg(10)`； +7. `compare_values(actual, expected)`。 + +### 1.6 pytest 契约（`tests/test_dsl_suite.py`） + +| 符号 | 精确名称 | 说明 | +|------|----------|------| +| 收集常量 | `ALL_CASES: tuple[CaseSpec, ...]` | 模块导入期 `discover_cases()`；0 用例则 `pytest.fail("no cases discovered")` | +| fixture | `suite_runner`（session） | `DSLSuiteRunner(verbose=False)` | +| fixture | `compile_results`（session） | `dict[str, CompileOutcome]`，逐个用例编译一次并缓存 | +| fixture | `asm_results`（session） | `dict[str, AssembleOutcome]` | +| 测试 | `test_meta_contract(case: CaseSpec)` | `case.meta_errors == ()` 且 `xfail.owner/reason` 非空；不可 xfail | +| 测试 | `test_compile_ok(case, compile_results)` | 断言 `compile.ok` | +| 测试 | `test_asm_encodable(case, asm_results)` | 断言 `assemble.ok` | +| 测试 | `test_instruction_budget(case, asm_results)` | `max_instructions is None` → `pytest.skip`；否则断言 `<=` | +| 测试 | `test_semantic_golden(case, compile_results)` | `oracle=none` → `pytest.skip`；否则断言 `semantic.ok` | +| 标记助手 | `marks_for(case: CaseSpec, stage: str) -> list[pytest.MarkDecorator]` | `stage in case.xfail.stages` 时返回 `pytest.mark.xfail(reason=..., strict=...)` | +| 参数化 | `@pytest.mark.parametrize("case", ALL_CASES, ids=[c.pytest_id for c in ALL_CASES])` | node id 例：`tests/test_dsl_suite.py::test_asm_encodable[cases-013_for_sum]` | + +### 1.7 `meta.json` 字段契约（实现必须逐字段照做） + +| 字段 | 类型 | 必填 | 默认 | 校验（违规 → `meta_errors`） | +|------|------|------|------|------------------------------| +| `schema_version` | int | 否 | 1 | `!= 1` 报错 | +| `description` | str | 是 | — | 空串报错 | +| `category` | str | 否 | 推断 | 不在 `{arith,nn,control,complex,const,stress}` 报错 | +| `flow` | str | 否 | `infer_flow` | 与推断不一致报错 | +| `oracle` | str | 是 | — | 非三类之一报错；`interpreter` 且 flow≠linear 报错 | +| `inputs` | object | 是 | `{}` | 非 object 报错；值类型限 int/float/list[number] | +| `input_registers` | object | `oracle=execution` 且 inputs 非空时必填 | `{}` | 键集与 `inputs` 不一致、值不匹配 `^(a[0-7]\|s[0-9]\|s1[01])$` 报错 | +| `expected_output_type` | str | 否 | `"return_value"` | 本期仅接受 `return_value` | +| `expected_return` | number/list | `oracle!=none` 时必填（interpreter 可用 `.expected` 回退） | — | 缺期望值报错 | +| `rtol` / `atol` | float | 否 | `1e-3` / `1e-6` | 负数报错 | +| `assertions` | list[str] | 否 | 全部四项 | 含未知断言名报错 | +| `max_instructions` | int | 否 | `None` | `<= 0` 报错 | +| `timeout_s` | float | 否 | `30.0` | `<= 0` 报错 | +| `xfail` | object | `oracle=none` 时必填 | `None` | `stages` 为 `["compile","assemble","budget","semantic"]` 的非空子集（允许单字符串）；`reason`/`owner` 非空；`strict` bool；同一根因导致的阶段失败写进同一 `stages` | + +未知字段不报错（WARNING 级，写入 `CaseSpec.meta_errors` 之外的 `warnings` 列表，报告可见）。 + +### 1.8 退出码契约 + +| 场景 | 退出码 | +|------|--------| +| 无硬失败（pass/xfail/xpass/skip 任意组合） | 0 | +| 存在 `failed`（含 `meta_contract` 违规） | 1 | +| 配置错误（roots 不存在、发现 0 用例、参数非法） | 2 | + +--- + +## 二、runner 实现方案 + +### 2.1 模块分层 + +``` +benchmarks/dsl_suite.py + ├─ 常量与异常（DSLSuiteError / CaseSpecError） + ├─ 数据类（§1.3） + ├─ meta 层：_read_json / _validate_meta / load_case_spec / infer_flow + ├─ 发现层：discover_cases + ├─ 执行层：DSLSuiteRunner.compile_case / assemble_asm + │ check_instruction_budget / evaluate_semantics + ├─ oracle：ExecutionOracle / _interpreter_semantics + ├─ 比较：compare_values / count_asm_instructions + └─ 报告：SuiteReport.to_dict / to_markdown / save_* +``` + +### 2.2 `discover_cases` 实现步骤 + +1. 规范化 roots 为 `Path`；不存在的 root 记 WARNING 并跳过，全部不存在 → 调用方按配置错误处理。 +2. 每个 root `rglob("*.dsl")`，过滤：文件名以 `_` 开头、任一父目录名为 `fixtures`；若存在 `{name}.skip`（stem 同名，如 `001_simple_add.skip`），读取其内容作为 `skip_reason` 并保留该用例（报告为 skip），不静默丢弃。 +3. 排序键 `f"{root.name}/{dsl.relative_to(root).as_posix()}"`。 +4. 逐个 `load_case_spec`；返回列表长度 0 由调用方判断。 + +### 2.3 `load_case_spec` 实现步骤 + +1. `name = dsl.stem`；`pytest_id = f"{root.name}-{relative_no_suffix.replace('/', '-')}"`。 +2. 读取 meta（缺失 → `meta_errors += ("meta.json not found",)`，继续用回退数据以便报告仍生成）。 +3. 读取 `.expected`/`.desc` 作为回退。 +4. `flow = infer_flow(source)`；若 meta 有 `flow` 且不同 → 报错。 +5. `oracle` 缺省规则：meta 未给时，linear → `interpreter`，control → `execution`（并在报告 WARNING 中提示“oracle 为推断值，建议显式声明”）。 +6. `expected_return` 缺省规则：meta > `.expected`（仅 interpreter）。 +7. 执行 §1.7 全部校验，返回 `CaseSpec`。 + +### 2.4 `compile_case` 实现步骤 + +1. `out_path = self.workdir / f"{spec.pytest_id}.s"`。 +2. `driver = CompilerDriver(CompilerConfig(backend=self.backend, optimize_level=self.optimize_level, reg_alloc=self.reg_alloc))`。 +3. `t0`；`result = driver.compile(str(spec.dsl_path), str(out_path))`；捕获全部异常为 `error`。 +4. `ir_instruction_count`：单独 parse 源码（与 bench_runner 同规则选择 `ExtendedDSLParser`/`DSLParser`），对 `program.functions -> blocks -> instructions` 计数；parse 失败不影响 compile 判定（记 0 + error 备注）。 +5. 返回 `CompileOutcome(ok=result.success and bool(result.output_text), asm_text=result.output_text, ...)`。 +6. 失败时把 `"\n".join(result.errors)` 写入 `error`；不得吞错。 + +### 2.5 `assemble_asm` 实现步骤 + +1. `t0`；`binary = assemble_to_binary(asm_text)`；异常捕获为 `error`（保留异常类型与消息）。 +2. `ok = bool(binary)`；`binary_len = len(binary)`。 +3. `instruction_count = count_asm_instructions(asm_text)`（即使编码失败也统计，便于报告与预算诊断）。 +4. `duration_s`。 + +### 2.6 `evaluate_semantics` 实现步骤 + +1. `oracle == none` → `SemanticOutcome(ok=None, blocked_reason="oracle=none", ...)`。 +2. `oracle == interpreter`： + - `inputs = spec.inputs` 若为空则 `default_inputs(source)`； + - `_interpreter_semantics`：构造 `DSLInterpreter().run(source, inputs)`；对 ndarray 用 `np.array2string(precision=6, suppress_small=True)` 字符串化； + - `expected = spec.expected_return` 或 `spec.expected_text`；`compare_values`。 +3. `oracle == execution`： + - 若 `assemble` 未成功（由 `run_case` 判断）→ `blocked_reason="blocked by assemble stage"`； + - 否则 `ExecutionOracle.execute(...)`； + - 输入映射缺失（契约已挡）不再兜底推断。 +4. 捕获异常写入 `error`，`ok=False`。 + +### 2.7 `run_case` 状态合成 + +``` +outcome.compile = compile_case(spec) # STAGE_COMPILE +if not outcome.compile.ok: -> status=fail 或 xfail(compile ∈ xfail.stages) +assemble = assemble_asm(...) # STAGE_ASSEMBLE +if not assemble.ok: -> status=fail 或 xfail(assemble ∈ xfail.stages) +budget = check_instruction_budget(...) # STAGE_BUDGET +if budget.ok is False: -> status=fail 或 xfail(budget ∈ xfail.stages) +semantic = evaluate_semantics(...) # STAGE_SEMANTIC +if semantic.ok is False: -> status=fail 或 xfail(semantic ∈ xfail.stages) +命中 xfail.stages 且该阶段失败 -> status=xfail，error_stage 记录首错阶段 +``` + +硬失败优先：同一用例若出现多个失败，`error_stage` 取最先发生的阶段；`error` 保留该阶段最小证据（首错指令行/期望值/实际值/异常）。阶段依赖规则：`budget` 与 `execution` 语义依赖 `assemble` 成功，汇编失败时二者记为 `blocked` 并按 `xfail.stages` 判定；`interpreter` 语义不依赖汇编，编译成功后即执行（`009_maxpool` 因此在 `asm_encodable` 失败的同时语义真实通过）。 + +### 2.8 报告与性能 + +- `run_all()` 逐用例调用 `run_case`，汇总 `SuiteReport`；`run_suite.py` 负责写盘。 +- 预计耗时 < 60s：25 次编译为主（实测单用例编译 0.01–0.05s），session fixture 缓存后 pytest 只会编译一遍。 +- `workdir` 缺省临时目录；`cleanup()` 幂等；CI 无需清理仓库。 + +--- + +## 三、case → pytest 参数化实现 + +### 3.1 骨架 + +```python +# tests/test_dsl_suite.py +from __future__ import annotations + +import pytest + +from benchmarks.dsl_suite import ( + DSLSuiteRunner, CaseSpec, discover_cases, compare_values, + STAGE_ASSEMBLE, STAGE_BUDGET, STAGE_COMPILE, STAGE_SEMANTIC, +) + +ALL_CASES: tuple[CaseSpec, ...] = tuple(discover_cases()) +if not ALL_CASES: + pytest.fail("dsl suite discovery found 0 cases", pytrace=False) + + +def marks_for(case: CaseSpec, stage: str): + if case.xfail is not None and stage in case.xfail.stages: + return [pytest.mark.xfail(reason=case.xfail.reason, strict=case.xfail.strict)] + return [] + + +def params_for(stage: str | None): + """生成每个用例独立的 pytest.param（marks 必须逐参数注入）。""" + return [ + pytest.param(case, id=case.pytest_id, + marks=marks_for(case, stage) if stage else []) + for case in ALL_CASES + ] + + +@pytest.fixture(scope="session") +def suite_runner() -> DSLSuiteRunner: + runner = DSLSuiteRunner(verbose=False) + yield runner + runner.cleanup() + + +@pytest.fixture(scope="session") +def compile_results(suite_runner): + return {c.case_id: suite_runner.compile_case(c) for c in ALL_CASES} + + +@pytest.fixture(scope="session") +def asm_results(compile_results): + return {cid: suite_runner.assemble_asm(out.asm_text) + for cid, out in compile_results.items()} + + +@pytest.mark.parametrize("case", params_for(None)) +def test_meta_contract(case: CaseSpec) -> None: + assert case.meta_errors == (), f"meta contract violations: {case.meta_errors}" + + +@pytest.mark.parametrize("case", params_for(None)) +def test_compile_ok(case: CaseSpec, compile_results) -> None: + outcome = compile_results[case.case_id] + assert outcome.ok, f"compile failed: {outcome.error}" + + +@pytest.mark.parametrize("case", params_for(STAGE_ASSEMBLE)) +def test_asm_encodable(case: CaseSpec, asm_results) -> None: + outcome = asm_results[case.case_id] + assert outcome.ok, f"assemble failed: {outcome.error}" + + +@pytest.mark.parametrize("case", params_for(STAGE_BUDGET)) +def test_instruction_budget(case: CaseSpec, asm_results) -> None: + if case.max_instructions is None: + pytest.skip("no max_instructions declared") + outcome = asm_results[case.case_id] + assert outcome.ok, "blocked by assemble stage" + assert outcome.instruction_count <= case.max_instructions, ( + f"{outcome.instruction_count} > {case.max_instructions}") + + +@pytest.mark.parametrize("case", params_for(STAGE_SEMANTIC)) +def test_semantic_golden(case: CaseSpec, compile_results) -> None: + if case.oracle == "none": + pytest.skip("oracle=none") + runner = DSLSuiteRunner(verbose=False) + outcome = runner.evaluate_semantics(case, compile_results[case.case_id]) + assert outcome.ok, outcome.blocked_reason or outcome.error +``` + +> 注意两点： +> 1. `marks` 必须通过 `pytest.param(..., marks=...)` 逐参数注入（直接给 `parametrize(marks=...)` 会给全部参数打同一标记）； +> 2. `test_instruction_budget` 与 `test_semantic_golden` 在汇编失败时断言失败，因此 11 个汇编失败用例的 `xfail.stages` 必须同时包含 `budget`，控制流用例还要包含 `semantic`（同一根因，同一 `reason`）。`009_maxpool` 例外：它使用 interpreter oracle，语义不依赖汇编，`xfail.stages` 只含 `assemble` 与 `budget`，其语义测试必须真实通过； +> 3. 各**执行类**测试函数入口统一处理 `case.skip_reason`：`if case.skip_reason: pytest.skip(case.skip_reason)`；`test_meta_contract` 不受 skip 影响（契约不可规避，元数据违规不得用 `.skip` 掩盖）。 + +### 3.2 运行与 node id 示例 + +```bash +python3.12 -m pytest tests/test_dsl_suite.py -v +# tests/test_dsl_suite.py::test_asm_encodable[cases-001_simple_add] PASSED +# tests/test_dsl_suite.py::test_asm_encodable[cases-013_for_sum] XFAIL (RISCVAEncoder cannot encode ...) +# 单阶段/单用例过滤： +python3.12 -m pytest "tests/test_dsl_suite.py::test_semantic_golden[cases-013_for_sum]" -v +``` + +--- + +## 四、失败 case 处置清单（实测，2026-09-14） + +### 4.1 复现方法（实现阶段照抄执行） + +```bash +# 1) 编译 + 汇编/编码判定（进程内，25 用例，不改仓库） +python3.11 - <<'EOF' +import sys; sys.path.insert(0, ".") +from pathlib import Path +from scratchv.compiler import CompilerDriver, CompilerConfig +from scratchv.backend.riscv_encoder import assemble_to_binary +for root in ("benchmarks/cases", "tests/stress"): + for dsl in sorted(Path(root).glob("*.dsl")): + out = Path("/tmp/asm") / (dsl.stem + ".s") + out.parent.mkdir(exist_ok=True) + r = CompilerDriver(CompilerConfig(backend="riscv", optimize_level="all")).compile(str(dsl), str(out)) + try: + binary = assemble_to_binary(r.output_text) if r.success else b"" + print(dsl.stem, "compile", r.success, "asm", bool(binary)) + except Exception as e: + print(dsl.stem, "compile", r.success, "asm FAIL", type(e).__name__, e) +EOF + +# 2) 现有 bench_runner 的绿（对照用） +python3 benchmarks/bench_runner.py benchmarks/cases --quiet # 实测 23/23 PASS（含空转绿） + +# 3) 首错指令定位（单用例） +python3.11 - <<'EOF' +from scratchv.backend.riscv_encoder import RISCVAEncoder +asm = open("/tmp/asm/013_for_sum.s").read() +enc = RISCVAEncoder() +for i, line in enumerate(asm.splitlines()): + s = line.strip() + if s and not s.startswith((".", "#")) and ":" not in s: + try: enc.assemble(s + "\n") + except Exception as e: + print("first bad line", i + 1, repr(s), type(e).__name__, e); break +EOF +``` + +### 4.2 25 用例实测总表 + +配置：`backend=riscv, optimize_level=all, reg_alloc=linear`。`语义(I)` = interpreter oracle 判定；`语义(X)` = execution oracle 判定（本期均不可达/失败）。 + +| 用例 | compile | asm | asm 文本指令数 | 语义(I) | 语义(X) | 预计套件状态 | xfail stages / 根因 | +|------|:-------:|:---:|:--------------:|:-------:|:-------:|:------------:|---------------------| +| 001_simple_add | ✅ | ✅ | 3 | ✅ | 探测通过（a0=2,a1=3→5） | pass | — | +| 002_simple_mul | ✅ | ✅ | 3 | ✅ | 未探测 | pass | — | +| 003_sub_div | ✅ | ✅ | 4 | ✅（整数 div 盲区） | 不可信（C5） | pass | interpreter 盲区见 C5 | +| 004_relu | ✅ | ✅ | 3 | ✅ | 未探测 | pass | — | +| 005_gelu | ✅ | ✅ | 5 | ✅ | 不可信（C5） | pass | — | +| 006_softmax | ✅ | ✅ | 3 | ✅（C4 盲区） | 错（直通） | pass | interpreter 盲区见 C4 | +| 007_matmul | ✅ | ✅ | 3 | ✅（C4 盲区） | 错（单条 mul） | pass | interpreter 盲区见 C4 | +| 008_dot | ✅ | ✅ | 3 | ✅（C4 盲区） | 错（单条 mul） | pass | interpreter 盲区见 C4 | +| 009_maxpool | ✅ | ❌ | 7 | ✅ | 阻塞 | xfail | `["assemble","budget"]` / C1（`bnez a1` 第 4 行） | +| 010_exp_neg | ✅ | ✅ | 5 | ✅ | 不可信（C5） | pass | — | +| 011_multi_op_chain | ✅ | ✅ | 5 | ✅ | 不可信 | pass | — | +| 012_nn_pipeline | ✅ | ✅ | 5 | ✅（C4 盲区） | 错（mul+add） | pass | interpreter 盲区见 C4 | +| 013_for_sum | ✅ | ❌ | 7 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1（`bge a3, 4` 第 6 行） | +| 014_for_dot | ✅ | ❌ | 7 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1 | +| 015_for_relu | ✅ | ❌ | 8 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1 | +| 016_if_simple | ✅ | ❌ | 6 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1（`bnez a1` 第 3 行）+ D1 | +| 017_while_sum | ✅ | ❌ | 7 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1（`j` 第 3 行）+ D1/D2 | +| 018_nested_if | ✅ | ❌ | 11 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1 + D1 | +| 019_nested_loop | ✅ | ❌ | 11 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1 | +| 020_constant_propagation | ✅ | ✅ | 3 | ✅ | 错（执行得 0，期望 20） | pass | interpreter 盲区见 C2 | +| 021_dsl_if_else | ✅ | ❌ | 10 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1 + D1 | +| 022_dsl_while_sum | ✅ | ❌ | 8 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1 + D1/D2 | +| 023_large_chain | ✅ | ✅ | 10 | ✅ | 不可信 | pass | — | +| stress/reg_pressure_32 | ✅ | ✅ | 77 | —（oracle=execution） | 错（178，期望 64） | xfail | `["semantic"]` / C3 | +| stress/reg_pressure_loop | ✅ | ❌ | 13 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1（`bge a3, 10` 第 6 行） | + +**统计**：compile 25/25；asm 14/25；interpreter 语义（14 个线性用例）14/14；预计**用例级** 14 pass / 11 xfail / 0 fail；**pytest 测试级** 92 passed / 33 xfailed（测试总数 125）。 + +**“当前会失败”的准确清单**： + +1. **汇编不可编码（11 个）**：`009`、`013`、`014`、`015`、`016`、`017`、`018`、`019`、`021`、`022`、`reg_pressure_loop`。首错指令见设计文档附录 5.2。处置：`meta.json` 声明 `xfail.stages=["assemble","budget","semantic"]`（owner=`backend/asm-encoder`，reason 引用 C1 并带首错指令）；`009_maxpool` 例外——它是 linear + interpreter oracle，语义不依赖汇编，`xfail.stages=["assemble","budget"]`，语义必须真实通过。 +2. **执行语义错误（1 个）**：`reg_pressure_32` 实测 178 ≠ 64。处置：`xfail.stages=["semantic"]`，owner=`backend/regalloc`，reason 引用 C3。 +3. **数据缺口（5 个）**：`016`、`017`、`018`、`021`、`022` 无 golden。处置：本课题补齐（§4.4 表 1），补齐后 `test_meta_contract` 必须转绿。 +4. **数据缺陷（2 个）**：`017`、`022` 循环无自增。处置：本课题修用例源码（§4.4 表 2）。 +5. **解释器盲区（C2/C4/C5，10 个线性用例）**：`003/005/006/007/008/010/011/012/020/023` 的 interpreter 语义通过，但掩盖后端缺失（如 007 汇编实为单条 `mul`）。处置：保持 `pass`，同时在报告 `warnings` 与本文档登记；后续由 execution oracle 接管（不在本期验收）。 + +### 4.3 缺陷登记（与设计文档 §2.8 编号一一对应） + +| 编号 | owner（非本课题） | 套件中的处理 | 触发转正的信号 | +|------|------------------|--------------|----------------| +| C1 | `backend/asm-encoder` | 11 个用例 xfail（stages 见 §4.2） | `assemble_to_binary` 对分支目标成功 → XPASS | +| C2 | `backend/asm-emitter` | 020 保持 interpreter；未来执行 oracle xfail | 020 执行得 20 | +| C3 | `backend/regalloc` | reg_pressure_32 xfail(semantic) | 执行得 64 | +| C4 | `backend/op-lowering` | 006/007/008/009/012 的 interpreter 结果不删除 | 增加 execution oracle 后转正 | +| C5 | `backend/数值语义` | 浮点用例维持 interpreter | 后端支持浮点语义 | +| C6 | `backend/regalloc` | `input_registers` 声明式校准 | 分配器变更需重新校准（测试失败即信号） | +| B1 | `verification` | 控制流用例禁用 interpreter oracle | `DSLInterpreter` 支持控制流后可对线性/控制统一 oracle | +| D1 | 课题 06 | 补 5 份 meta（§4.4） | `test_meta_contract` 绿 | +| D2 | 课题 06 | 修 2 个 DSL（补自增）+ 循环用例标量 golden 写入 meta | 执行 oracle 修复 C1 后按新 golden 判定 | + +### 4.4 数据补齐与修复清单（本课题执行） + +**表 1：补齐 5 个用例的 golden（写入 `meta.json`）** + +| 用例 | `inputs` | `input_registers` | `expected_return` | 推导 | +|------|----------|-------------------|-------------------|------| +| 016_if_simple | `{"a": 4, "b": 1}` | `{"a": "a0", "b": "a1"}` | `5` | `a>b` 真 → `add(a,b)` | +| 017_while_sum | `{"i": 0, "x": 1, "acc": 0}` | `{"i": "a0", "x": "a1", "acc": "a2"}` | `10` | 自增后 10 次迭代，每次 `acc += x` | +| 018_nested_if | `{"a": 4, "b": 1}` | `{"a": "a0", "b": "a1"}` | `5` | 外层真 + 内层 `a>0` 真 → `add` | +| 021_dsl_if_else | `{"a": 4, "b": 1}` | `{"a": "a0", "b": "a1"}` | `10` | `t1=5, t2=10, relu(10)=10` | +| 022_dsl_while_sum | `{"i": 0, "x": 2, "y": 3, "acc": 0}` | `{"i": "a0", "x": "a1", "y": "a2", "acc": "a3"}` | `30` | 自增后 5 次迭代 × `mul(2,3)` | + +**表 2：修复 2 个数据缺陷用例（源码改动，仅用例数据，不触碰编译器）** + +```diff + # 017_while_sum.dsl + while (i < 10): + acc = add(acc, x) ++ i = add(i, 1) + endwhile + return acc + + # 022_dsl_while_sum.dsl + while (i < 5): + t1 = mul(x, y) + acc = add(acc, t1) ++ i = add(i, 1) + endwhile + return acc +``` + +**表 3：循环用例真实语义 golden（写入 meta，legacy `.expected` 保持不动）** + +| 用例 | `inputs` | `input_registers` | `expected_return` | 推导 | +|------|----------|-------------------|-------------------|------| +| 013_for_sum | `{"x": 1, "acc": 0}` | `{"x": "a0", "acc": "a1"}` | `4` | 4 次迭代 × `acc += 1` | +| 014_for_dot | `{"a": 2, "b": 3, "acc": 0}` | `{"a": "a0", "b": "a1", "acc": "a2"}` | `24` | 4 次迭代 × `acc += 6` | +| 015_for_relu | `{"x": 1, "y": 0}` | `{"x": "a0", "y": "a1"}` | `4` | 4 次迭代 × `acc += relu(1)` | +| 019_nested_loop | `{"x": 1, "y": 1, "acc": 0}` | `{"x": "a0", "y": "a1", "acc": "a2"}` | `8` | 4×2 次迭代 × `acc += 1` | +| reg_pressure_loop | `{"x": 1, "acc": 0}` | `{"x": "a0", "acc": "a1"}` | `210` | 10 次迭代 × 21（既有 meta 的 210 不变，补 `acc` 与 `input_registers`） | + +（注：这些用例 oracle=execution 且当前 xfail（C1），golden 修正不改变本期红绿；修正的目的是 C1 修复后套件立即具备正确判定，避免“被 xfail 掩盖的错误数据”二次污染。legacy `.expected`（如 `013=[1,2,3,4]`）是 bench_runner 的自洽数据，**不得改动**，否则 `tests/test_bench_runner.py` 回归将转红。） + +### 4.5 `input_registers` 校准方法（实现阶段一次性执行） + +1. 取一个可判别用例（`012_nn_pipeline` 或 `001_simple_add`），用互异值探测：例如 `a=100,b=1`，观察哪个候选映射能得到与 DSL 语义一致的 `a0` 返回值； +2. 对同一用例的候选声明逐个试跑 `ExecutionOracle.execute`（约 2–3 次），记录命中映射； +3. 把结果写入该用例 meta 的 `input_registers`，并在 commit message 中注明“校准用探针用例与数值”； +4. 校准只在**新增 execution 用例**或**分配器变更后**执行；测试失败信息会给出“期望/实际值”便于识别映射漂移。 + +### 4.6 xfail 填写规范（示例，013） + +```json +"xfail": { + "stages": ["assemble", "budget", "semantic"], + "reason": "RISCVAEncoder cannot encode symbolic branch targets; first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false +} +``` + +`reason` 必须包含：现象 + 首错证据；`owner` 必须是模块/课题标识；`stages` 只列真实失败或同根因阻塞的阶段（可通过“临时移除该 stage 后测试是否转红”验证）；不得写“TODO”“unknown”“可能”等无效描述。 + +--- + +## 五、CI 接线点（精确位置） + +### 5.1 `test` job：新增门禁步骤 + +**位置**：`.github/workflows/ci.yml` 第 80–85 行 `Run all topic tests` 步骤之后、第 87 行 `Run assembly-beautifier regressions` 步骤之前（当前第 86 行为空行）。 + +```yaml + # ── 课题06：DSL 基准套件（硬门禁，xfail 不红灯） ───────────────── + - name: Run DSL benchmark suite + run: | + python3.12 -m pytest tests/test_dsl_suite.py -v --tb=short +``` + +约束：不添加 `continue-on-error`，不添加 `|| echo` 兜底；pytest 收集期发现 0 用例即 collection error（红灯）。 + +### 5.2 `benchmark` job：新增报告步骤 + +**位置**：`.github/workflows/ci.yml` 第 202–209 行 `DSL case compilation benchmarks` 步骤之后、第 211 行 `CNN RISC-V compilation & estimation` 注释之前。 + +```yaml + # ── 课题06：DSL 基准套件 JSON/MD 报告（同时作为第二道门禁） ─────── + - name: DSL benchmark suite report + run: | + python3.12 benchmarks/run_suite.py \ + --json benchmark_reports/dsl_suite.json \ + --markdown benchmark_reports/dsl_suite.md +``` + +退出码语义见 §1.8；xpass/xfail 不红灯。产物由既有 `Upload benchmark reports`（ci.yml:287-293）自动上传。 + +### 5.3 `Write job summary`：追加摘要 + +**位置**：`.github/workflows/ci.yml` 第 338–361 行步骤内，`Register Allocation Benchmarks` 段落（第 357–360 行）之后、步骤结束之前追加： + +```yaml + echo "" >> $GITHUB_STEP_SUMMARY + echo "### DSL Benchmark Suite" >> $GITHUB_STEP_SUMMARY + if [ -f benchmark_reports/dsl_suite.md ]; then + cat benchmark_reports/dsl_suite.md >> $GITHUB_STEP_SUMMARY + fi +``` + +### 5.4 （可选）Makefile 页面 + +**位置**：`Makefile` 第 45–51 行 `bench:` 目标之后追加（非验收必需）： + +```make +bench-suite: + python3 -m pytest tests/test_dsl_suite.py -v --tb=short + python3 benchmarks/run_suite.py --json benchmark_reports/dsl_suite.json \ + --markdown benchmark_reports/dsl_suite.md +``` + +### 5.5 同仓 PR 的既有约束 + +- `ci.yml` 第 5 行 push 分支含 `main, wjy_dev, jzj_dev`，PR 目标为 `main`；新增步骤需在两个触发路径下都可用（步骤内命令不依赖 self-hosted 专有路径）。 +- PR 侧 runner 为 `ubuntu-latest`，依赖安装走 `pip install -e ".[all]"`（ci.yml:74-78），tinyfive 已含；无新增依赖。 + +--- + +## 六、验收标准 + +### 6.1 命令与预期 + +```bash +# 主验收：pytest 套件 +python3.12 -m pytest tests/test_dsl_suite.py -v --tb=short +# 预期：0 failed, 0 errors；约 92 passed, 33 xfailed（数量以实现后冻结为准） + +# 报告验收 +python3.12 benchmarks/run_suite.py \ + --json /tmp/dsl_suite.json --markdown /tmp/dsl_suite.md +echo $? # 预期 0 +python3.12 -m json.tool /tmp/dsl_suite.json > /dev/null && echo JSON_OK + +# 回归：既有套件不受影响 +python3.12 -m pytest tests/test_bench_runner.py -v # 预期全绿 +make test # 预期全绿 +``` + +### 6.2 判定矩阵 + +| 检查项 | 通过条件 | +|--------|----------| +| `pytest tests/test_dsl_suite.py` | `failed=0` 且 `errors=0`；xfail 必须全部带 `reason`（pytest 输出可见） | +| `test_meta_contract` | 25/25 通过（数据缺口 D1 全部补齐） | +| `test_compile_ok` | 25/25 通过 | +| `test_asm_encodable` | 14 passed + 11 xfailed（C1） | +| `test_instruction_budget` | 有 `max_instructions` 的用例 14 passed + 11 xfailed；未声明者 skip | +| `test_semantic_golden` | 14 passed（interpreter）+ 11 xfailed（execution 阻塞/C3） | +| `run_suite.py` | 退出码 0；JSON schema 与设计文档 §2.6 一致；summary 计数与 pytest 一致 | +| CI | 两处新步骤均执行且为绿（有 xfail 仍绿）；人为制造一个硬失败（如把某用例 `max_instructions` 改小 1）时门禁红灯 | +| 范围边界 | `git diff --stat` 不包含任何 `scratchv/**` 文件 | + +### 6.3 反验收（不允许的“绿”） + +- 用 `continue-on-error`/`|| echo` 把 CI 步骤粉饰为绿； +- 把尚未定位、偶发、或数据缺失导致的失败直接标 xfail； +- 为了让套件变绿而修改 `scratchv/**` 或删除/跳过失败用例； +- `test_meta_contract` 被 xfail（契约不可 xfail）； +- 发现 0 个用例仍返回 0。 + +--- + +## 七、实施顺序与风险回退 + +### 7.1 建议提交拆分（每步可独立验证） + +| 步骤 | 内容 | 验证 | +|------|------|------| +| S1 | `benchmarks/dsl_suite.py` + `benchmarks/run_suite.py`（核心与 CLI） | `run_suite.py --list` 输出 25 用例；对临时伪用例跑负例 | +| S2 | `tests/test_dsl_suite.py` + 23 份新增 meta.json + 2 份修订（含 §4.4 数据修复） | `pytest tests/test_dsl_suite.py` 达到预期计数 | +| S3 | `tests/test_bench_runner.py` 回归 + `make test` | 全绿 | +| S4 | `ci.yml` 三处接线 + （可选）Makefile 页面 | 本地模拟两条命令；PR CI 实测 | + +### 7.2 风险与缓解 + +| 风险 | 概率 | 影响 | 缓解/回退 | +|------|------|------|-----------| +| 输入寄存器映射（C6）随分配器变化 | 中 | execution 用例误判 | 声明式 `input_registers` + 校准流程；失败信息含期望/实际值 | +| pytest 收集期 `discover_cases` 异常导致整套 collection error | 低 | 门禁红灯但难定位 | `load_case_spec` 全异常捕获；0 用例显式 `pytest.fail` | +| 25 次编译拖慢 `test` job | 低 | CI 时长 | session fixture 缓存；实测编译 0.01–0.05s/用例；必要时 `--ignore` 报告步骤重复 | +| 循环用例 golden 数据修正被误认为“改测试就绿” | 低 | 评审质疑 | 修正写入 meta（legacy `.expected` 不动），仅在 execution oracle 下生效且当前 xfail；变更单列（§4.4 表 2/3）并在 commit message 说明 | +| `meta.json` 字段演进导致旧数据失效 | 低 | 数据维护 | `schema_version` + 未知字段容忍（WARNING） | +| `run_suite.py` 与 pytest 判定不一致 | 低 | 两处红绿矛盾 | 共用 `DSLSuiteRunner`/`compare_values`；验收要求 summary 计数一致 | + +### 7.3 回退方案 + +- **整体回退**：`git revert `（所有改动均为新增文件 + meta 数据 + 3 处 YAML/Makefile 片段）。 +- **仅关闭门禁**：删除 §5.1/§5.2 两个步骤即可，文件与数据保留供离线使用。 +- **仅关闭执行 oracle**：把相关用例 `oracle` 改为 `none` 并补 `xfail`（`stages` 含 `semantic`）；不影响 compile/asm/budget 门禁。 +- **进程隔离兜底**：若发现同进程编译状态污染，为 `compile_case` 增加 `--isolate` 子进程模式（接口不变，仅实现切换）。 + +--- + +## 八、附录 + +### 8.1 复现命令全集（本文件所有实测量） + +```bash +# 25 用例编译/汇编总表 +python3.11 - <<'EOF' +import sys; sys.path.insert(0, ".") +from pathlib import Path +from scratchv.compiler import CompilerDriver, CompilerConfig +from scratchv.backend.riscv_encoder import assemble_to_binary +from scratchv.backend._asm_parser import parse_asm +for root in ("benchmarks/cases", "tests/stress"): + for dsl in sorted(Path(root).glob("*.dsl")): + out = Path("/tmp/asm") / (dsl.stem + ".s"); out.parent.mkdir(exist_ok=True) + r = CompilerDriver(CompilerConfig(backend="riscv", optimize_level="all")).compile(str(dsl), str(out)) + n = sum(1 for l in parse_asm(r.output_text or "") if l.opcode and not l.is_directive) + try: + ok = bool(assemble_to_binary(r.output_text)) if r.success else False + except Exception as e: + ok = f"FAIL {type(e).__name__}" + print(f"{dsl.stem:<26} compile={r.success} asm={ok} insts={n}") +EOF + +# 现有 bench_runner 对照 +python3 benchmarks/bench_runner.py benchmarks/cases --quiet + +# 执行 oracle 探针（001） +python3.11 - <<'EOF' +import sys; sys.path.insert(0, ".") +from scratchv.backend.riscv_encoder import assemble_to_binary +from scratchv.simulator.tinyfive import ProfiledMachine +asm = open("/tmp/asm/001_simple_add.s").read() +binary = assemble_to_binary(asm) +words = [int.from_bytes(binary[i:i+4], "little") for i in range(0, len(binary), 4)] +m = ProfiledMachine(mem_size=128*1024*1024); m.load_binary(words, origin=0) +m.set_reg(10, 2); m.set_reg(11, 3) +m.run(instructions=len(words), start=0, strict=True) +print("a0 =", m.get_reg(10)) # 实测 5 +EOF +``` + +### 8.2 评审条目对照表 + +| 历史评审条目（`课题06-性能测试套件-commit-review.md`） | 本方案对应 | +|-------------------------------------------------------|------------| +| ISSUE-01 报告函数 5 对重复（~800 行） | 套件只保留一组 `SuiteReport.to_dict` / `to_markdown` 生成器；旧 `run_tests.py` 不迁入 | +| ISSUE-02 Windows 路径硬编码 | 报告只写相对/`case_id`；不含绝对路径与预生成产物 | +| ISSUE-03 `infer_initial_registers` 依赖 IR 文本 | 取消推断，改为 meta 声明式 `input_registers`（§4.5 校准） | +| ISSUE-04 README powershell 标注 | 不在本课题（文档不在交付物内） | +| ISSUE-05 `values_equal` 张量处理 | `compare_values` 支持标量/向量/嵌套列表数值比较 | +| §6 建议“pytest 化 + 分批启用 + CI 集成” | 本方案即该建议的落地：pytest 门禁 + xfail 分级 + CI 双步骤 | +| §2 “13/23 PASS 根因：branch 超时 / reduction 返回 0 / tensor 未实现” | 对应 C1/C3/C4，均在 §4.3 登记并 xfail | +| §7.3 “去掉参考解释器是正确的” | 保持：不新增第二解释器；控制流用例等待 execution oracle | + +### 8.3 关联文件 + +- 设计文档：`./设计文档.md` +- 课题文档：`/root/Lab/ScratchV/docs/topics/06-性能基准套件.md` +- 现有编排：`/root/Lab/ScratchV/benchmarks/bench_runner.py` +- 历史评审：`/root/Lab/GaoMD/ScratchV/SPEC&review/CommitReview/课题06-性能测试套件-commit-review.md` +- CI：`/root/Lab/ScratchV/.github/workflows/ci.yml` + +--- + +## 实现结果（2026-09-14 集成） + +> **集成 commit**：`cae2c08`（`feat(topic06): add pytest-based DSL benchmark suite with xfail policy`） +> **集成位置**：`Seven_big_summary` 上 `d146515` 之后的第 1 个 topic commit（全分支共 13 个 commit） +> **集成后全量**：`PYTHONPATH=. python3.11 -m pytest tests/ -q` → **1011 passed / 13 xfailed / 20 xpassed / 0 failed** + +### 实现文件与要点 + +| 文件 | 要点 | +|------|------| +| `benchmarks/dsl_suite.py` | 套件核心：`discover_cases` / `load_case_spec` / `DSLSuiteRunner` / `ExecutionOracle` / `SuiteReport`，按 §1.2–§1.8 契约实现 | +| `benchmarks/run_suite.py` | CLI（`--json` / `--markdown`）与 §1.8 退出码契约 | +| `tests/test_dsl_suite.py` | session fixture 缓存 + 逐 stage `pytest.param(marks=...)` 注入 xfail | +| `benchmarks/cases/*.meta.json` ×23 | 新增元数据（含 §4.4 表 1/表 3 数据补齐） | +| `tests/stress/*.meta.json` ×2 | 修订接入统一 schema | +| `benchmarks/cases/017_while_sum.dsl`、`022_dsl_while_sum.dsl` | §4.4 表 2 补自增（仅用例数据，不触碰编译器） | +| `.github/workflows/ci.yml` | 三处接线：`test` job 门禁（§5.1）、`benchmark` job 报告（§5.2）、job summary 摘要（§5.3） | + +### 测试数字 + +| 口径 | 结果 | +|------|------| +| 定向（`tests/test_dsl_suite.py`） | 96 passed / 33 xfailed / 0 failed | +| 分支全量（cherry-pick 前） | 661 passed | +| 集成后全量 | 1011 passed / 13 xfailed / 20 xpassed / 0 failed | + +### 与本文档的偏差 / 未完成项 + +- 11 个分支指令失败案例（009/013/014/015/016/017/018/019/021/022/`reg_pressure_loop`）**全部 xfail**，根因登记为 backend 编码器缺陷 C1–C6，按课题边界**未修**。 +- 本地无 pyyaml，CI YAML 只做了人工 diff 核对，未做语法解析校验。 +- `017/022` 的预算冻结值待 C1 修复后复测收紧。 + +### 已知限制 + +- 33 个 xfail 是缺陷的真实登记，不得为了全绿而删除或放宽。 +- execution oracle 依赖 tinyfive；缺失时按 blocked 处理。 +- `017/022` 的 legacy `.expected` 是 bench_runner 的自洽数据，按要求保持不动。 + +### 关联集成修复（`fix(integration)`，不计入本 topic 实现 commit） + +- `76d852f`：`reg_pressure_32` 预算 77 → 103（topic17 linear-scan 重构后实测；语义同时由失败转正确）。 +- `58dd2b4`：`gelu` 预算 5 → 6、`large_chain` 10 → 11（topic29 常量物化修复 R 型立即数编码后的合理指令数增加）。 diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000..b6f9244 --- /dev/null +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,617 @@ +# ScratchV 编译器性能基准套件技术设计文档 + +> 文档版本：v1.0 +> 编写日期：2026-09-14 +> 涉及模块：`benchmarks/dsl_suite.py`（新增，套件核心）、`benchmarks/run_suite.py`（新增，CLI 报告入口）、`tests/test_dsl_suite.py`（新增，pytest 门禁）、`benchmarks/cases/*.meta.json` 与 `tests/stress/*.meta.json`（用例元数据迁移/补齐）、`.github/workflows/ci.yml`（CI 接线） +> 功能范围：23 个 `benchmarks/cases` DSL 用例 + 2 个 `tests/stress` 用例的 pytest 化基准套件；编译成功 / 汇编可编码 / 指令数上限 / 语义 golden 四级断言；硬失败与 xfail 分级；JSON 报告；CI 门禁 + +--- + +## 一、功能介绍 + +### 1.1 功能概述 + +#### 背景现状（2026-09-14 实测，代码基线 HEAD `d146515`，Python 3.11 + tinyfive 已安装） + +课题 06 的文档（`docs/topics/06-性能基准套件.md`）标记为“✅ 已完成”，但仓库实际状态与目标形态存在五类断层： + +1. **编排脚本存在，但绿得不可信**。`benchmarks/bench_runner.py` 的语义判定走 `DSLInterpreter`（`scratchv/verification/verifier.py:140`），而后者**不实现控制流**：`for ... / endfor` 直接跳过、循环体只执行一次（`verifier.py:169-175`）、`if/else/while` 的行无 `=` 被当作无关行忽略，两个分支体都会顺序执行。因此： + - 5 个控制流用例（`016_if_simple`、`017_while_sum`、`018_nested_if`、`021_dsl_if_else`、`022_dsl_while_sum`）`.expected` 为空，`run_case` 在无期望文件时“只要不抛异常就 PASS”（`bench_runner.py:470-472`）； + - 循环用例 `.expected`（如 `013_for_sum` = `[1. 2. 3. 4.]`）是按“只执行一次循环体”的残缺解释器校准出来的，与 DSL 真实语义（4 次迭代应为 `[4. 8. 12. 16.]`）不符； + - 结论：现状 `23/23 PASS` 中有大量**空转绿**。 + +2. **真实编译/汇编/执行路径大面积失败，但无人红灯**。本次对 25 个用例（`benchmarks/cases` 23 个 + `tests/stress` 2 个）实测： + + | 阶段 | 通过 | 失败 | 失败清单 | + |------|------|------|----------| + | 编译（parse → optimize → codegen） | 25 | 0 | — | + | 汇编可编码（`assemble_to_binary`） | 14 | 11 | `009_maxpool`、`013_for_sum`、`014_for_dot`、`015_for_relu`、`016_if_simple`、`017_while_sum`、`018_nested_if`、`019_nested_loop`、`021_dsl_if_else`、`022_dsl_while_sum`、`reg_pressure_loop` | + | 语义 golden（真实 RISC-V 执行） | 仅 1 例端到端探测成功（`001`，加法交换律下映射仍不可判别） | 其余不可信或阻塞 | 输入寄存器映射无契约、常量 `mv rd, imm` 非法发射、溢出/栈未初始化、NN 算子未 lowering、分支汇编不可编码 | + + 11 个汇编失败的首错指令全部是**控制流指令**（`bnez` / `j` / `bge`），根因是 `AsmEmitter` 把分支目标写进注释、操作数缺失，而 `RISCVAEncoder` 需要符号目标/偏移（详见 §2.8 C1、附录 5.2）。 + +3. **交付目录只剩产物**。`ScratchV-topic06-deliverable/` 现仅存 `build/*.s` 共 23 个汇编产物；`run_tests.py` 已不存在。历史版本（commit `82860b4`）中的 23 个 `.dsl + .meta.json` 用例采用了“scalar 输入 + `expected_return` + TinyFive 真执行”的正确格式，可作为本课题元数据 schema 的参考来源，但**不纳入本期验收范围**（见 §4.5 可选步骤）。 + +4. **`tests/stress/` 两个用例无 runner**。`reg_pressure_32`、`reg_pressure_loop` 已带 `.meta.json`（`inputs` + `expected_return`），但它们既不在 `bench_runner` 的发现路径，也不在 pytest 收集范围，实际处于“提交后从未被任何 CI 执行”的状态。 + +5. **CI 无 DSL 套件门禁**。`.github/workflows/ci.yml` 的 `test` job 没有 DSL 用例步骤；`benchmark` job 的 `DSL case compilation benchmarks`（ci.yml:202-209）只调用自证式 `bench_runner.py`，且该 job 与 `test` job 相互独立、报告步骤失败不阻塞功能测试门禁。本方案把硬门禁放进 `test` job（§2.7），确保失败即红灯。 + +#### 本课题定义 + +把 25 个用例变成**可参数化、可定位、可门禁**的 pytest 套件，落地三件事： + +1. **核心 runner**：新增 `benchmarks/dsl_suite.py`，负责用例发现、`meta.json` 解析与校验、编译/汇编/语义/指令预算四阶段执行、结果与 xfail 判定、JSON/Markdown 报告生成。 +2. **pytest 门禁**：新增 `tests/test_dsl_suite.py`，把 `CaseSpec` 参数化为 pytest 用例，每个阶段独立成一个测试函数，失败可精确定位到“哪个用例的哪个阶段”。 +3. **CI 接线**：`test` job 增加门禁步骤；`benchmark` job 增加报告步骤；两者失败即红灯，xfail 不红灯。 + +同时**记录而不修复**所有暴露出的编译器缺陷（§2.8），保证套件诚实：声明为 xfail 的用例必须携带明确的 `reason` 与 `owner`。 + +#### 明确不做（范围边界） + +- 不修改 DSL 语义、解析器、IR、优化器、后端、汇编发射器、编码器（`scratchv/frontend/*`、`scratchv/ir/*`、`scratchv/optimizer/*`、`scratchv/backend/*` 零改动）。 +- 不修复用例暴露的编译器缺陷（C1–C6）。暴露即登记 + xfail，修复归属其他课题。 +- 不修改 Spike/TinyFive（课题 24）。语义执行只使用既有 API：`scratchv.backend.riscv_encoder.assemble_to_binary`、`scratchv.simulator.tinyfive.ProfiledMachine`；不需要 `initial_registers`/`return_value` 之类的新接口（`ProfiledMachine.get_reg/set_reg` 在 main 已存在）。 +- 不做性能回归门禁：计时只写入报告，不作为断言；指令数断言只在 meta 显式声明 `max_instructions` 时生效。 +- 不修改、不删除 `benchmarks/bench_runner.py` 及其测试（兼容保留，二者并存）。 +- 不实现新的 DSL 求值器（避免与编译器语义漂移，评审记录 §5.1 已确认“去掉参考解释器”是正确决策）；修复 `DSLInterpreter` 控制流（B1）不属本课题。 + +### 1.2 设计目标 + +- **可门禁**：`pytest tests/test_dsl_suite.py` 一条命令决定红绿；未声明的失败必须红灯。 +- **诚实**：禁止“吞错式 PASS”。任何阶段异常都落到 `CaseOutcome.error`，不得以空字符串或 `## SIM ERROR` 之类字符串掩盖。 +- **可定位**：失败输出精确到 `case_id + stage + 首错指令/期望值/实际值`。 +- **可扩展**：新增用例 = 新增 `{name}.dsl` + `{name}.meta.json`，零代码改动即可被发现。 +- **零编译器侵入**：所有新增能力位于 `benchmarks/` 与 `tests/`，不触碰 `scratchv/`。 +- **可回退**：全部改动为新增文件 + 数据文件 + 两处 workflow 片段，`git revert` 即完整回退。 +- **报告机器可读**：JSON 供 dashboard/历史对比；Markdown 供 PR/CI summary。 + +--- + +## 二、设计规范 + +### 2.1 套件结构 + +#### 2.1.1 文件布局 + +``` +benchmarks/ + dsl_suite.py # 新增：套件核心（发现/执行/断言/报告） + run_suite.py # 新增：CLI 入口（生成 JSON/MD、退出码） + cases/ # 既有：23 个 .dsl + .expected + .desc + *.meta.json # 新增：每个用例一份元数据 + bench_runner.py # 既有：保留不动 +tests/ + test_dsl_suite.py # 新增：pytest 门禁 + stress/ + reg_pressure_32.dsl + .meta.json # 既有：仅接入 + reg_pressure_loop.dsl + .meta.json # 既有：仅接入 +``` + +#### 2.1.2 用例发现规则 + +- **根目录（roots）**：默认 `("benchmarks/cases", "tests/stress")`，可用 `DSLSuiteRunner(roots=...)` 或 `run_suite.py --roots` 覆盖。 +- **匹配**：对每个 root 递归 `rglob("*.dsl")`，按 `root 名 / 相对路径` 的 POSIX 字符串排序，保证顺序稳定。 +- **case_id**：`"{root_dir_name}/{相对路径去后缀}"`，例如 `cases/001_simple_add`、`stress/reg_pressure_32`。 +- **pytest id**：`case_id` 中 `/` 替换为 `-`，例如 `cases-001_simple_add`。 +- **跳过规则**：以 `_` 开头的文件、位于 `fixtures/` 目录下的文件、以及同目录存在 `{name}.skip` 文件时跳过（`skip` 文件内容作为原因，报告 `status="skip"`）。 +- **空发现即配置错误**：`discover()` 返回 0 条时，CLI 以错误码 2 退出，pytest 以 collection error 处理（不允许“没有用例所以全绿”）。 + +#### 2.1.3 元数据解析优先级 + +同名旁路文件，优先级从高到低： + +1. `{name}.meta.json`（规范来源，字段见 §2.2）； +2. `{name}.expected`（仅当 oracle=`interpreter` 且 meta 未给 `expected_return` 时作为期望值回退）； +3. `{name}.desc`（仅当 meta 未给 `description` 时回退）。 + +`meta.json` 一旦存在即为权威：`description`、`inputs`、`expected_return` 均以 meta 为准，`.expected` 只是线性用例的兼容缺省。实现阶段为 23 个 legacy 用例新增 meta、为 2 个 stress 用例修订 meta（见 §4.5），此后残缺 meta 属契约违规。 + +#### 2.1.4 `CaseSpec`（不可变，发现期产物） + +| 字段 | 类型 | 说明 | +|------|------|------| +| `case_id` | `str` | `cases/001_simple_add` 形式的稳定标识 | +| `pytest_id` | `str` | `cases-001_simple_add` | +| `name` | `str` | 文件名去后缀 | +| `dsl_path` | `Path` | `.dsl` 绝对/相对路径 | +| `meta_path` | `Path \| None` | `.meta.json` 路径 | +| `expected_path` | `Path \| None` | 兼容 `.expected` | +| `description` | `str` | 用例描述 | +| `category` | `str` | `arith`/`nn`/`control`/`complex`/`const`/`stress`，来自 meta 或按目录/前缀推断 | +| `flow` | `str` | `linear` 或 `control`（发现期由源码关键字推断，meta 可覆盖但必须一致） | +| `oracle` | `str` | `interpreter` / `execution` / `none` | +| `inputs` | `dict[str, object]` | 变量名 → 标量/列表 | +| `input_registers` | `dict[str, str]` | 变量名 → `a0`…（仅 execution oracle 需要） | +| `expected_return` | `object \| None` | 数值或列表 | +| `expected_text` | `str` | `.expected` 原文（可为空） | +| `rtol` / `atol` | `float` | 默认 `1e-3` / `1e-6` | +| `assertions` | `tuple[str, ...]` | 断言集合，默认全部四级 | +| `max_instructions` | `int \| None` | 汇编指令数上限（文本指令口径，见 §2.4） | +| `timeout_s` | `float` | 默认 30.0 | +| `xfail` | `XFailSpec \| None` | `stages` + `reason` + `owner` + `strict`（默认 False） | +| `skip_reason` | `str \| None` | 发现 `{name}.skip` 时的跳过原因（非空则执行类断言测试 `skip`；`test_meta_contract` 不受影响） | +| `meta_errors` | `tuple[str, ...]` | 契约违规列表，非空即硬失败 | + +### 2.2 `meta.json` schema + +#### 2.2.1 字段表 + +| 字段 | 类型 | 必填 | 默认 | 约束 | +|------|------|------|------|------| +| `schema_version` | int | 否 | `1` | 目前只接受 `1` | +| `description` | str | 是 | — | 非空 | +| `category` | str | 否 | 目录名/前缀推断 | `arith`/`nn`/`control`/`complex`/`const`/`stress` | +| `flow` | str | 否 | 源码推断 | `linear`/`control`；与推断冲突时硬失败 | +| `oracle` | str | 是 | — | `interpreter`/`execution`/`none`；`interpreter` 仅允许 `flow=linear` | +| `inputs` | object | 是 | `{}` | 值可为 int/float/一维 list | +| `input_registers` | object | 条件必填 | `{}` | 当 `oracle=execution` 且 `inputs` 非空时必填；键必须与 `inputs` 一致；值匹配 `^(a[0-7]|s[0-9]|s1[01])$` | +| `expected_output_type` | str | 否 | `return_value` | 与旧交付格式兼容，仅允许 `return_value`（本期） | +| `expected_return` | number \| list | 条件必填 | — | `oracle != none` 时必填（interpreter 可由 `.expected` 回退） | +| `rtol` / `atol` | float | 否 | `1e-3` / `1e-6` | 均非负 | +| `assertions` | list[str] | 否 | 全部 | 取值 `compile_ok`/`asm_encodable`/`inst_budget`/`semantic_golden` | +| `max_instructions` | int | 否 | `None` | >0；仅当 `inst_budget` 在 `assertions` 中生效 | +| `timeout_s` | float | 否 | `30.0` | >0 | +| `xfail` | object | 否 | `None` | `{stages, reason, owner, strict?}`；`stages` 为 `compile/assemble/budget/semantic` 的非空子集（允许单字符串写法）；仅当多个阶段的失败属于同一根因时才把多阶段一并列入；`reason`、`owner` 非空 | + +未知字段：记录 WARNING 到 `meta_errors` 之外（不参与硬失败），保证向前兼容。JSON 解析失败、类型错误、条件必填缺失：进入 `meta_errors`，`test_meta_contract` 硬失败。 + +#### 2.2.2 合法示例（线性用例） + +```json +{ + "schema_version": 1, + "description": "Basic addition of two vectors", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [2.0, 4.0, 6.0, 8.0], + "max_instructions": 3 +} +``` + +#### 2.2.3 合法示例（控制流 + 执行 oracle + xfail） + +```json +{ + "schema_version": 1, + "description": "Simple if-else branch (extended parser)", + "category": "control", + "flow": "control", + "oracle": "execution", + "inputs": {"a": 4, "b": 1}, + "input_registers": {"a": "a0", "b": "a1"}, + "expected_return": 5, + "assertions": ["compile_ok", "asm_encodable", "inst_budget", "semantic_golden"], + "max_instructions": 8, + "xfail": { + "stages": ["assemble", "budget", "semantic"], + "reason": "RISCVAEncoder cannot encode symbolic branch targets (bnez/j/bge emitted with target in comment); first bad line: bnez a1", + "owner": "backend/asm-encoder", + "strict": false + } +} +``` + +#### 2.2.4 非法示例（必须硬失败） + +| 例子 | 违规 | +|------|------| +| `{"oracle": "interpreter"}` 且源码含 `while (` | oracle 与 flow 冲突 | +| `oracle=execution` 且有 `inputs` 但无 `input_registers` | 条件必填缺失 | +| `oracle=interpreter`、无 `expected_return`、无 `.expected` | 缺期望值 | +| `{"oracle": "none"}` 且无 `xfail` | none 必须显式声明阻塞原因 | +| `"max_instructions": 0` | 约束违规 | + +### 2.3 执行器接口 + +#### 2.3.1 流水线与精确 API + +核心类：`benchmarks.dsl_suite.DSLSuiteRunner`（避免与既有 `BenchmarkRunner` 重名）。 + +``` +discover() -> list[CaseSpec] + └─ run_case(spec) -> CaseOutcome + ├─ compile_case(spec) -> CompileOutcome # 阶段 1 + ├─ assemble_asm(asm_text) -> AssembleOutcome # 阶段 2 + ├─ check_instruction_budget(spec, asm_text) -> BudgetOutcome # 阶段 3（可选） + └─ evaluate_semantics(spec, compile_outcome) -> SemanticOutcome # 阶段 4 + └─ run_all() -> SuiteReport +``` + +| 方法 | 签名 | 说明 | +|------|------|------| +| `__init__` | `(roots=("benchmarks/cases","tests/stress"), *, workdir: Path\|None=None, backend="riscv", optimize_level="all", reg_alloc="linear", timeout_s=30.0, verbose=False)` | `workdir` 缺省为 `tempfile.mkdtemp(prefix="dsl_suite_")`，汇编产物写入其中，不污染仓库 | +| `discover` | `() -> list[CaseSpec]` | §2.1.2 规则；结果按 `case_id` 排序 | +| `compile_case` | `(spec) -> CompileOutcome` | 以 `CompilerDriver(CompilerConfig(backend=..., optimize_level=..., reg_alloc=...))` 编译，`compile(dsl_path, out_s)`；捕获异常；记录 `ir_instruction_count`（单独 parse 计数）与耗时 | +| `assemble_asm` | `(asm_text: str) -> AssembleOutcome` | `assemble_to_binary(asm_text)`；同时用 `parse_asm` 计数字面指令（非 label、非 directive），得到 `asm_instruction_count` | +| `check_instruction_budget` | `(spec, asm_text) -> BudgetOutcome` | `max_instructions is None` 时 `skipped=True` | +| `evaluate_semantics` | `(spec, compile_outcome) -> SemanticOutcome` | 按 `oracle` 分派：`interpreter` → `DSLInterpreter`；`execution` → `ExecutionOracle`；`none` → `blocked` | +| `run_case` | `(spec) -> CaseOutcome` | 逐阶段执行，合成 `status`（见 §2.5） | +| `run_all` | `() -> SuiteReport` | 汇总计数、写出报告用结构 | + +#### 2.3.2 语义 oracle 三种模式 + +| oracle | 实现 | 适用 | 局限（必须在报告中声明） | +|--------|------|------|--------------------------| +| `interpreter` | `scratchv.verification.verifier.DSLInterpreter.run(source, inputs)`，输出与 `expected_return`/`.expected` 做数值比较 | `flow=linear` 用例 | 不经过代码生成，**不验证后端语义**；对向量用例使用默认输入 `[1,2,3,4]`（与 bench_runner 一致） | +| `execution` | `ExecutionOracle.execute(asm, input_registers, inputs, timeout_s)`：`assemble_to_binary` → `ProfiledMachine.load_binary` → `set_reg` 注入 → `run(instructions=len(words), start=0, strict=True)` → `get_reg(10)` 作为返回值 → 数值比较 | `flow=control`、压力用例、需要验证真实代码路径的用例 | 依赖汇编成功；输入→寄存器映射为声明式（不推断）；tinyfive 不可用时 `blocked` | +| `none` | 不执行，返回 `blocked` | 已知无法执行的用例 | 必须携带 `xfail`（`stages` 含 `semantic`），否则硬失败 | + +`interpreter` 模式对向量输入沿用现状默认值 `np.array([1.0,2.0,3.0,4.0])`；后续用例可在 `inputs` 中显式给出向量，逐步替代隐式缺省。 + +#### 2.3.3 输入寄存器契约（不推断） + +历史实现（交付目录 `run_tests.py` 的 `infer_initial_registers`）从 IR 文本正则推断变量→寄存器映射，评审记录 ISSUE-03 已判定其脆弱（依赖 IR 打印格式与分配器行为）。本设计**取消推断**： + +- `oracle=execution` 的用例必须在 `meta.json` 中用 `input_registers` 显式声明 `变量 → a0/a1/...`； +- 变量赋值按声明进行 `ProfiledMachine.set_reg(idx, value)`（int 直接注入；float 若非整数值则先报告 `blocked`，因为当前后端寄存器为整数语义）； +- 声明缺失即 `meta_errors`（硬失败），不尝试猜测； +- 映射由实现阶段一次性校准（用 001/012 这类可判别用例做探针，方法见开发文档 §4.5），后端分配器变化导致映射失效时，测试以“输入值不匹配”失败并触发重新校准，这是显式契约成本而非隐式耦合。 + +#### 2.3.4 超时与隔离 + +- 每个用例四阶段总耗时受 `timeout_s` 约束（默认 30s）。执行 oracle 的 `ProfiledMachine.run` 以 `instructions=len(words)` 限定步数，天然防死循环； +- 编译/汇编在进程内执行，异常全部捕获为 `CaseOutcome.error`； +- 若未来发现优化器全局状态污染（同进程连续编译相互影响），回退方案为每用例子进程隔离（设计预留 `--isolate`，见开发文档 §7）。 + +### 2.4 断言类型 + +| 断言名 | 常量 | 阶段 | 判定 | 失败级别 | +|--------|------|------|------|----------| +| 编译成功 | `ASSERT_COMPILE_OK = "compile_ok"` | compile | `CompileResult.success` 为真且 `output_text` 非空 | 硬失败（除 `xfail.stages` 含 `compile`） | +| 汇编可编码 | `ASSERT_ASM_ENCODABLE = "asm_encodable"` | assemble | `assemble_to_binary(asm)` 返回非空字节序列 | 硬失败（除 `xfail.stages` 含 `assemble`） | +| 指令数上限 | `ASSERT_INST_BUDGET = "inst_budget"` | budget | `asm_instruction_count <= max_instructions`；口径 = `parse_asm` 后 opcode 非空且非 directive 的行数 | 硬失败（除 `xfail.stages` 含 `budget`） | +| 语义 golden | `ASSERT_SEMANTIC_GOLDEN = "semantic_golden"` | semantic | oracle 输出与期望值在 `rtol`/`atol` 内相等 | 硬失败（除 `xfail.stages` 含 `semantic`） | +| 元数据契约 | `ASSERT_META_CONTRACT = "meta_contract"` | 发现期 | `meta_errors` 为空 | 硬失败（不可 xfail） | + +补充规则： + +- **口径一致性**：`asm_instruction_count` 使用 `scratchv.backend._asm_parser.parse_asm`，与 `benchmarks/run_benchmark.py::_count_asm_instructions` 同口径，排除 label 与 `.directive`；不得用“二进制字节数/4”替代（伪指令展开会放大，实测 `023_large_chain` 文本 10 条 vs 二进制 16 字）。 +- **向量比较**：复用 bench_runner 的数值化比较算法（去 `[]`、按 `,;空格` 切分、`np.allclose`），实现为套件内公开函数 `compare_values(actual, expected, rtol, atol) -> bool`；标量直接数值比较。 +- **计时**：`compile_time_s`、`assemble_time_s`、`semantic_time_s` 写入报告，但**不参与断言**。 +- **预算断言启用条件**：`max_instructions` 未声明时 `inst_budget` 不生效，pytest 侧 `skip("no max_instructions declared")`；声明后即硬断言。 +- **阶段依赖与 xfail 声明**：`assemble` 失败必然导致 `budget` 无法计数、`execution` 语义被阻塞，三者属同一根因，写在同一个 `xfail.stages` 列表中；`interpreter` 语义不依赖汇编，汇编失败时其语义断言照常执行（例如 `009_maxpool` 的 `xfail.stages=["assemble","budget"]`，语义仍须真实通过）。 + +### 2.5 失败分级 + +#### 2.5.1 硬失败（必须红灯） + +- 任一启用断言失败，且该阶段未在 `meta.json` 的 `xfail.stages` 中声明； +- `meta_errors` 非空（契约违规，`meta_contract` 不可 xfail）； +- 发现 0 个用例、roots 不存在、报告无法写出； +- `oracle=none` 但未声明 xfail。 + +#### 2.5.2 xfail 判据（允许绿但必须留痕） + +同时满足才允许写入 `xfail`： + +1. **确定性**：失败可稳定复现，不依赖机器负载/时钟/环境随机性； +2. **已定位**：能给出首错阶段与最小证据（首错指令、期望/实际值、异常类型）； +3. **有归属**：`owner` 指向负责修复的课题/模块（backend、encoder、op-lowering、verification、或本套件的数据修复）； +4. **不在本课题修复范围**：编译器缺陷、解释器缺陷、后端缺失（本课题只修自己的用例数据缺陷）。 + +禁止 xfail 的情形：偶发/不稳定失败（应修 runner 或标记 skip 并说明环境依赖）、尚未定位的失败、缺少期望值这类数据缺口（必须先补数据，否则属硬失败）。 + +#### 2.5.3 xpass 与 strict 策略 + +- `XFailSpec.strict` 默认 `false`：被 xfail 的用例若因其他课题修复而转绿，记为 `XPASS`，报告计数并在 CI summary 显示，**不红灯**（避免跨课题阻塞）； +- 提供 `run_suite.py --strict-xfail` 将全部 xfail 置为 strict，用于发布前清账； +- 任何 xpass 必须在 30 天内转正（移除 xfail/降低 `max_instructions`），由清理任务跟踪（当前以报告 `xpass` 计数暴露）。 + +### 2.6 报告 schema + +#### 2.6.1 JSON（`run_suite.py --json PATH`） + +顶层字段： + +| 字段 | 类型 | 说明 | +|------|------|------| +| `schema_version` | int | `1` | +| `suite` | str | 固定 `"dsl-benchmark"` | +| `generated_at` | str | ISO8601 | +| `roots` | list[str] | 实际使用的 roots | +| `compiler` | object | `{backend, optimize_level, reg_alloc}` | +| `summary` | object | `{total, passed, xfailed, xpassed, failed, skipped}` | +| `results` | list[object] | 每用例一条，见下 | + +`results[i]` 字段： + +| 字段 | 类型 | 说明 | +|------|------|------| +| `case_id` / `pytest_id` | str | 稳定标识 | +| `category` / `flow` / `oracle` | str | 分类与策略 | +| `status` | str | `pass`/`fail`/`xfail`/`xpass`/`skip` | +| `stages` | object | `{compile_ok, asm_encodable, inst_budget_ok, semantic_ok}`，未执行阶段为 `null` | +| `metrics` | object | `{ir_instructions, asm_instructions, compile_time_s, assemble_time_s, semantic_time_s}` | +| `expected` / `actual` | any | 语义比较值（字符串化） | +| `xfail` | object\|null | 命中的 xfail 声明 | +| `error_stage` / `error` | str\|null | 首错阶段与消息（含首错指令行） | + +#### 2.6.2 Markdown（`run_suite.py --markdown PATH`） + +- 头部汇总（total/passed/xfailed/xpassed/failed）； +- 用例表格：`case_id | status | compile | asm | inst | semantic | oracle | xfail reason`； +- 失败/阻塞清单：按 `error_stage` 分组； +- 末尾“数据缺口与缺陷登记”小节（链接 §2.8 编号）。 + +### 2.7 CI 集成约定 + +| 约定项 | 内容 | +|--------|------| +| 门禁位置 | `.github/workflows/ci.yml` 的 `test` job，紧随 `Run all topic tests`（ci.yml:80-85）之后新增 `Run DSL benchmark suite` 步骤 | +| 门禁命令 | `python3.12 -m pytest tests/test_dsl_suite.py -v --tb=short`，**不加** `continue-on-error`、不加 `\|\| echo` 兜底 | +| 报告位置 | `benchmark` job，紧随 `DSL case compilation benchmarks`（ci.yml:202-209）之后新增 `DSL benchmark suite report` 步骤，输出 `benchmark_reports/dsl_suite.json` / `.md` | +| 退出码 | `run_suite.py`：0 = 无硬失败（xfail/xpass 不算失败）；1 = 存在硬失败；2 = 配置错误（roots 无效/0 用例/参数非法） | +| 红灯语义 | xfail 不红灯；xpass 不红灯；硬失败红灯；`meta_contract` 违规红灯 | +| 产物 | 复用既有 `Upload benchmark reports`（ci.yml:287-293）上传 `benchmark_reports/`；`Write job summary`（ci.yml:338-361）追加 `dsl_suite.md` 摘要 | +| 运行时长预算 | 全套件预计 < 60s（25 用例，编译为主）；pytest 步骤纳入 `test` job 既有 20 分钟超时 | +| 依赖 | 仅使用既有依赖（pytest、numpy、tinyfive 属 `.[all]`）；不引入新第三方库 | + +### 2.8 缺陷登记与数据缺口（套件处置依据） + +本课题只记录，不修复。编号供 `xfail.owner`/报告引用： + +| 编号 | 现象（实测证据） | 归属（非本课题） | 套件处置 | +|------|------------------|------------------|----------| +| C1 | 分支/跳转汇编不可编码：`bnez a1 # label`、`j # label`、`bge a3, 4 # label` 操作数为空或符号目标在注释；`assemble_to_binary` 抛 `IndexError: list index out of range`（11 例，附录 5.2） | backend/asm 发射器与编码器 | `asm_encodable`/`semantic` xfail（owner=`backend/asm-encoder`） | +| C2 | 常量物化发射 `li a0, 20` 后又发 `mv a0, 20`（`mv` 立即数非法）；`020_constant_propagation` 实测执行结果为 0（期望 20） | backend/asm 发射器 | 用例 oracle 保持 interpreter；执行 oracle 用例 xfail 并引用 C2 | +| C3 | 溢出/栈路径未初始化栈帧：`reg_pressure_32`（x=1，期望 64）实测 178 | backend/regalloc | `execution` xfail（owner=`backend/regalloc`） | +| C4 | NN 算子未 lowering：`007_matmul` 汇编仅 `mul a2,a1,a0`；`006_softmax` 为 `mv` 直通；`008_dot`/`009_maxpool` 同类 | backend/op-lowering | `execution` xfail（owner=`backend/op-lowering`） | +| C5 | 浮点语义丢失：除法/激活按整数指令执行（`003_sub_div` 整数 div、`010_exp_neg` 实测 -2 而期望 -2.718…） | backend/数值语义 | `execution` xfail；interpreter 用例不受影响 | +| C6 | 输入→寄存器映射无公开契约（由分配器顺序决定，如 `012_nn_pipeline` 实测 x→a0、W→a1、b→a2；`001`/`003` 无法从 IR 首用顺序稳定推出） | backend/regalloc + 接口 | 声明式 `input_registers`；变化即重新校准 | +| B1 | `DSLInterpreter` 不支持控制流：跳过 `for ... / endfor`、循环体至多执行一次、`if/else` 两分支都执行（`verifier.py:140-192`） | verification | 控制流用例禁用 interpreter oracle；B1 修复后不改变本套件契约 | +| D1 | 数据缺口：`016_if_simple`、`017_while_sum`、`018_nested_if`、`021_dsl_if_else`、`022_dsl_while_sum` 无 golden（`.expected` 为空） | 本课题（数据） | 实现阶段补齐 `meta.json`（§4.5） | +| D2 | 数据缺陷：`017`/`022` 的 `while` 循环变量无自增（按语义为死循环）；`013`/`014`/`015`/`019` 的 `.expected` 按“单次迭代”校准，与真实语义不符 | 本课题（数据） | 修用例源码（补 `i = add(i, 1)`）；循环用例真实标量 golden 写入 meta（legacy `.expected` 不动） | + +--- + +## 三、测试设计 + +以下用例同时是本套件自身的验收测试与设计示例。所有“实测”数据来自 2026-09-14 在本仓库的实跑（编译配置 `backend=riscv, optimize_level=all, reg_alloc=linear`）。 + +### 测试用例 1：`cases/001_simple_add` —— 全绿路径（四阶段全通过） + +**输入**（`benchmarks/cases/001_simple_add.dsl`）： + +``` +# Simple addition +c = add(a, b) +return c +``` + +**元数据**（实现阶段补齐，interpreter oracle）： + +```json +{ + "schema_version": 1, + "description": "Basic addition of two vectors", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "inputs": {}, + "expected_return": [2.0, 4.0, 6.0, 8.0], + "max_instructions": 3 +} +``` + +**预期输出/指标**： + +| 阶段 | 预期 | 实测 | +|------|------|------| +| compile | 成功；IR 2 条 | 成功（3 条汇编文本指令） | +| asm_encodable | 成功编码 | 成功（3 个字） | +| inst_budget | `3 <= 3` | 通过 | +| semantic | `DSLInterpreter` 输出 `[2. 4. 6. 8.]` | 通过 | + +**验证点**：四级断言全部执行且 `status=pass`；报告 `metrics.asm_instructions=3`、`semantic_time_s>=0`；该用例证明套件基础路径（发现 → meta 校验 → 编译 → 汇编 → 语义）端到端可用。 + +### 测试用例 2：`cases/013_for_sum` —— 汇编阶段 xfail（C1） + +**输入**（`benchmarks/cases/013_for_sum.dsl`）： + +``` +# Sum using for loop +for i = 0, 4 + acc = add(acc, x) +endfor +return acc +``` + +**元数据要点**：`flow=control`、`oracle=execution`、`inputs={"x":1,"acc":0}`、`input_registers={"x":"a0","acc":"a1"}`、`expected_return=4`（4 次迭代 ×1；legacy `.expected=[1,2,3,4]` 是按单次迭代校准的 D2 陈旧数据，**不修改**，以 meta 为准）、`max_instructions=9`（实测汇编文本 7 条，留 1.2 倍余量）、`xfail.stages=["assemble","budget","semantic"]`（owner=`backend/asm-encoder`，引用 C1）。 + +**预期输出/指标**： + +| 阶段 | 预期 | 实测 | +|------|------|------| +| compile | 成功（IR 4 条） | 成功 | +| asm_encodable | **xfail**：`IndexError` at `bge a3, 4 # .Lloop_exit_3`（汇编文件第 6 行） | 与预期一致 | +| semantic | **xfail**（阻塞于汇编；报告 `error_stage=assemble`） | 与预期一致 | +| inst_budget | **xfail**（依赖汇编计数） | 与预期一致 | + +**验证点**：pytest 输出 `XFAIL`，原因为 meta 中的 `reason`；报告 JSON 中 `status=xfail`、`error_stage=assemble`、`error` 包含首错指令文本；移除编译器缺陷 C1 后此用例应转为 `XPASS` 并可在 30 天内转正。 + +### 测试用例 3：`stress/reg_pressure_32` —— 执行 oracle 与溢出路径（C3） + +**输入**（`tests/stress/reg_pressure_32.dsl`）：32 个 `vNN = add(x, x)` + 六级归约树；`tests/stress/reg_pressure_32.meta.json` 声明 `inputs={"x":1}`、`expected_return=64`。 + +**新增/修订元数据要点**：`category=stress`、`flow=linear`、`oracle=execution`、`input_registers={"x":"a0"}`、`max_instructions=77`（实测汇编文本 77 条）、`xfail.stages=["semantic"]`（owner=`backend/regalloc`，引用 C3）。 + +**预期输出/指标**： + +| 阶段 | 预期 | 实测 | +|------|------|------| +| compile | 成功 | 成功 | +| asm_encodable | 成功（77 字） | 成功 | +| inst_budget | `77 <= 77` | 通过 | +| semantic（执行 oracle） | 期望 64 | **178**（C3：栈/溢出未初始化）→ xfail | + +**验证点**：该用例是唯一能覆盖“32 > 19/27 寄存器上限触发 spill/reload”语义的用例；它证明读取 `get_reg(10)` 的端到端执行 oracle 可用，并把 C3 转化为可追踪的 xfail；一旦 C3 修复即 `XPASS`。 + +### 测试用例 4：`cases/016_if_simple` —— 数据补齐 + 多阶段 xfail（D1 + C1） + +**输入**（既有 DSL，无 golden）：`if (a > b): c = add(a, b) else: c = mul(a, b) endif; return c` + +**元数据**（实现阶段新增）：`flow=control`、`oracle=execution`、`inputs={"a":4,"b":1}`、`input_registers={"a":"a0","b":"a1"}`、`expected_return=5`、`max_instructions=8`（实测汇编文本 6 条）、`xfail.stages=["assemble","budget","semantic"]`（C1）。 + +**预期输出/指标**：compile 成功（IR 6 条）；asm 在 `bnez a1 # if_then1`（第 3 行）失败 → xfail；semantic 阻塞 → xfail；`test_meta_contract` 必须**通过**（证明数据缺口已补齐）。 + +**验证点**：`test_meta_contract[016]` 绿而执行类测试 xfail，二者分离体现“数据质量”与“编译器能力”是两笔账；若实现阶段忘记补 golden，`meta_contract` 立即红灯。 + +### 测试用例 5：契约负例（`test_meta_contract` 自检测） + +**输入**：临时目录中的 3 个伪用例——(a) `oracle=interpreter` 且源码含 `while (`；(b) `oracle=execution` 有 `inputs` 无 `input_registers`；(c) `oracle=none` 无 `xfail`。 + +**预期输出**：`load_case_spec` 返回的 `meta_errors` 分别包含 `oracle/flow conflict`、`input_registers missing`、`oracle=none requires xfail`；对应 pytest 用例失败（硬失败）。 + +**验证点**：负例测试使用 `tmp_path` 构造仓库外数据，证明契约校验真实生效，而非仅对现有 25 个用例“恰好通过”。 + +--- + +## 四、修改模块与实现步骤 + +### 4.1 涉及文件 + +| 文件 | 操作 | 说明 | +|------|------|------| +| `benchmarks/dsl_suite.py` | 新增 | 套件核心：发现、meta 契约、四阶段执行、报告 | +| `benchmarks/run_suite.py` | 新增 | CLI：`--json/--markdown/--roots/--strict-xfail/--list/--filter` | +| `tests/test_dsl_suite.py` | 新增 | pytest 参数化门禁（5 个测试函数） | +| `benchmarks/cases/*.meta.json` | 新增 23 份 | 迁移元数据；循环用例的标量 golden/inputs 写入 meta（legacy `.expected` 不动，保证 `bench_runner` 回归仍绿） | +| `benchmarks/cases/017_while_sum.dsl`、`022_dsl_while_sum.dsl` | 修改 2 行 | 补循环变量自增（D2 数据缺陷） | +| `tests/stress/*.meta.json` | 修改 2 份 | 补 `category/oracle/input_registers/max_instructions/xfail` 字段 | +| `.github/workflows/ci.yml` | 修改 2 处 + summary 1 处 | 门禁、报告、摘要 | +| `tests/test_bench_runner.py`、`benchmarks/bench_runner.py` | 不动 | 兼容保留；回归保证其 23/23 行为不变 | + +（注：实现阶段若发现 `benchmarks/cases/const_merge_feature.asm` 之类非 `.dsl` 文件，发现规则天然忽略，无需处理。） + +### 4.2 实现 `benchmarks/dsl_suite.py` + +1. **数据模型**：定义 `XFailSpec`、`CaseSpec`、`CompileOutcome`、`AssembleOutcome`、`BudgetOutcome`、`SemanticOutcome`、`CaseOutcome`、`SuiteReport`（字段见开发文档“接口契约”）。 +2. **发现**：`discover(roots)` 按 §2.1.2 实现；解析 meta 并合并 `.expected`/`.desc` 回退；`flow` 由源码关键字推断（`if (`、`else:`、`endif`、`while (`、`endwhile`、`for `、`endfor`）；执行 §2.2 校验，违规写入 `meta_errors`。 +3. **编译**：`CompilerConfig(backend, optimize_level, reg_alloc)`，输出到 `workdir/{pytest_id}.s`；用 `IRPrinter`/parse 单独计 IR 指令数。 +4. **汇编**：`assemble_to_binary(asm_text)`；空结果视为失败；`parse_asm` 计文本指令数。 +5. **语义**：`compare_values(actual, expected, rtol, atol)`；interpreter 分支复用 bench_runner 的输入推断逻辑（提取为套件内显式函数 `default_inputs(source)`）；execution 分支实现 `ExecutionOracle`。 +6. **状态合成**：任一启用断言硬失败 → `fail`；所有启用断言通过 → `pass`；失败命中 `xfail.stages` 中对应阶段 → 整体 `xfail`；xpass 按阶段分别记录（pytest 侧处理，报告侧按“是否存在任何 xpass”给出）。 +7. **报告**：`SuiteReport.to_dict/save_json/to_markdown/save_markdown`；JSON 与 Markdown 使用同一结果对象，保证数值一致。 + +### 4.3 实现 `tests/test_dsl_suite.py` + +- 收集期：`ALL_CASES = tuple(discover_cases())`（默认 roots），逐用例 `pytest.param(case, id=case.pytest_id, marks=...)` 参数化（marks 必须逐参数注入）。 +- session 级 fixture：`suite_runner`、`compile_results`（dict 缓存编译结果，避免 5 个测试重复编译）、`asm_results`、`semantic_results`。 +- 测试函数：`test_meta_contract`、`test_compile_ok`、`test_asm_encodable`、`test_instruction_budget`、`test_semantic_golden`。 +- xfail 注入：`_marks_for(case, stage)` 读取 `case.xfail`，当 `stage in case.xfail.stages` 时返回 `pytest.mark.xfail(reason=..., strict=...)`。 +- 预算测试在 `max_instructions is None` 时 `pytest.skip("no max_instructions declared")`。 + +### 4.4 实现 `benchmarks/run_suite.py` + +- argparse：`--roots`（可重复，默认两个 root）、`--json`、`--markdown`、`--backend`、`--optimize-level`、`--reg-alloc`、`--timeout`、`--filter`（子串匹配 case_id）、`--list`、`--strict-xfail`、`--quiet`。 +- 行为：执行 `DSLSuiteRunner.run_all()`，打印摘要与失败清单；写报告；退出码按 §2.7。 + +### 4.5 元数据迁移与数据修复（本课题唯一“修改用例数据”的部分） + +1. 为 23 个 `benchmarks/cases/*.dsl` 补齐 `*.meta.json`（oracle 按 §2.3.2 规则：14 个线性用例 `interpreter`；9 个控制流用例 `execution`）。 +2. 补齐 5 个无 golden 用例的 `inputs` + `expected_return`（控制流 oracle=execution，xfail 引用 C1）： + + | 用例 | inputs | expected_return（真实语义） | + |------|--------|------------------------------| + | `016_if_simple` | `{a:4, b:1}` | `5`（then 分支 add） | + | `017_while_sum` | `{i:0, x:1, acc:0}` | `10`（10 次自增迭代后 ×1） | + | `018_nested_if` | `{a:4, b:1}` | `5`（外层 then + 内层 then） | + | `021_dsl_if_else` | `{a:4, b:1}` | `10`（t1=5, t2=10, relu(10)） | + | `022_dsl_while_sum` | `{i:0, x:2, y:3, acc:0}` | `30`（5 次 ×6） | + +3. 修复 D2 数据缺陷：`017`、`022` 在循环体补 `i = add(i, 1)`；循环用例的**真实语义标量 golden** 写入 meta（legacy `.expected` 保持不动，避免破坏 bench_runner 既有回归）：`013`（inputs `{x:1, acc:0}`）→ `4`、`014`（`{a:2, b:3, acc:0}`）→ `24`、`015`（`{x:1, y:0}`）→ `4`、`019`（`{x:1, y:1, acc:0}`，8 次迭代）→ `8`、`reg_pressure_loop`（`{x:1, acc:0}`）→ `210`。 +4. 冻结 `max_instructions`：asm 通过的用例取实测值；汇编失败用例取实测文本指令数的 1.2 倍向上取整（`009→9, 013→9, 014→9, 015→10, 016→8, 017→9, 018→14, 019→14, 021→12, 022→10, reg_pressure_loop→16`），C1 修复后收紧为实测值。 +5. （可选，后续）从 `82860b4:ScratchV-topic06-deliverable/tests_main/` 迁回交付版 23 个 NN/控制流用例作第二个 root（`benchmarks/cases_nn/`），格式已兼容，不属于本期验收。 + +### 4.6 集成与回归测试 + +- `pytest tests/test_dsl_suite.py -v`：预计 `92 passed, 33 xfailed, 0 failed`（**pytest 测试级**：25 meta + 25 compile + 25 asm + 25 budget + 25 semantic；其中 asm/budget 各 11 个 xfail，semantic 11 个 xfail）。报告 JSON 的 `summary` 是**用例级**：`14 passed, 11 xfailed`（控制流用例的多个阶段失败合并为一个 xfail 状态）。 +- `pytest tests/test_bench_runner.py -v`：保持全绿（套件不改 bench_runner，且不改 legacy `.expected`）。 +- `make test`：全量 pytest 仍绿。 +- `python benchmarks/run_suite.py --json /tmp/dsl_suite.json --markdown /tmp/dsl_suite.md`：退出码 0；JSON 可被 `python -m json.tool` 解析。 +- `python .claude/harness/verify/run.py --level L2`（若环境具备）：作为提交前验证。 + +--- + +## 五、附录 + +### 5.1 报告 JSON 示例（节选：1 个全绿 + 1 个 xfail） + +```json +{ + "schema_version": 1, + "suite": "dsl-benchmark", + "generated_at": "2026-09-14T10:30:00", + "roots": ["benchmarks/cases", "tests/stress"], + "compiler": {"backend": "riscv", "optimize_level": "all", "reg_alloc": "linear"}, + "summary": {"total": 25, "passed": 14, "xfailed": 11, "xpassed": 0, "failed": 0, "skipped": 0}, + "results": [ + { + "case_id": "cases/001_simple_add", + "pytest_id": "cases-001_simple_add", + "category": "arith", + "flow": "linear", + "oracle": "interpreter", + "status": "pass", + "stages": {"compile_ok": true, "asm_encodable": true, "inst_budget_ok": true, "semantic_ok": true}, + "metrics": {"ir_instructions": 2, "asm_instructions": 3, + "compile_time_s": 0.0121, "assemble_time_s": 0.0004, "semantic_time_s": 0.0002}, + "expected": "[2. 4. 6. 8.]", + "actual": "[2. 4. 6. 8.]", + "xfail": null, + "error_stage": null, + "error": null + }, + { + "case_id": "cases/013_for_sum", + "pytest_id": "cases-013_for_sum", + "category": "control", + "flow": "control", + "oracle": "execution", + "status": "xfail", + "stages": {"compile_ok": true, "asm_encodable": false, "inst_budget_ok": null, "semantic_ok": null}, + "metrics": {"ir_instructions": 4, "asm_instructions": 7, + "compile_time_s": 0.0130, "assemble_time_s": 0.0003, "semantic_time_s": 0.0}, + "expected": 4, + "actual": null, + "xfail": { + "stages": ["assemble", "budget", "semantic"], + "reason": "RISCVAEncoder cannot encode symbolic branch targets; first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/asm-encoder", + "strict": false + }, + "error_stage": "assemble", + "error": "IndexError: list index out of range at line 6: 'bge a3, 4 # .Lloop_exit_3'" + } + ] +} +``` + +### 5.2 首错指令登记表（2026-09-14 实测，11 例汇编失败） + +| 用例 | 汇编行号 | 首错指令 | 编码器异常 | +|------|----------|----------|------------| +| `009_maxpool` | 4 | `bnez a1 # .Lmp_gt_1` | `IndexError: list index out of range` | +| `013_for_sum` | 6 | `bge a3, 4 # .Lloop_exit_3` | 同上 | +| `014_for_dot` | 6 | `bge a3, 4 # .Lloop_exit_3` | 同上 | +| `015_for_relu` | 6 | `bge a3, 4 # .Lloop_exit_3` | 同上 | +| `016_if_simple` | 3 | `bnez a1 # if_then1` | 同上 | +| `017_while_sum` | 3 | `j # while_hdr1` | 同上 | +| `018_nested_if` | 3 | `bnez a1 # if_then1` | 同上 | +| `019_nested_loop` | 6 | `bge a3, 4 # .Lloop_exit_3` | 同上 | +| `021_dsl_if_else` | 3 | `bnez a2 # if_then1` | 同上 | +| `022_dsl_while_sum` | 3 | `j # while_hdr1` | 同上 | +| `reg_pressure_loop` | 6 | `bge a3, 10 # .Lloop_exit_3` | 同上 | + +汇编文本指令数实测（`parse_asm` 口径，供 `max_instructions` 基线）：`001=3, 002=3, 003=4, 004=3, 005=5, 006=3, 007=3, 008=3, 010=5, 011=5, 012=5, 020=3, 023=10, reg_pressure_32=77`；编码失败但文本可计数的 11 例：`009=7, 013=7, 014=7, 015=8, 016=6, 017=7, 018=11, 019=11, 021=10, 022=8, reg_pressure_loop=13`。 + +### 5.3 参考资料 + +- `docs/topics/06-性能基准套件.md` —— 课题目标形态（本文档描述从现状到目标的落地路径） +- `benchmarks/bench_runner.py`、`tests/test_bench_runner.py` —— 现有编排与单测（兼容保留） +- `/root/Lab/GaoMD/ScratchV/SPEC&review/CommitReview/课题06-性能测试套件-commit-review.md` —— 历史交付评审（13/23 PASS、报告代码重复、`infer_initial_registers` 脆弱性等结论） +- commit `82860b4` —— 交付版 `.dsl + .meta.json` 用例与 `run_tests.py`（仅作 schema 参考） +- `scratchv/verification/verifier.py:140`（`DSLInterpreter`）、`scratchv/simulator/tinyfive.py:35`（`ProfiledMachine`）、`scratchv/backend/riscv_encoder.py:538`（`assemble_to_binary`）、`scratchv/compiler.py:205`（`CompilerDriver`）、`scratchv/backend/_asm_parser.py:31`（`ParsedAsmLine`） +- `设计文档模板.md` —— 本文档遵循的结构模板 From 75fa4babc71ebd0542205d2c5bf2cc3ea5f9e437 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:55:31 +0800 Subject: [PATCH 3/6] fix(topic06): correct execution oracle, blocked status and xfail attribution - ExecutionOracle: step instruction-by-instruction until the code region is left (ra sentinel halts jalr zero, ra) with a real wall-clock timeout and a step ceiling, instead of truncating loops at len(words) static words. - Blocked stages not covered by xfail.stages are now hard failures, matching the pytest gate instead of reporting a false pass. - Preserve oracle timeout/unsupported-instruction errors in semantic outcomes. - run_suite: xpassed>0 prints a warning; --timeout help documents its scope. - Attribute the 11 branch-target assemble xfails to the linear-scan emitter (backend/regalloc), not the encoder; greedy path encodes the same branches. --- benchmarks/cases/009_maxpool.meta.json | 4 +- benchmarks/cases/013_for_sum.meta.json | 4 +- benchmarks/cases/014_for_dot.meta.json | 4 +- benchmarks/cases/015_for_relu.meta.json | 4 +- benchmarks/cases/016_if_simple.meta.json | 4 +- benchmarks/cases/017_while_sum.meta.json | 4 +- benchmarks/cases/018_nested_if.meta.json | 4 +- benchmarks/cases/019_nested_loop.meta.json | 4 +- benchmarks/cases/021_dsl_if_else.meta.json | 4 +- benchmarks/cases/022_dsl_while_sum.meta.json | 4 +- benchmarks/dsl_suite.py | 108 +++++++++++++++++-- benchmarks/run_suite.py | 10 +- tests/stress/reg_pressure_loop.meta.json | 4 +- 13 files changed, 131 insertions(+), 31 deletions(-) diff --git a/benchmarks/cases/009_maxpool.meta.json b/benchmarks/cases/009_maxpool.meta.json index 8e8cc4a..67bc05b 100644 --- a/benchmarks/cases/009_maxpool.meta.json +++ b/benchmarks/cases/009_maxpool.meta.json @@ -15,8 +15,8 @@ "assemble", "budget" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a1 # .Lmp_gt_1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bnez a1 # .Lmp_gt_1", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/013_for_sum.meta.json b/benchmarks/cases/013_for_sum.meta.json index 6ad84e6..84d0e8d 100644 --- a/benchmarks/cases/013_for_sum.meta.json +++ b/benchmarks/cases/013_for_sum.meta.json @@ -20,8 +20,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/014_for_dot.meta.json b/benchmarks/cases/014_for_dot.meta.json index b985177..2013033 100644 --- a/benchmarks/cases/014_for_dot.meta.json +++ b/benchmarks/cases/014_for_dot.meta.json @@ -22,8 +22,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/015_for_relu.meta.json b/benchmarks/cases/015_for_relu.meta.json index 4b3a036..80aac0e 100644 --- a/benchmarks/cases/015_for_relu.meta.json +++ b/benchmarks/cases/015_for_relu.meta.json @@ -20,8 +20,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/016_if_simple.meta.json b/benchmarks/cases/016_if_simple.meta.json index 78ace3c..eec7eb7 100644 --- a/benchmarks/cases/016_if_simple.meta.json +++ b/benchmarks/cases/016_if_simple.meta.json @@ -20,8 +20,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a1 # if_then1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bnez a1 # if_then1", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/017_while_sum.meta.json b/benchmarks/cases/017_while_sum.meta.json index 47f3831..48f5bec 100644 --- a/benchmarks/cases/017_while_sum.meta.json +++ b/benchmarks/cases/017_while_sum.meta.json @@ -22,8 +22,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: j # while_hdr1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: j # while_hdr1", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/018_nested_if.meta.json b/benchmarks/cases/018_nested_if.meta.json index 0faa34b..5d49a68 100644 --- a/benchmarks/cases/018_nested_if.meta.json +++ b/benchmarks/cases/018_nested_if.meta.json @@ -20,8 +20,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a1 # if_then1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bnez a1 # if_then1", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/019_nested_loop.meta.json b/benchmarks/cases/019_nested_loop.meta.json index 94afc15..b3bba99 100644 --- a/benchmarks/cases/019_nested_loop.meta.json +++ b/benchmarks/cases/019_nested_loop.meta.json @@ -22,8 +22,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 4 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/021_dsl_if_else.meta.json b/benchmarks/cases/021_dsl_if_else.meta.json index f0077a3..cbbb68d 100644 --- a/benchmarks/cases/021_dsl_if_else.meta.json +++ b/benchmarks/cases/021_dsl_if_else.meta.json @@ -20,8 +20,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bnez a2 # if_then1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bnez a2 # if_then1", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/cases/022_dsl_while_sum.meta.json b/benchmarks/cases/022_dsl_while_sum.meta.json index 0a78272..a79b0eb 100644 --- a/benchmarks/cases/022_dsl_while_sum.meta.json +++ b/benchmarks/cases/022_dsl_while_sum.meta.json @@ -24,8 +24,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: j # while_hdr1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: j # while_hdr1", + "owner": "backend/regalloc", "strict": false } } diff --git a/benchmarks/dsl_suite.py b/benchmarks/dsl_suite.py index 2fa119a..6fba4fe 100644 --- a/benchmarks/dsl_suite.py +++ b/benchmarks/dsl_suite.py @@ -345,8 +345,9 @@ def to_markdown(self) -> str: "", "## 数据缺口与缺陷登记", "", - "- C1 backend/asm-encoder: symbolic branch targets are emitted in " - "comments; affected assemble semantics stages are xfailed.", + "- C1 backend/regalloc: the linear-scan emitter keeps branch " + "targets in comments instead of operands (greedy path emits them " + "correctly); affected assemble/semantic stages are xfailed.", "- C2 backend/asm-emitter: constant materialisation emits invalid " "`mv rd, imm`.", "- C3 backend/regalloc: spill/reload stack slots read uninitialised " @@ -814,6 +815,12 @@ def load_case_spec(dsl_path: Path, root: Path) -> CaseSpec: errors.append( "oracle/flow conflict: oracle=interpreter requires flow=linear" ) + if oracle == ORACLE_INTERPRETER: + warnings.append( + "interpreter oracle validates DSL semantics only; generated " + "code is not executed (blind spots: C2 constant materialisation, " + "C4 op lowering, C5 float semantics)" + ) inputs = _validate_inputs(meta.get("inputs", {}), errors) input_registers = _validate_input_registers( @@ -984,11 +991,42 @@ def _broken_spec(dsl_path: Path, root: Path, exc: Exception) -> CaseSpec: } +EXECUTION_MAX_STEPS: int = 100_000_000 + + +def _machine_pc(machine: Any) -> int: + """Read the PC, tolerating scalar and one-element-array TinyFive PCs. + + ``ProfiledMachine.pc`` assumes the PC is indexable, which breaks after a + ``jalr`` stores a NumPy scalar (pre-existing simulator adapter issue). + The suite reads the raw attribute instead of touching ``scratchv/**``. + """ + raw = machine._machine.pc + if hasattr(raw, "__len__"): + return int(raw[0]) + return int(raw) + + class ExecutionOracle: - """Execute assembled RISC-V and read the ``a0`` return register.""" + """Execute assembled RISC-V and read the ``a0`` return register. - def __init__(self, *, mem_size: int = 128 * 1024 * 1024) -> None: + Execution is stepped instruction by instruction so that loops run to + completion (the previous ``instructions=len(words)`` bound truncated any + dynamic execution longer than the static code size) while a real wall + clock ``timeout_s`` still bounds runaway programs. The return address + register ``ra`` is preloaded with a sentinel just past the code, so the + compiler's ``jalr zero, ra`` epilogue halts the machine instead of + jumping back to address 0. + """ + + def __init__( + self, + *, + mem_size: int = 128 * 1024 * 1024, + max_steps: int = EXECUTION_MAX_STEPS, + ) -> None: self.mem_size = mem_size + self.max_steps = max_steps def available(self) -> bool: try: @@ -1047,7 +1085,54 @@ def elapsed() -> float: error=None, ) machine.set_reg(_REG_NUMS[register], int(value)) - machine.run(instructions=len(words), start=0, strict=True) + code_end = len(words) * 4 + machine.set_reg(1, code_end) + steps = 0 + while 0 <= _machine_pc(machine) < code_end: + if steps >= self.max_steps: + return SemanticOutcome( + ok=False, oracle=ORACLE_EXECUTION, expected=None, + actual=machine.get_reg(10), duration_s=elapsed(), + blocked_reason=None, + error=( + "execution step limit exceeded " + f"({self.max_steps} instructions)" + ), + ) + if elapsed() >= timeout_s: + return SemanticOutcome( + ok=False, oracle=ORACLE_EXECUTION, expected=None, + actual=machine.get_reg(10), duration_s=elapsed(), + blocked_reason=None, + error=( + f"execution timeout after {timeout_s:g}s " + f"({steps} instructions executed)" + ), + ) + pc_before = _machine_pc(machine) + machine.run(instructions=1, start=pc_before, strict=True) + steps += 1 + if _machine_pc(machine) == pc_before: + return SemanticOutcome( + ok=False, oracle=ORACLE_EXECUTION, expected=None, + actual=machine.get_reg(10), duration_s=elapsed(), + blocked_reason=None, + error=( + "unsupported instruction at " + f"pc={pc_before:#x} (decoder made no progress)" + ), + ) + final_pc = _machine_pc(machine) + if final_pc != code_end: + return SemanticOutcome( + ok=False, oracle=ORACLE_EXECUTION, expected=None, + actual=machine.get_reg(10), duration_s=elapsed(), + blocked_reason=None, + error=( + f"pc left the code region: {final_pc:#x} " + f"(code_end={code_end:#x})" + ), + ) actual = machine.get_reg(10) return SemanticOutcome( ok=None, oracle=ORACLE_EXECUTION, expected=None, @@ -1313,12 +1398,14 @@ def _execution_semantics( compile_outcome.asm_text, spec.input_registers, spec.inputs, - timeout_s=spec.timeout_s, + timeout_s=min(spec.timeout_s, self.timeout_s), ) if outcome.ok is None and ( outcome.blocked_reason or outcome.error ): return replace(outcome, expected=spec.expected_return) + if outcome.ok is False and outcome.error: + return replace(outcome, expected=spec.expected_return) ok = compare_values( outcome.actual, spec.expected_return, rtol=spec.rtol, atol=spec.atol, @@ -1480,14 +1567,19 @@ def run_case(self, spec: CaseSpec) -> CaseOutcome: (stage, message) for stage, message in blocked if stage in xfail_stages ] + blocked_uncovered = [ + (stage, message) + for stage, message in blocked if stage not in xfail_stages + ] status = STATUS_PASS error_stage: str | None = None error: str | None = None - if hard: + if hard or blocked_uncovered: status = STATUS_FAIL error_stage, error = min( - hard, key=lambda item: stage_order[item[0]], + hard + blocked_uncovered, + key=lambda item: stage_order[item[0]], ) elif covered or blocked_covered: status = STATUS_XFAIL diff --git a/benchmarks/run_suite.py b/benchmarks/run_suite.py index 5a7e8fd..a6db2e6 100644 --- a/benchmarks/run_suite.py +++ b/benchmarks/run_suite.py @@ -59,7 +59,8 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--timeout", type=float, default=30.0, - help="Per-case timeout budget in seconds (default: 30)", + help="Per-case timeout budget in seconds for the execution oracle " + "(default: 30); caps each case's meta timeout_s", ) parser.add_argument( "--filter", default=None, @@ -138,6 +139,13 @@ def main(argv: list[str] | None = None) -> int: def _print_summary(report: SuiteReport, *, quiet: bool) -> None: summary = report.summary() + if summary["xpassed"]: + print( + f"warning: {summary['xpassed']} xpassed case(s) — declared " + "xfail stages now pass; remove the obsolete declaration or run " + "with --strict-xfail", + file=sys.stderr, + ) if quiet: return print( diff --git a/tests/stress/reg_pressure_loop.meta.json b/tests/stress/reg_pressure_loop.meta.json index 57ad01f..a0edf74 100644 --- a/tests/stress/reg_pressure_loop.meta.json +++ b/tests/stress/reg_pressure_loop.meta.json @@ -21,8 +21,8 @@ "budget", "semantic" ], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (C1); first bad line: bge a3, 10 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: the linear-scan emitter keeps the branch target in the instruction comment and emits no operand, so assemble_to_binary fails; greedy path encodes the same branch. first bad line: bge a3, 10 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false } } From 54863ed922c8fda16ae0f6d3b17a06aaa5ef9c00 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:55:37 +0800 Subject: [PATCH 4/6] test(topic06): add regression tests for oracle, blocked and compile xfail - F1: manual 4-iteration loop must return 4 (was truncated at 2). - F1/F3: green codegen+simulation path through ExecutionOracle. - F6: infinite loop times out; runner budget caps meta timeout_s. - F4: blocked semantic without xfail is a hard failure. - F5: xfail(compile) reaches the compile test node via params_for. - F2: 11 assemble xfails must blame backend/regalloc; greedy path encodes the same branch targets. - F3/F8: interpreter blind-spot warnings in the report; xpass warning. --- tests/test_dsl_suite.py | 275 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 274 insertions(+), 1 deletion(-) diff --git a/tests/test_dsl_suite.py b/tests/test_dsl_suite.py index 0fd01a9..77ec44d 100644 --- a/tests/test_dsl_suite.py +++ b/tests/test_dsl_suite.py @@ -9,6 +9,8 @@ from __future__ import annotations import json +import sys +import time from pathlib import Path import pytest @@ -19,14 +21,23 @@ ORACLE_NONE, STAGE_ASSEMBLE, STAGE_BUDGET, + STAGE_COMPILE, STAGE_SEMANTIC, + STATUS_FAIL, + STATUS_XFAIL, + STATUS_XPASS, + CaseOutcome, CaseSpec, + CompileOutcome, DSLSuiteRunner, + ExecutionOracle, SemanticOutcome, + SuiteReport, compare_values, discover_cases, load_case_spec, ) +from benchmarks.run_suite import _print_summary ALL_CASES: tuple[CaseSpec, ...] = tuple(discover_cases()) if not ALL_CASES: @@ -117,7 +128,7 @@ def test_meta_contract(case: CaseSpec) -> None: assert case.xfail.reason.strip(), "xfail.reason must be non-empty" -@pytest.mark.parametrize("case", params_for(None)) +@pytest.mark.parametrize("case", params_for(STAGE_COMPILE)) def test_compile_ok(case: CaseSpec, compile_results: dict) -> None: if case.skip_reason: pytest.skip(case.skip_reason) @@ -233,3 +244,265 @@ def test_meta_contract_rejects_invalid_metadata( assert any(fragment in error for error in spec.meta_errors), ( f"expected {fragment!r} in {spec.meta_errors}" ) + + +MANUAL_LOOP_ASM = ( + "addi a0, zero, 0\n" + "addi t0, zero, 0\n" + "addi t1, zero, 4\n" + "loop:\n" + "addi a0, a0, 1\n" + "addi t0, t0, 1\n" + "blt t0, t1, loop\n" + "jalr zero, ra\n" +) + +INFINITE_LOOP_ASM = ( + "addi t0, zero, 0\n" + "loop:\n" + "addi t0, t0, 1\n" + "j loop\n" +) + + +def _require_tinyfive() -> None: + if not ExecutionOracle().available(): + pytest.skip("tinyfive not installed") + + +def test_execution_oracle_completes_manual_loop() -> None: + """F1 regression: static word count must not truncate dynamic loops.""" + _require_tinyfive() + outcome = ExecutionOracle().execute(MANUAL_LOOP_ASM, {}, {}, timeout_s=5.0) + assert outcome.error is None, outcome.error + assert outcome.blocked_reason is None + assert outcome.actual == 4 + + +def test_execution_oracle_validates_compiled_code( + tmp_path: Path, suite_runner: DSLSuiteRunner, +) -> None: + """F1/F3 regression: green codegen + simulation path for the oracle.""" + dsl_path = _write_pseudo_case( + tmp_path, + "execution_add", + "c = add(a, b)\nreturn c\n", + { + "description": "codegen+execution probe", + "oracle": "execution", + "inputs": {"a": 2, "b": 3}, + "input_registers": {"a": "a0", "b": "a1"}, + "expected_return": 5, + }, + ) + spec = load_case_spec(dsl_path, tmp_path) + assert spec.meta_errors == (), spec.meta_errors + _require_tinyfive() + compile_outcome = suite_runner.compile_case(spec) + assert compile_outcome.ok, compile_outcome.error + outcome = suite_runner.evaluate_semantics(spec, compile_outcome) + assert outcome.ok is True, outcome.blocked_reason or outcome.error + + +def test_execution_oracle_times_out() -> None: + """F6 regression: timeout_s must bound a non-terminating execution.""" + _require_tinyfive() + started = time.perf_counter() + outcome = ExecutionOracle().execute( + INFINITE_LOOP_ASM, {}, {}, timeout_s=0.1, + ) + elapsed = time.perf_counter() - started + assert outcome.ok is False + assert outcome.error is not None and "timeout" in outcome.error + assert elapsed < 5.0 + + +def test_runner_timeout_budget_caps_execution(tmp_path: Path) -> None: + """F6 regression: the runner/CLI budget reaches the execution oracle.""" + dsl_path = _write_pseudo_case( + tmp_path, + "timeout_case", + "c = add(a, b)\nreturn c\n", + { + "description": "timeout budget probe", + "oracle": "execution", + "inputs": {"a": 1, "b": 2}, + "input_registers": {"a": "a0", "b": "a1"}, + "expected_return": 3, + "timeout_s": 30.0, + }, + ) + spec = load_case_spec(dsl_path, tmp_path) + assert spec.meta_errors == (), spec.meta_errors + _require_tinyfive() + runner = DSLSuiteRunner(roots=(tmp_path,), timeout_s=0.1) + try: + synthetic = CompileOutcome( + ok=True, asm_text=INFINITE_LOOP_ASM, output_path=None, + ir_instruction_count=0, duration_s=0.0, error=None, + ) + outcome = runner.evaluate_semantics(spec, synthetic) + assert outcome.ok is False + assert outcome.error is not None and "timeout" in outcome.error + finally: + runner.cleanup() + + +def test_blocked_semantic_without_xfail_is_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """F4 regression: uncovered blocked stages must be hard failures.""" + dsl_path = _write_pseudo_case( + tmp_path, + "blocked_execution", + "c = add(a, b)\nreturn c\n", + { + "description": "blocked semantic probe", + "oracle": "execution", + "inputs": {"a": 1, "b": 2}, + "input_registers": {"a": "a0", "b": "a1"}, + "expected_return": 3, + }, + ) + spec = load_case_spec(dsl_path, tmp_path) + assert spec.meta_errors == (), spec.meta_errors + assert spec.xfail is None + monkeypatch.setattr(ExecutionOracle, "available", lambda self: False) + runner = DSLSuiteRunner(roots=(tmp_path,)) + try: + outcome = runner.run_case(spec) + finally: + runner.cleanup() + assert outcome.status == STATUS_FAIL + assert outcome.error_stage == STAGE_SEMANTIC + + +def test_compile_stage_xfail_is_injected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + """F5 regression: xfail(compile) must reach the compile test node.""" + dsl_path = _write_pseudo_case( + tmp_path, + "compile_xfail", + "this is not a valid dsl program !!!\n", + { + "description": "compile-stage xfail probe", + "category": "arith", + "oracle": "interpreter", + "expected_return": 0, + "xfail": { + "stages": ["compile"], + "reason": "probe: compile stage fails", + "owner": "tests", + }, + }, + ) + spec = load_case_spec(dsl_path, tmp_path) + assert spec.meta_errors == (), spec.meta_errors + module = sys.modules[__name__] + monkeypatch.setattr(module, "ALL_CASES", (spec,)) + params = params_for(STAGE_COMPILE) + assert params and params[0].marks, "compile-stage xfail mark missing" + assert any(mark.name == "xfail" for mark in params[0].marks) + assert not params_for(None)[0].marks + runner = DSLSuiteRunner(roots=(tmp_path,)) + try: + outcome = runner.run_case(spec) + finally: + runner.cleanup() + assert outcome.status == STATUS_XFAIL + assert outcome.error_stage == STAGE_COMPILE + + +BRANCH_TARGET_XFAIL_CASES = frozenset({ + "cases/009_maxpool", + "cases/013_for_sum", + "cases/014_for_dot", + "cases/015_for_relu", + "cases/016_if_simple", + "cases/017_while_sum", + "cases/018_nested_if", + "cases/019_nested_loop", + "cases/021_dsl_if_else", + "cases/022_dsl_while_sum", + "stress/reg_pressure_loop", +}) + + +def test_branch_target_xfails_blame_linear_scan() -> None: + """F2 regression: assemble xfails must blame the linear-scan emitter.""" + discovered = {case.case_id for case in ALL_CASES} + missing = BRANCH_TARGET_XFAIL_CASES - discovered + assert not missing, f"cases disappeared: {sorted(missing)}" + for case in ALL_CASES: + if case.case_id not in BRANCH_TARGET_XFAIL_CASES: + continue + assert case.xfail is not None + assert STAGE_ASSEMBLE in case.xfail.stages + assert case.xfail.owner == "backend/regalloc", case.case_id + assert "linear" in case.xfail.reason, case.case_id + assert "first bad line" in case.xfail.reason, case.case_id + + +@pytest.mark.parametrize( + "case_id", ["cases/013_for_sum", "cases/016_if_simple"], +) +def test_greedy_allocator_encodes_branch_targets(case_id: str) -> None: + """F2 regression: same encoder encodes branch targets on greedy path.""" + spec = next(case for case in ALL_CASES if case.case_id == case_id) + runner = DSLSuiteRunner(roots=(spec.root,), reg_alloc="greedy") + try: + compile_outcome = runner.compile_case(spec) + assert compile_outcome.ok, compile_outcome.error + assemble_outcome = runner.assemble_asm(compile_outcome.asm_text) + assert assemble_outcome.ok, assemble_outcome.error + finally: + runner.cleanup() + + +def test_interpreter_blind_spot_reported_as_warning( + suite_runner: DSLSuiteRunner, +) -> None: + """F3 regression: interpreter blind spots must surface in the report.""" + spec = next( + case for case in ALL_CASES + if case.case_id == "cases/020_constant_propagation" + ) + assert any("interpreter" in warning for warning in spec.warnings) + outcome = suite_runner.run_case(spec) + report = SuiteReport( + results=[outcome], + roots=("benchmarks/cases",), + compiler=suite_runner.compiler_info, + generated_at="test", + specs={spec.case_id: spec}, + ) + warnings = report.to_dict()["results"][0]["warnings"] + assert warnings, "interpreter blind spot warning missing from report" + assert any("interpreter" in warning for warning in warnings) + + +def test_xpass_warning_is_printed(capsys: pytest.CaptureFixture) -> None: + """F8 regression: unexpected passes must produce a visible warning.""" + result = CaseOutcome( + case_id="pseudo/xpass", + pytest_id="pseudo-xpass", + status=STATUS_XPASS, + compile=CompileOutcome( + ok=True, asm_text="", output_path=None, + ir_instruction_count=0, duration_s=0.0, error=None, + ), + assemble=None, + budget=None, + semantic=None, + error_stage=None, + error=None, + xfail=None, + ) + report = SuiteReport( + results=[result], roots=("pseudo",), compiler={}, + generated_at="test", + ) + _print_summary(report, quiet=True) + captured = capsys.readouterr() + assert "xpass" in captured.err.lower() From 7b2a09b2bc29bbf2cada7abcfa045b34fdd0eb80 Mon Sep 17 00:00:00 2001 From: Seven Gao <799889633@qq.com> Date: Mon, 14 Sep 2026 22:55:43 +0800 Subject: [PATCH 5/6] docs(topic06): align suite docs with fixed oracle and corrected C1 owner - Replace the 'len(words) caps execution' design claim with the stepwise execution + real timeout_s description. - C1 root cause/owner: linear-scan emitter (backend/regalloc); the encoder encodes branch targets (greedy path proof). - Refresh measured numbers: 13 pass/12 xfail case level, 107 passed/33 xfailed pytest level (140 nodes), asm counts 017=8, 022=9, full suite 778 passed/33 xfailed. - Remove references to integration-only commits not present on this branch. --- ...00\345\217\221\346\226\207\346\241\243.md" | 74 ++++++++++--------- ...76\350\256\241\346\226\207\346\241\243.md" | 26 ++++--- 2 files changed, 55 insertions(+), 45 deletions(-) diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" index 33bae9d..65d6ef0 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -244,7 +244,7 @@ class ExecutionOracle: 3. `m = ProfiledMachine(mem_size=...)`；`available` 为假 → `blocked_reason="tinyfive not installed"`； 4. `m.load_binary(words, origin=0)`； 5. 对每个输入：`value = int(inputs[name])`（非整数 float → `blocked_reason="non-integer input not supported by integer register semantics"`）；`m.set_reg(register_number(name), value)`，寄存器名→编号用本地 `_REG_NUMS` 表（`a0`=10 起）； -6. `m.run(instructions=len(words), start=0, strict=True)`；`actual = m.get_reg(10)`； +6. `machine.set_reg(1, code_end)` 预置 `ra` 哨兵（代码区末尾），随后按指令单步推进：每步检查墙钟 `timeout_s`（真实生效，取 meta 与 runner 预算的较小值）与 1e8 步硬上限，`pc` 跳出 `[0, code_end)` 即视为返回（`jalr zero, ra` 停机）；超时/步数超限/解码无进展分别返回显式语义错误；`actual = m.get_reg(10)`； 7. `compare_values(actual, expected)`。 ### 1.6 pytest 契约（`tests/test_dsl_suite.py`） @@ -439,7 +439,7 @@ def test_meta_contract(case: CaseSpec) -> None: assert case.meta_errors == (), f"meta contract violations: {case.meta_errors}" -@pytest.mark.parametrize("case", params_for(None)) +@pytest.mark.parametrize("case", params_for(STAGE_COMPILE)) def test_compile_ok(case: CaseSpec, compile_results) -> None: outcome = compile_results[case.case_id] assert outcome.ok, f"compile failed: {outcome.error}" @@ -480,7 +480,7 @@ def test_semantic_golden(case: CaseSpec, compile_results) -> None: ```bash python3.12 -m pytest tests/test_dsl_suite.py -v # tests/test_dsl_suite.py::test_asm_encodable[cases-001_simple_add] PASSED -# tests/test_dsl_suite.py::test_asm_encodable[cases-013_for_sum] XFAIL (RISCVAEncoder cannot encode ...) +# tests/test_dsl_suite.py::test_asm_encodable[cases-013_for_sum] XFAIL (C1 backend/regalloc: linear-scan emitter ...) # 单阶段/单用例过滤： python3.12 -m pytest "tests/test_dsl_suite.py::test_semantic_golden[cases-013_for_sum]" -v ``` @@ -559,21 +559,21 @@ EOF | stress/reg_pressure_32 | ✅ | ✅ | 77 | —（oracle=execution） | 错（178，期望 64） | xfail | `["semantic"]` / C3 | | stress/reg_pressure_loop | ✅ | ❌ | 13 | 禁用（B1） | 阻塞 | xfail | `["assemble","budget","semantic"]` / C1（`bge a3, 10` 第 6 行） | -**统计**：compile 25/25；asm 14/25；interpreter 语义（14 个线性用例）14/14；预计**用例级** 14 pass / 11 xfail / 0 fail；**pytest 测试级** 92 passed / 33 xfailed（测试总数 125）。 +**统计**：compile 25/25；asm 14/25；interpreter 语义（14 个线性用例）14/14；**用例级** 13 pass / 12 xfail / 0 fail；**pytest 测试级**（修复轮后）107 passed / 33 xfailed（测试总数 140）。 **“当前会失败”的准确清单**： -1. **汇编不可编码（11 个）**：`009`、`013`、`014`、`015`、`016`、`017`、`018`、`019`、`021`、`022`、`reg_pressure_loop`。首错指令见设计文档附录 5.2。处置：`meta.json` 声明 `xfail.stages=["assemble","budget","semantic"]`（owner=`backend/asm-encoder`，reason 引用 C1 并带首错指令）；`009_maxpool` 例外——它是 linear + interpreter oracle，语义不依赖汇编，`xfail.stages=["assemble","budget"]`，语义必须真实通过。 +1. **汇编不可编码（11 个）**：`009`、`013`、`014`、`015`、`016`、`017`、`018`、`019`、`021`、`022`、`reg_pressure_loop`。首错指令见设计文档附录 5.2。处置：`meta.json` 声明 `xfail.stages=["assemble","budget","semantic"]`（owner=`backend/regalloc`，reason 引用 C1 并带首错指令）；`009_maxpool` 例外——它是 linear + interpreter oracle，语义不依赖汇编，`xfail.stages=["assemble","budget"]`，语义必须真实通过。根因在 linear-scan 发射器（detail 见 §4.3 C1 行）；greedy 路径同一 encoder 下可编码，回归测试 `test_greedy_allocator_encodes_branch_targets` 固定这一归因。 2. **执行语义错误（1 个）**：`reg_pressure_32` 实测 178 ≠ 64。处置：`xfail.stages=["semantic"]`，owner=`backend/regalloc`，reason 引用 C3。 3. **数据缺口（5 个）**：`016`、`017`、`018`、`021`、`022` 无 golden。处置：本课题补齐（§4.4 表 1），补齐后 `test_meta_contract` 必须转绿。 4. **数据缺陷（2 个）**：`017`、`022` 循环无自增。处置：本课题修用例源码（§4.4 表 2）。 -5. **解释器盲区（C2/C4/C5，10 个线性用例）**：`003/005/006/007/008/010/011/012/020/023` 的 interpreter 语义通过，但掩盖后端缺失（如 007 汇编实为单条 `mul`）。处置：保持 `pass`，同时在报告 `warnings` 与本文档登记；后续由 execution oracle 接管（不在本期验收）。 +5. **解释器盲区（C2/C4/C5，10 个线性用例）**：`003/005/006/007/008/010/011/012/020/023` 的 interpreter 语义通过，但掩盖后端缺失（如 007 汇编实为单条 `mul`）。处置：保持 `pass`，同时把“generated code is not executed”盲区 WARNING 写入每个 interpreter 用例的报告 `warnings`（回归测试见 `test_interpreter_blind_spot_reported_as_warning`）；后续由 execution oracle 接管（不在本期验收）。 ### 4.3 缺陷登记（与设计文档 §2.8 编号一一对应） | 编号 | owner（非本课题） | 套件中的处理 | 触发转正的信号 | |------|------------------|--------------|----------------| -| C1 | `backend/asm-encoder` | 11 个用例 xfail（stages 见 §4.2） | `assemble_to_binary` 对分支目标成功 → XPASS | +| C1 | `backend/regalloc`（linear-scan 发射器） | 11 个用例 xfail（stages 见 §4.2）；greedy 路径对照测试证明 encoder 无缺陷 | linear-scan 发射器写回分支目标 → XPASS | | C2 | `backend/asm-emitter` | 020 保持 interpreter；未来执行 oracle xfail | 020 执行得 20 | | C3 | `backend/regalloc` | reg_pressure_32 xfail(semantic) | 执行得 64 | | C4 | `backend/op-lowering` | 006/007/008/009/012 的 interpreter 结果不删除 | 增加 execution oracle 后转正 | @@ -638,8 +638,8 @@ EOF ```json "xfail": { "stages": ["assemble", "budget", "semantic"], - "reason": "RISCVAEncoder cannot encode symbolic branch targets; first bad line: bge a3, 4 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: linear-scan emitter keeps the branch target in the comment and emits no operand; first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false } ``` @@ -715,7 +715,7 @@ bench-suite: ```bash # 主验收：pytest 套件 python3.12 -m pytest tests/test_dsl_suite.py -v --tb=short -# 预期：0 failed, 0 errors；约 92 passed, 33 xfailed（数量以实现后冻结为准） +# 预期：0 failed, 0 errors；107 passed, 33 xfailed（修复轮实测，数量变化以冻结为准） # 报告验收 python3.12 benchmarks/run_suite.py \ @@ -810,18 +810,24 @@ EOF # 现有 bench_runner 对照 python3 benchmarks/bench_runner.py benchmarks/cases --quiet -# 执行 oracle 探针（001） +# 执行 oracle 探针（001 codegen+仿真绿色路径） python3.11 - <<'EOF' import sys; sys.path.insert(0, ".") -from scratchv.backend.riscv_encoder import assemble_to_binary -from scratchv.simulator.tinyfive import ProfiledMachine +from benchmarks.dsl_suite import ExecutionOracle asm = open("/tmp/asm/001_simple_add.s").read() -binary = assemble_to_binary(asm) -words = [int.from_bytes(binary[i:i+4], "little") for i in range(0, len(binary), 4)] -m = ProfiledMachine(mem_size=128*1024*1024); m.load_binary(words, origin=0) -m.set_reg(10, 2); m.set_reg(11, 3) -m.run(instructions=len(words), start=0, strict=True) -print("a0 =", m.get_reg(10)) # 实测 5 +outcome = ExecutionOracle().execute( + asm, {"a": "a0", "b": "a1"}, {"a": 2, "b": 3}, timeout_s=5.0) +print("a0 =", outcome.actual, "error =", outcome.error) # 实测 5, None +EOF + +# 循环执行探针（F1：动态步数不受静态指令数截断） +python3.11 - <<'EOF' +import sys; sys.path.insert(0, ".") +from benchmarks.dsl_suite import ExecutionOracle +asm = ("addi a0, zero, 0\naddi t0, zero, 0\naddi t1, zero, 4\nloop:\n" + "addi a0, a0, 1\naddi t0, t0, 1\nblt t0, t1, loop\njalr zero, ra\n") +outcome = ExecutionOracle().execute(asm, {}, {}, timeout_s=5.0) +print("a0 =", outcome.actual, "error =", outcome.error) # 实测 4, None EOF ``` @@ -848,20 +854,19 @@ EOF --- -## 实现结果（2026-09-14 集成） +## 实现结果（2026-09-14，分支 `impl/topic06`） -> **集成 commit**：`cae2c08`（`feat(topic06): add pytest-based DSL benchmark suite with xfail policy`） -> **集成位置**：`Seven_big_summary` 上 `d146515` 之后的第 1 个 topic commit（全分支共 13 个 commit） -> **集成后全量**：`PYTHONPATH=. python3.11 -m pytest tests/ -q` → **1011 passed / 13 xfailed / 20 xpassed / 0 failed** +> **实现 commit**：`d5b79ac`（`feat(topic06): add pytest-based DSL benchmark suite with xfail policy`）、`6c85290`（文档）+ 本修复轮 commit +> **分支基线**：main `73c3926`（含 `d146515`）；分支全量实测（修复轮后）：`PYTHONPATH=. python3.11 -m pytest tests/ -q` → **778 passed / 33 xfailed / 0 failed**（本环境含 tinyfive；无 tinyfive 时新增的 execution oracle 测试按 `skip` 处理） ### 实现文件与要点 | 文件 | 要点 | |------|------| -| `benchmarks/dsl_suite.py` | 套件核心：`discover_cases` / `load_case_spec` / `DSLSuiteRunner` / `ExecutionOracle` / `SuiteReport`，按 §1.2–§1.8 契约实现 | -| `benchmarks/run_suite.py` | CLI（`--json` / `--markdown`）与 §1.8 退出码契约 | -| `tests/test_dsl_suite.py` | session fixture 缓存 + 逐 stage `pytest.param(marks=...)` 注入 xfail | -| `benchmarks/cases/*.meta.json` ×23 | 新增元数据（含 §4.4 表 1/表 3 数据补齐） | +| `benchmarks/dsl_suite.py` | 套件核心：`discover_cases` / `load_case_spec` / `DSLSuiteRunner` / `ExecutionOracle` / `SuiteReport`，按 §1.2–§1.8 契约实现；执行 oracle 单步执行 + 真实 `timeout_s`（见 §1.5） | +| `benchmarks/run_suite.py` | CLI（`--json` / `--markdown`）与 §1.8 退出码契约；`xpassed>0` 时输出警告（F8） | +| `tests/test_dsl_suite.py` | session fixture 缓存 + 逐 stage `pytest.param(marks=...)` 注入 xfail；compile 阶段同样注入（F5）；新增 11 个回归测试 | +| `benchmarks/cases/*.meta.json` ×23 | 新增元数据（含 §4.4 表 1/表 3 数据补齐）；11 份汇编失败用例的 `xfail.owner/reason` 指向 `backend/regalloc`（F2） | | `tests/stress/*.meta.json` ×2 | 修订接入统一 schema | | `benchmarks/cases/017_while_sum.dsl`、`022_dsl_while_sum.dsl` | §4.4 表 2 补自增（仅用例数据，不触碰编译器） | | `.github/workflows/ci.yml` | 三处接线：`test` job 门禁（§5.1）、`benchmark` job 报告（§5.2）、job summary 摘要（§5.3） | @@ -870,23 +875,26 @@ EOF | 口径 | 结果 | |------|------| -| 定向（`tests/test_dsl_suite.py`） | 96 passed / 33 xfailed / 0 failed | -| 分支全量（cherry-pick 前） | 661 passed | -| 集成后全量 | 1011 passed / 13 xfailed / 20 xpassed / 0 failed | +| 定向（`tests/test_dsl_suite.py`） | 107 passed / 33 xfailed / 0 failed | +| 分支全量 | 778 passed / 33 xfailed / 0 failed | +| 套件 CLI（`run_suite.py`） | exit 0；用例级 13 passed / 12 xfailed / 0 xpassed / 0 failed | ### 与本文档的偏差 / 未完成项 -- 11 个分支指令失败案例（009/013/014/015/016/017/018/019/021/022/`reg_pressure_loop`）**全部 xfail**，根因登记为 backend 编码器缺陷 C1–C6，按课题边界**未修**。 +- 11 个分支指令失败案例（009/013/014/015/016/017/018/019/021/022/`reg_pressure_loop`）**全部 xfail**；根因按实测修正为 `backend/regalloc`（linear-scan 发射器不写回 comment 中的分支目标，C1），encoder 本身无缺陷（greedy 路径对照测试为证）；按课题边界**未修**。 +- `reg_pressure_32`（C3）仍实测 178 ≠ 64，semantic xfail 保持。 - 本地无 pyyaml，CI YAML 只做了人工 diff 核对，未做语法解析校验。 - `017/022` 的预算冻结值待 C1 修复后复测收紧。 ### 已知限制 - 33 个 xfail 是缺陷的真实登记，不得为了全绿而删除或放宽。 -- execution oracle 依赖 tinyfive；缺失时按 blocked 处理。 +- execution oracle 依赖 tinyfive；缺失时按 blocked 处理，且未被 `xfail.stages` 覆盖的 blocked 阶段现在是硬失败（F4）。 - `017/022` 的 legacy `.expected` 是 bench_runner 的自洽数据，按要求保持不动。 -### 关联集成修复（`fix(integration)`，不计入本 topic 实现 commit） +### 关联集成分支修复（**不在本分支历史上，不得当作本分支成果引用**） + +`76d852f`、`58dd2b4` 经 `git merge-base --is-ancestor` 校验均不是本分支祖先： - `76d852f`：`reg_pressure_32` 预算 77 → 103（topic17 linear-scan 重构后实测；语义同时由失败转正确）。 - `58dd2b4`：`gelu` 预算 5 → 6、`large_chain` 10 → 11（topic29 常量物化修复 R 型立即数编码后的合理指令数增加）。 diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" index b6f9244..d8ad5ad 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -28,7 +28,7 @@ | 汇编可编码（`assemble_to_binary`） | 14 | 11 | `009_maxpool`、`013_for_sum`、`014_for_dot`、`015_for_relu`、`016_if_simple`、`017_while_sum`、`018_nested_if`、`019_nested_loop`、`021_dsl_if_else`、`022_dsl_while_sum`、`reg_pressure_loop` | | 语义 golden（真实 RISC-V 执行） | 仅 1 例端到端探测成功（`001`，加法交换律下映射仍不可判别） | 其余不可信或阻塞 | 输入寄存器映射无契约、常量 `mv rd, imm` 非法发射、溢出/栈未初始化、NN 算子未 lowering、分支汇编不可编码 | - 11 个汇编失败的首错指令全部是**控制流指令**（`bnez` / `j` / `bge`），根因是 `AsmEmitter` 把分支目标写进注释、操作数缺失，而 `RISCVAEncoder` 需要符号目标/偏移（详见 §2.8 C1、附录 5.2）。 + 11 个汇编失败的首错指令全部是**控制流指令**（`bnez` / `j` / `bge`），根因是 linear-scan 发射路径（`regalloc_linear.py` 的 `to_asm`）只把分支目标作为注释输出、操作数缺失；同 encoder 下 greedy 路径（`asm_emit.py` 会把注释还原为操作数）可以编码成功，`RISCVAEncoder` 本身能编码符号目标（详见 §2.8 C1、附录 5.2）。 3. **交付目录只剩产物**。`ScratchV-topic06-deliverable/` 现仅存 `build/*.s` 共 23 个汇编产物；`run_tests.py` 已不存在。历史版本（commit `82860b4`）中的 23 个 `.dsl + .meta.json` 用例采用了“scalar 输入 + `expected_return` + TinyFive 真执行”的正确格式，可作为本课题元数据 schema 的参考来源，但**不纳入本期验收范围**（见 §4.5 可选步骤）。 @@ -186,8 +186,8 @@ tests/ "max_instructions": 8, "xfail": { "stages": ["assemble", "budget", "semantic"], - "reason": "RISCVAEncoder cannot encode symbolic branch targets (bnez/j/bge emitted with target in comment); first bad line: bnez a1", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: linear-scan emitter keeps the branch target in the comment and emits no operand, so assemble fails; first bad line: bnez a1 # if_then1", + "owner": "backend/regalloc", "strict": false } } @@ -235,7 +235,7 @@ discover() -> list[CaseSpec] | oracle | 实现 | 适用 | 局限（必须在报告中声明） | |--------|------|------|--------------------------| | `interpreter` | `scratchv.verification.verifier.DSLInterpreter.run(source, inputs)`，输出与 `expected_return`/`.expected` 做数值比较 | `flow=linear` 用例 | 不经过代码生成，**不验证后端语义**；对向量用例使用默认输入 `[1,2,3,4]`（与 bench_runner 一致） | -| `execution` | `ExecutionOracle.execute(asm, input_registers, inputs, timeout_s)`：`assemble_to_binary` → `ProfiledMachine.load_binary` → `set_reg` 注入 → `run(instructions=len(words), start=0, strict=True)` → `get_reg(10)` 作为返回值 → 数值比较 | `flow=control`、压力用例、需要验证真实代码路径的用例 | 依赖汇编成功；输入→寄存器映射为声明式（不推断）；tinyfive 不可用时 `blocked` | +| `execution` | `ExecutionOracle.execute(asm, input_registers, inputs, timeout_s)`：`assemble_to_binary` → `ProfiledMachine.load_binary` → `set_reg` 注入 → 单步执行至跳出代码区（`ra` 预置哨兵令 `jalr zero, ra` 停机）或 `timeout_s` 超时 → `get_reg(10)` 作为返回值 → 数值比较 | `flow=control`、压力用例、需要验证真实代码路径的用例 | 依赖汇编成功；输入→寄存器映射为声明式（不推断）；tinyfive 不可用时 `blocked` | | `none` | 不执行，返回 `blocked` | 已知无法执行的用例 | 必须携带 `xfail`（`stages` 含 `semantic`），否则硬失败 | `interpreter` 模式对向量输入沿用现状默认值 `np.array([1.0,2.0,3.0,4.0])`；后续用例可在 `inputs` 中显式给出向量，逐步替代隐式缺省。 @@ -251,7 +251,7 @@ discover() -> list[CaseSpec] #### 2.3.4 超时与隔离 -- 每个用例四阶段总耗时受 `timeout_s` 约束（默认 30s）。执行 oracle 的 `ProfiledMachine.run` 以 `instructions=len(words)` 限定步数，天然防死循环； +- `timeout_s`（取 meta `timeout_s` 与 runner/CLI 预算的较小值，默认 30s）在执行 oracle 中真实生效：执行按指令单步推进并逐步检查墙钟超时，另有 1e8 步硬上限兜底；失控程序返回语义失败（`timeout after ...`）而非静默截断。编译/汇编阶段为进程内调用，不单独中断； - 编译/汇编在进程内执行，异常全部捕获为 `CaseOutcome.error`； - 若未来发现优化器全局状态污染（同进程连续编译相互影响），回退方案为每用例子进程隔离（设计预留 `--isolate`，见开发文档 §7）。 @@ -354,7 +354,7 @@ discover() -> list[CaseSpec] | 编号 | 现象（实测证据） | 归属（非本课题） | 套件处置 | |------|------------------|------------------|----------| -| C1 | 分支/跳转汇编不可编码：`bnez a1 # label`、`j # label`、`bge a3, 4 # label` 操作数为空或符号目标在注释；`assemble_to_binary` 抛 `IndexError: list index out of range`（11 例，附录 5.2） | backend/asm 发射器与编码器 | `asm_encodable`/`semantic` xfail（owner=`backend/asm-encoder`） | +| C1 | 分支/跳转汇编不可编码：linear-scan 发射路径（`regalloc_linear.py` 的 `to_asm`）把分支目标（存于 comment）只作为 `# ...` 输出、操作数为空，`assemble_to_binary` 抛 `IndexError: list index out of range`（11 例，附录 5.2）；同 encoder 下 greedy 路径（`asm_emit.py` 将 comment 还原为目标操作数）编码成功 | backend/regalloc（linear-scan 发射器） | `asm_encodable`/`semantic` xfail（owner=`backend/regalloc`） | | C2 | 常量物化发射 `li a0, 20` 后又发 `mv a0, 20`（`mv` 立即数非法）；`020_constant_propagation` 实测执行结果为 0（期望 20） | backend/asm 发射器 | 用例 oracle 保持 interpreter；执行 oracle 用例 xfail 并引用 C2 | | C3 | 溢出/栈路径未初始化栈帧：`reg_pressure_32`（x=1，期望 64）实测 178 | backend/regalloc | `execution` xfail（owner=`backend/regalloc`） | | C4 | NN 算子未 lowering：`007_matmul` 汇编仅 `mul a2,a1,a0`；`006_softmax` 为 `mv` 直通；`008_dot`/`009_maxpool` 同类 | backend/op-lowering | `execution` xfail（owner=`backend/op-lowering`） | @@ -418,7 +418,7 @@ endfor return acc ``` -**元数据要点**：`flow=control`、`oracle=execution`、`inputs={"x":1,"acc":0}`、`input_registers={"x":"a0","acc":"a1"}`、`expected_return=4`（4 次迭代 ×1；legacy `.expected=[1,2,3,4]` 是按单次迭代校准的 D2 陈旧数据，**不修改**，以 meta 为准）、`max_instructions=9`（实测汇编文本 7 条，留 1.2 倍余量）、`xfail.stages=["assemble","budget","semantic"]`（owner=`backend/asm-encoder`，引用 C1）。 +**元数据要点**：`flow=control`、`oracle=execution`、`inputs={"x":1,"acc":0}`、`input_registers={"x":"a0","acc":"a1"}`、`expected_return=4`（4 次迭代 ×1；legacy `.expected=[1,2,3,4]` 是按单次迭代校准的 D2 陈旧数据，**不修改**，以 meta 为准）、`max_instructions=9`（实测汇编文本 7 条，留 1.2 倍余量）、`xfail.stages=["assemble","budget","semantic"]`（owner=`backend/regalloc`，引用 C1）。 **预期输出/指标**： @@ -527,7 +527,7 @@ return acc ### 4.6 集成与回归测试 -- `pytest tests/test_dsl_suite.py -v`：预计 `92 passed, 33 xfailed, 0 failed`（**pytest 测试级**：25 meta + 25 compile + 25 asm + 25 budget + 25 semantic；其中 asm/budget 各 11 个 xfail，semantic 11 个 xfail）。报告 JSON 的 `summary` 是**用例级**：`14 passed, 11 xfailed`（控制流用例的多个阶段失败合并为一个 xfail 状态）。 +- `pytest tests/test_dsl_suite.py -v`：本分支实测（修复轮后）`107 passed, 33 xfailed, 0 failed`（**pytest 测试级**：25 meta + 25 compile + 25 asm + 25 budget + 25 semantic + 15 辅助/回归测试 = 140 节点；其中 asm/budget/semantic 各 11 个 xfail）。报告 JSON 的 `summary` 是**用例级**：`13 passed, 12 xfailed`（控制流用例的多个阶段失败合并为一个 xfail 状态）。 - `pytest tests/test_bench_runner.py -v`：保持全绿（套件不改 bench_runner，且不改 legacy `.expected`）。 - `make test`：全量 pytest 仍绿。 - `python benchmarks/run_suite.py --json /tmp/dsl_suite.json --markdown /tmp/dsl_suite.md`：退出码 0；JSON 可被 `python -m json.tool` 解析。 @@ -546,7 +546,7 @@ return acc "generated_at": "2026-09-14T10:30:00", "roots": ["benchmarks/cases", "tests/stress"], "compiler": {"backend": "riscv", "optimize_level": "all", "reg_alloc": "linear"}, - "summary": {"total": 25, "passed": 14, "xfailed": 11, "xpassed": 0, "failed": 0, "skipped": 0}, + "summary": {"total": 25, "passed": 13, "xfailed": 12, "xpassed": 0, "failed": 0, "skipped": 0}, "results": [ { "case_id": "cases/001_simple_add", @@ -578,8 +578,8 @@ return acc "actual": null, "xfail": { "stages": ["assemble", "budget", "semantic"], - "reason": "RISCVAEncoder cannot encode symbolic branch targets; first bad line: bge a3, 4 # .Lloop_exit_3", - "owner": "backend/asm-encoder", + "reason": "C1 backend/regalloc: linear-scan emitter keeps the branch target in the comment and emits no operand; first bad line: bge a3, 4 # .Lloop_exit_3", + "owner": "backend/regalloc", "strict": false }, "error_stage": "assemble", @@ -605,7 +605,9 @@ return acc | `022_dsl_while_sum` | 3 | `j # while_hdr1` | 同上 | | `reg_pressure_loop` | 6 | `bge a3, 10 # .Lloop_exit_3` | 同上 | -汇编文本指令数实测（`parse_asm` 口径，供 `max_instructions` 基线）：`001=3, 002=3, 003=4, 004=3, 005=5, 006=3, 007=3, 008=3, 010=5, 011=5, 012=5, 020=3, 023=10, reg_pressure_32=77`；编码失败但文本可计数的 11 例：`009=7, 013=7, 014=7, 015=8, 016=6, 017=7, 018=11, 019=11, 021=10, 022=8, reg_pressure_loop=13`。 +汇编文本指令数实测（`parse_asm` 口径，供 `max_instructions` 基线）：`001=3, 002=3, 003=4, 004=3, 005=5, 006=3, 007=3, 008=3, 010=5, 011=5, 012=5, 020=3, 023=10, reg_pressure_32=77`；编码失败但文本可计数的 11 例：`009=7, 013=7, 014=7, 015=8, 016=6, 017=8, 018=11, 019=11, 021=10, 022=9, reg_pressure_loop=13`。 + +（上表 `IndexError` 均由 linear-scan 发射路径缺失分支目标操作数触发；greedy 路径在同一 encoder 下编码成功，根因归属 `backend/regalloc`，见 §2.8 C1 与开发文档 §4.3。） ### 5.3 参考资料 From a11e4c9d7190408f2b782fcfef5faa24f7fa2fcd Mon Sep 17 00:00:00 2001 From: opencode Date: Tue, 15 Sep 2026 00:59:17 +0800 Subject: [PATCH 6/6] feat(topic06): add suite report contract tests and stage/owner aggregates --- .github/workflows/ci.yml | 7 +- benchmarks/dsl_suite.py | 169 ++++++++-- tests/test_suite_report_contract.py | 486 ++++++++++++++++++++++++++++ 3 files changed, 641 insertions(+), 21 deletions(-) create mode 100644 tests/test_suite_report_contract.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 644db5f..10b7df4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,10 +85,13 @@ jobs: --junit-xml=benchmark_reports/test_results.xml \ --ignore=tests/test_simulator.py - # ── 课题06：DSL 基准套件（硬门禁，xfail 不红灯） ───────────────── + # ── 课题06：DSL 基准套件（硬门禁，xfail 不红灯）+ 报告契约 ─────── - name: Run DSL benchmark suite run: | - python3.12 -m pytest tests/test_dsl_suite.py -v --tb=short + python3.12 -m pytest \ + tests/test_dsl_suite.py \ + tests/test_suite_report_contract.py \ + -v --tb=short - name: Run assembly-beautifier regressions run: | diff --git a/benchmarks/dsl_suite.py b/benchmarks/dsl_suite.py index 6fba4fe..34e99ec 100644 --- a/benchmarks/dsl_suite.py +++ b/benchmarks/dsl_suite.py @@ -46,6 +46,13 @@ STAGE_SEMANTIC, ) +_STAGE_RESULT_KEYS: dict[str, str] = { + STAGE_COMPILE: "compile_ok", + STAGE_ASSEMBLE: "asm_encodable", + STAGE_BUDGET: "inst_budget_ok", + STAGE_SEMANTIC: "semantic_ok", +} + ASSERT_COMPILE_OK: str = "compile_ok" ASSERT_ASM_ENCODABLE: str = "asm_encodable" ASSERT_INST_BUDGET: str = "inst_budget" @@ -256,6 +263,68 @@ def summary(self) -> dict[str, int]: "skipped": self.skip_count, } + def stage_summary(self) -> dict[str, dict[str, int]]: + """Per-stage verdict aggregation with xfail attribution. + + ``passed`` counts true verdicts not excused by an unexpected pass, + ``xfail`` counts failures excused for that stage by the case's + declaration, ``failed`` counts unexcused failures, ``xpass`` counts + declared stages of a case that unexpectedly passed, and ``blocked`` + counts stages that never executed (skipped case, earlier failure, or + the assertion was disabled). + """ + counts: dict[str, dict[str, int]] = { + stage: { + "passed": 0, "xfail": 0, "failed": 0, + "xpass": 0, "blocked": 0, + } + for stage in ALL_STAGES + } + for r in self.results: + declared = set(r.xfail.stages) if r.xfail else set() + stages = _stage_map(r) + for stage, key in _STAGE_RESULT_KEYS.items(): + bucket = counts[stage] + verdict = stages[key] + declared_here = stage in declared + if verdict is None: + bucket["blocked"] += 1 + elif verdict is False: + if declared_here and r.status == STATUS_XFAIL: + bucket["xfail"] += 1 + else: + bucket["failed"] += 1 + elif declared_here and r.status == STATUS_XPASS: + bucket["xpass"] += 1 + else: + bucket["passed"] += 1 + return counts + + def owner_summary(self) -> dict[str, dict[str, int]]: + """Per-owner xfail ledger, sorted by owner for stable output.""" + owners: dict[str, dict[str, int]] = {} + for r in self.results: + if r.xfail is None: + continue + bucket = owners.setdefault( + r.xfail.owner, + {"declared": 0, "xfailed": 0, "xpassed": 0, "failed": 0}, + ) + bucket["declared"] += 1 + if r.status == STATUS_XFAIL: + bucket["xfailed"] += 1 + elif r.status == STATUS_XPASS: + bucket["xpassed"] += 1 + elif r.status == STATUS_FAIL: + bucket["failed"] += 1 + return dict(sorted(owners.items())) + + def xpassed_cases(self) -> list[str]: + """Case ids whose declared xfail stages unexpectedly passed.""" + return sorted( + r.case_id for r in self.results if r.status == STATUS_XPASS + ) + def to_dict(self) -> dict[str, Any]: return { "schema_version": SCHEMA_VERSION, @@ -264,6 +333,9 @@ def to_dict(self) -> dict[str, Any]: "roots": list(self.roots), "compiler": dict(self.compiler), "summary": self.summary(), + "stage_summary": self.stage_summary(), + "owner_summary": self.owner_summary(), + "xpassed_cases": self.xpassed_cases(), "results": [self._result_to_dict(r) for r in self.results], } @@ -288,6 +360,45 @@ def to_markdown(self) -> str: "|-------|--------|---------|---------|--------|---------|", "| {total} | {passed} | {xfailed} | {xpassed} | {failed} | " "{skipped} |".format(**summary), + ] + if summary["xpassed"]: + lines += [ + "", + f"**WARNING: {summary['xpassed']} unexpected pass(es) " + "(xpass)** — declared xfail stages now pass; remove the " + "obsolete declaration or run with `--strict-xfail`.", + ] + lines += [ + "", + "## Stage summary", + "", + "| stage | passed | xfail | failed | xpass | blocked |", + "|-------|--------|-------|--------|-------|---------|", + ] + for stage, counts in self.stage_summary().items(): + lines.append( + f"| {stage} | {counts['passed']} | {counts['xfail']} | " + f"{counts['failed']} | {counts['xpass']} | " + f"{counts['blocked']} |" + ) + owners = self.owner_summary() + lines += [ + "", + "## Owner summary", + "", + "| owner | declared | xfailed | xpassed | failed |", + "|-------|----------|---------|---------|--------|", + ] + if owners: + for owner, counts in owners.items(): + lines.append( + f"| {owner} | {counts['declared']} | " + f"{counts['xfailed']} | {counts['xpassed']} | " + f"{counts['failed']} |" + ) + else: + lines.append("| - | - | - | - | - |") + lines += [ "", "## Cases", "", @@ -326,18 +437,28 @@ def to_markdown(self) -> str: else: lines.append("No hard failures.") - blocked_or_xfail = [ - r for r in self.results - if r.status in (STATUS_XFAIL, STATUS_XPASS) + xpassed = [r for r in self.results if r.status == STATUS_XPASS] + expected_failures = [ + r for r in self.results if r.status == STATUS_XFAIL ] - lines += ["", "## Expected failures / unexpected passes", ""] - if blocked_or_xfail: - for r in blocked_or_xfail: - which = ( - r.error_stage if r.status == STATUS_XFAIL - else "declared stages now pass" + lines += ["", "## Unexpected passes (xpass — action required)", ""] + if xpassed: + for result in xpassed: + stages = ( + ", ".join(result.xfail.stages) if result.xfail else "-" + ) + owner = result.xfail.owner if result.xfail else "-" + lines.append( + f"- **{result.case_id}**: declared stages now pass " + f"(owner={owner}, stages={stages})" ) - lines.append(f"- {r.case_id} [{r.status}] ({which})") + else: + lines.append("None.") + lines += ["", "## Expected failures (xfail)", ""] + if expected_failures: + for result in expected_failures: + which = result.error_stage or "unknown" + lines.append(f"- {result.case_id} ({which})") else: lines.append("None.") @@ -399,15 +520,7 @@ def _result_to_dict(self, r: CaseOutcome) -> dict[str, Any]: "flow": spec.flow if spec else "unknown", "oracle": spec.oracle if spec else "unknown", "status": r.status, - "stages": { - "compile_ok": None if skipped else r.compile.ok, - "asm_encodable": None if skipped or r.assemble is None - else r.assemble.ok, - "inst_budget_ok": None if skipped or r.budget is None - else r.budget.ok, - "semantic_ok": None if skipped or r.semantic is None - else r.semantic.ok, - }, + "stages": _stage_map(r), "metrics": { "ir_instructions": None if skipped else r.compile.ir_instruction_count, @@ -433,6 +546,24 @@ def _mark(value: bool | None) -> str: return "ok" if value else "FAIL" +def _stage_map(r: CaseOutcome) -> dict[str, bool | None]: + """Per-stage verdicts with the report's ``null`` semantics. + + Skipped cases expose ``None`` for every stage so that the JSON schema and + the aggregate summaries agree on what "did not execute" means. + """ + skipped = r.status == STATUS_SKIP + return { + "compile_ok": None if skipped else r.compile.ok, + "asm_encodable": None if skipped or r.assemble is None + else r.assemble.ok, + "inst_budget_ok": None if skipped or r.budget is None + else r.budget.ok, + "semantic_ok": None if skipped or r.semantic is None + else r.semantic.ok, + } + + def _xfail_to_dict(spec: XFailSpec | None) -> dict[str, Any] | None: if spec is None: return None diff --git a/tests/test_suite_report_contract.py b/tests/test_suite_report_contract.py new file mode 100644 index 0000000..2612a2c --- /dev/null +++ b/tests/test_suite_report_contract.py @@ -0,0 +1,486 @@ +"""Hermetic contract tests for the DSL benchmark suite report (topic 06). + +``SuiteReport`` produces the CI artifacts consumed by the job summary +(``benchmark_reports/dsl_suite.md``) and by dashboards/history +(``benchmark_reports/dsl_suite.json``). These tests pin that contract +without running the 25 real cases: outcomes and case metadata are +constructed (or loaded from a temporary one-case root), so the tests stay +fast and independent of compiler progress. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +from benchmarks.dsl_suite import ( + ALL_STAGES, + SCHEMA_VERSION, + STAGE_ASSEMBLE, + STAGE_SEMANTIC, + STATUS_FAIL, + STATUS_PASS, + STATUS_SKIP, + STATUS_XFAIL, + STATUS_XPASS, + SUITE_NAME, + AssembleOutcome, + BudgetOutcome, + CaseOutcome, + CaseSpec, + CompileOutcome, + SemanticOutcome, + SuiteReport, + load_case_spec, +) + +GENERATED_AT = "2026-09-15T00:00:00" + +TOP_LEVEL_KEYS = frozenset({ + "schema_version", "suite", "generated_at", "roots", "compiler", + "summary", "stage_summary", "owner_summary", "xpassed_cases", "results", +}) +RESULT_KEYS = frozenset({ + "case_id", "pytest_id", "category", "flow", "oracle", "status", + "stages", "metrics", "expected", "actual", "xfail", "error_stage", + "error", "warnings", +}) +STAGE_KEYS = frozenset({ + "compile_ok", "asm_encodable", "inst_budget_ok", "semantic_ok", +}) +METRIC_KEYS = frozenset({ + "ir_instructions", "asm_instructions", "compile_time_s", + "assemble_time_s", "semantic_time_s", +}) +STAGE_SUMMARY_KEYS = frozenset({ + "passed", "xfail", "failed", "xpass", "blocked", +}) +OWNER_SUMMARY_KEYS = frozenset({ + "declared", "xfailed", "xpassed", "failed", +}) + + +def _make_spec( + root: Path, name: str, *, xfail: dict | None = None, +) -> CaseSpec: + """Write and load one minimal valid interpreter-oracle case.""" + root.mkdir(parents=True, exist_ok=True) + dsl_path = root / f"{name}.dsl" + dsl_path.write_text("c = add(a, b)\nreturn c\n") + meta: dict = { + "description": f"report contract probe {name}", + "category": "arith", + "oracle": "interpreter", + "expected_return": 3, + } + if xfail is not None: + meta["xfail"] = xfail + dsl_path.with_suffix(".meta.json").write_text(json.dumps(meta)) + spec = load_case_spec(dsl_path, root) + assert spec.meta_errors == (), spec.meta_errors + return spec + + +def _outcome( + spec: CaseSpec, + status: str, + *, + compile_ok: bool = True, + assemble_ok: bool | None = True, + budget_ok: bool | None = True, + semantic_ok: bool | None = True, + error_stage: str | None = None, + error: str | None = None, + actual: object | None = None, +) -> CaseOutcome: + """Construct one case outcome with the requested per-stage verdicts.""" + compile_outcome = CompileOutcome( + ok=compile_ok, + asm_text="addi a0, zero, 3\n" if compile_ok else "", + output_path=None, + ir_instruction_count=1 if compile_ok else 0, + duration_s=0.0, + error=None if compile_ok else "compile failed", + ) + assemble = ( + None if assemble_ok is None + else AssembleOutcome( + ok=assemble_ok, + binary_len=4 if assemble_ok else 0, + instruction_count=1 if assemble_ok else None, + duration_s=0.0, + error=None if assemble_ok else "assemble failed", + ) + ) + budget = ( + None if budget_ok is None + else BudgetOutcome( + ok=budget_ok, + instruction_count=1, + limit=2, + skipped=False, + error=None if budget_ok else "1 > 2", + ) + ) + semantic = ( + None if semantic_ok is None + else SemanticOutcome( + ok=semantic_ok, + oracle=spec.oracle, + expected=3, + actual=actual if actual is not None else (3 if semantic_ok else 4), + duration_s=0.0, + blocked_reason=None, + error=None if semantic_ok else "semantic mismatch", + ) + ) + return CaseOutcome( + case_id=spec.case_id, + pytest_id=spec.pytest_id, + status=status, + compile=compile_outcome, + assemble=assemble, + budget=budget, + semantic=semantic, + error_stage=error_stage, + error=error, + xfail=spec.xfail, + ) + + +def _report_from( + results: list[CaseOutcome], specs: list[CaseSpec], +) -> SuiteReport: + return SuiteReport( + results=results, + roots=("probe",), + compiler={ + "backend": "riscv", + "optimize_level": "all", + "reg_alloc": "linear", + }, + generated_at=GENERATED_AT, + specs={spec.case_id: spec for spec in specs}, + ) + + +def _build_mixed_report(tmp_path: Path) -> SuiteReport: + """One pass, one xfail, one xpass, one hard fail and one skip.""" + root = tmp_path / "probe" + pass_spec = _make_spec(root, "pass_case") + xfail_spec = _make_spec(root, "xfail_case", xfail={ + "stages": ["assemble", "budget"], + "reason": "C1 probe: linear emitter drops branch targets", + "owner": "backend/regalloc", + }) + xpass_spec = _make_spec(root, "xpass_case", xfail={ + "stages": ["semantic"], + "reason": "C4 probe: op lowering now works", + "owner": "backend/op-lowering", + }) + fail_spec = _make_spec(root, "fail_case") + skip_spec = _make_spec(root, "skip_case") + results = [ + _outcome( + pass_spec, STATUS_PASS, actual=np.array([1.0, 2.0, 3.0]), + ), + _outcome( + xfail_spec, STATUS_XFAIL, + assemble_ok=False, budget_ok=None, semantic_ok=None, + error_stage=STAGE_ASSEMBLE, + error="C1 probe: linear emitter drops branch targets", + ), + _outcome(xpass_spec, STATUS_XPASS), + _outcome( + fail_spec, STATUS_FAIL, + assemble_ok=False, budget_ok=None, semantic_ok=None, + error_stage=STAGE_ASSEMBLE, error="assemble failed", + ), + _outcome( + skip_spec, STATUS_SKIP, + compile_ok=False, assemble_ok=None, + budget_ok=None, semantic_ok=None, + ), + ] + return _report_from(results, [ + pass_spec, xfail_spec, xpass_spec, fail_spec, skip_spec, + ]) + + +def _recomputed_summary(report: SuiteReport) -> dict[str, int]: + statuses = [r.status for r in report.results] + return { + "total": len(statuses), + "passed": statuses.count(STATUS_PASS), + "xfailed": statuses.count(STATUS_XFAIL), + "xpassed": statuses.count(STATUS_XPASS), + "failed": statuses.count(STATUS_FAIL), + "skipped": statuses.count(STATUS_SKIP), + } + + +def assert_report_consistent(report: SuiteReport) -> None: + """Gate: summary/aggregates must be recomputable from raw outcomes.""" + summary = report.summary() + recomputed = _recomputed_summary(report) + assert summary == recomputed, ( + f"summary() drifted from raw case statuses: {summary} != {recomputed}" + ) + assert summary["total"] == sum( + summary[key] + for key in ("passed", "xfailed", "xpassed", "failed", "skipped") + ), summary + for stage, counts in report.stage_summary().items(): + assert set(counts) == STAGE_SUMMARY_KEYS, (stage, counts) + assert sum(counts.values()) == summary["total"], (stage, counts) + for owner, counts in report.owner_summary().items(): + assert set(counts) == OWNER_SUMMARY_KEYS, (owner, counts) + data = report.to_dict() + assert data["summary"] == summary + assert data["stage_summary"] == report.stage_summary() + assert data["owner_summary"] == report.owner_summary() + assert data["xpassed_cases"] == report.xpassed_cases() + + +# --------------------------------------------------------------------------- +# summary() counts and boundaries +# --------------------------------------------------------------------------- + + +def test_summary_counts_match_constructed_outcomes(tmp_path: Path) -> None: + report = _build_mixed_report(tmp_path) + assert report.summary() == { + "total": 5, "passed": 1, "xfailed": 1, + "xpassed": 1, "failed": 1, "skipped": 1, + } + assert report.pass_count == 1 + assert report.xfail_count == 1 + assert report.xpass_count == 1 + assert report.fail_count == 1 + assert report.skip_count == 1 + + +def test_summary_all_pass_and_empty_suite(tmp_path: Path) -> None: + root = tmp_path / "probe_all_pass" + specs = [_make_spec(root, f"pass_{i}") for i in range(3)] + report = _report_from( + [_outcome(spec, STATUS_PASS) for spec in specs], specs, + ) + assert report.summary() == { + "total": 3, "passed": 3, "xfailed": 0, + "xpassed": 0, "failed": 0, "skipped": 0, + } + + empty = SuiteReport( + results=[], roots=(), compiler={}, generated_at=GENERATED_AT, + ) + assert empty.summary() == { + "total": 0, "passed": 0, "xfailed": 0, + "xpassed": 0, "failed": 0, "skipped": 0, + } + assert set(empty.stage_summary()) == set(ALL_STAGES) + assert all( + sum(counts.values()) == 0 + for counts in empty.stage_summary().values() + ) + assert empty.owner_summary() == {} + assert empty.xpassed_cases() == [] + assert_report_consistent(empty) + + +def test_summary_single_status_xfail_and_xpass(tmp_path: Path) -> None: + root = tmp_path / "probe_single" + xfail_spec = _make_spec(root, "xfail_only", xfail={ + "stages": ["assemble"], + "reason": "probe: assemble stage fails", + "owner": "backend/regalloc", + }) + xpass_spec = _make_spec(root, "xpass_only", xfail={ + "stages": ["semantic"], + "reason": "probe: semantic stage now passes", + "owner": "backend/op-lowering", + }) + + xfail_report = _report_from( + [_outcome( + xfail_spec, STATUS_XFAIL, + assemble_ok=False, budget_ok=None, semantic_ok=None, + error_stage=STAGE_ASSEMBLE, error="assemble failed", + )], + [xfail_spec], + ) + assert xfail_report.summary() == { + "total": 1, "passed": 0, "xfailed": 1, + "xpassed": 0, "failed": 0, "skipped": 0, + } + assert_report_consistent(xfail_report) + + xpass_report = _report_from( + [_outcome(xpass_spec, STATUS_XPASS)], [xpass_spec], + ) + assert xpass_report.summary() == { + "total": 1, "passed": 0, "xfailed": 0, + "xpassed": 1, "failed": 0, "skipped": 0, + } + assert xpass_report.stage_summary()[STAGE_SEMANTIC]["xpass"] == 1 + assert_report_consistent(xpass_report) + + +# --------------------------------------------------------------------------- +# stage / owner aggregation +# --------------------------------------------------------------------------- + + +def test_stage_and_owner_aggregation(tmp_path: Path) -> None: + report = _build_mixed_report(tmp_path) + + assert report.stage_summary() == { + "compile": { + "passed": 4, "xfail": 0, "failed": 0, "xpass": 0, "blocked": 1, + }, + "assemble": { + "passed": 2, "xfail": 1, "failed": 1, "xpass": 0, "blocked": 1, + }, + "budget": { + "passed": 2, "xfail": 0, "failed": 0, "xpass": 0, "blocked": 3, + }, + "semantic": { + "passed": 1, "xfail": 0, "failed": 0, "xpass": 1, "blocked": 3, + }, + } + assert report.owner_summary() == { + "backend/op-lowering": { + "declared": 1, "xfailed": 0, "xpassed": 1, "failed": 0, + }, + "backend/regalloc": { + "declared": 1, "xfailed": 1, "xpassed": 0, "failed": 0, + }, + } + assert report.xpassed_cases() == ["probe/xpass_case"] + assert_report_consistent(report) + + +# --------------------------------------------------------------------------- +# JSON schema +# --------------------------------------------------------------------------- + + +def test_to_dict_schema_is_stable_and_json_serializable( + tmp_path: Path, +) -> None: + report = _build_mixed_report(tmp_path) + data = report.to_dict() + + assert data["schema_version"] == SCHEMA_VERSION + assert data["suite"] == SUITE_NAME + assert data["generated_at"] == GENERATED_AT + assert TOP_LEVEL_KEYS <= set(data) + assert len(data["results"]) == 5 + + payload = json.dumps(data, sort_keys=True) + assert json.loads(payload) == data + + for entry in data["results"]: + assert RESULT_KEYS <= set(entry), entry["case_id"] + assert set(entry["stages"]) == STAGE_KEYS, entry["case_id"] + assert set(entry["metrics"]) == METRIC_KEYS, entry["case_id"] + assert entry["case_id"] in report.specs + skipped = data["results"][4] + assert skipped["status"] == STATUS_SKIP + assert all(value is None for value in skipped["stages"].values()) + + passed = data["results"][0] + assert isinstance(passed["actual"], str), ( + "numpy actual values must be stringified for JSON transport" + ) + + +def test_save_helpers_write_renderer_output(tmp_path: Path) -> None: + report = _build_mixed_report(tmp_path) + json_path = tmp_path / "out" / "suite.json" + md_path = tmp_path / "out" / "suite.md" + report.save_json(json_path) + report.save_markdown(md_path) + + assert json.loads(json_path.read_text()) == report.to_dict() + assert md_path.read_text() == report.to_markdown() + + +# --------------------------------------------------------------------------- +# Markdown rendering +# --------------------------------------------------------------------------- + + +def test_to_markdown_has_summary_row_per_case_and_prominent_xpass( + tmp_path: Path, +) -> None: + report = _build_mixed_report(tmp_path) + markdown = report.to_markdown() + + assert ( + "| total | passed | xfailed | xpassed | failed | skipped |" + ) in markdown + assert "| 5 | 1 | 1 | 1 | 1 | 1 |" in markdown + for r in report.results: + assert markdown.count(f"| {r.case_id} |") == 1, r.case_id + + assert "**WARNING: 1 unexpected pass(es) (xpass)**" in markdown + assert "## Unexpected passes (xpass — action required)" in markdown + assert "- **probe/xpass_case**: declared stages now pass" in markdown + assert "owner=backend/op-lowering" in markdown + assert "## Expected failures (xfail)" in markdown + assert "- probe/xfail_case (assemble)" in markdown + + assert "## Stage summary" in markdown + assert "| compile | 4 | 0 | 0 | 0 | 1 |" in markdown + assert "| assemble | 2 | 1 | 1 | 0 | 1 |" in markdown + assert "| budget | 2 | 0 | 0 | 0 | 3 |" in markdown + assert "| semantic | 1 | 0 | 0 | 1 | 3 |" in markdown + + assert "## Owner summary" in markdown + assert "| backend/op-lowering | 1 | 0 | 1 | 0 |" in markdown + assert "| backend/regalloc | 1 | 1 | 0 | 0 |" in markdown + + +def test_to_markdown_omits_xpass_warning_when_clean(tmp_path: Path) -> None: + root = tmp_path / "probe_clean" + spec = _make_spec(root, "pass_case") + report = _report_from([_outcome(spec, STATUS_PASS)], [spec]) + markdown = report.to_markdown() + assert "**WARNING:" not in markdown + assert "## Unexpected passes (xpass — action required)" in markdown + assert "None." in markdown + + +# --------------------------------------------------------------------------- +# Determinism and the anti-vacuity gate +# --------------------------------------------------------------------------- + + +def test_render_is_byte_stable_for_equal_inputs(tmp_path: Path) -> None: + first = _build_mixed_report(tmp_path) + second = _build_mixed_report(tmp_path) + + assert first.to_dict() == second.to_dict() + assert json.dumps(first.to_dict(), sort_keys=True) == ( + json.dumps(second.to_dict(), sort_keys=True) + ) + assert first.to_markdown() == second.to_markdown() + assert first.to_markdown().encode() == first.to_markdown().encode() + + +def test_report_gate_is_not_vacuous( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + report = _build_mixed_report(tmp_path) + assert_report_consistent(report) + + monkeypatch.setattr( + SuiteReport, "pass_count", property(lambda self: 99), + ) + assert report.summary()["passed"] == 99 + with pytest.raises(AssertionError): + assert_report_consistent(report)