diff --git a/.github/workflows/topic06-benchmark.yml b/.github/workflows/topic06-benchmark.yml new file mode 100644 index 0000000..8f47ebd --- /dev/null +++ b/.github/workflows/topic06-benchmark.yml @@ -0,0 +1,48 @@ +name: Topic 06 Compiler Benchmark + +on: + push: + pull_request: + +jobs: + topic06-benchmark: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install ScratchV and benchmark dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[topic06]" + + - name: Verify Topic 06 integration contracts + run: python -m pytest tests/topic06/test_integration.py -q + + - name: Run supported correctness gates + run: | + python scripts/run_topic06_benchmarks.py --category activation --fail-on-test-failure + python scripts/run_topic06_benchmarks.py --category elementwise --fail-on-test-failure + python scripts/run_topic06_benchmarks.py --category loop --fail-on-test-failure + + - name: Generate full diagnostic benchmark report + run: python scripts/run_topic06_benchmarks.py --benchmark 3 + + - name: Upload Topic 06 reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: topic06-benchmark-reports + path: | + benchmark_reports/topic06/report.md + benchmark_reports/topic06/report.json + benchmark_reports/topic06/cases/ + benchmark_reports/topic06/failures/ + benchmarks/topic06/baseline.json diff --git a/Makefile b/Makefile index 031f739..fc51a5a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # ScratchV developer makefile .POSIX: -.PHONY: quick-start install test bench bench-cnn clean lint +.PHONY: quick-start install test bench bench-topic06 bench-cnn clean lint # ── Beginner quick-start ───────────────────────────────────────────────────── @@ -48,6 +48,11 @@ bench: --output-json benchmark_reports/dsl_bench.json \ --output-html benchmark_reports/dsl_bench.html +# ── Topic 06 DSL correctness + TinyFive benchmark ───────────────────────── + +bench-topic06: + python3 scripts/run_topic06_benchmarks.py --benchmark 3 + # ── CNN RISC-V 编译 + 估算 ──────────────────────────────────────────────── bench-cnn: diff --git a/README.md b/README.md index a0f32ab..ec299b7 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ ```bash make quick-start # 打印新手引导 make test # 运行全部测试 +make bench-topic06 # 运行课题 06 TinyFive 性能测试套件 make bench-cnn # 编译 CNN 模型 + 性能估算 make bench-ci # 完整 CI 对比 (ScratchV vs LLVM) make bench-reports # 生成 Dashboard + 优化历史 @@ -62,9 +63,9 @@ ScratchV/ │ ├── 00~04-*.md ← 新手入门 5 篇 │ ├── topics/ ← 30 个模块详解 │ └── topics/html/ ← 🌐 交互式课程站点 -├── benchmarks/ ← 23 个 DSL 基准用例 -├── tests/ ← 348 个单元测试 -├── scripts/ ← 工具脚本 +├── benchmarks/ ← 通用基准与课题 06 性能基线 +├── tests/ ← 单元测试及课题 06 的 23 个 DSL 用例 +├── scripts/ ← 工具脚本及课题 06 测试驱动 └── models/ ← 测试用 ONNX 模型 ``` @@ -104,6 +105,7 @@ ONNX 模型 (.onnx) | [📖 文档导航](docs/INDEX.md) | 全部 Markdown 文档索引 | | [🏗️ 架构总览](docs/ARCHITECTURE.md) | ONNX→RISC-V 双路径详解 | | [📊 性能仪表盘](https://scratchv-compiler.github.io/ScratchV/dashboard.html) | LLVM vs ScratchV 对比 | +| [课题 06 测试套件](docs/topics/06-性能测试套件使用说明.md) | TinyFive 正确性验证、Benchmark 与回归报告 | | [📢 项目海报](https://scratchv-compiler.github.io/ScratchV/ScratchV.html) | 招募信息 + 3个月学习路线 + 课题精选 | --- diff --git a/benchmark_reports/topic06/cases/activation-add_relu_relu.json b/benchmark_reports/topic06/cases/activation-add_relu_relu.json new file mode 100644 index 0000000..24250bc --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-add_relu_relu.json @@ -0,0 +1,153 @@ +{ + "mode": "benchmark", + "name": "add_relu_relu", + "category": "activation", + "path": "tests/topic06/cases/activation/add_relu_relu.dsl", + "status": "PASS", + "description": "Add input and bias, then apply ReLU twice.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 7, + "actual": 7, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 7.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12910449993796647, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -3, + "t1": 10 + }, + "register_map": "build/topic06/add_relu_relu.registers.json", + "backend": "tinyfive", + "instr_count": 7, + "static_asm_instruction_count": 5, + "machine_code_instruction_count": 11, + "code_size_bytes": 44, + "perf_counters": { + "total": 7, + "load": 0, + "store": 0, + "mul": 0, + "add": 4, + "madd": 0, + "branch": 2 + }, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 11, + "code_size_bytes": 44, + "dynamic_instructions": 7.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 2.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 11, + "baseline": 11, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 44, + "baseline": 44, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 7.0, + "baseline": 7.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\add_relu_relu.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_relu_relu.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_relu_relu.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08510669996030629, + "simulation_time_sec": 0.14671280002221465, + "total_time_sec": 0.8289841000223532, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 7.0, + "min_instr_count": 7, + "max_instr_count": 7, + "ci95_instr_count": 0.0, + "baseline_instr_count": 7.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_relu_relu.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-add_relu_relu.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-add_relu_relu.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/activation-add_relu_relu.md b/benchmark_reports/topic06/cases/activation-add_relu_relu.md new file mode 100644 index 0000000..b14ddb3 --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-add_relu_relu.md @@ -0,0 +1,65 @@ +# add_relu_relu 测试详情 + +## 基本信息 + +- 类别: activation +- DSL: `tests/topic06/cases/activation/add_relu_relu.dsl` +- 描述: Add input and bias, then apply ReLU twice. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 7.0 | True | null | null | +| TinyFive | PASS | 7 | True | null | null | + +- 期望输出: 7 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 5 +- 编码后机器指令数: 11 +- 代码大小(bytes): 44 +- TinyFive 动态执行指令数: 7 +- TinyFive 分类计数: {'total': 7, 'load': 0, 'store': 0, 'mul': 0, 'add': 4, 'madd': 0, 'branch': 2} +- 编译耗时(s): 0.085107 +- 解释器耗时(s): 0.129104 +- TinyFive 模拟耗时(s): 0.146713 +- 总耗时(s): 0.828984 +- 基线动态指令数: 7.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 5, 'machine_instructions': 11, 'code_size_bytes': 44, 'dynamic_instructions': 7.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 4.0, 'dynamic_madd': 0.0, 'dynamic_branch': 2.0} +- Cost model 对比: {'static_asm_instructions': {'current': 5, 'baseline': 5, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 11, 'baseline': 11, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 44, 'baseline': 44, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 7.0, 'baseline': 7.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 4.0, 'baseline': 4.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 2.0, 'baseline': 2.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\add_relu_relu.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_relu_relu.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_relu_relu.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/add_relu_relu.registers.json` +- 汇编文件: `build/topic06/add_relu_relu.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + max t3, t2, 0 + max t4, t3, 0 + mv a0, t4 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/activation-relu_add.json b/benchmark_reports/topic06/cases/activation-relu_add.json new file mode 100644 index 0000000..5dfecd5 --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-relu_add.json @@ -0,0 +1,153 @@ +{ + "mode": "benchmark", + "name": "relu_add", + "category": "activation", + "path": "tests/topic06/cases/activation/relu_add.dsl", + "status": "PASS", + "description": "Add input and bias, then apply one ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 3, + "actual": 3, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 3.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12399170000571758, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -2, + "t1": 5 + }, + "register_map": "build/topic06/relu_add.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 1.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 7, + "baseline": 7, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 28, + "baseline": 28, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 1.0, + "baseline": 1.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\relu_add.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_add.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_add.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08786199998576194, + "simulation_time_sec": 0.15414490003604442, + "total_time_sec": 0.8028744000475854, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/relu_add.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-relu_add.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-relu_add.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/activation-relu_add.md b/benchmark_reports/topic06/cases/activation-relu_add.md new file mode 100644 index 0000000..6a5fb8f --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-relu_add.md @@ -0,0 +1,64 @@ +# relu_add 测试详情 + +## 基本信息 + +- 类别: activation +- DSL: `tests/topic06/cases/activation/relu_add.dsl` +- 描述: Add input and bias, then apply one ReLU. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 3.0 | True | null | null | +| TinyFive | PASS | 3 | True | null | null | + +- 期望输出: 3 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 1} +- 编译耗时(s): 0.087862 +- 解释器耗时(s): 0.123992 +- TinyFive 模拟耗时(s): 0.154145 +- 总耗时(s): 0.802874 +- 基线动态指令数: 5.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 7, 'code_size_bytes': 28, 'dynamic_instructions': 5.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 3.0, 'dynamic_madd': 0.0, 'dynamic_branch': 1.0} +- Cost model 对比: {'static_asm_instructions': {'current': 4, 'baseline': 4, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 7, 'baseline': 7, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 28, 'baseline': 28, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 5.0, 'baseline': 5.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 3.0, 'baseline': 3.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 1.0, 'baseline': 1.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_add.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\relu_add.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\relu_add.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/relu_add.registers.json` +- 汇编文件: `build/topic06/relu_add.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + max t3, t2, 0 + mv a0, t3 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/activation-relu_only.json b/benchmark_reports/topic06/cases/activation-relu_only.json new file mode 100644 index 0000000..a63496d --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-relu_only.json @@ -0,0 +1,152 @@ +{ + "mode": "benchmark", + "name": "relu_only", + "category": "activation", + "path": "tests/topic06/cases/activation/relu_only.dsl", + "status": "PASS", + "description": "Apply ReLU directly to a single input value.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": 0, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 0.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12315600004512817, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -5 + }, + "register_map": "build/topic06/relu_only.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 6, + "code_size_bytes": 24, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 6, + "code_size_bytes": 24, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 2.0, + "dynamic_madd": 0.0, + "dynamic_branch": 1.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 3, + "baseline": 3, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 6, + "baseline": 6, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 24, + "baseline": 24, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 1.0, + "baseline": 1.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\relu_only.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_only.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_only.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08596989989746362, + "simulation_time_sec": 0.15454419993329793, + "total_time_sec": 0.8122592000290751, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/relu_only.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-relu_only.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-relu_only.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/activation-relu_only.md b/benchmark_reports/topic06/cases/activation-relu_only.md new file mode 100644 index 0000000..d28f6dc --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-relu_only.md @@ -0,0 +1,63 @@ +# relu_only 测试详情 + +## 基本信息 + +- 类别: activation +- DSL: `tests/topic06/cases/activation/relu_only.dsl` +- 描述: Apply ReLU directly to a single input value. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 0.0 | True | null | null | +| TinyFive | PASS | 0 | True | null | null | + +- 期望输出: 0 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 3 +- 编码后机器指令数: 6 +- 代码大小(bytes): 24 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 2, 'madd': 0, 'branch': 1} +- 编译耗时(s): 0.085970 +- 解释器耗时(s): 0.123156 +- TinyFive 模拟耗时(s): 0.154544 +- 总耗时(s): 0.812259 +- 基线动态指令数: 5.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 3, 'machine_instructions': 6, 'code_size_bytes': 24, 'dynamic_instructions': 5.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 2.0, 'dynamic_madd': 0.0, 'dynamic_branch': 1.0} +- Cost model 对比: {'static_asm_instructions': {'current': 3, 'baseline': 3, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 6, 'baseline': 6, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 24, 'baseline': 24, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 5.0, 'baseline': 5.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 2.0, 'baseline': 2.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 1.0, 'baseline': 1.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_only.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\relu_only.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\relu_only.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/relu_only.registers.json` +- 汇编文件: `build/topic06/relu_only.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + max t1, t0, 0 + mv a0, t1 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/activation-relu_twice.json b/benchmark_reports/topic06/cases/activation-relu_twice.json new file mode 100644 index 0000000..65d7b5b --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-relu_twice.json @@ -0,0 +1,152 @@ +{ + "mode": "benchmark", + "name": "relu_twice", + "category": "activation", + "path": "tests/topic06/cases/activation/relu_twice.dsl", + "status": "PASS", + "description": "Apply ReLU twice to the same activation path.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 4, + "actual": 4, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 4.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12500800006091595, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 4 + }, + "register_map": "build/topic06/relu_twice.registers.json", + "backend": "tinyfive", + "instr_count": 6, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 10, + "code_size_bytes": 40, + "perf_counters": { + "total": 6, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 2 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 10, + "code_size_bytes": 40, + "dynamic_instructions": 6.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 2.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 10, + "baseline": 10, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 40, + "baseline": 40, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 6.0, + "baseline": 6.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\relu_twice.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_twice.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_twice.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08641380001790822, + "simulation_time_sec": 0.15094910003244877, + "total_time_sec": 0.8119307000888512, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 6.0, + "min_instr_count": 6, + "max_instr_count": 6, + "ci95_instr_count": 0.0, + "baseline_instr_count": 6.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/relu_twice.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-relu_twice.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-relu_twice.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/activation-relu_twice.md b/benchmark_reports/topic06/cases/activation-relu_twice.md new file mode 100644 index 0000000..02115ba --- /dev/null +++ b/benchmark_reports/topic06/cases/activation-relu_twice.md @@ -0,0 +1,64 @@ +# relu_twice 测试详情 + +## 基本信息 + +- 类别: activation +- DSL: `tests/topic06/cases/activation/relu_twice.dsl` +- 描述: Apply ReLU twice to the same activation path. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 4.0 | True | null | null | +| TinyFive | PASS | 4 | True | null | null | + +- 期望输出: 4 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 10 +- 代码大小(bytes): 40 +- TinyFive 动态执行指令数: 6 +- TinyFive 分类计数: {'total': 6, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 2} +- 编译耗时(s): 0.086414 +- 解释器耗时(s): 0.125008 +- TinyFive 模拟耗时(s): 0.150949 +- 总耗时(s): 0.811931 +- 基线动态指令数: 6.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 10, 'code_size_bytes': 40, 'dynamic_instructions': 6.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 3.0, 'dynamic_madd': 0.0, 'dynamic_branch': 2.0} +- Cost model 对比: {'static_asm_instructions': {'current': 4, 'baseline': 4, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 10, 'baseline': 10, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 40, 'baseline': 40, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 6.0, 'baseline': 6.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 3.0, 'baseline': 3.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 2.0, 'baseline': 2.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_twice.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\relu_twice.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\relu_twice.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/relu_twice.registers.json` +- 汇编文件: `build/topic06/relu_twice.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + max t1, t0, 0 + max t2, t1, 0 + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/branch-if_else.json b/benchmark_reports/topic06/cases/branch-if_else.json new file mode 100644 index 0000000..661d7e6 --- /dev/null +++ b/benchmark_reports/topic06/cases/branch-if_else.json @@ -0,0 +1,75 @@ +{ + "mode": "benchmark", + "name": "if_else", + "category": "branch", + "path": "tests/topic06/cases/branch/if_else.dsl", + "status": "FAIL", + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": null, + "matched": false, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: if", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 7.140007801353931e-05, + "backend_outputs_match": null, + "tinyfive_status": "TIMEOUT", + "tinyfive_failure_kind": "simulation_timeout", + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": "simulation timeout after 5s", + "initial_registers": { + "t1": 0, + "t2": 9, + "t3": 4 + }, + "register_map": "build/topic06/if_else.registers.json", + "backend": "timeout", + "instr_count": 0, + "static_asm_instruction_count": 12, + "machine_code_instruction_count": null, + "code_size_bytes": null, + "perf_counters": {}, + "cost_model": { + "static_asm_instructions": 12, + "machine_instructions": null, + "code_size_bytes": null, + "dynamic_instructions": null, + "dynamic_load": null, + "dynamic_store": null, + "dynamic_mul": null, + "dynamic_add": null, + "dynamic_madd": null, + "dynamic_branch": null + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\branch\\if_else.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_else.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_else.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08768309990409762, + "simulation_time_sec": 5.024043000070378, + "total_time_sec": 5.117446599993855, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: initial simulation timeout", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/if_else.s", + "case_report_md": "benchmark_reports/topic06/cases/branch-if_else.md", + "case_report_json": "benchmark_reports/topic06/cases/branch-if_else.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/branch-if_else.md b/benchmark_reports/topic06/cases/branch-if_else.md new file mode 100644 index 0000000..0d06ff9 --- /dev/null +++ b/benchmark_reports/topic06/cases/branch-if_else.md @@ -0,0 +1,75 @@ +# if_else 测试详情 + +## 基本信息 + +- 类别: branch +- DSL: `tests/topic06/cases/branch/if_else.dsl` +- 描述: if/else branch returns subtraction result when flag is zero. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | UNSUPPORTED | None | False | interpreter_control_flow_unsupported | unsupported control flow: if | +| TinyFive | TIMEOUT | None | False | simulation_timeout | simulation timeout after 5s | + +- 期望输出: 5 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 12 +- 编码后机器指令数: null +- 代码大小(bytes): null +- TinyFive 动态执行指令数: 0 +- TinyFive 分类计数: {} +- 编译耗时(s): 0.087683 +- 解释器耗时(s): 0.000071 +- TinyFive 模拟耗时(s): 5.024043 +- 总耗时(s): 5.117447 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 12, 'machine_instructions': None, 'code_size_bytes': None, 'dynamic_instructions': None, 'dynamic_load': None, 'dynamic_store': None, 'dynamic_mul': None, 'dynamic_add': None, 'dynamic_madd': None, 'dynamic_branch': None} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\branch\if_else.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\if_else.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\if_else.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/if_else.registers.json` +- 汇编文件: `build/topic06/if_else.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + li t0, 0 # const 0 + bnez t1, if_then1 + j if_else2 +.if_then1: + add t4, t2, t3 + mv a0, t4 # return value + jalr zero, ra # ret + j if_end3 +.if_else2: + sub t5, t2, t3 + mv a0, t5 # return value + jalr zero, ra # ret + j if_end3 +.if_end3: + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/branch-if_relu.json b/benchmark_reports/topic06/cases/branch-if_relu.json new file mode 100644 index 0000000..dde6fee --- /dev/null +++ b/benchmark_reports/topic06/cases/branch-if_relu.json @@ -0,0 +1,73 @@ +{ + "mode": "benchmark", + "name": "if_relu", + "category": "branch", + "path": "tests/topic06/cases/branch/if_relu.dsl", + "status": "FAIL", + "description": "if/else branch combined with add and relu.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": null, + "matched": false, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: if", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 0.00010579999070614576, + "backend_outputs_match": null, + "tinyfive_status": "TIMEOUT", + "tinyfive_failure_kind": "simulation_timeout", + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": "simulation timeout after 5s", + "initial_registers": { + "t1": 1 + }, + "register_map": "build/topic06/if_relu.registers.json", + "backend": "timeout", + "instr_count": 0, + "static_asm_instruction_count": 11, + "machine_code_instruction_count": null, + "code_size_bytes": null, + "perf_counters": {}, + "cost_model": { + "static_asm_instructions": 11, + "machine_instructions": null, + "code_size_bytes": null, + "dynamic_instructions": null, + "dynamic_load": null, + "dynamic_store": null, + "dynamic_mul": null, + "dynamic_add": null, + "dynamic_madd": null, + "dynamic_branch": null + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\branch\\if_relu.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_relu.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_relu.registers.json'", + "compile_log": null, + "compile_time_sec": 0.09591350005939603, + "simulation_time_sec": 5.023449000087567, + "total_time_sec": 5.1205431000562385, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: initial simulation timeout", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/if_relu.s", + "case_report_md": "benchmark_reports/topic06/cases/branch-if_relu.md", + "case_report_json": "benchmark_reports/topic06/cases/branch-if_relu.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/branch-if_relu.md b/benchmark_reports/topic06/cases/branch-if_relu.md new file mode 100644 index 0000000..f31f643 --- /dev/null +++ b/benchmark_reports/topic06/cases/branch-if_relu.md @@ -0,0 +1,74 @@ +# if_relu 测试详情 + +## 基本信息 + +- 类别: branch +- DSL: `tests/topic06/cases/branch/if_relu.dsl` +- 描述: if/else branch combined with add and relu. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | UNSUPPORTED | None | False | interpreter_control_flow_unsupported | unsupported control flow: if | +| TinyFive | TIMEOUT | None | False | simulation_timeout | simulation timeout after 5s | + +- 期望输出: 0 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 11 +- 编码后机器指令数: null +- 代码大小(bytes): null +- TinyFive 动态执行指令数: 0 +- TinyFive 分类计数: {} +- 编译耗时(s): 0.095914 +- 解释器耗时(s): 0.000106 +- TinyFive 模拟耗时(s): 5.023449 +- 总耗时(s): 5.120543 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 11, 'machine_instructions': None, 'code_size_bytes': None, 'dynamic_instructions': None, 'dynamic_load': None, 'dynamic_store': None, 'dynamic_mul': None, 'dynamic_add': None, 'dynamic_madd': None, 'dynamic_branch': None} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\branch\if_relu.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\if_relu.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\if_relu.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/if_relu.registers.json` +- 汇编文件: `build/topic06/if_relu.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + li t0, 0 # const 0 + bnez t1, if_then1 + j if_else2 +.if_then1: + max t3, t2, 0 + mv a0, t3 # return value + jalr zero, ra # ret + j if_end3 +.if_else2: + mv a0, t2 # return value + jalr zero, ra # ret + j if_end3 +.if_end3: + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/branch-if_then.json b/benchmark_reports/topic06/cases/branch-if_then.json new file mode 100644 index 0000000..e2018c0 --- /dev/null +++ b/benchmark_reports/topic06/cases/branch-if_then.json @@ -0,0 +1,75 @@ +{ + "mode": "benchmark", + "name": "if_then", + "category": "branch", + "path": "tests/topic06/cases/branch/if_then.dsl", + "status": "FAIL", + "description": "if/else branch returns add result when flag is non-zero.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 13, + "actual": null, + "matched": false, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: if", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 7.11000757291913e-05, + "backend_outputs_match": null, + "tinyfive_status": "TIMEOUT", + "tinyfive_failure_kind": "simulation_timeout", + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": "simulation timeout after 5s", + "initial_registers": { + "t1": 1, + "t2": 9, + "t3": 4 + }, + "register_map": "build/topic06/if_then.registers.json", + "backend": "timeout", + "instr_count": 0, + "static_asm_instruction_count": 12, + "machine_code_instruction_count": null, + "code_size_bytes": null, + "perf_counters": {}, + "cost_model": { + "static_asm_instructions": 12, + "machine_instructions": null, + "code_size_bytes": null, + "dynamic_instructions": null, + "dynamic_load": null, + "dynamic_store": null, + "dynamic_mul": null, + "dynamic_add": null, + "dynamic_madd": null, + "dynamic_branch": null + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\branch\\if_then.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_then.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_then.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08916259999386966, + "simulation_time_sec": 5.01582570001483, + "total_time_sec": 5.1061101000523195, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: initial simulation timeout", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/if_then.s", + "case_report_md": "benchmark_reports/topic06/cases/branch-if_then.md", + "case_report_json": "benchmark_reports/topic06/cases/branch-if_then.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/branch-if_then.md b/benchmark_reports/topic06/cases/branch-if_then.md new file mode 100644 index 0000000..11f6249 --- /dev/null +++ b/benchmark_reports/topic06/cases/branch-if_then.md @@ -0,0 +1,75 @@ +# if_then 测试详情 + +## 基本信息 + +- 类别: branch +- DSL: `tests/topic06/cases/branch/if_then.dsl` +- 描述: if/else branch returns add result when flag is non-zero. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | UNSUPPORTED | None | False | interpreter_control_flow_unsupported | unsupported control flow: if | +| TinyFive | TIMEOUT | None | False | simulation_timeout | simulation timeout after 5s | + +- 期望输出: 13 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 12 +- 编码后机器指令数: null +- 代码大小(bytes): null +- TinyFive 动态执行指令数: 0 +- TinyFive 分类计数: {} +- 编译耗时(s): 0.089163 +- 解释器耗时(s): 0.000071 +- TinyFive 模拟耗时(s): 5.015826 +- 总耗时(s): 5.106110 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 12, 'machine_instructions': None, 'code_size_bytes': None, 'dynamic_instructions': None, 'dynamic_load': None, 'dynamic_store': None, 'dynamic_mul': None, 'dynamic_add': None, 'dynamic_madd': None, 'dynamic_branch': None} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\branch\if_then.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\if_then.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\if_then.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/if_then.registers.json` +- 汇编文件: `build/topic06/if_then.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + li t0, 0 # const 0 + bnez t1, if_then1 + j if_else2 +.if_then1: + add t4, t2, t3 + mv a0, t4 # return value + jalr zero, ra # ret + j if_end3 +.if_else2: + sub t5, t2, t3 + mv a0, t5 # return value + jalr zero, ra # ret + j if_end3 +.if_end3: + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/elementwise-add_chain.json b/benchmark_reports/topic06/cases/elementwise-add_chain.json new file mode 100644 index 0000000..a95cfe4 --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_chain.json @@ -0,0 +1,154 @@ +{ + "mode": "benchmark", + "name": "add_chain", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_chain.dsl", + "status": "PASS", + "description": "Add a and b, then add c to the intermediate result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 9, + "actual": 9, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 9, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12835399992763996, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3, + "t3": 4 + }, + "register_map": "build/topic06/add_chain.registers.json", + "backend": "tinyfive", + "instr_count": 4, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 4, + "code_size_bytes": 16, + "perf_counters": { + "total": 4, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 16, + "baseline": 16, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_chain.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08758619998116046, + "simulation_time_sec": 0.1567648999625817, + "total_time_sec": 0.8240788999246433, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 4.0, + "min_instr_count": 4, + "max_instr_count": 4, + "ci95_instr_count": 0.0, + "baseline_instr_count": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_chain.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_chain.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_chain.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/elementwise-add_chain.md b/benchmark_reports/topic06/cases/elementwise-add_chain.md new file mode 100644 index 0000000..5fed0d2 --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_chain.md @@ -0,0 +1,64 @@ +# add_chain 测试详情 + +## 基本信息 + +- 类别: elementwise +- DSL: `tests/topic06/cases/elementwise/add_chain.dsl` +- 描述: Add a and b, then add c to the intermediate result. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 9 | True | null | null | +| TinyFive | PASS | 9 | True | null | null | + +- 期望输出: 9 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 4 +- 代码大小(bytes): 16 +- TinyFive 动态执行指令数: 4 +- TinyFive 分类计数: {'total': 4, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.087586 +- 解释器耗时(s): 0.128354 +- TinyFive 模拟耗时(s): 0.156765 +- 总耗时(s): 0.824079 +- 基线动态指令数: 4.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 4, 'code_size_bytes': 16, 'dynamic_instructions': 4.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 3.0, 'dynamic_madd': 0.0, 'dynamic_branch': 0.0} +- Cost model 对比: {'static_asm_instructions': {'current': 4, 'baseline': 4, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 4, 'baseline': 4, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 16, 'baseline': 16, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 4.0, 'baseline': 4.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 3.0, 'baseline': 3.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\elementwise\add_chain.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_chain.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_chain.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/add_chain.registers.json` +- 汇编文件: `build/topic06/add_chain.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + add t4, t2, t3 + mv a0, t4 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/elementwise-add_chain_3.json b/benchmark_reports/topic06/cases/elementwise-add_chain_3.json new file mode 100644 index 0000000..a6a691c --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_chain_3.json @@ -0,0 +1,155 @@ +{ + "mode": "benchmark", + "name": "add_chain_3", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_chain_3.dsl", + "status": "PASS", + "description": "Chain three add operations across four symbolic inputs.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 14, + "actual": 14, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 14, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.121873400057666, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3, + "t3": 4, + "t5": 5 + }, + "register_map": "build/topic06/add_chain_3.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 5, + "machine_code_instruction_count": 5, + "code_size_bytes": 20, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 4, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 5, + "code_size_bytes": 20, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 20, + "baseline": 20, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_chain_3.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain_3.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain_3.registers.json'", + "compile_log": null, + "compile_time_sec": 0.0855520999757573, + "simulation_time_sec": 0.1468493000138551, + "total_time_sec": 0.8014750999864191, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_chain_3.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_chain_3.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_chain_3.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/elementwise-add_chain_3.md b/benchmark_reports/topic06/cases/elementwise-add_chain_3.md new file mode 100644 index 0000000..343d2ce --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_chain_3.md @@ -0,0 +1,65 @@ +# add_chain_3 测试详情 + +## 基本信息 + +- 类别: elementwise +- DSL: `tests/topic06/cases/elementwise/add_chain_3.dsl` +- 描述: Chain three add operations across four symbolic inputs. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 14 | True | null | null | +| TinyFive | PASS | 14 | True | null | null | + +- 期望输出: 14 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 5 +- 编码后机器指令数: 5 +- 代码大小(bytes): 20 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 4, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.085552 +- 解释器耗时(s): 0.121873 +- TinyFive 模拟耗时(s): 0.146849 +- 总耗时(s): 0.801475 +- 基线动态指令数: 5.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 5, 'machine_instructions': 5, 'code_size_bytes': 20, 'dynamic_instructions': 5.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 4.0, 'dynamic_madd': 0.0, 'dynamic_branch': 0.0} +- Cost model 对比: {'static_asm_instructions': {'current': 5, 'baseline': 5, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 5, 'baseline': 5, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 20, 'baseline': 20, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 5.0, 'baseline': 5.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 4.0, 'baseline': 4.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\elementwise\add_chain_3.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_chain_3.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_chain_3.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/add_chain_3.registers.json` +- 汇编文件: `build/topic06/add_chain_3.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + add t4, t2, t3 + add t6, t4, t5 + mv a0, t6 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/elementwise-add_fan_in_4.json b/benchmark_reports/topic06/cases/elementwise-add_fan_in_4.json new file mode 100644 index 0000000..5484e24 --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_fan_in_4.json @@ -0,0 +1,155 @@ +{ + "mode": "benchmark", + "name": "add_fan_in_4", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_fan_in_4.dsl", + "status": "PASS", + "description": "Compute two independent adds and then merge them with a final add.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 10, + "actual": 10, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 10, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.1248123999685049, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 1, + "t1": 2, + "t3": 3, + "t4": 4 + }, + "register_map": "build/topic06/add_fan_in_4.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 5, + "machine_code_instruction_count": 5, + "code_size_bytes": 20, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 4, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 5, + "code_size_bytes": 20, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 20, + "baseline": 20, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_fan_in_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_fan_in_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_fan_in_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08502920006867498, + "simulation_time_sec": 0.1628414000151679, + "total_time_sec": 0.8474700000369921, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_fan_in_4.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_fan_in_4.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_fan_in_4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/elementwise-add_fan_in_4.md b/benchmark_reports/topic06/cases/elementwise-add_fan_in_4.md new file mode 100644 index 0000000..ec9b897 --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_fan_in_4.md @@ -0,0 +1,65 @@ +# add_fan_in_4 测试详情 + +## 基本信息 + +- 类别: elementwise +- DSL: `tests/topic06/cases/elementwise/add_fan_in_4.dsl` +- 描述: Compute two independent adds and then merge them with a final add. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 10 | True | null | null | +| TinyFive | PASS | 10 | True | null | null | + +- 期望输出: 10 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 5 +- 编码后机器指令数: 5 +- 代码大小(bytes): 20 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 4, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.085029 +- 解释器耗时(s): 0.124812 +- TinyFive 模拟耗时(s): 0.162841 +- 总耗时(s): 0.847470 +- 基线动态指令数: 5.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 5, 'machine_instructions': 5, 'code_size_bytes': 20, 'dynamic_instructions': 5.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 4.0, 'dynamic_madd': 0.0, 'dynamic_branch': 0.0} +- Cost model 对比: {'static_asm_instructions': {'current': 5, 'baseline': 5, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 5, 'baseline': 5, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 20, 'baseline': 20, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 5.0, 'baseline': 5.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 4.0, 'baseline': 4.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\elementwise\add_fan_in_4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_fan_in_4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_fan_in_4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/add_fan_in_4.registers.json` +- 汇编文件: `build/topic06/add_fan_in_4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + add t5, t3, t4 + add t6, t2, t5 + mv a0, t6 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/elementwise-add_reuse.json b/benchmark_reports/topic06/cases/elementwise-add_reuse.json new file mode 100644 index 0000000..228d29e --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_reuse.json @@ -0,0 +1,153 @@ +{ + "mode": "benchmark", + "name": "add_reuse", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_reuse.dsl", + "status": "PASS", + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 10, + "actual": 10, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 10, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.16349690000060946, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "register_map": "build/topic06/add_reuse.registers.json", + "backend": "tinyfive", + "instr_count": 4, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 4, + "code_size_bytes": 16, + "perf_counters": { + "total": 4, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 16, + "baseline": 16, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_reuse.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_reuse.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_reuse.registers.json'", + "compile_log": null, + "compile_time_sec": 0.10644100001081824, + "simulation_time_sec": 0.16117799992207438, + "total_time_sec": 0.8809606999857351, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 4.0, + "min_instr_count": 4, + "max_instr_count": 4, + "ci95_instr_count": 0.0, + "baseline_instr_count": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_reuse.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_reuse.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_reuse.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/elementwise-add_reuse.md b/benchmark_reports/topic06/cases/elementwise-add_reuse.md new file mode 100644 index 0000000..44f58c6 --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-add_reuse.md @@ -0,0 +1,64 @@ +# add_reuse 测试详情 + +## 基本信息 + +- 类别: elementwise +- DSL: `tests/topic06/cases/elementwise/add_reuse.dsl` +- 描述: Reuse the same intermediate add result on both operands of a second add. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 10 | True | null | null | +| TinyFive | PASS | 10 | True | null | null | + +- 期望输出: 10 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 4 +- 代码大小(bytes): 16 +- TinyFive 动态执行指令数: 4 +- TinyFive 分类计数: {'total': 4, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.106441 +- 解释器耗时(s): 0.163497 +- TinyFive 模拟耗时(s): 0.161178 +- 总耗时(s): 0.880961 +- 基线动态指令数: 4.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 4, 'code_size_bytes': 16, 'dynamic_instructions': 4.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 3.0, 'dynamic_madd': 0.0, 'dynamic_branch': 0.0} +- Cost model 对比: {'static_asm_instructions': {'current': 4, 'baseline': 4, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 4, 'baseline': 4, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 16, 'baseline': 16, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 4.0, 'baseline': 4.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 3.0, 'baseline': 3.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\elementwise\add_reuse.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_reuse.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\add_reuse.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/add_reuse.registers.json` +- 汇编文件: `build/topic06/add_reuse.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + add t3, t2, t2 + mv a0, t3 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/elementwise-vector_add.json b/benchmark_reports/topic06/cases/elementwise-vector_add.json new file mode 100644 index 0000000..4ad1f6f --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-vector_add.json @@ -0,0 +1,153 @@ +{ + "mode": "benchmark", + "name": "vector_add", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/vector_add.dsl", + "status": "PASS", + "description": "Single add over two symbolic vector inputs.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": 5, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 5, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12906479998491704, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "register_map": "build/topic06/vector_add.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 0, + "add": 2, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 2.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 3, + "baseline": 3, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 3, + "baseline": 3, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 12, + "baseline": 12, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\vector_add.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\vector_add.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\vector_add.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08543810003902763, + "simulation_time_sec": 0.14759199996478856, + "total_time_sec": 0.818663900019601, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 3.0, + "min_instr_count": 3, + "max_instr_count": 3, + "ci95_instr_count": 0.0, + "baseline_instr_count": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/vector_add.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-vector_add.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-vector_add.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/elementwise-vector_add.md b/benchmark_reports/topic06/cases/elementwise-vector_add.md new file mode 100644 index 0000000..07138a6 --- /dev/null +++ b/benchmark_reports/topic06/cases/elementwise-vector_add.md @@ -0,0 +1,63 @@ +# vector_add 测试详情 + +## 基本信息 + +- 类别: elementwise +- DSL: `tests/topic06/cases/elementwise/vector_add.dsl` +- 描述: Single add over two symbolic vector inputs. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 5 | True | null | null | +| TinyFive | PASS | 5 | True | null | null | + +- 期望输出: 5 +- 两后端输出一致: True +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 动态执行指令数: 3 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 0, 'add': 2, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.085438 +- 解释器耗时(s): 0.129065 +- TinyFive 模拟耗时(s): 0.147592 +- 总耗时(s): 0.818664 +- 基线动态指令数: 3.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 3, 'machine_instructions': 3, 'code_size_bytes': 12, 'dynamic_instructions': 3.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 2.0, 'dynamic_madd': 0.0, 'dynamic_branch': 0.0} +- Cost model 对比: {'static_asm_instructions': {'current': 3, 'baseline': 3, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 3, 'baseline': 3, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 12, 'baseline': 12, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 3.0, 'baseline': 3.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 2.0, 'baseline': 2.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\elementwise\vector_add.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\vector_add.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\vector_add.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/vector_add.registers.json` +- 汇编文件: `build/topic06/vector_add.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/loop-loop_add_4.json b/benchmark_reports/topic06/cases/loop-loop_add_4.json new file mode 100644 index 0000000..620e972 --- /dev/null +++ b/benchmark_reports/topic06/cases/loop-loop_add_4.json @@ -0,0 +1,153 @@ +{ + "mode": "benchmark", + "name": "loop_add_4", + "category": "loop", + "path": "tests/topic06/cases/loop/loop_add_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": 5, + "matched": true, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: for", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 9.869993664324284e-05, + "backend_outputs_match": null, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "register_map": "build/topic06/loop_add_4.registers.json", + "backend": "tinyfive", + "instr_count": 22, + "static_asm_instruction_count": 7, + "machine_code_instruction_count": 8, + "code_size_bytes": 32, + "perf_counters": { + "total": 22, + "load": 0, + "store": 0, + "mul": 0, + "add": 12, + "madd": 0, + "branch": 5 + }, + "cost_model": { + "static_asm_instructions": 7, + "machine_instructions": 8, + "code_size_bytes": 32, + "dynamic_instructions": 22.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 12.0, + "dynamic_madd": 0.0, + "dynamic_branch": 5.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 7, + "baseline": 7, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 8, + "baseline": 8, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 32, + "baseline": 32, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 22.0, + "baseline": 22.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 12.0, + "baseline": 12.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\loop\\loop_add_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.0912627000361681, + "simulation_time_sec": 0.15588609990663826, + "total_time_sec": 0.7474795000161976, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 22.0, + "min_instr_count": 22, + "max_instr_count": 22, + "ci95_instr_count": 0.0, + "baseline_instr_count": 22.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/loop_add_4.s", + "case_report_md": "benchmark_reports/topic06/cases/loop-loop_add_4.md", + "case_report_json": "benchmark_reports/topic06/cases/loop-loop_add_4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/loop-loop_add_4.md b/benchmark_reports/topic06/cases/loop-loop_add_4.md new file mode 100644 index 0000000..e41ebea --- /dev/null +++ b/benchmark_reports/topic06/cases/loop-loop_add_4.md @@ -0,0 +1,70 @@ +# loop_add_4 测试详情 + +## 基本信息 + +- 类别: loop +- DSL: `tests/topic06/cases/loop/loop_add_4.dsl` +- 描述: Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | UNSUPPORTED | None | False | interpreter_control_flow_unsupported | unsupported control flow: for | +| TinyFive | PASS | 5 | True | null | null | + +- 期望输出: 5 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 7 +- 编码后机器指令数: 8 +- 代码大小(bytes): 32 +- TinyFive 动态执行指令数: 22 +- TinyFive 分类计数: {'total': 22, 'load': 0, 'store': 0, 'mul': 0, 'add': 12, 'madd': 0, 'branch': 5} +- 编译耗时(s): 0.091263 +- 解释器耗时(s): 0.000099 +- TinyFive 模拟耗时(s): 0.155886 +- 总耗时(s): 0.747480 +- 基线动态指令数: 22.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 7, 'machine_instructions': 8, 'code_size_bytes': 32, 'dynamic_instructions': 22.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 12.0, 'dynamic_madd': 0.0, 'dynamic_branch': 5.0} +- Cost model 对比: {'static_asm_instructions': {'current': 7, 'baseline': 7, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 8, 'baseline': 8, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 32, 'baseline': 32, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 22.0, 'baseline': 22.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 12.0, 'baseline': 12.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 5.0, 'baseline': 5.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\loop\loop_add_4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\loop_add_4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\loop_add_4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/loop_add_4.registers.json` +- 汇编文件: `build/topic06/loop_add_4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + li t3, 0 # loop init +.Lloop_header_1: + bge t3, 4, .Lloop_exit_3 +.Lloop_body_2: + addi t3, t3, 1 # loop inc + j .Lloop_header_1 +.Lloop_exit_3: + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/loop-loop_add_chain_4.json b/benchmark_reports/topic06/cases/loop-loop_add_chain_4.json new file mode 100644 index 0000000..9f86738 --- /dev/null +++ b/benchmark_reports/topic06/cases/loop-loop_add_chain_4.json @@ -0,0 +1,154 @@ +{ + "mode": "benchmark", + "name": "loop_add_chain_4", + "category": "loop", + "path": "tests/topic06/cases/loop/loop_add_chain_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 9, + "actual": 9, + "matched": true, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: for", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 8.849997539073229e-05, + "backend_outputs_match": null, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3, + "t4": 4 + }, + "register_map": "build/topic06/loop_add_chain_4.registers.json", + "backend": "tinyfive", + "instr_count": 26, + "static_asm_instruction_count": 8, + "machine_code_instruction_count": 9, + "code_size_bytes": 36, + "perf_counters": { + "total": 26, + "load": 0, + "store": 0, + "mul": 0, + "add": 16, + "madd": 0, + "branch": 5 + }, + "cost_model": { + "static_asm_instructions": 8, + "machine_instructions": 9, + "code_size_bytes": 36, + "dynamic_instructions": 26.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 16.0, + "dynamic_madd": 0.0, + "dynamic_branch": 5.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 8, + "baseline": 8, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 9, + "baseline": 9, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 36, + "baseline": 36, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 26.0, + "baseline": 26.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 16.0, + "baseline": 16.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\loop\\loop_add_chain_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_chain_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_chain_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08660579996649176, + "simulation_time_sec": 0.15727660001721233, + "total_time_sec": 0.7031202999642119, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 26.0, + "min_instr_count": 26, + "max_instr_count": 26, + "ci95_instr_count": 0.0, + "baseline_instr_count": 26.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/loop_add_chain_4.s", + "case_report_md": "benchmark_reports/topic06/cases/loop-loop_add_chain_4.md", + "case_report_json": "benchmark_reports/topic06/cases/loop-loop_add_chain_4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/loop-loop_add_chain_4.md b/benchmark_reports/topic06/cases/loop-loop_add_chain_4.md new file mode 100644 index 0000000..d6ca8ee --- /dev/null +++ b/benchmark_reports/topic06/cases/loop-loop_add_chain_4.md @@ -0,0 +1,71 @@ +# loop_add_chain_4 测试详情 + +## 基本信息 + +- 类别: loop +- DSL: `tests/topic06/cases/loop/loop_add_chain_4.dsl` +- 描述: Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | UNSUPPORTED | None | False | interpreter_control_flow_unsupported | unsupported control flow: for | +| TinyFive | PASS | 9 | True | null | null | + +- 期望输出: 9 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 8 +- 编码后机器指令数: 9 +- 代码大小(bytes): 36 +- TinyFive 动态执行指令数: 26 +- TinyFive 分类计数: {'total': 26, 'load': 0, 'store': 0, 'mul': 0, 'add': 16, 'madd': 0, 'branch': 5} +- 编译耗时(s): 0.086606 +- 解释器耗时(s): 0.000088 +- TinyFive 模拟耗时(s): 0.157277 +- 总耗时(s): 0.703120 +- 基线动态指令数: 26.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 8, 'machine_instructions': 9, 'code_size_bytes': 36, 'dynamic_instructions': 26.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 16.0, 'dynamic_madd': 0.0, 'dynamic_branch': 5.0} +- Cost model 对比: {'static_asm_instructions': {'current': 8, 'baseline': 8, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 9, 'baseline': 9, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 36, 'baseline': 36, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 26.0, 'baseline': 26.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 16.0, 'baseline': 16.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 5.0, 'baseline': 5.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\loop\loop_add_chain_4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\loop_add_chain_4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\loop_add_chain_4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/loop_add_chain_4.registers.json` +- 汇编文件: `build/topic06/loop_add_chain_4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + li t3, 0 # loop init +.Lloop_header_1: + bge t3, 4, .Lloop_exit_3 +.Lloop_body_2: + add t5, t2, t4 + addi t3, t3, 1 # loop inc + j .Lloop_header_1 +.Lloop_exit_3: + mv a0, t5 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/loop-loop_relu_add_4.json b/benchmark_reports/topic06/cases/loop-loop_relu_add_4.json new file mode 100644 index 0000000..516cc98 --- /dev/null +++ b/benchmark_reports/topic06/cases/loop-loop_relu_add_4.json @@ -0,0 +1,153 @@ +{ + "mode": "benchmark", + "name": "loop_relu_add_4", + "category": "loop", + "path": "tests/topic06/cases/loop/loop_relu_add_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 2, + "actual": 2, + "matched": true, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: for", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 7.280008867383003e-05, + "backend_outputs_match": null, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -4, + "t1": 6 + }, + "register_map": "build/topic06/loop_relu_add_4.registers.json", + "backend": "tinyfive", + "instr_count": 30, + "static_asm_instruction_count": 8, + "machine_code_instruction_count": 12, + "code_size_bytes": 48, + "perf_counters": { + "total": 30, + "load": 0, + "store": 0, + "mul": 0, + "add": 16, + "madd": 0, + "branch": 9 + }, + "cost_model": { + "static_asm_instructions": 8, + "machine_instructions": 12, + "code_size_bytes": 48, + "dynamic_instructions": 30.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 16.0, + "dynamic_madd": 0.0, + "dynamic_branch": 9.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 8, + "baseline": 8, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 12, + "baseline": 12, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 48, + "baseline": 48, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 30.0, + "baseline": 30.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 16.0, + "baseline": 16.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 9.0, + "baseline": 9.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\loop\\loop_relu_add_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_relu_add_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_relu_add_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08553029992617667, + "simulation_time_sec": 0.15667920000851154, + "total_time_sec": 0.7070331000722945, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 30.0, + "min_instr_count": 30, + "max_instr_count": 30, + "ci95_instr_count": 0.0, + "baseline_instr_count": 30.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/loop_relu_add_4.s", + "case_report_md": "benchmark_reports/topic06/cases/loop-loop_relu_add_4.md", + "case_report_json": "benchmark_reports/topic06/cases/loop-loop_relu_add_4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/loop-loop_relu_add_4.md b/benchmark_reports/topic06/cases/loop-loop_relu_add_4.md new file mode 100644 index 0000000..2b6254a --- /dev/null +++ b/benchmark_reports/topic06/cases/loop-loop_relu_add_4.md @@ -0,0 +1,71 @@ +# loop_relu_add_4 测试详情 + +## 基本信息 + +- 类别: loop +- DSL: `tests/topic06/cases/loop/loop_relu_add_4.dsl` +- 描述: Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result. +- 总体状态: PASS +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | UNSUPPORTED | None | False | interpreter_control_flow_unsupported | unsupported control flow: for | +| TinyFive | PASS | 2 | True | null | null | + +- 期望输出: 2 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: True +- TinyFive 输入 ABI 说明: null + +## 性能指标 + +- 静态汇编指令数: 8 +- 编码后机器指令数: 12 +- 代码大小(bytes): 48 +- TinyFive 动态执行指令数: 30 +- TinyFive 分类计数: {'total': 30, 'load': 0, 'store': 0, 'mul': 0, 'add': 16, 'madd': 0, 'branch': 9} +- 编译耗时(s): 0.085530 +- 解释器耗时(s): 0.000073 +- TinyFive 模拟耗时(s): 0.156679 +- 总耗时(s): 0.707033 +- 基线动态指令数: 30.0 +- 动态指令变化率(%): 0.0 +- 是否退化: False + +- Cost model 指标: {'static_asm_instructions': 8, 'machine_instructions': 12, 'code_size_bytes': 48, 'dynamic_instructions': 30.0, 'dynamic_load': 0.0, 'dynamic_store': 0.0, 'dynamic_mul': 0.0, 'dynamic_add': 16.0, 'dynamic_madd': 0.0, 'dynamic_branch': 9.0} +- Cost model 对比: {'static_asm_instructions': {'current': 8, 'baseline': 8, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'machine_instructions': {'current': 12, 'baseline': 12, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'code_size_bytes': {'current': 48, 'baseline': 48, 'delta': 0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_instructions': {'current': 30.0, 'baseline': 30.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_load': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_store': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_mul': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_add': {'current': 16.0, 'baseline': 16.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_madd': {'current': 0.0, 'baseline': 0.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}, 'dynamic_branch': {'current': 9.0, 'baseline': 9.0, 'delta': 0.0, 'delta_pct': 0.0, 'regressed': False}} +- Cost model 是否退化: False + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\loop\loop_relu_add_4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\loop_relu_add_4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\loop_relu_add_4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/loop_relu_add_4.registers.json` +- 汇编文件: `build/topic06/loop_relu_add_4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + add t2, t0, t1 + li t3, 0 # loop init +.Lloop_header_1: + bge t3, 4, .Lloop_exit_3 +.Lloop_body_2: + max t4, t2, 0 + addi t3, t3, 1 # loop inc + j .Lloop_header_1 +.Lloop_exit_3: + mv a0, t4 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/reduction-dot_4.json b/benchmark_reports/topic06/cases/reduction-dot_4.json new file mode 100644 index 0000000..62341e6 --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_4.json @@ -0,0 +1,79 @@ +{ + "mode": "benchmark", + "name": "dot_4", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_4.dsl", + "status": "FAIL", + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 70, + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 70, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12432019994594157, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_4.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08576739998534322, + "simulation_time_sec": 0.14933500008191913, + "total_time_sec": 0.3608566999901086, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_4.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_4.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/reduction-dot_4.md b/benchmark_reports/topic06/cases/reduction-dot_4.md new file mode 100644 index 0000000..5745120 --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_4.md @@ -0,0 +1,63 @@ +# dot_4 测试详情 + +## 基本信息 + +- 类别: reduction +- DSL: `tests/topic06/cases/reduction/dot_4.dsl` +- 描述: Compute the dot product of two symbolic vectors of length 4. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 70 | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: 70 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: a, b + +## 性能指标 + +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 动态执行指令数: 3 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.085767 +- 解释器耗时(s): 0.124320 +- TinyFive 模拟耗时(s): 0.149335 +- 总耗时(s): 0.360857 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 3, 'machine_instructions': 3, 'code_size_bytes': 12, 'dynamic_instructions': 3, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 1, 'dynamic_madd': 0, 'dynamic_branch': 0} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\reduction\dot_4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/dot_4.registers.json` +- 汇编文件: `build/topic06/dot_4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # dot: a * b + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/reduction-dot_8.json b/benchmark_reports/topic06/cases/reduction-dot_8.json new file mode 100644 index 0000000..81e1b84 --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_8.json @@ -0,0 +1,79 @@ +{ + "mode": "benchmark", + "name": "dot_8", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_8.dsl", + "status": "FAIL", + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 36, + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 36, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12384599994402379, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_8.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_8.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_8.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_8.registers.json'", + "compile_log": null, + "compile_time_sec": 0.083566699991934, + "simulation_time_sec": 0.15027460001874715, + "total_time_sec": 0.35855530004482716, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_8.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_8.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_8.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/reduction-dot_8.md b/benchmark_reports/topic06/cases/reduction-dot_8.md new file mode 100644 index 0000000..382737c --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_8.md @@ -0,0 +1,63 @@ +# dot_8 测试详情 + +## 基本信息 + +- 类别: reduction +- DSL: `tests/topic06/cases/reduction/dot_8.dsl` +- 描述: Compute the dot product of two symbolic vectors of length 8. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 36 | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: 36 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: a, b + +## 性能指标 + +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 动态执行指令数: 3 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.083567 +- 解释器耗时(s): 0.123846 +- TinyFive 模拟耗时(s): 0.150275 +- 总耗时(s): 0.358555 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 3, 'machine_instructions': 3, 'code_size_bytes': 12, 'dynamic_instructions': 3, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 1, 'dynamic_madd': 0, 'dynamic_branch': 0} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\reduction\dot_8.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_8.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_8.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/dot_8.registers.json` +- 汇编文件: `build/topic06/dot_8.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # dot: a * b + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/reduction-dot_relu_4.json b/benchmark_reports/topic06/cases/reduction-dot_relu_4.json new file mode 100644 index 0000000..c9d9219 --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_relu_4.json @@ -0,0 +1,79 @@ +{ + "mode": "benchmark", + "name": "dot_relu_4", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_relu_4.dsl", + "status": "FAIL", + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": 0, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 0.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12336369999684393, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_relu_4.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 1 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_relu_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08426539995707572, + "simulation_time_sec": 0.1483513000421226, + "total_time_sec": 0.3573680999688804, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_relu_4.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_relu_4.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_relu_4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/reduction-dot_relu_4.md b/benchmark_reports/topic06/cases/reduction-dot_relu_4.md new file mode 100644 index 0000000..b1dc85a --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_relu_4.md @@ -0,0 +1,64 @@ +# dot_relu_4 测试详情 + +## 基本信息 + +- 类别: reduction +- DSL: `tests/topic06/cases/reduction/dot_relu_4.dsl` +- 描述: Compute a length-4 dot product and pass it through ReLU. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 0.0 | True | null | null | +| TinyFive | UNSUPPORTED | 0 | True | input_abi_unsupported | null | + +- 期望输出: 0 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: a, b + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 1} +- 编译耗时(s): 0.084265 +- 解释器耗时(s): 0.123364 +- TinyFive 模拟耗时(s): 0.148351 +- 总耗时(s): 0.357368 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 7, 'code_size_bytes': 28, 'dynamic_instructions': 5, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 2, 'dynamic_madd': 0, 'dynamic_branch': 1} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\reduction\dot_relu_4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_relu_4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_relu_4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/dot_relu_4.registers.json` +- 汇编文件: `build/topic06/dot_relu_4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # dot: a * b + max t3, t2, 0 + mv a0, t3 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/reduction-dot_relu_8.json b/benchmark_reports/topic06/cases/reduction-dot_relu_8.json new file mode 100644 index 0000000..c131860 --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_relu_8.json @@ -0,0 +1,79 @@ +{ + "mode": "benchmark", + "name": "dot_relu_8", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_relu_8.dsl", + "status": "FAIL", + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 8, + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 8.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12608389998786151, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_relu_8.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 1 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_relu_8.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_8.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_8.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08828250004444271, + "simulation_time_sec": 0.1534100000280887, + "total_time_sec": 0.3687144999857992, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_relu_8.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_relu_8.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_relu_8.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/reduction-dot_relu_8.md b/benchmark_reports/topic06/cases/reduction-dot_relu_8.md new file mode 100644 index 0000000..32cef5b --- /dev/null +++ b/benchmark_reports/topic06/cases/reduction-dot_relu_8.md @@ -0,0 +1,64 @@ +# dot_relu_8 测试详情 + +## 基本信息 + +- 类别: reduction +- DSL: `tests/topic06/cases/reduction/dot_relu_8.dsl` +- 描述: Compute a length-8 dot product and pass it through ReLU. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | 8.0 | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: 8 +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: a, b + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 1} +- 编译耗时(s): 0.088283 +- 解释器耗时(s): 0.126084 +- TinyFive 模拟耗时(s): 0.153410 +- 总耗时(s): 0.368714 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 7, 'code_size_bytes': 28, 'dynamic_instructions': 5, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 2, 'dynamic_madd': 0, 'dynamic_branch': 1} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\reduction\dot_relu_8.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_relu_8.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\dot_relu_8.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/dot_relu_8.registers.json` +- 汇编文件: `build/topic06/dot_relu_8.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # dot: a * b + max t3, t2, 0 + mv a0, t3 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/tensor-matmul_2x2.json b/benchmark_reports/topic06/cases/tensor-matmul_2x2.json new file mode 100644 index 0000000..68a1eb5 --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_2x2.json @@ -0,0 +1,100 @@ +{ + "mode": "benchmark", + "name": "matmul_2x2", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_2x2.dsl", + "status": "FAIL", + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 19, + 22 + ], + [ + 43, + 50 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 19, + 22 + ], + [ + 43, + 50 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.1250815000385046, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_2x2.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_2x2.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_2x2.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_2x2.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08626819995697588, + "simulation_time_sec": 0.15745349996723235, + "total_time_sec": 0.37013469997327775, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_2x2.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_2x2.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_2x2.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/tensor-matmul_2x2.md b/benchmark_reports/topic06/cases/tensor-matmul_2x2.md new file mode 100644 index 0000000..78f425e --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_2x2.md @@ -0,0 +1,63 @@ +# matmul_2x2 测试详情 + +## 基本信息 + +- 类别: tensor +- DSL: `tests/topic06/cases/tensor/matmul_2x2.dsl` +- 描述: Compute a symbolic 2x2 by 2x2 matrix multiplication. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | [[19, 22], [43, 50]] | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: [[19, 22], [43, 50]] +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: A, B + +## 性能指标 + +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 动态执行指令数: 3 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.086268 +- 解释器耗时(s): 0.125082 +- TinyFive 模拟耗时(s): 0.157453 +- 总耗时(s): 0.370135 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 3, 'machine_instructions': 3, 'code_size_bytes': 12, 'dynamic_instructions': 3, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 1, 'dynamic_madd': 0, 'dynamic_branch': 0} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\tensor\matmul_2x2.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_2x2.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_2x2.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/matmul_2x2.registers.json` +- 汇编文件: `build/topic06/matmul_2x2.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # matmul: a * b + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/tensor-matmul_4x4.json b/benchmark_reports/topic06/cases/tensor-matmul_4x4.json new file mode 100644 index 0000000..c9da202 --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_4x4.json @@ -0,0 +1,132 @@ +{ + "mode": "benchmark", + "name": "matmul_4x4", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_4x4.dsl", + "status": "FAIL", + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 4, + 4 + ], + "expected": [ + [ + 1, + 2, + 3, + 4 + ], + [ + 5, + 6, + 7, + 8 + ], + [ + 9, + 10, + 11, + 12 + ], + [ + 13, + 14, + 15, + 16 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 1, + 2, + 3, + 4 + ], + [ + 5, + 6, + 7, + 8 + ], + [ + 9, + 10, + 11, + 12 + ], + [ + 13, + 14, + 15, + 16 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12329240003600717, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_4x4.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_4x4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_4x4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_4x4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08791819994803518, + "simulation_time_sec": 0.1479027000023052, + "total_time_sec": 0.3600894999690354, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_4x4.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_4x4.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_4x4.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/tensor-matmul_4x4.md b/benchmark_reports/topic06/cases/tensor-matmul_4x4.md new file mode 100644 index 0000000..ba5e0b4 --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_4x4.md @@ -0,0 +1,63 @@ +# matmul_4x4 测试详情 + +## 基本信息 + +- 类别: tensor +- DSL: `tests/topic06/cases/tensor/matmul_4x4.dsl` +- 描述: Compute a symbolic 4x4 by 4x4 matrix multiplication. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: A, B + +## 性能指标 + +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 动态执行指令数: 3 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.087918 +- 解释器耗时(s): 0.123292 +- TinyFive 模拟耗时(s): 0.147903 +- 总耗时(s): 0.360089 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 3, 'machine_instructions': 3, 'code_size_bytes': 12, 'dynamic_instructions': 3, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 1, 'dynamic_madd': 0, 'dynamic_branch': 0} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\tensor\matmul_4x4.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_4x4.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_4x4.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/matmul_4x4.registers.json` +- 汇编文件: `build/topic06/matmul_4x4.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # matmul: a * b + mv a0, t2 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/tensor-matmul_add_2x2.json b/benchmark_reports/topic06/cases/tensor-matmul_add_2x2.json new file mode 100644 index 0000000..b2e3eca --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_add_2x2.json @@ -0,0 +1,100 @@ +{ + "mode": "benchmark", + "name": "matmul_add_2x2", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_add_2x2.dsl", + "status": "FAIL", + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 20, + 23 + ], + [ + 44, + 51 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 20, + 23 + ], + [ + 44, + 51 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12155430007260293, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B, bias", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_add_2x2.registers.json", + "backend": "tinyfive", + "instr_count": 4, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 4, + "code_size_bytes": 16, + "perf_counters": { + "total": 4, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_add_2x2.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_add_2x2.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_add_2x2.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08429349993821234, + "simulation_time_sec": 0.15792770008556545, + "total_time_sec": 0.36477210000157356, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_add_2x2.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_add_2x2.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_add_2x2.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/tensor-matmul_add_2x2.md b/benchmark_reports/topic06/cases/tensor-matmul_add_2x2.md new file mode 100644 index 0000000..6fe61f6 --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_add_2x2.md @@ -0,0 +1,64 @@ +# matmul_add_2x2 测试详情 + +## 基本信息 + +- 类别: tensor +- DSL: `tests/topic06/cases/tensor/matmul_add_2x2.dsl` +- 描述: Compute a 2x2 matmul and then add a symbolic bias term. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | [[20, 23], [44, 51]] | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: [[20, 23], [44, 51]] +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: A, B, bias + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 4 +- 代码大小(bytes): 16 +- TinyFive 动态执行指令数: 4 +- TinyFive 分类计数: {'total': 4, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 0} +- 编译耗时(s): 0.084293 +- 解释器耗时(s): 0.121554 +- TinyFive 模拟耗时(s): 0.157928 +- 总耗时(s): 0.364772 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 4, 'code_size_bytes': 16, 'dynamic_instructions': 4, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 2, 'dynamic_madd': 0, 'dynamic_branch': 0} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\tensor\matmul_add_2x2.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_add_2x2.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_add_2x2.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/matmul_add_2x2.registers.json` +- 汇编文件: `build/topic06/matmul_add_2x2.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # matmul: a * b + add t4, t2, t3 + mv a0, t4 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.json b/benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.json new file mode 100644 index 0000000..4623f89 --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.json @@ -0,0 +1,100 @@ +{ + "mode": "benchmark", + "name": "matmul_relu_2x2", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_relu_2x2.dsl", + "status": "FAIL", + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 0, + 2 + ], + [ + 0, + 4 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 0.0, + 2.0 + ], + [ + 0.0, + 4.0 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12728470005095005, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_relu_2x2.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 1 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_relu_2x2.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_relu_2x2.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_relu_2x2.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08514879993163049, + "simulation_time_sec": 0.15872700000181794, + "total_time_sec": 0.3722879000706598, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_relu_2x2.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.json" +} \ No newline at end of file diff --git a/benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.md b/benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.md new file mode 100644 index 0000000..dd632a9 --- /dev/null +++ b/benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.md @@ -0,0 +1,64 @@ +# matmul_relu_2x2 测试详情 + +## 基本信息 + +- 类别: tensor +- DSL: `tests/topic06/cases/tensor/matmul_relu_2x2.dsl` +- 描述: Compute a 2x2 matmul and then apply ReLU to its result. +- 总体状态: FAIL +- 验证模式: both + +## 后端能力矩阵 + +| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 | +|---|---|---|---|---|---| +| DSLInterpreter | PASS | [[0.0, 2.0], [0.0, 4.0]] | True | null | null | +| TinyFive | UNSUPPORTED | 0 | False | input_abi_unsupported | null | + +- 期望输出: [[0, 2], [0, 4]] +- 两后端输出一致: null +- TinyFive 输入 ABI 可用: False +- TinyFive 输入 ABI 说明: non-scalar TinyFive input ABI is unavailable: A, B + +## 性能指标 + +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 动态执行指令数: 5 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 1} +- 编译耗时(s): 0.085149 +- 解释器耗时(s): 0.127285 +- TinyFive 模拟耗时(s): 0.158727 +- 总耗时(s): 0.372288 +- 基线动态指令数: null +- 动态指令变化率(%): null +- 是否退化: null + +- Cost model 指标: {'static_asm_instructions': 4, 'machine_instructions': 7, 'code_size_bytes': 28, 'dynamic_instructions': 5, 'dynamic_load': 0, 'dynamic_store': 0, 'dynamic_mul': 1, 'dynamic_add': 2, 'dynamic_madd': 0, 'dynamic_branch': 1} +- Cost model 对比: {} +- Cost model 是否退化: null + +## 编译信息 + +- 命令: `'D:\anaconda3\python.exe' -m scratchv.main 'D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\tensor\matmul_relu_2x2.dsl' -o 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_relu_2x2.s' --optimize all --emit-register-map 'D:\PycharmProjects\ScratchV\ScratchV\build\topic06\matmul_relu_2x2.registers.json'` +- 返回码: 0 +- 编译错误: null +- 失败日志: null +- 寄存器映射: `build/topic06/matmul_relu_2x2.registers.json` +- 汇编文件: `build/topic06/matmul_relu_2x2.s` + +## 生成汇编 + +```asm +.text +.align 2 + .globl main + .type main, @function +main: +.entry: + mul t2, t0, t1 # matmul: a * b + max t3, t2, 0 + mv a0, t3 # return value + jalr zero, ra # ret +``` diff --git a/benchmark_reports/topic06/report.html b/benchmark_reports/topic06/report.html new file mode 100644 index 0000000..474f72b --- /dev/null +++ b/benchmark_reports/topic06/report.html @@ -0,0 +1,314 @@ + + + + + ScratchV 测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

Schema 版本：2，运行模式：benchmark

+

类别筛选：null，名称筛选：null

+

用例总数：23，通过：12，失败：11

+

性能退化阈值：5.00%

+
+ 课程版指令数图表 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
用例类别状态解释器状态TinyFive 状态失败类型编译返回码编译日志模拟后端指令数Benchmark 次数Benchmark 停止原因平均指令数95% 置信区间最小最大编译耗时(s)模拟耗时(s)总耗时(s)基线变化量变化率(%)退化阈值(%)是否退化预期输出TinyFive 输出输出匹配汇编文件
add_relu_reluactivationPASSPASSPASSnull0nulltinyfive73null7.00±0.00770.08510.14670.82907.000.000.005.00False77Truebuild/topic06/add_relu_relu.s
relu_addactivationPASSPASSPASSnull0nulltinyfive53null5.00±0.00550.08790.15410.80295.000.000.005.00False33Truebuild/topic06/relu_add.s
relu_onlyactivationPASSPASSPASSnull0nulltinyfive53null5.00±0.00550.08600.15450.81235.000.000.005.00False00Truebuild/topic06/relu_only.s
relu_twiceactivationPASSPASSPASSnull0nulltinyfive63null6.00±0.00660.08640.15090.81196.000.000.005.00False44Truebuild/topic06/relu_twice.s
if_elsebranchFAILUNSUPPORTEDTIMEOUTsimulation_timeout0nulltimeout00benchmark skipped: initial simulation timeoutnullnullnullnull0.08775.02405.1174nullnullnull5.00null5NoneFalsebuild/topic06/if_else.s
if_relubranchFAILUNSUPPORTEDTIMEOUTsimulation_timeout0nulltimeout00benchmark skipped: initial simulation timeoutnullnullnullnull0.09595.02345.1205nullnullnull5.00null0NoneFalsebuild/topic06/if_relu.s
if_thenbranchFAILUNSUPPORTEDTIMEOUTsimulation_timeout0nulltimeout00benchmark skipped: initial simulation timeoutnullnullnullnull0.08925.01585.1061nullnullnull5.00null13NoneFalsebuild/topic06/if_then.s
add_chainelementwisePASSPASSPASSnull0nulltinyfive43null4.00±0.00440.08760.15680.82414.000.000.005.00False99Truebuild/topic06/add_chain.s
add_chain_3elementwisePASSPASSPASSnull0nulltinyfive53null5.00±0.00550.08560.14680.80155.000.000.005.00False1414Truebuild/topic06/add_chain_3.s
add_fan_in_4elementwisePASSPASSPASSnull0nulltinyfive53null5.00±0.00550.08500.16280.84755.000.000.005.00False1010Truebuild/topic06/add_fan_in_4.s
add_reuseelementwisePASSPASSPASSnull0nulltinyfive43null4.00±0.00440.10640.16120.88104.000.000.005.00False1010Truebuild/topic06/add_reuse.s
vector_addelementwisePASSPASSPASSnull0nulltinyfive33null3.00±0.00330.08540.14760.81873.000.000.005.00False55Truebuild/topic06/vector_add.s
loop_add_4loopPASSUNSUPPORTEDPASSnull0nulltinyfive223null22.00±0.0022220.09130.15590.747522.000.000.005.00False55Truebuild/topic06/loop_add_4.s
loop_add_chain_4loopPASSUNSUPPORTEDPASSnull0nulltinyfive263null26.00±0.0026260.08660.15730.703126.000.000.005.00False99Truebuild/topic06/loop_add_chain_4.s
loop_relu_add_4loopPASSUNSUPPORTEDPASSnull0nulltinyfive303null30.00±0.0030300.08550.15670.707030.000.000.005.00False22Truebuild/topic06/loop_relu_add_4.s
dot_4reductionFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive30benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08580.14930.3609nullnullnull5.00null700Falsebuild/topic06/dot_4.s
dot_8reductionFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive30benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08360.15030.3586nullnullnull5.00null360Falsebuild/topic06/dot_8.s
dot_relu_4reductionFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive50benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08430.14840.3574nullnullnull5.00null00Truebuild/topic06/dot_relu_4.s
dot_relu_8reductionFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive50benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08830.15340.3687nullnullnull5.00null80Falsebuild/topic06/dot_relu_8.s
matmul_2x2tensorFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive30benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08630.15750.3701nullnullnull5.00null[[19, 22], [43, 50]]0Falsebuild/topic06/matmul_2x2.s
matmul_4x4tensorFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive30benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08790.14790.3601nullnullnull5.00null[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]0Falsebuild/topic06/matmul_4x4.s
matmul_add_2x2tensorFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive40benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08430.15790.3648nullnullnull5.00null[[20, 23], [44, 51]]0Falsebuild/topic06/matmul_add_2x2.s
matmul_relu_2x2tensorFAILPASSUNSUPPORTEDinput_abi_unsupported0nulltinyfive50benchmark skipped: TinyFive input ABI unsupportednullnullnullnull0.08510.15870.3723nullnullnull5.00null[[0, 2], [0, 4]]0Falsebuild/topic06/matmul_relu_2x2.s
+ + \ No newline at end of file diff --git a/benchmark_reports/topic06/report.json b/benchmark_reports/topic06/report.json new file mode 100644 index 0000000..529f8a1 --- /dev/null +++ b/benchmark_reports/topic06/report.json @@ -0,0 +1,2845 @@ +{ + "schema_version": 2, + "mode": "benchmark", + "report_level": "full", + "regression_threshold_pct": 5.0, + "selection": { + "category": null, + "filter": null + }, + "generated_at": "2026-09-14T22:48:51", + "summary": { + "total": 23, + "passed": 12, + "failed": 11 + }, + "backend_matrix": { + "interpreter": { + "PASS": 17, + "UNSUPPORTED": 6 + }, + "tinyfive": { + "PASS": 12, + "TIMEOUT": 3, + "UNSUPPORTED": 8 + }, + "cross_backend": { + "comparable": 9, + "matched": 9, + "mismatched": 0 + } + }, + "results": [ + { + "mode": "benchmark", + "name": "add_relu_relu", + "category": "activation", + "path": "tests/topic06/cases/activation/add_relu_relu.dsl", + "status": "PASS", + "description": "Add input and bias, then apply ReLU twice.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 7, + "actual": 7, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 7.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12910449993796647, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -3, + "t1": 10 + }, + "register_map": "build/topic06/add_relu_relu.registers.json", + "backend": "tinyfive", + "instr_count": 7, + "static_asm_instruction_count": 5, + "machine_code_instruction_count": 11, + "code_size_bytes": 44, + "perf_counters": { + "total": 7, + "load": 0, + "store": 0, + "mul": 0, + "add": 4, + "madd": 0, + "branch": 2 + }, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 11, + "code_size_bytes": 44, + "dynamic_instructions": 7.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 2.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 11, + "baseline": 11, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 44, + "baseline": 44, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 7.0, + "baseline": 7.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\add_relu_relu.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_relu_relu.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_relu_relu.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08510669996030629, + "simulation_time_sec": 0.14671280002221465, + "total_time_sec": 0.8289841000223532, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 7.0, + "min_instr_count": 7, + "max_instr_count": 7, + "ci95_instr_count": 0.0, + "baseline_instr_count": 7.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_relu_relu.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-add_relu_relu.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-add_relu_relu.json" + }, + { + "mode": "benchmark", + "name": "relu_add", + "category": "activation", + "path": "tests/topic06/cases/activation/relu_add.dsl", + "status": "PASS", + "description": "Add input and bias, then apply one ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 3, + "actual": 3, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 3.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12399170000571758, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -2, + "t1": 5 + }, + "register_map": "build/topic06/relu_add.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 1.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 7, + "baseline": 7, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 28, + "baseline": 28, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 1.0, + "baseline": 1.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\relu_add.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_add.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_add.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08786199998576194, + "simulation_time_sec": 0.15414490003604442, + "total_time_sec": 0.8028744000475854, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/relu_add.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-relu_add.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-relu_add.json" + }, + { + "mode": "benchmark", + "name": "relu_only", + "category": "activation", + "path": "tests/topic06/cases/activation/relu_only.dsl", + "status": "PASS", + "description": "Apply ReLU directly to a single input value.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": 0, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 0.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12315600004512817, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -5 + }, + "register_map": "build/topic06/relu_only.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 6, + "code_size_bytes": 24, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 6, + "code_size_bytes": 24, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 2.0, + "dynamic_madd": 0.0, + "dynamic_branch": 1.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 3, + "baseline": 3, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 6, + "baseline": 6, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 24, + "baseline": 24, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 1.0, + "baseline": 1.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\relu_only.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_only.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_only.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08596989989746362, + "simulation_time_sec": 0.15454419993329793, + "total_time_sec": 0.8122592000290751, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/relu_only.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-relu_only.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-relu_only.json" + }, + { + "mode": "benchmark", + "name": "relu_twice", + "category": "activation", + "path": "tests/topic06/cases/activation/relu_twice.dsl", + "status": "PASS", + "description": "Apply ReLU twice to the same activation path.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 4, + "actual": 4, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 4.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12500800006091595, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 4 + }, + "register_map": "build/topic06/relu_twice.registers.json", + "backend": "tinyfive", + "instr_count": 6, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 10, + "code_size_bytes": 40, + "perf_counters": { + "total": 6, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 2 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 10, + "code_size_bytes": 40, + "dynamic_instructions": 6.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 2.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 10, + "baseline": 10, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 40, + "baseline": 40, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 6.0, + "baseline": 6.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\activation\\relu_twice.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_twice.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\relu_twice.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08641380001790822, + "simulation_time_sec": 0.15094910003244877, + "total_time_sec": 0.8119307000888512, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 6.0, + "min_instr_count": 6, + "max_instr_count": 6, + "ci95_instr_count": 0.0, + "baseline_instr_count": 6.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/relu_twice.s", + "case_report_md": "benchmark_reports/topic06/cases/activation-relu_twice.md", + "case_report_json": "benchmark_reports/topic06/cases/activation-relu_twice.json" + }, + { + "mode": "benchmark", + "name": "if_else", + "category": "branch", + "path": "tests/topic06/cases/branch/if_else.dsl", + "status": "FAIL", + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": null, + "matched": false, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: if", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 7.140007801353931e-05, + "backend_outputs_match": null, + "tinyfive_status": "TIMEOUT", + "tinyfive_failure_kind": "simulation_timeout", + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": "simulation timeout after 5s", + "initial_registers": { + "t1": 0, + "t2": 9, + "t3": 4 + }, + "register_map": "build/topic06/if_else.registers.json", + "backend": "timeout", + "instr_count": 0, + "static_asm_instruction_count": 12, + "machine_code_instruction_count": null, + "code_size_bytes": null, + "perf_counters": {}, + "cost_model": { + "static_asm_instructions": 12, + "machine_instructions": null, + "code_size_bytes": null, + "dynamic_instructions": null, + "dynamic_load": null, + "dynamic_store": null, + "dynamic_mul": null, + "dynamic_add": null, + "dynamic_madd": null, + "dynamic_branch": null + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\branch\\if_else.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_else.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_else.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08768309990409762, + "simulation_time_sec": 5.024043000070378, + "total_time_sec": 5.117446599993855, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: initial simulation timeout", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/if_else.s", + "case_report_md": "benchmark_reports/topic06/cases/branch-if_else.md", + "case_report_json": "benchmark_reports/topic06/cases/branch-if_else.json" + }, + { + "mode": "benchmark", + "name": "if_relu", + "category": "branch", + "path": "tests/topic06/cases/branch/if_relu.dsl", + "status": "FAIL", + "description": "if/else branch combined with add and relu.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": null, + "matched": false, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: if", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 0.00010579999070614576, + "backend_outputs_match": null, + "tinyfive_status": "TIMEOUT", + "tinyfive_failure_kind": "simulation_timeout", + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": "simulation timeout after 5s", + "initial_registers": { + "t1": 1 + }, + "register_map": "build/topic06/if_relu.registers.json", + "backend": "timeout", + "instr_count": 0, + "static_asm_instruction_count": 11, + "machine_code_instruction_count": null, + "code_size_bytes": null, + "perf_counters": {}, + "cost_model": { + "static_asm_instructions": 11, + "machine_instructions": null, + "code_size_bytes": null, + "dynamic_instructions": null, + "dynamic_load": null, + "dynamic_store": null, + "dynamic_mul": null, + "dynamic_add": null, + "dynamic_madd": null, + "dynamic_branch": null + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\branch\\if_relu.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_relu.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_relu.registers.json'", + "compile_log": null, + "compile_time_sec": 0.09591350005939603, + "simulation_time_sec": 5.023449000087567, + "total_time_sec": 5.1205431000562385, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: initial simulation timeout", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/if_relu.s", + "case_report_md": "benchmark_reports/topic06/cases/branch-if_relu.md", + "case_report_json": "benchmark_reports/topic06/cases/branch-if_relu.json" + }, + { + "mode": "benchmark", + "name": "if_then", + "category": "branch", + "path": "tests/topic06/cases/branch/if_then.dsl", + "status": "FAIL", + "description": "if/else branch returns add result when flag is non-zero.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 13, + "actual": null, + "matched": false, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: if", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 7.11000757291913e-05, + "backend_outputs_match": null, + "tinyfive_status": "TIMEOUT", + "tinyfive_failure_kind": "simulation_timeout", + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": "simulation timeout after 5s", + "initial_registers": { + "t1": 1, + "t2": 9, + "t3": 4 + }, + "register_map": "build/topic06/if_then.registers.json", + "backend": "timeout", + "instr_count": 0, + "static_asm_instruction_count": 12, + "machine_code_instruction_count": null, + "code_size_bytes": null, + "perf_counters": {}, + "cost_model": { + "static_asm_instructions": 12, + "machine_instructions": null, + "code_size_bytes": null, + "dynamic_instructions": null, + "dynamic_load": null, + "dynamic_store": null, + "dynamic_mul": null, + "dynamic_add": null, + "dynamic_madd": null, + "dynamic_branch": null + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\branch\\if_then.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_then.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\if_then.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08916259999386966, + "simulation_time_sec": 5.01582570001483, + "total_time_sec": 5.1061101000523195, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: initial simulation timeout", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/if_then.s", + "case_report_md": "benchmark_reports/topic06/cases/branch-if_then.md", + "case_report_json": "benchmark_reports/topic06/cases/branch-if_then.json" + }, + { + "mode": "benchmark", + "name": "add_chain", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_chain.dsl", + "status": "PASS", + "description": "Add a and b, then add c to the intermediate result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 9, + "actual": 9, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 9, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12835399992763996, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3, + "t3": 4 + }, + "register_map": "build/topic06/add_chain.registers.json", + "backend": "tinyfive", + "instr_count": 4, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 4, + "code_size_bytes": 16, + "perf_counters": { + "total": 4, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 16, + "baseline": 16, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_chain.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08758619998116046, + "simulation_time_sec": 0.1567648999625817, + "total_time_sec": 0.8240788999246433, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 4.0, + "min_instr_count": 4, + "max_instr_count": 4, + "ci95_instr_count": 0.0, + "baseline_instr_count": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_chain.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_chain.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_chain.json" + }, + { + "mode": "benchmark", + "name": "add_chain_3", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_chain_3.dsl", + "status": "PASS", + "description": "Chain three add operations across four symbolic inputs.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 14, + "actual": 14, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 14, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.121873400057666, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3, + "t3": 4, + "t5": 5 + }, + "register_map": "build/topic06/add_chain_3.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 5, + "machine_code_instruction_count": 5, + "code_size_bytes": 20, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 4, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 5, + "code_size_bytes": 20, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 20, + "baseline": 20, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_chain_3.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain_3.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_chain_3.registers.json'", + "compile_log": null, + "compile_time_sec": 0.0855520999757573, + "simulation_time_sec": 0.1468493000138551, + "total_time_sec": 0.8014750999864191, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_chain_3.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_chain_3.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_chain_3.json" + }, + { + "mode": "benchmark", + "name": "add_fan_in_4", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_fan_in_4.dsl", + "status": "PASS", + "description": "Compute two independent adds and then merge them with a final add.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 10, + "actual": 10, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 10, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.1248123999685049, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 1, + "t1": 2, + "t3": 3, + "t4": 4 + }, + "register_map": "build/topic06/add_fan_in_4.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 5, + "machine_code_instruction_count": 5, + "code_size_bytes": 20, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 0, + "add": 4, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 5, + "code_size_bytes": 20, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 5, + "baseline": 5, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 20, + "baseline": 20, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_fan_in_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_fan_in_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_fan_in_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08502920006867498, + "simulation_time_sec": 0.1628414000151679, + "total_time_sec": 0.8474700000369921, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 5.0, + "min_instr_count": 5, + "max_instr_count": 5, + "ci95_instr_count": 0.0, + "baseline_instr_count": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_fan_in_4.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_fan_in_4.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_fan_in_4.json" + }, + { + "mode": "benchmark", + "name": "add_reuse", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/add_reuse.dsl", + "status": "PASS", + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 10, + "actual": 10, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 10, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.16349690000060946, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "register_map": "build/topic06/add_reuse.registers.json", + "backend": "tinyfive", + "instr_count": 4, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 4, + "code_size_bytes": 16, + "perf_counters": { + "total": 4, + "load": 0, + "store": 0, + "mul": 0, + "add": 3, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 4, + "baseline": 4, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 16, + "baseline": 16, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 4.0, + "baseline": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\add_reuse.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_reuse.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\add_reuse.registers.json'", + "compile_log": null, + "compile_time_sec": 0.10644100001081824, + "simulation_time_sec": 0.16117799992207438, + "total_time_sec": 0.8809606999857351, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 4.0, + "min_instr_count": 4, + "max_instr_count": 4, + "ci95_instr_count": 0.0, + "baseline_instr_count": 4.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/add_reuse.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-add_reuse.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-add_reuse.json" + }, + { + "mode": "benchmark", + "name": "vector_add", + "category": "elementwise", + "path": "tests/topic06/cases/elementwise/vector_add.dsl", + "status": "PASS", + "description": "Single add over two symbolic vector inputs.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": 5, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 5, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12906479998491704, + "backend_outputs_match": true, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "register_map": "build/topic06/vector_add.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 0, + "add": 2, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 2.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 3, + "baseline": 3, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 3, + "baseline": 3, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 12, + "baseline": 12, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 3.0, + "baseline": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 2.0, + "baseline": 2.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\elementwise\\vector_add.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\vector_add.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\vector_add.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08543810003902763, + "simulation_time_sec": 0.14759199996478856, + "total_time_sec": 0.818663900019601, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 3.0, + "min_instr_count": 3, + "max_instr_count": 3, + "ci95_instr_count": 0.0, + "baseline_instr_count": 3.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/vector_add.s", + "case_report_md": "benchmark_reports/topic06/cases/elementwise-vector_add.md", + "case_report_json": "benchmark_reports/topic06/cases/elementwise-vector_add.json" + }, + { + "mode": "benchmark", + "name": "loop_add_4", + "category": "loop", + "path": "tests/topic06/cases/loop/loop_add_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": 5, + "matched": true, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: for", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 9.869993664324284e-05, + "backend_outputs_match": null, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "register_map": "build/topic06/loop_add_4.registers.json", + "backend": "tinyfive", + "instr_count": 22, + "static_asm_instruction_count": 7, + "machine_code_instruction_count": 8, + "code_size_bytes": 32, + "perf_counters": { + "total": 22, + "load": 0, + "store": 0, + "mul": 0, + "add": 12, + "madd": 0, + "branch": 5 + }, + "cost_model": { + "static_asm_instructions": 7, + "machine_instructions": 8, + "code_size_bytes": 32, + "dynamic_instructions": 22.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 12.0, + "dynamic_madd": 0.0, + "dynamic_branch": 5.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 7, + "baseline": 7, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 8, + "baseline": 8, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 32, + "baseline": 32, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 22.0, + "baseline": 22.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 12.0, + "baseline": 12.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\loop\\loop_add_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.0912627000361681, + "simulation_time_sec": 0.15588609990663826, + "total_time_sec": 0.7474795000161976, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 22.0, + "min_instr_count": 22, + "max_instr_count": 22, + "ci95_instr_count": 0.0, + "baseline_instr_count": 22.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/loop_add_4.s", + "case_report_md": "benchmark_reports/topic06/cases/loop-loop_add_4.md", + "case_report_json": "benchmark_reports/topic06/cases/loop-loop_add_4.json" + }, + { + "mode": "benchmark", + "name": "loop_add_chain_4", + "category": "loop", + "path": "tests/topic06/cases/loop/loop_add_chain_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 9, + "actual": 9, + "matched": true, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: for", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 8.849997539073229e-05, + "backend_outputs_match": null, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": 2, + "t1": 3, + "t4": 4 + }, + "register_map": "build/topic06/loop_add_chain_4.registers.json", + "backend": "tinyfive", + "instr_count": 26, + "static_asm_instruction_count": 8, + "machine_code_instruction_count": 9, + "code_size_bytes": 36, + "perf_counters": { + "total": 26, + "load": 0, + "store": 0, + "mul": 0, + "add": 16, + "madd": 0, + "branch": 5 + }, + "cost_model": { + "static_asm_instructions": 8, + "machine_instructions": 9, + "code_size_bytes": 36, + "dynamic_instructions": 26.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 16.0, + "dynamic_madd": 0.0, + "dynamic_branch": 5.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 8, + "baseline": 8, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 9, + "baseline": 9, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 36, + "baseline": 36, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 26.0, + "baseline": 26.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 16.0, + "baseline": 16.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 5.0, + "baseline": 5.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\loop\\loop_add_chain_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_chain_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_add_chain_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08660579996649176, + "simulation_time_sec": 0.15727660001721233, + "total_time_sec": 0.7031202999642119, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 26.0, + "min_instr_count": 26, + "max_instr_count": 26, + "ci95_instr_count": 0.0, + "baseline_instr_count": 26.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/loop_add_chain_4.s", + "case_report_md": "benchmark_reports/topic06/cases/loop-loop_add_chain_4.md", + "case_report_json": "benchmark_reports/topic06/cases/loop-loop_add_chain_4.json" + }, + { + "mode": "benchmark", + "name": "loop_relu_add_4", + "category": "loop", + "path": "tests/topic06/cases/loop/loop_relu_add_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 2, + "actual": 2, + "matched": true, + "verification_backend": "both", + "interpreter_status": "UNSUPPORTED", + "interpreter_actual": null, + "interpreter_matched": false, + "interpreter_error": "unsupported control flow: for", + "interpreter_failure_kind": "interpreter_control_flow_unsupported", + "interpreter_time_sec": 7.280008867383003e-05, + "backend_outputs_match": null, + "tinyfive_status": "PASS", + "tinyfive_failure_kind": null, + "tinyfive_input_abi_supported": true, + "tinyfive_input_abi_reason": null, + "simulation_error": null, + "initial_registers": { + "t0": -4, + "t1": 6 + }, + "register_map": "build/topic06/loop_relu_add_4.registers.json", + "backend": "tinyfive", + "instr_count": 30, + "static_asm_instruction_count": 8, + "machine_code_instruction_count": 12, + "code_size_bytes": 48, + "perf_counters": { + "total": 30, + "load": 0, + "store": 0, + "mul": 0, + "add": 16, + "madd": 0, + "branch": 9 + }, + "cost_model": { + "static_asm_instructions": 8, + "machine_instructions": 12, + "code_size_bytes": 48, + "dynamic_instructions": 30.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 16.0, + "dynamic_madd": 0.0, + "dynamic_branch": 9.0 + }, + "cost_model_comparison": { + "static_asm_instructions": { + "current": 8, + "baseline": 8, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "machine_instructions": { + "current": 12, + "baseline": 12, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "code_size_bytes": { + "current": 48, + "baseline": 48, + "delta": 0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_instructions": { + "current": 30.0, + "baseline": 30.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_load": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_store": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_mul": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_add": { + "current": 16.0, + "baseline": 16.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_madd": { + "current": 0.0, + "baseline": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + }, + "dynamic_branch": { + "current": 9.0, + "baseline": 9.0, + "delta": 0.0, + "delta_pct": 0.0, + "regressed": false + } + }, + "cost_model_regressed": false, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\loop\\loop_relu_add_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_relu_add_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\loop_relu_add_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08553029992617667, + "simulation_time_sec": 0.15667920000851154, + "total_time_sec": 0.7070331000722945, + "benchmark_runs": 3, + "benchmark_stopped_reason": null, + "avg_instr_count": 30.0, + "min_instr_count": 30, + "max_instr_count": 30, + "ci95_instr_count": 0.0, + "baseline_instr_count": 30.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": 5.0, + "regressed": false, + "asm": "build/topic06/loop_relu_add_4.s", + "case_report_md": "benchmark_reports/topic06/cases/loop-loop_relu_add_4.md", + "case_report_json": "benchmark_reports/topic06/cases/loop-loop_relu_add_4.json" + }, + { + "mode": "benchmark", + "name": "dot_4", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_4.dsl", + "status": "FAIL", + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 70, + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 70, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12432019994594157, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_4.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08576739998534322, + "simulation_time_sec": 0.14933500008191913, + "total_time_sec": 0.3608566999901086, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_4.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_4.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_4.json" + }, + { + "mode": "benchmark", + "name": "dot_8", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_8.dsl", + "status": "FAIL", + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 36, + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 36, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12384599994402379, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_8.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_8.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_8.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_8.registers.json'", + "compile_log": null, + "compile_time_sec": 0.083566699991934, + "simulation_time_sec": 0.15027460001874715, + "total_time_sec": 0.35855530004482716, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_8.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_8.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_8.json" + }, + { + "mode": "benchmark", + "name": "dot_relu_4", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_relu_4.dsl", + "status": "FAIL", + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": 0, + "matched": true, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 0.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12336369999684393, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_relu_4.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 1 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_relu_4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08426539995707572, + "simulation_time_sec": 0.1483513000421226, + "total_time_sec": 0.3573680999688804, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_relu_4.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_relu_4.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_relu_4.json" + }, + { + "mode": "benchmark", + "name": "dot_relu_8", + "category": "reduction", + "path": "tests/topic06/cases/reduction/dot_relu_8.dsl", + "status": "FAIL", + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 8, + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": 8.0, + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12608389998786151, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: a, b", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/dot_relu_8.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 1 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\reduction\\dot_relu_8.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_8.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\dot_relu_8.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08828250004444271, + "simulation_time_sec": 0.1534100000280887, + "total_time_sec": 0.3687144999857992, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/dot_relu_8.s", + "case_report_md": "benchmark_reports/topic06/cases/reduction-dot_relu_8.md", + "case_report_json": "benchmark_reports/topic06/cases/reduction-dot_relu_8.json" + }, + { + "mode": "benchmark", + "name": "matmul_2x2", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_2x2.dsl", + "status": "FAIL", + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 19, + 22 + ], + [ + 43, + 50 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 19, + 22 + ], + [ + 43, + 50 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.1250815000385046, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_2x2.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_2x2.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_2x2.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_2x2.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08626819995697588, + "simulation_time_sec": 0.15745349996723235, + "total_time_sec": 0.37013469997327775, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_2x2.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_2x2.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_2x2.json" + }, + { + "mode": "benchmark", + "name": "matmul_4x4", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_4x4.dsl", + "status": "FAIL", + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 4, + 4 + ], + "expected": [ + [ + 1, + 2, + 3, + 4 + ], + [ + 5, + 6, + 7, + 8 + ], + [ + 9, + 10, + 11, + 12 + ], + [ + 13, + 14, + 15, + 16 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 1, + 2, + 3, + 4 + ], + [ + 5, + 6, + 7, + 8 + ], + [ + 9, + 10, + 11, + 12 + ], + [ + 13, + 14, + 15, + 16 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12329240003600717, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_4x4.registers.json", + "backend": "tinyfive", + "instr_count": 3, + "static_asm_instruction_count": 3, + "machine_code_instruction_count": 3, + "code_size_bytes": 12, + "perf_counters": { + "total": 3, + "load": 0, + "store": 0, + "mul": 1, + "add": 1, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 1, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_4x4.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_4x4.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_4x4.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08791819994803518, + "simulation_time_sec": 0.1479027000023052, + "total_time_sec": 0.3600894999690354, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_4x4.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_4x4.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_4x4.json" + }, + { + "mode": "benchmark", + "name": "matmul_add_2x2", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_add_2x2.dsl", + "status": "FAIL", + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 20, + 23 + ], + [ + 44, + 51 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 20, + 23 + ], + [ + 44, + 51 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12155430007260293, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B, bias", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_add_2x2.registers.json", + "backend": "tinyfive", + "instr_count": 4, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 4, + "code_size_bytes": 16, + "perf_counters": { + "total": 4, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 0 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 0 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_add_2x2.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_add_2x2.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_add_2x2.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08429349993821234, + "simulation_time_sec": 0.15792770008556545, + "total_time_sec": 0.36477210000157356, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_add_2x2.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_add_2x2.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_add_2x2.json" + }, + { + "mode": "benchmark", + "name": "matmul_relu_2x2", + "category": "tensor", + "path": "tests/topic06/cases/tensor/matmul_relu_2x2.dsl", + "status": "FAIL", + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 0, + 2 + ], + [ + 0, + 4 + ] + ], + "actual": 0, + "matched": false, + "verification_backend": "both", + "interpreter_status": "PASS", + "interpreter_actual": [ + [ + 0.0, + 2.0 + ], + [ + 0.0, + 4.0 + ] + ], + "interpreter_matched": true, + "interpreter_error": null, + "interpreter_failure_kind": null, + "interpreter_time_sec": 0.12728470005095005, + "backend_outputs_match": null, + "tinyfive_status": "UNSUPPORTED", + "tinyfive_failure_kind": "input_abi_unsupported", + "tinyfive_input_abi_supported": false, + "tinyfive_input_abi_reason": "non-scalar TinyFive input ABI is unavailable: A, B", + "simulation_error": null, + "initial_registers": {}, + "register_map": "build/topic06/matmul_relu_2x2.registers.json", + "backend": "tinyfive", + "instr_count": 5, + "static_asm_instruction_count": 4, + "machine_code_instruction_count": 7, + "code_size_bytes": 28, + "perf_counters": { + "total": 5, + "load": 0, + "store": 0, + "mul": 1, + "add": 2, + "madd": 0, + "branch": 1 + }, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5, + "dynamic_load": 0, + "dynamic_store": 0, + "dynamic_mul": 1, + "dynamic_add": 2, + "dynamic_madd": 0, + "dynamic_branch": 1 + }, + "cost_model_comparison": {}, + "cost_model_regressed": null, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_command": "'D:\\anaconda3\\python.exe' -m scratchv.main 'D:\\PycharmProjects\\ScratchV\\ScratchV\\tests\\topic06\\cases\\tensor\\matmul_relu_2x2.dsl' -o 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_relu_2x2.s' --optimize all --emit-register-map 'D:\\PycharmProjects\\ScratchV\\ScratchV\\build\\topic06\\matmul_relu_2x2.registers.json'", + "compile_log": null, + "compile_time_sec": 0.08514879993163049, + "simulation_time_sec": 0.15872700000181794, + "total_time_sec": 0.3722879000706598, + "benchmark_runs": 0, + "benchmark_stopped_reason": "benchmark skipped: TinyFive input ABI unsupported", + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": 5.0, + "regressed": null, + "asm": "build/topic06/matmul_relu_2x2.s", + "case_report_md": "benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.md", + "case_report_json": "benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.json" + } + ] +} \ No newline at end of file diff --git a/benchmark_reports/topic06/report.md b/benchmark_reports/topic06/report.md new file mode 100644 index 0000000..d7b8b9f --- /dev/null +++ b/benchmark_reports/topic06/report.md @@ -0,0 +1,1048 @@ +# ScratchV DSL 编译器性能测试报告 + +## 测试概览 + +- Schema 版本: 2 +- 运行模式: benchmark +- 类别筛选: null +- 名称筛选: null +- 生成时间: 2026-09-14 22:48:51 +- 用例总数: 23 +- 通过数量: 12 +- 失败数量: 11 +- 通过率: 52.2% +- 测试目录: `tests/topic06/cases` +- 汇编输出目录: `build/topic06` +- 性能基线文件: `benchmarks/topic06/baseline.json` +- 性能退化阈值: 5.00% +- 单次编译超时: 30s +- 单次模拟超时: 5s + +## 测试结果 + +| 用例 | 类别 | 状态 | 编译返回码 | 编译日志 | 模拟后端 | 指令数 | Benchmark 次数 | Benchmark 停止原因 | 平均指令数 | 95% 置信区间 | 最小 | 最大 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 基线 | 变化量 | 变化率(%) | 退化阈值(%) | 是否退化 | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 | +|---|---|---|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---| +| add_relu_relu | activation | PASS | 0 | null | tinyfive | 7 | 3 | null | 7.00 | ±0.00 | 7 | 7 | 0.0851 | 0.1467 | 0.8290 | 7.00 | 0.00 | 0.00 | 5.00 | False | 7 | 7 | True | build/topic06/add_relu_relu.s | +| relu_add | activation | PASS | 0 | null | tinyfive | 5 | 3 | null | 5.00 | ±0.00 | 5 | 5 | 0.0879 | 0.1541 | 0.8029 | 5.00 | 0.00 | 0.00 | 5.00 | False | 3 | 3 | True | build/topic06/relu_add.s | +| relu_only | activation | PASS | 0 | null | tinyfive | 5 | 3 | null | 5.00 | ±0.00 | 5 | 5 | 0.0860 | 0.1545 | 0.8123 | 5.00 | 0.00 | 0.00 | 5.00 | False | 0 | 0 | True | build/topic06/relu_only.s | +| relu_twice | activation | PASS | 0 | null | tinyfive | 6 | 3 | null | 6.00 | ±0.00 | 6 | 6 | 0.0864 | 0.1509 | 0.8119 | 6.00 | 0.00 | 0.00 | 5.00 | False | 4 | 4 | True | build/topic06/relu_twice.s | +| if_else | branch | FAIL | 0 | null | timeout | 0 | 0 | benchmark skipped: initial simulation timeout | null | null | null | null | 0.0877 | 5.0240 | 5.1174 | null | null | null | 5.00 | null | 5 | None | False | build/topic06/if_else.s | +| if_relu | branch | FAIL | 0 | null | timeout | 0 | 0 | benchmark skipped: initial simulation timeout | null | null | null | null | 0.0959 | 5.0234 | 5.1205 | null | null | null | 5.00 | null | 0 | None | False | build/topic06/if_relu.s | +| if_then | branch | FAIL | 0 | null | timeout | 0 | 0 | benchmark skipped: initial simulation timeout | null | null | null | null | 0.0892 | 5.0158 | 5.1061 | null | null | null | 5.00 | null | 13 | None | False | build/topic06/if_then.s | +| add_chain | elementwise | PASS | 0 | null | tinyfive | 4 | 3 | null | 4.00 | ±0.00 | 4 | 4 | 0.0876 | 0.1568 | 0.8241 | 4.00 | 0.00 | 0.00 | 5.00 | False | 9 | 9 | True | build/topic06/add_chain.s | +| add_chain_3 | elementwise | PASS | 0 | null | tinyfive | 5 | 3 | null | 5.00 | ±0.00 | 5 | 5 | 0.0856 | 0.1468 | 0.8015 | 5.00 | 0.00 | 0.00 | 5.00 | False | 14 | 14 | True | build/topic06/add_chain_3.s | +| add_fan_in_4 | elementwise | PASS | 0 | null | tinyfive | 5 | 3 | null | 5.00 | ±0.00 | 5 | 5 | 0.0850 | 0.1628 | 0.8475 | 5.00 | 0.00 | 0.00 | 5.00 | False | 10 | 10 | True | build/topic06/add_fan_in_4.s | +| add_reuse | elementwise | PASS | 0 | null | tinyfive | 4 | 3 | null | 4.00 | ±0.00 | 4 | 4 | 0.1064 | 0.1612 | 0.8810 | 4.00 | 0.00 | 0.00 | 5.00 | False | 10 | 10 | True | build/topic06/add_reuse.s | +| vector_add | elementwise | PASS | 0 | null | tinyfive | 3 | 3 | null | 3.00 | ±0.00 | 3 | 3 | 0.0854 | 0.1476 | 0.8187 | 3.00 | 0.00 | 0.00 | 5.00 | False | 5 | 5 | True | build/topic06/vector_add.s | +| loop_add_4 | loop | PASS | 0 | null | tinyfive | 22 | 3 | null | 22.00 | ±0.00 | 22 | 22 | 0.0913 | 0.1559 | 0.7475 | 22.00 | 0.00 | 0.00 | 5.00 | False | 5 | 5 | True | build/topic06/loop_add_4.s | +| loop_add_chain_4 | loop | PASS | 0 | null | tinyfive | 26 | 3 | null | 26.00 | ±0.00 | 26 | 26 | 0.0866 | 0.1573 | 0.7031 | 26.00 | 0.00 | 0.00 | 5.00 | False | 9 | 9 | True | build/topic06/loop_add_chain_4.s | +| loop_relu_add_4 | loop | PASS | 0 | null | tinyfive | 30 | 3 | null | 30.00 | ±0.00 | 30 | 30 | 0.0855 | 0.1567 | 0.7070 | 30.00 | 0.00 | 0.00 | 5.00 | False | 2 | 2 | True | build/topic06/loop_relu_add_4.s | +| dot_4 | reduction | FAIL | 0 | null | tinyfive | 3 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0858 | 0.1493 | 0.3609 | null | null | null | 5.00 | null | 70 | 0 | False | build/topic06/dot_4.s | +| dot_8 | reduction | FAIL | 0 | null | tinyfive | 3 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0836 | 0.1503 | 0.3586 | null | null | null | 5.00 | null | 36 | 0 | False | build/topic06/dot_8.s | +| dot_relu_4 | reduction | FAIL | 0 | null | tinyfive | 5 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0843 | 0.1484 | 0.3574 | null | null | null | 5.00 | null | 0 | 0 | True | build/topic06/dot_relu_4.s | +| dot_relu_8 | reduction | FAIL | 0 | null | tinyfive | 5 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0883 | 0.1534 | 0.3687 | null | null | null | 5.00 | null | 8 | 0 | False | build/topic06/dot_relu_8.s | +| matmul_2x2 | tensor | FAIL | 0 | null | tinyfive | 3 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0863 | 0.1575 | 0.3701 | null | null | null | 5.00 | null | [[19, 22], [43, 50]] | 0 | False | build/topic06/matmul_2x2.s | +| matmul_4x4 | tensor | FAIL | 0 | null | tinyfive | 3 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0879 | 0.1479 | 0.3601 | null | null | null | 5.00 | null | [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] | 0 | False | build/topic06/matmul_4x4.s | +| matmul_add_2x2 | tensor | FAIL | 0 | null | tinyfive | 4 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0843 | 0.1579 | 0.3648 | null | null | null | 5.00 | null | [[20, 23], [44, 51]] | 0 | False | build/topic06/matmul_add_2x2.s | +| matmul_relu_2x2 | tensor | FAIL | 0 | null | tinyfive | 5 | 0 | benchmark skipped: TinyFive input ABI unsupported | null | null | null | null | 0.0851 | 0.1587 | 0.3723 | null | null | null | 5.00 | null | [[0, 2], [0, 4]] | 0 | False | build/topic06/matmul_relu_2x2.s | + +## 双后端能力矩阵 + +| 用例 | 类别 | DSLInterpreter | TinyFive | TinyFive 失败类型 | 两后端输出一致 | 单用例报告 | +|---|---|---|---|---|---|---| +| add_relu_relu | activation | PASS | PASS | null | True | benchmark_reports/topic06/cases/activation-add_relu_relu.md | +| relu_add | activation | PASS | PASS | null | True | benchmark_reports/topic06/cases/activation-relu_add.md | +| relu_only | activation | PASS | PASS | null | True | benchmark_reports/topic06/cases/activation-relu_only.md | +| relu_twice | activation | PASS | PASS | null | True | benchmark_reports/topic06/cases/activation-relu_twice.md | +| if_else | branch | UNSUPPORTED | TIMEOUT | simulation_timeout | null | benchmark_reports/topic06/cases/branch-if_else.md | +| if_relu | branch | UNSUPPORTED | TIMEOUT | simulation_timeout | null | benchmark_reports/topic06/cases/branch-if_relu.md | +| if_then | branch | UNSUPPORTED | TIMEOUT | simulation_timeout | null | benchmark_reports/topic06/cases/branch-if_then.md | +| add_chain | elementwise | PASS | PASS | null | True | benchmark_reports/topic06/cases/elementwise-add_chain.md | +| add_chain_3 | elementwise | PASS | PASS | null | True | benchmark_reports/topic06/cases/elementwise-add_chain_3.md | +| add_fan_in_4 | elementwise | PASS | PASS | null | True | benchmark_reports/topic06/cases/elementwise-add_fan_in_4.md | +| add_reuse | elementwise | PASS | PASS | null | True | benchmark_reports/topic06/cases/elementwise-add_reuse.md | +| vector_add | elementwise | PASS | PASS | null | True | benchmark_reports/topic06/cases/elementwise-vector_add.md | +| loop_add_4 | loop | UNSUPPORTED | PASS | null | null | benchmark_reports/topic06/cases/loop-loop_add_4.md | +| loop_add_chain_4 | loop | UNSUPPORTED | PASS | null | null | benchmark_reports/topic06/cases/loop-loop_add_chain_4.md | +| loop_relu_add_4 | loop | UNSUPPORTED | PASS | null | null | benchmark_reports/topic06/cases/loop-loop_relu_add_4.md | +| dot_4 | reduction | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/reduction-dot_4.md | +| dot_8 | reduction | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/reduction-dot_8.md | +| dot_relu_4 | reduction | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/reduction-dot_relu_4.md | +| dot_relu_8 | reduction | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/reduction-dot_relu_8.md | +| matmul_2x2 | tensor | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/tensor-matmul_2x2.md | +| matmul_4x4 | tensor | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/tensor-matmul_4x4.md | +| matmul_add_2x2 | tensor | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/tensor-matmul_add_2x2.md | +| matmul_relu_2x2 | tensor | PASS | UNSUPPORTED | input_abi_unsupported | null | benchmark_reports/topic06/cases/tensor-matmul_relu_2x2.md | + +## 性能图表 + +![课程版指令数图表](course_report_instructions.png) + + +## 用例详情 + +### add_relu_relu + +- 类别: activation +- 描述: Add input and bias, then apply ReLU twice. +- 预期输出 (scalar): 7 +- TinyFive 输出: 7 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 7.0 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': -3, 't1': 10} +- 模拟后端: tinyfive +- 指令数: 7 +- 静态汇编指令数: 5 +- 编码后机器指令数: 11 +- 代码大小(bytes): 44 +- TinyFive 分类计数: {'total': 7, 'load': 0, 'store': 0, 'mul': 0, 'add': 4, 'madd': 0, 'branch': 2} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 7.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 7 +- 最大指令数: 7 +- 编译耗时(s): 0.0851 +- 模拟耗时(s): 0.1467 +- 总耗时(s): 0.8290 +- 基线指令数: 7.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/add_relu_relu.s + +### relu_add + +- 类别: activation +- 描述: Add input and bias, then apply one ReLU. +- 预期输出 (scalar): 3 +- TinyFive 输出: 3 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 3.0 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': -2, 't1': 5} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 1} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 5.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 5 +- 最大指令数: 5 +- 编译耗时(s): 0.0879 +- 模拟耗时(s): 0.1541 +- 总耗时(s): 0.8029 +- 基线指令数: 5.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/relu_add.s + +### relu_only + +- 类别: activation +- 描述: Apply ReLU directly to a single input value. +- 预期输出 (scalar): 0 +- TinyFive 输出: 0 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 0.0 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': -5} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 3 +- 编码后机器指令数: 6 +- 代码大小(bytes): 24 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 2, 'madd': 0, 'branch': 1} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 5.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 5 +- 最大指令数: 5 +- 编译耗时(s): 0.0860 +- 模拟耗时(s): 0.1545 +- 总耗时(s): 0.8123 +- 基线指令数: 5.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/relu_only.s + +### relu_twice + +- 类别: activation +- 描述: Apply ReLU twice to the same activation path. +- 预期输出 (scalar): 4 +- TinyFive 输出: 4 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 4.0 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': 4} +- 模拟后端: tinyfive +- 指令数: 6 +- 静态汇编指令数: 4 +- 编码后机器指令数: 10 +- 代码大小(bytes): 40 +- TinyFive 分类计数: {'total': 6, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 2} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 6.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 6 +- 最大指令数: 6 +- 编译耗时(s): 0.0864 +- 模拟耗时(s): 0.1509 +- 总耗时(s): 0.8119 +- 基线指令数: 6.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/relu_twice.s + +### if_else + +- 类别: branch +- 描述: if/else branch returns subtraction result when flag is zero. +- 预期输出 (scalar): 5 +- TinyFive 输出: None +- 输出是否匹配: False +- DSLInterpreter 状态: UNSUPPORTED +- DSLInterpreter 输出: None +- DSLInterpreter 错误: unsupported control flow: if +- TinyFive 状态: TIMEOUT +- TinyFive 失败类型: simulation_timeout +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: null +- TinyFive 初始寄存器: {'t1': 0, 't2': 9, 't3': 4} +- 模拟后端: timeout +- 指令数: 0 +- 静态汇编指令数: 12 +- 编码后机器指令数: null +- 代码大小(bytes): null +- TinyFive 分类计数: {} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: initial simulation timeout +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0877 +- 模拟耗时(s): 5.0240 +- 总耗时(s): 5.1174 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/if_else.s + +### if_relu + +- 类别: branch +- 描述: if/else branch combined with add and relu. +- 预期输出 (scalar): 0 +- TinyFive 输出: None +- 输出是否匹配: False +- DSLInterpreter 状态: UNSUPPORTED +- DSLInterpreter 输出: None +- DSLInterpreter 错误: unsupported control flow: if +- TinyFive 状态: TIMEOUT +- TinyFive 失败类型: simulation_timeout +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: null +- TinyFive 初始寄存器: {'t1': 1} +- 模拟后端: timeout +- 指令数: 0 +- 静态汇编指令数: 11 +- 编码后机器指令数: null +- 代码大小(bytes): null +- TinyFive 分类计数: {} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: initial simulation timeout +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0959 +- 模拟耗时(s): 5.0234 +- 总耗时(s): 5.1205 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/if_relu.s + +### if_then + +- 类别: branch +- 描述: if/else branch returns add result when flag is non-zero. +- 预期输出 (scalar): 13 +- TinyFive 输出: None +- 输出是否匹配: False +- DSLInterpreter 状态: UNSUPPORTED +- DSLInterpreter 输出: None +- DSLInterpreter 错误: unsupported control flow: if +- TinyFive 状态: TIMEOUT +- TinyFive 失败类型: simulation_timeout +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: null +- TinyFive 初始寄存器: {'t1': 1, 't2': 9, 't3': 4} +- 模拟后端: timeout +- 指令数: 0 +- 静态汇编指令数: 12 +- 编码后机器指令数: null +- 代码大小(bytes): null +- TinyFive 分类计数: {} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: initial simulation timeout +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0892 +- 模拟耗时(s): 5.0158 +- 总耗时(s): 5.1061 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/if_then.s + +### add_chain + +- 类别: elementwise +- 描述: Add a and b, then add c to the intermediate result. +- 预期输出 (scalar): 9 +- TinyFive 输出: 9 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 9 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3, 't3': 4} +- 模拟后端: tinyfive +- 指令数: 4 +- 静态汇编指令数: 4 +- 编码后机器指令数: 4 +- 代码大小(bytes): 16 +- TinyFive 分类计数: {'total': 4, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 4.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 4 +- 最大指令数: 4 +- 编译耗时(s): 0.0876 +- 模拟耗时(s): 0.1568 +- 总耗时(s): 0.8241 +- 基线指令数: 4.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/add_chain.s + +### add_chain_3 + +- 类别: elementwise +- 描述: Chain three add operations across four symbolic inputs. +- 预期输出 (scalar): 14 +- TinyFive 输出: 14 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 14 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3, 't3': 4, 't5': 5} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 5 +- 编码后机器指令数: 5 +- 代码大小(bytes): 20 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 4, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 5.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 5 +- 最大指令数: 5 +- 编译耗时(s): 0.0856 +- 模拟耗时(s): 0.1468 +- 总耗时(s): 0.8015 +- 基线指令数: 5.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/add_chain_3.s + +### add_fan_in_4 + +- 类别: elementwise +- 描述: Compute two independent adds and then merge them with a final add. +- 预期输出 (scalar): 10 +- TinyFive 输出: 10 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 10 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': 1, 't1': 2, 't3': 3, 't4': 4} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 5 +- 编码后机器指令数: 5 +- 代码大小(bytes): 20 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 0, 'add': 4, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 5.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 5 +- 最大指令数: 5 +- 编译耗时(s): 0.0850 +- 模拟耗时(s): 0.1628 +- 总耗时(s): 0.8475 +- 基线指令数: 5.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/add_fan_in_4.s + +### add_reuse + +- 类别: elementwise +- 描述: Reuse the same intermediate add result on both operands of a second add. +- 预期输出 (scalar): 10 +- TinyFive 输出: 10 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 10 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3} +- 模拟后端: tinyfive +- 指令数: 4 +- 静态汇编指令数: 4 +- 编码后机器指令数: 4 +- 代码大小(bytes): 16 +- TinyFive 分类计数: {'total': 4, 'load': 0, 'store': 0, 'mul': 0, 'add': 3, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 4.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 4 +- 最大指令数: 4 +- 编译耗时(s): 0.1064 +- 模拟耗时(s): 0.1612 +- 总耗时(s): 0.8810 +- 基线指令数: 4.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/add_reuse.s + +### vector_add + +- 类别: elementwise +- 描述: Single add over two symbolic vector inputs. +- 预期输出 (scalar): 5 +- TinyFive 输出: 5 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 5 +- DSLInterpreter 错误: null +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3} +- 模拟后端: tinyfive +- 指令数: 3 +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 0, 'add': 2, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 3.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 3 +- 最大指令数: 3 +- 编译耗时(s): 0.0854 +- 模拟耗时(s): 0.1476 +- 总耗时(s): 0.8187 +- 基线指令数: 3.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/vector_add.s + +### loop_add_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result. +- 预期输出 (scalar): 5 +- TinyFive 输出: 5 +- 输出是否匹配: True +- DSLInterpreter 状态: UNSUPPORTED +- DSLInterpreter 输出: None +- DSLInterpreter 错误: unsupported control flow: for +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: null +- TinyFive 初始寄存器: {'t0': 2, 't1': 3} +- 模拟后端: tinyfive +- 指令数: 22 +- 静态汇编指令数: 7 +- 编码后机器指令数: 8 +- 代码大小(bytes): 32 +- TinyFive 分类计数: {'total': 22, 'load': 0, 'store': 0, 'mul': 0, 'add': 12, 'madd': 0, 'branch': 5} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 22.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 22 +- 最大指令数: 22 +- 编译耗时(s): 0.0913 +- 模拟耗时(s): 0.1559 +- 总耗时(s): 0.7475 +- 基线指令数: 22.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/loop_add_4.s + +### loop_add_chain_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result. +- 预期输出 (scalar): 9 +- TinyFive 输出: 9 +- 输出是否匹配: True +- DSLInterpreter 状态: UNSUPPORTED +- DSLInterpreter 输出: None +- DSLInterpreter 错误: unsupported control flow: for +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: null +- TinyFive 初始寄存器: {'t0': 2, 't1': 3, 't4': 4} +- 模拟后端: tinyfive +- 指令数: 26 +- 静态汇编指令数: 8 +- 编码后机器指令数: 9 +- 代码大小(bytes): 36 +- TinyFive 分类计数: {'total': 26, 'load': 0, 'store': 0, 'mul': 0, 'add': 16, 'madd': 0, 'branch': 5} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 26.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 26 +- 最大指令数: 26 +- 编译耗时(s): 0.0866 +- 模拟耗时(s): 0.1573 +- 总耗时(s): 0.7031 +- 基线指令数: 26.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/loop_add_chain_4.s + +### loop_relu_add_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result. +- 预期输出 (scalar): 2 +- TinyFive 输出: 2 +- 输出是否匹配: True +- DSLInterpreter 状态: UNSUPPORTED +- DSLInterpreter 输出: None +- DSLInterpreter 错误: unsupported control flow: for +- TinyFive 状态: PASS +- TinyFive 失败类型: null +- TinyFive 输入 ABI 可用: True +- 两后端输出一致: null +- TinyFive 初始寄存器: {'t0': -4, 't1': 6} +- 模拟后端: tinyfive +- 指令数: 30 +- 静态汇编指令数: 8 +- 编码后机器指令数: 12 +- 代码大小(bytes): 48 +- TinyFive 分类计数: {'total': 30, 'load': 0, 'store': 0, 'mul': 0, 'add': 16, 'madd': 0, 'branch': 9} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 3 +- Benchmark 停止原因: null +- 平均指令数: 30.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 30 +- 最大指令数: 30 +- 编译耗时(s): 0.0855 +- 模拟耗时(s): 0.1567 +- 总耗时(s): 0.7070 +- 基线指令数: 30.00 +- 性能变化量: 0.00 +- 性能变化率(%): 0.00 +- 性能退化阈值(%): 5.00 +- 是否性能退化: False +- Cost model 是否退化: False +- 汇编文件: build/topic06/loop_relu_add_4.s + +### dot_4 + +- 类别: reduction +- 描述: Compute the dot product of two symbolic vectors of length 4. +- 预期输出 (scalar): 70 +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 70 +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0858 +- 模拟耗时(s): 0.1493 +- 总耗时(s): 0.3609 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/dot_4.s + +### dot_8 + +- 类别: reduction +- 描述: Compute the dot product of two symbolic vectors of length 8. +- 预期输出 (scalar): 36 +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 36 +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0836 +- 模拟耗时(s): 0.1503 +- 总耗时(s): 0.3586 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/dot_8.s + +### dot_relu_4 + +- 类别: reduction +- 描述: Compute a length-4 dot product and pass it through ReLU. +- 预期输出 (scalar): 0 +- TinyFive 输出: 0 +- 输出是否匹配: True +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 0.0 +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 1} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0843 +- 模拟耗时(s): 0.1484 +- 总耗时(s): 0.3574 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/dot_relu_4.s + +### dot_relu_8 + +- 类别: reduction +- 描述: Compute a length-8 dot product and pass it through ReLU. +- 预期输出 (scalar): 8 +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: 8.0 +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 1} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0883 +- 模拟耗时(s): 0.1534 +- 总耗时(s): 0.3687 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/dot_relu_8.s + +### matmul_2x2 + +- 类别: tensor +- 描述: Compute a symbolic 2x2 by 2x2 matrix multiplication. +- 预期输出 (tensor): [[19, 22], [43, 50]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: [[19, 22], [43, 50]] +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0863 +- 模拟耗时(s): 0.1575 +- 总耗时(s): 0.3701 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/matmul_2x2.s + +### matmul_4x4 + +- 类别: tensor +- 描述: Compute a symbolic 4x4 by 4x4 matrix multiplication. +- 预期输出 (tensor): [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 静态汇编指令数: 3 +- 编码后机器指令数: 3 +- 代码大小(bytes): 12 +- TinyFive 分类计数: {'total': 3, 'load': 0, 'store': 0, 'mul': 1, 'add': 1, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0879 +- 模拟耗时(s): 0.1479 +- 总耗时(s): 0.3601 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/matmul_4x4.s + +### matmul_add_2x2 + +- 类别: tensor +- 描述: Compute a 2x2 matmul and then add a symbolic bias term. +- 预期输出 (tensor): [[20, 23], [44, 51]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: [[20, 23], [44, 51]] +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 4 +- 静态汇编指令数: 4 +- 编码后机器指令数: 4 +- 代码大小(bytes): 16 +- TinyFive 分类计数: {'total': 4, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 0} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0843 +- 模拟耗时(s): 0.1579 +- 总耗时(s): 0.3648 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/matmul_add_2x2.s + +### matmul_relu_2x2 + +- 类别: tensor +- 描述: Compute a 2x2 matmul and then apply ReLU to its result. +- 预期输出 (tensor): [[0, 2], [0, 4]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- DSLInterpreter 状态: PASS +- DSLInterpreter 输出: [[0.0, 2.0], [0.0, 4.0]] +- DSLInterpreter 错误: null +- TinyFive 状态: UNSUPPORTED +- TinyFive 失败类型: input_abi_unsupported +- TinyFive 输入 ABI 可用: False +- 两后端输出一致: null +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 5 +- 静态汇编指令数: 4 +- 编码后机器指令数: 7 +- 代码大小(bytes): 28 +- TinyFive 分类计数: {'total': 5, 'load': 0, 'store': 0, 'mul': 1, 'add': 2, 'madd': 0, 'branch': 1} +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: 0 +- Benchmark 停止原因: benchmark skipped: TinyFive input ABI unsupported +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0851 +- 模拟耗时(s): 0.1587 +- 总耗时(s): 0.3723 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): 5.00 +- 是否性能退化: null +- Cost model 是否退化: null +- 汇编文件: build/topic06/matmul_relu_2x2.s diff --git a/benchmarks/topic06/baseline.json b/benchmarks/topic06/baseline.json new file mode 100644 index 0000000..322a990 --- /dev/null +++ b/benchmarks/topic06/baseline.json @@ -0,0 +1,215 @@ +{ + "schema_version": 2, + "primary_metric": "dynamic_instructions", + "generated_at": "2026-09-14T15:58:42", + "environment": { + "python": "3.11.7", + "platform": "Windows-10-10.0.26200-SP0" + }, + "cases": { + "add_relu_relu": { + "category": "activation", + "avg_instr_count": 7.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 11, + "code_size_bytes": 44, + "dynamic_instructions": 7.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 2.0 + } + }, + "relu_add": { + "category": "activation", + "avg_instr_count": 5.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 7, + "code_size_bytes": 28, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 1.0 + } + }, + "relu_only": { + "category": "activation", + "avg_instr_count": 5.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 6, + "code_size_bytes": 24, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 2.0, + "dynamic_madd": 0.0, + "dynamic_branch": 1.0 + } + }, + "relu_twice": { + "category": "activation", + "avg_instr_count": 6.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 10, + "code_size_bytes": 40, + "dynamic_instructions": 6.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 2.0 + } + }, + "add_chain": { + "category": "elementwise", + "avg_instr_count": 4.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + } + }, + "add_chain_3": { + "category": "elementwise", + "avg_instr_count": 5.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 5, + "code_size_bytes": 20, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + } + }, + "add_fan_in_4": { + "category": "elementwise", + "avg_instr_count": 5.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 5, + "machine_instructions": 5, + "code_size_bytes": 20, + "dynamic_instructions": 5.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 4.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + } + }, + "add_reuse": { + "category": "elementwise", + "avg_instr_count": 4.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 4, + "machine_instructions": 4, + "code_size_bytes": 16, + "dynamic_instructions": 4.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 3.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + } + }, + "vector_add": { + "category": "elementwise", + "avg_instr_count": 3.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 3, + "code_size_bytes": 12, + "dynamic_instructions": 3.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 2.0, + "dynamic_madd": 0.0, + "dynamic_branch": 0.0 + } + }, + "loop_add_4": { + "category": "loop", + "avg_instr_count": 22.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 7, + "machine_instructions": 8, + "code_size_bytes": 32, + "dynamic_instructions": 22.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 12.0, + "dynamic_madd": 0.0, + "dynamic_branch": 5.0 + } + }, + "loop_add_chain_4": { + "category": "loop", + "avg_instr_count": 26.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 8, + "machine_instructions": 9, + "code_size_bytes": 36, + "dynamic_instructions": 26.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 16.0, + "dynamic_madd": 0.0, + "dynamic_branch": 5.0 + } + }, + "loop_relu_add_4": { + "category": "loop", + "avg_instr_count": 30.0, + "runs": 3, + "cost_model": { + "static_asm_instructions": 8, + "machine_instructions": 12, + "code_size_bytes": 48, + "dynamic_instructions": 30.0, + "dynamic_load": 0.0, + "dynamic_store": 0.0, + "dynamic_mul": 0.0, + "dynamic_add": 16.0, + "dynamic_madd": 0.0, + "dynamic_branch": 9.0 + } + } + } +} diff --git "a/docs/topics/06-\345\217\214\345\220\216\347\253\257\346\234\252\351\200\232\350\277\207\347\224\250\344\276\213\345\210\206\346\236\220.md" "b/docs/topics/06-\345\217\214\345\220\216\347\253\257\346\234\252\351\200\232\350\277\207\347\224\250\344\276\213\345\210\206\346\236\220.md" new file mode 100644 index 0000000..a4fd6fe --- /dev/null +++ "b/docs/topics/06-\345\217\214\345\220\216\347\253\257\346\234\252\351\200\232\350\277\207\347\224\250\344\276\213\345\210\206\346\236\220.md" @@ -0,0 +1,192 @@ +# ScratchV 课题 06 双后端未通过用例分析 + +> 分析日期：2026-09-14 +> +> 测试范围：`tests/topic06/cases/` 中全部 23 个 DSL 用例 +> +> 测试模式：DSLInterpreter 与真实 TinyFive 双后端验证 +> +> 数据来源：`benchmark_reports/topic06/report.json` + +## 一、结论摘要 + +本次测试不存在“两个后端都成功执行但计算结果不同”的用例。9 个可同时有效执行的用例结果全部一致。 + +| 项目 | 结果 | +|---|---:| +| 用例总数 | 23 | +| 两后端都可有效比较 | 9 | +| 两后端结果一致 | 9 | +| 两后端结果不一致 | 0 | +| DSLInterpreter 未通过 | 6 | +| TinyFive 未通过 | 11 | +| 至少一个后端未通过的不同用例 | 14 | + +未通过主要分为三类： + +1. DSLInterpreter 尚未实现循环和条件分支语义，共 6 例。 +2. 分支汇编标签与跳转目标命名不一致，导致 TinyFive 执行死循环并超时，共 3 例。 +3. 数组/矩阵输入输出 ABI 和后端降低逻辑未接通，TinyFive 无法有效验证，共 8 例。 + +DSLInterpreter 是 ScratchV 原仓库中的正式实现，位于 `scratchv/verification/verifier.py`。Topic 06 测试套件只负责调用、隔离和报告其结果，没有重新实现解释器，也不会用解释器结果替代 TinyFive 结果。 + +## 二、DSLInterpreter 未通过用例 + +### 2.1 循环类：3例 + +| 用例 | 解释器状态 | TinyFive状态 | 直接原因 | +|---|---|---|---| +| `loop_add_4` | `UNSUPPORTED` | `PASS` | 解释器没有真正执行 `for/endfor` 循环 | +| `loop_add_chain_4` | `UNSUPPORTED` | `PASS` | 解释器没有真正执行 `for/endfor` 循环 | +| `loop_relu_add_4` | `UNSUPPORTED` | `PASS` | 解释器没有真正执行 `for/endfor` 循环 | + +当前 `DSLInterpreter.run()` 遇到 `for` 和 `endfor` 时只跳过控制语句，没有根据循环上下界重复执行循环体。如果直接使用该结果，循环体只会运行一次，可能产生错误答案。测试套件因此在执行前识别控制流并明确标记为 `UNSUPPORTED`，避免假通过。 + +这 3 个用例在 TinyFive 中均能正确返回期望值，说明当前编译器生成的循环汇编和 TinyFive 循环执行路径可以工作。 + +**问题归属**：ScratchV DSLInterpreter 控制流语义。 + +**建议处理**：解释器需要构建或遍历结构化语句块，按照 `start/end` 重复执行循环体，不能继续逐行跳过 `for/endfor`。 + +### 2.2 分支类：3例 + +| 用例 | 解释器状态 | TinyFive状态 | 解释器直接原因 | +|---|---|---|---| +| `if_else` | `UNSUPPORTED` | `TIMEOUT` | 没有实现 `if/else/endif` 条件语义 | +| `if_relu` | `UNSUPPORTED` | `TIMEOUT` | 没有实现 `if/else/endif` 条件语义 | +| `if_then` | `UNSUPPORTED` | `TIMEOUT` | 没有实现 `if/else/endif` 条件语义 | + +当前解释器不会根据条件只执行 then 或 else 分支。若不做前置能力检查，它可能顺序处理两个分支中的普通赋值和返回语句，得到不可信结果。因此 Topic 06 Runner 将含 `if/else/endif` 的用例标记为 `UNSUPPORTED`。 + +**问题归属**：ScratchV DSLInterpreter 条件控制流语义。 + +**建议处理**：解释器需要解析条件表达式，建立 then/else 语句块，并只执行被选中的分支。 + +## 三、TinyFive未通过用例 + +### 3.1 分支类超时：3例 + +| 用例 | 状态 | 失败类型 | 现象 | +|---|---|---|---| +| `if_else` | `TIMEOUT` | `simulation_timeout` | TinyFive执行超过5秒未返回 | +| `if_relu` | `TIMEOUT` | `simulation_timeout` | TinyFive执行超过5秒未返回 | +| `if_then` | `TIMEOUT` | `simulation_timeout` | TinyFive执行超过5秒未返回 | + +TinyFive支持这些用例使用的 `bnez`、`j` 和 `jalr` 指令。问题不是TinyFive缺少分支指令，而是编译器生成的标签名称不一致。例如： + +```asm +bnez t1, if_then1 +j if_else2 + +.if_then1: + ... +.if_else2: + ... +``` + +跳转目标是 `if_then1`、`if_else2`，实际标签却是 `.if_then1`、`.if_else2`。汇编编码器找不到目标标签时使用当前指令位置作为默认目标，最终产生偏移量为 0 的跳转： + +```text +PC=8 -> 执行 j -> PC仍为8 -> 再次执行 j -> 无限循环 +``` + +具体原因链： + +1. `InstructionSelector._select_function()` 给基本块标签增加 `.` 前缀。 +2. `_select_br()` 和 `_select_br_if()` 使用未加 `.` 的 IR 基本块名称作为跳转目标。 +3. `RISCVAEncoder._apply_fixup()` 找不到标签时静默回退到当前指令，而不是报错。 +4. TinyFive忠实执行偏移量为0的跳转，最终由测试套件5秒超时终止。 + +**问题归属**：ScratchV 指令选择和 RISC-V 汇编编码，不是 TinyFive 指令支持问题。 + +**建议处理**：统一基本块标签定义和引用的名称；编码器遇到未知标签时立即报错，禁止静默生成零偏移跳转。 + +### 3.2 Reduction数组类：4例 + +| 用例 | 解释器状态 | TinyFive状态 | 期望输出 | TinyFive观测值 | +|---|---|---|---|---| +| `dot_4` | `PASS` | `UNSUPPORTED` | `70` | `0` | +| `dot_8` | `PASS` | `UNSUPPORTED` | `36` | `0` | +| `dot_relu_4` | `PASS` | `UNSUPPORTED` | `0` | `0` | +| `dot_relu_8` | `PASS` | `UNSUPPORTED` | `8` | `0` | + +失败类型统一为 `input_abi_unsupported`。解释器可以直接把 `.meta.json` 中的数组转换为 NumPy 数组并执行 `numpy.dot()`。TinyFive 路径目前只能把整数标量写入物理寄存器，尚未完成： + +1. 将数组元素写入 TinyFive 模拟内存； +2. 把数组首地址、长度和结果地址传给汇编； +3. 生成逐元素加载、乘法、累加和结果写回； +4. 从返回寄存器或结果内存读取最终值。 + +此外，当前 `_select_dot()` 只生成单条 `mul`，没有使用 DSL 中的 `len` 属性，也没有生成数组遍历和累加逻辑。因此只在测试套件中写入数组内存仍不足以使该类用例通过。 + +`dot_relu_4` 的期望值与未初始化输入产生的返回值都恰好为 `0`，但这属于偶然相等。新版报告将其标记为 `UNSUPPORTED`，不计为 TinyFive PASS，也不写入性能基线。 + +**问题归属**：数组 ABI、ScratchV dot 后端降低和测试套件内存输入输出适配。 + +### 3.3 Tensor矩阵类：4例 + +| 用例 | 解释器状态 | TinyFive状态 | TinyFive观测值 | +|---|---|---|---| +| `matmul_2x2` | `PASS` | `UNSUPPORTED` | `0` | +| `matmul_4x4` | `PASS` | `UNSUPPORTED` | `0` | +| `matmul_add_2x2` | `PASS` | `UNSUPPORTED` | `0` | +| `matmul_relu_2x2` | `PASS` | `UNSUPPORTED` | `0` | + +失败类型同样为 `input_abi_unsupported`。解释器使用 NumPy 的矩阵乘法，可以直接处理二维输入。TinyFive 路径尚未定义矩阵的内存布局、地址传递和结果读取方式；当前 `_select_matmul()` 也只生成单条 `mul`，没有根据 `m/n/k` 生成三层循环、加载、乘加及存储指令。 + +`matmul_add_2x2` 还包含数组形式的 `bias`，除了矩阵乘法外还需要明确逐元素广播或同形加法语义。 + +**问题归属**：张量 ABI、ScratchV matmul 后端降低和测试套件内存输入输出适配。 + +## 四、后端差异汇总 + +| 类别 | 用例数 | DSLInterpreter | TinyFive | 当前结论 | +|---|---:|---|---|---| +| `activation` | 4 | 全部PASS | 全部PASS | 两后端有效一致 | +| `elementwise` | 5 | 全部PASS | 全部PASS | 两后端有效一致 | +| `loop` | 3 | 不支持循环 | 全部PASS | 需要完善解释器 | +| `branch` | 3 | 不支持分支 | 全部超时 | 解释器和编译器分别存在问题 | +| `reduction` | 4 | 全部PASS | 输入ABI不支持 | 需要数组ABI和dot降低 | +| `tensor` | 4 | 全部PASS | 输入ABI不支持 | 需要张量ABI和matmul降低 | + +`backend_outputs_match: null` 表示至少一个后端不具备有效执行条件，不能进行结果比较；它不表示两个有效结果不相等。当前没有 `backend_outputs_match: false` 的用例。 + +## 五、建议处理顺序 + +### P0：修复分支标签 + +统一基本块标签名称，并让编码器对未知标签立即报错。该问题范围明确，修复后可直接验证 3 个 branch 用例。 + +### P1：补充解释器控制流 + +实现 `for/endfor` 和 `if/else/endif` 的真实执行语义，使解释器能够覆盖 6 个控制流用例，并与 TinyFive 结果交叉验证。 + +### P2：设计数组和张量ABI + +先明确内存布局、参数寄存器、shape/length传递及结果返回约定，再实现 `dot` 和 `matmul` 的 RISC-V 降低，最后补充 Topic 06 的 TinyFive 内存注入与读取。 + +数组和张量问题涉及编译器 ABI 与后端代码生成，不能仅通过修改测试期望值或解释器结果解决。 + +## 六、复现方式 + +运行全部双后端测试： + +```bash +python scripts/run_topic06_benchmarks.py --benchmark 3 --verification-backend both --full-report +``` + +分别定位三类问题： + +```bash +python scripts/run_topic06_benchmarks.py --category loop --verification-backend both +python scripts/run_topic06_benchmarks.py --category branch --verification-backend both +python scripts/run_topic06_benchmarks.py --category reduction --verification-backend both +python scripts/run_topic06_benchmarks.py --category tensor --verification-backend both +``` + +详细结果位于： + +```text +benchmark_reports/topic06/report.json +benchmark_reports/topic06/cases/ +``` diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" index 3637b6d..1cd63cf 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" @@ -1,200 +1,25 @@ -# 课题6：编译器性能测试套件 +# 课题 06：编译器性能测试套件 -> **难度**：中 | **类型**：项目实战 | **源文件**：`benchmarks/bench_runner.py` | **行数**：~400 -> **状态**：✅ 已完成 +课题 06 测试套件已经集成到 ScratchV 仓库正式目录。它使用 23 个 DSL 用例分别验证 DSLInterpreter 语义路径与 `DSL -> ScratchV -> RISC-V -> TinyFive` 编译路径，并支持指令数统计、耗时测量、Benchmark、cost-model 基线、退化检测和 CI 报告。两条路径的状态独立记录，解释器不会作为 TinyFive 的失败回退。 ---- +- [使用说明](06-性能测试套件使用说明.md) +- [设计文档](06-性能测试套件设计文档.md) +- [双后端未通过用例分析](06-双后端未通过用例分析.md) -## 概述 +主要入口： -设计一组基准测试程序（DSL用例），自动化执行编译、模拟运行、对比预期输出，生成性能报告。该套件用于持续验证编译器的正确性和性能变化。 - ---- - -## 理解背景 - -### 是什么？ - -性能基准套件（Benchmark Suite）自动化测试 ScratchV 编译器的**正确性**和**性能**。它收集了 23 个 DSL 测试用例，覆盖算术、神经网络算子、控制流等各类场景，每次运行自动生成 JSON/HTML/Markdown 报告。 - -### 为什么？ - -编译器每次改动都可能引入 bug 或性能衰退。手工测试既不全面也不可靠。自动化的基准测试套件让你： -- **回归检测**：改了代码后跑一遍，立即知道有没有破坏现有功能 -- **性能追踪**：记录每次运行的时间，发现性能衰退 -- **CI 集成**：每次 push 自动运行，阻止有问题的代码合入 - -### 核心概念 - -#### 测试用例格式 - -每个测试用例最多 2 个文件（放在 `benchmarks/cases/`）： -``` -001_simple_add.dsl ← DSL 源代码（必需） -001_simple_add.expected ← 期望输出（可选） -``` - -#### DSL 语法要点 - -数字可以直接内联，变量名自动解析： -```python -# 正确写法 -c = add(a, b) # a, b 作为输入变量 -t1 = add(2.0, 3.0) # 数字直接写 -return c - -# 错误写法（没有 const 操作符！） -a = const(3) # ❌ 不支持 -``` - -支持的算子：`add`, `sub`, `mul`, `div`, `neg`, `exp`, `relu`, `gelu`, `dot`, `matmul`, `softmax`, `maxpool` - -#### 23 个用例覆盖范围 - -| 类别 | 数量 | 示例 | -|------|------|------| -| 算术 | 5 | add, mul, sub, div, chained | -| 神经网络 | 6 | relu, gelu, softmax, matmul, dot, maxpool | -| 控制流 | 7 | for-loop, if/else, while, nested | -| 复杂场景 | 3 | NN pipeline, large chain | -| 常量 | 1 | constant propagation | - -#### 报告格式 - -| 格式 | 用途 | -|------|------| -| **终端输出** | 开发时快速查看 | -| **JSON** | CI dashboard 数据源 | -| **HTML** | 可视化展示 | -| **Markdown** | 文档、PR review | - ---- - -## 代码走读 - -### 运行全部测试 - -```bash -# 基础运行（指定用例目录） -python3 benchmarks/bench_runner.py benchmarks/cases - -# 生成 JSON + HTML 报告 -python3 benchmarks/bench_runner.py benchmarks/cases \ - --output-json /tmp/dsl_bench.json \ - --output-html /tmp/dsl_bench.html -``` - -实际输出（23/23 PASS）： -``` -Discovered 23 test case(s) in benchmarks/cases ------------------------------------------------------------- - [1/23] 001_simple_add ... PASS (0.008s parse, 2 inst) - [2/23] 002_simple_mul ... PASS (0.000s parse, 2 inst) - ... - [23/23] 023_large_chain ... PASS (0.000s parse, 7 inst) -``` - -### 用 Makefile 一键运行 - -```bash -make bench # 等价于上面的完整流程 +```text +scripts/run_topic06_benchmarks.py +tests/topic06/cases/ +benchmarks/topic06/baseline.json +.github/workflows/topic06-benchmark.yml ``` -### 添加新测试用例 +在仓库根目录运行： ```bash -cat > benchmarks/cases/024_my_test.dsl << 'EOF' -# 我的自定义测试: 10 + 20 - 5 -a = add(10, 20) -b = sub(a, 5) -return b -EOF - -echo "25" > benchmarks/cases/024_my_test.expected - -# 运行确认 -python3 benchmarks/bench_runner.py benchmarks/cases -``` - -> 💡 **关键**：数字直接内联使用（如 `10`, `20`），不需要 `const()` 包装。期望输出文件是纯文本，末尾不要有多余空行。 - -### Python API 用法 - -```python -from benchmarks.bench_runner import BenchmarkRunner - -# 创建 runner -runner = BenchmarkRunner( - test_dir="benchmarks/cases", - timeout=30.0, - verbose=True, -) - -# 发现测试用例 -cases = runner.discover_cases() -print(f"Found {len(cases)} test cases") - -# 运行全部（返回 BenchmarkReport） -report = runner.run_all() -report.print_summary() - -# 生成报告 -report.save_json("/tmp/results.json") -report.save_html("/tmp/results.html") -report.save_markdown("/tmp/results.md") +pip install -e ".[topic06]" +python scripts/run_topic06_benchmarks.py ``` -### CI 回归检测 - -```bash -# 生成基线 -python3 benchmarks/bench_runner.py benchmarks/cases --output-json /tmp/baseline.json - -# 修改代码后... -python3 benchmarks/bench_runner.py benchmarks/cases --output-json /tmp/current.json - -# 对比通过率 -python3 -c " -import json -baseline = json.load(open('/tmp/baseline.json')) -current = json.load(open('/tmp/current.json')) -if current['pass_rate'] < baseline['pass_rate']: - print('REGRESSION DETECTED!') - exit(1) -else: - print('OK: pass_rate stable') -" -``` - ---- - -## 动手练习 - -### 练习 1: 添加一个新测试用例 - -为你正在开发的编译器功能写一个测试用例，跑通并确认通过。 - -### 练习 2: 故意引入 bug - -修改代码引入一个 bug（比如把 `ADD` 映射成 `SUB`），跑 benchmark 看看哪些测试会失败。 - -### 练习 3: 分析性能趋势 - -连续跑 3 次 benchmark，对比 JSON 输出，看编译时间是否有波动。 - ---- - -## 常见坑 - -| 坑 | 说明 | -|----|------| -| **没有 `const` 操作符** | DSL 不支持 `a = const(3)`，直接用数字 `a = add(3, 5)` | -| **expected 文件格式** | .expected 文件应该是纯文本，末尾不要有多余换行 | -| **超时** | 某些测试可能死循环，BenchmarkRunner 有 timeout 保护 | - ---- - -## 进阶阅读 - -- [03-指标解读指南](../03-指标解读指南.md) — 如何解读 benchmark 数据 -- 相关课题: [课题19：Standalone RISC-V 编译器](19-Standalone-RISC-V编译器.md) | [课题30：CI Dashboard](30-CI-Dashboard.md) +当前双后端全量结果中，DSLInterpreter 为 17 个通过、6 个控制流不支持；TinyFive 为 12 个通过、3 个分支超时、8 个数组输入 ABI 不支持。CI 将当前稳定的 `activation`、`elementwise` 和 `loop` 类别作为阻塞门禁，其余用例保留在全量诊断报告中。 diff --git "a/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..8f8e08b --- /dev/null +++ "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,248 @@ +# ScratchV 课题 06 性能测试套件使用说明 + +本测试套件已经迁入 ScratchV 仓库正式目录，用于编译正确性与性能回归测试。它包含 23 个 DSL 测试用例、自动化测试脚本、性能基线、CI 工作流以及可重新生成的测试报告。 + +当前程序在原始自动编译、指令数统计、Benchmark、基线对比和报告功能上，进一步增加了 DSLInterpreter 与真实 TinyFive 双后端验证、元数据校验、编译与模拟超时、编译失败日志、统一 JSON schema、单用例报告、cost-model 指标、轻量/完整报告、可配置退化阈值和子集测试。 + +编译器通过 `--emit-register-map` 输出 JSON 格式的变量到物理寄存器映射，测试套件直接读取该文件注入标量输入，不再解析 `--dump-ir` 文本。 + +两条验证路径为： + +```text +DSL -> ScratchV 编译器 -> RISC-V 汇编 -> TinyFive -> a0/x10 返回值 -> 期望值对比 +DSL -> DSLInterpreter -> 解释器结果 -> 期望值对比 +``` + +最近一次全量运行中，DSLInterpreter 为 17 个通过、6 个控制流不支持；TinyFive 为 12 个通过、3 个 `branch` 模拟超时、8 个 `reduction`/`tensor` 数组输入 ABI 不支持。解释器结果不会替代 TinyFive 结果，因此数组输入恰好返回 0 的用例也不会被计为编译路径通过。 + +## 运行依赖 + +测试命令需要在 ScratchV 仓库根目录运行。`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 +调用真实 TinyFive 所需的配套接口位于 `scratchv/simulator/tinyfive.py`、 +`scratchv/backend/register_alloc.py`、`scratchv/compiler.py` 和 `scratchv/main.py`。 + +安装基础测试依赖： + +```powershell +pip install -e ".[topic06]" +``` + +基础模式需要以下依赖： + +- `tinyfive`：用于模拟执行生成的 RISC-V 汇编 +- `pytest`：用于测试支持 + +如需生成 HTML 和 PNG 完整报告，再安装可选依赖： + +```powershell +pip install -e ".[topic06-report]" +``` + +其中 `jinja2` 用于生成 HTML，`matplotlib` 用于生成性能图表。 + +## 目录结构 + +```text +scripts/run_topic06_benchmarks.py +tests/topic06/ + test_integration.py + cases/ + activation/ + branch/ + elementwise/ + loop/ + reduction/ + tensor/ +benchmarks/topic06/baseline.json +benchmark_reports/topic06/ # 运行时生成，不提交 + cases/ # 每个用例独立的 Markdown/JSON 报告 +build/topic06/ # 运行时生成，不提交 +.github/workflows/topic06-benchmark.yml +``` + +`tests/topic06/cases/` 下共有 23 个 DSL 用例。每个用例由 `.dsl` 文件和对应的 +`.meta.json` 文件组成，`.meta.json` 中定义输入、期望返回值和用例说明。 + +## 运行测试 + +在仓库根目录执行： + +```powershell +python scripts/run_topic06_benchmarks.py +``` + +默认同时执行解释器与 TinyFive，并分别记录状态。也可以显式选择： + +```powershell +python scripts/run_topic06_benchmarks.py --verification-backend both +python scripts/run_topic06_benchmarks.py --verification-backend interpreter +python scripts/run_topic06_benchmarks.py --verification-backend tinyfive +``` + +`both` 模式不会在 TinyFive 失败时使用解释器结果替代其判定。 + +测试脚本会自动完成以下步骤： + +- 遍历 `tests/topic06/cases/` 下的所有 `.dsl` 文件 +- 调用 ScratchV 编译器生成 RISC-V 汇编 +- 读取编译器生成的 `build/topic06/*.registers.json`，按变量名注入真实物理寄存器 +- 编译单个用例超过 30 秒时终止该编译进程，并继续运行后续用例 +- 编译失败或超时时将命令、返回码、stdout 和 stderr 保存到 `benchmark_reports/topic06/failures/*.compile.log` +- 通过 ScratchV 的 TinyFive 适配层验证生成的汇编 +- 在独立子进程中执行 TinyFive；单次模拟超过 5 秒时终止该用例并继续后续测试 +- TinyFive 未安装或执行失败时明确判定为 FAIL，不回退到 stub +- 使用 TinyFive 返回寄存器 `a0/x10` 与 `.meta.json` 中的期望值进行结果对比 +- 使用 `time.perf_counter()` 记录编译、模拟和总耗时 +- 统计 PASS/FAIL 和指令数 +- 默认在 `benchmark_reports/topic06/` 下生成 Markdown 和固定 schema 的 JSON 报告 + +## Benchmark 模式 + +重复运行每个用例并统计平均指令数： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +如果首次模拟已经 timeout，该用例不会再执行 Benchmark 重复；如果重复过程中发生 timeout 或模拟失败，脚本会立即停止剩余次数。失败运行不会作为 `0` 加入平均值，报告会记录实际完成次数和 `benchmark_stopped_reason`。 + +更新性能基线： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --update-baseline +``` + +性能退化阈值默认是 5%，可以按场景调整： + +```powershell +# 超过 2% 即判定为退化 +python scripts/run_topic06_benchmarks.py --benchmark 3 --regression-threshold 2 + +# 超过 10% 才判定为退化 +python scripts/run_topic06_benchmarks.py --benchmark 3 --regression-threshold 10 +``` + +阈值必须大于或等于 0，并会写入 Markdown、HTML 和 JSON 报告。 + +## 子集测试 + +按类别运行： + +```powershell +python scripts/run_topic06_benchmarks.py --category activation +``` + +按用例名称进行大小写不敏感的包含匹配： + +```powershell +python scripts/run_topic06_benchmarks.py --filter matmul +``` + +组合筛选： + +```powershell +python scripts/run_topic06_benchmarks.py --category tensor --filter relu +``` + +没有匹配用例时脚本返回退出码 2，并保留已有报告。对子集使用 `--update-baseline` 时，只更新匹配用例的基线，不会删除其他用例的基线。 + +CI 或其他需要将失败传递给调用方的场景可以增加： + +```text +python scripts/run_topic06_benchmarks.py --category activation --fail-on-test-failure +``` + +默认模式仍会完成全部用例并生成报告；`--fail-on-test-failure` 会在存在 FAIL 时返回退出码 1。 + +性能基线文件位于： + +```text +benchmarks/topic06/baseline.json +``` + +基线 schema 版本为 2。除兼容字段 `avg_instr_count` 外，每个有效 TinyFive 用例还保存静态汇编指令数、机器指令数、代码大小、动态指令数以及 load/store/mul/add/madd/branch 分类计数。数组输入 ABI 不支持或 TinyFive 执行失败的用例不会写入 cost-model 基线。 + +## 测试报告 + +默认运行后生成轻量报告： + +```text +benchmark_reports/topic06/report.md +benchmark_reports/topic06/report.json +benchmark_reports/topic06/cases/*.md +benchmark_reports/topic06/cases/*.json +``` + +需要 HTML 和 PNG 时运行： + +```powershell +python scripts/run_topic06_benchmarks.py --full-report +``` + +完整模式额外生成： + +```text +benchmark_reports/topic06/report.html +benchmark_reports/topic06/course_report_instructions.png +``` + +`report.md` 适合提交或归档，`report.json` 供 CI 或其他程序稳定解析，`report.html` 适合在浏览器中查看。普通模式和 Benchmark 模式使用相同字段，Benchmark 专属字段在普通模式下为 `null`。JSON 顶层的 `selection` 会记录 `--category` 和 `--filter`，未筛选时两项均为 `null`。轻量模式不会加载 `jinja2` 或 `matplotlib`，也不会刷新已有的 HTML、PNG 文件。 + +编译失败时，报告中的 `compile_returncode`、`compile_timed_out`、`compile_error` 和 `compile_log` 会记录错误摘要及独立日志位置。成功用例的错误和日志字段为 `null`。 + +## 添加测试用例 + +新增用例时添加两个文件： + +```text +tests/topic06/cases/{category}/{name}.dsl +tests/topic06/cases/{category}/{name}.meta.json +``` + +DSL 示例： + +```text +result = add(a, b) +return result +``` + +元数据示例： + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "scalar", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +矩阵输出需要声明类型、元素类型和形状： + +```json +{ + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "expected_return": [[19, 22], [43, 50]] +} +``` + +大型期望结果也可以使用 `expected_output_file` 引用同目录下的 JSON 文件。`expected_return` 与 `expected_output_file` 必须且只能填写一个。元数据中的类型、形状或元素值不合法时，仅当前用例标记为 FAIL，后续测试继续运行。 + +## CI 示例 + +GitHub Actions 示例位于： + +```text +.github/workflows/topic06-benchmark.yml +``` + +工作流位于仓库根目录。它会在 push 和 pull request 时运行 pytest 集成契约，对 activation、elementwise、loop 执行阻塞正确性检查，再运行全量诊断 Benchmark，并上传 Markdown、JSON、性能基线和失败日志。CI 默认使用轻量报告，不需要安装 `jinja2` 和 `matplotlib`。 + +## 详细设计 + +程序模块、报告字段、需求演进、Mentor Review 处理状态和当前失败原因见 [06-性能测试套件设计文档.md](06-性能测试套件设计文档.md)。 diff --git "a/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000..3831f44 --- /dev/null +++ "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,721 @@ +# ScratchV 课题 06 性能测试套件设计文档 + +> 文档版本：v1.8 +> 编写日期：2026-07-22 +> 更新日期：2026-09-14 +> 集成位置：ScratchV 仓库正式目录 +> 核心文件：`scripts/run_topic06_benchmarks.py`、`tests/topic06/`、`benchmarks/topic06/` +> 功能范围：DSL 用例管理、DSLInterpreter/TinyFive 双后端验证、元数据校验、编译与模拟超时保护、失败现场保存、指令数与耗时统计、Benchmark、cost-model 基线对比、单用例报告、子集测试、统一 Markdown/JSON/HTML 报告、CI 示例 + +--- + +## 一、功能介绍 + +### 1.1 功能概述 + +本测试套件用于验证 DSL 语义以及 ScratchV 编译器在多个 DSL 程序上的编译正确性、模拟结果和性能表现。测试脚本分别运行 DSLInterpreter 参考路径和真实 TinyFive 编译路径，两条结果独立记录且禁止互相回退。测试结束后会生成总报告、双后端能力矩阵和每个用例的独立报告。 + +当前套件包含 23 个 DSL 测试用例，覆盖以下类别： + +| 类别 | 说明 | +|---|---| +| `activation` | ReLU 等激活函数组合 | +| `elementwise` | 标量/向量加法、链式加法、复用输入 | +| `loop` | `for/endfor` 循环类用例 | +| `branch` | `if/else/endif` 分支类用例 | +| `reduction` | `dot` 等归约类用例 | +| `tensor` | `matmul` 等张量计算类用例 | + +测试输出不仅包含 PASS/FAIL，还包含编译状态、TinyFive 后端状态、期望值与实际值、指令数、编译耗时、模拟耗时、总耗时、Benchmark 平均值、95% 置信区间、性能基线对比和失败日志路径等信息。 + +### 1.2 设计目标 + +- **自动化**：一条命令完成所有用例的编译、模拟、结果比对和报告生成。 +- **真实性**：TinyFive 编译路径只由真实 TinyFive 结果判断；DSLInterpreter 作为独立语义参考，不允许替代、覆盖或伪装成 TinyFive 结果。 +- **可扩展**：新增测试只需要添加 `.dsl` 和 `.meta.json` 文件，不需要修改主测试逻辑。 +- **可解释**：报告中展示每个用例的状态、预期输出、实际输出、模拟后端、指令数和耗时。 +- **可对比**：支持保存 Benchmark 基线，并在后续运行中检查性能变化率。 +- **可集成**：普通模式和 Benchmark 模式采用统一 JSON schema，便于 CI 稳定解析。 +- **可选择**：支持按类别和名称运行子集，缩短定位单个模块问题时的等待时间。 +- **避免卡死**：编译阶段设置 30 秒超时，TinyFive 模拟阶段设置 5 秒超时，单个异常用例不会阻塞整套测试。 +- **便于诊断**：编译失败时保存命令、返回码、stdout、stderr 和耗时等现场信息。 +- **轻量运行**：默认报告不依赖绘图库和模板库，需要可视化时再通过 `--full-report` 启用。 + +### 1.3 需求演进与当前范围 + +项目最初按照 W1-W12 实现自动编译、模拟、结果对比、性能统计、报告和 CI。经过实际接入 TinyFive 以及 Mentor Review 后，程序增加了可靠性和工程化能力。新增内容不是另一套测试目标，而是保证原有目标能够在真实编译链路中稳定执行。 + +| 来源 | 当前实现内容 | 解决的问题 | +|---|---|---| +| 原始 W1-W4 | DSL 用例格式、自动调用编译器和模拟器、PASS/FAIL 对比 | 完成基本自动化正确性测试 | +| 原始 W5-W10 | 指令数、耗时、Benchmark、置信区间、基线和退化检测 | 完成性能测量与回归比较 | +| 原始 W11-W12 | GitHub Actions 示例和使用文档 | 支持 CI 和交付使用 | +| S1、A2 | 真实 TinyFive 直通验证，禁用 stub 回退，模拟子进程超时 | 防止模拟结果被替代，并避免模拟卡死 | +| S2、T1 | 编译超时和独立失败日志 | 防止编译卡死并保留崩溃现场 | +| A1 | 标量/张量期望值、dtype/shape 和外部期望文件校验 | 提高测试元数据的表达能力和错误可读性 | +| S3 | Markdown、HTML、JSON 统一字段，未执行数据使用 `null` | 避免普通模式和 Benchmark 模式的报告解析冲突 | +| I1 | 默认轻量报告与 `--full-report` 可选完整报告 | 降低日常测试和 CI 的依赖成本 | +| I2、I3 | 可配置退化阈值、Benchmark 失败短路 | 适应不同检查标准并避免重复等待失败用例 | +| T2 | `--category`、`--filter` 和子集基线合并 | 支持快速定位和增量测试 | +| P1 集成整改 | 根目录 pytest 契约测试、正式 GitHub Actions workflow、结构化寄存器映射 | 消除独立目录与 IR 文本格式造成的集成风险 | +| 正式目录迁移 | 脚本、用例、基线和文档分别归入仓库现有目录 | 消除独立交付目录，形成可直接合入的 local 测试套件 | + +因此，当前程序的交付边界不仅是“运行 23 个用例并生成性能报告”，还包括真实后端验证、异常隔离、稳定报告接口、失败诊断和子集执行。编译器后端本身的分支、数组及矩阵指令生成不属于测试套件的实现范围；测试套件负责真实暴露并记录这些问题。 + +--- + +## 二、目录结构 + +迁移后的仓库结构如下： + +```text +ScratchV/ + .github/ + workflows/ + topic06-benchmark.yml + scripts/ + run_topic06_benchmarks.py + tests/ + topic06/ + test_integration.py + cases/ + activation/ + branch/ + elementwise/ + loop/ + reduction/ + tensor/ + benchmarks/ + topic06/ + baseline.json + benchmark_reports/ + topic06/ + report.md + report.json + report.html + course_report_instructions.png + failures/ + build/ + topic06/ +``` + +其中： + +- `scripts/run_topic06_benchmarks.py` 是测试主程序。 +- `tests/topic06/cases/` 保存 DSL 测试用例，`tests/topic06/test_integration.py` 检查用例清单、编译器寄存器映射和报告路径契约。 +- `benchmarks/topic06/baseline.json` 是纳入版本控制的性能基线。 +- `benchmark_reports/topic06/` 保存可重新生成的测试报告与失败日志，不纳入版本控制。 +- `build/topic06/` 保存可重新生成的汇编与寄存器映射，不纳入版本控制。 +- `pyproject.toml` 的 `topic06` 和 `topic06-report` 可选依赖组分别支持轻量测试和完整报告。 +- `.github/workflows/topic06-benchmark.yml` 是 GitHub 实际识别并执行的 CI 配置。 + +原来的 `ScratchV-topic06-deliverable/` 独立目录在迁移后删除，仓库根目录的 `LICENSE` 和 `pyproject.toml` 统一承担许可与打包配置。 + +--- + +## 三、测试用例格式 + +每个测试用例由两个文件组成： + +```text +tests/topic06/cases/{category}/{name}.dsl +tests/topic06/cases/{category}/{name}.meta.json +``` + +示例 DSL： + +```text +result = add(a, b) +return result +``` + +示例元数据： + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "scalar", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +这里没有采用单独的 `.expected` 和 `.desc` 文件，而是使用 `.meta.json` 统一保存输入、描述和期望输出。这样结构更集中，也方便 Python 脚本直接解析。 + +`expected_output_type` 支持两种值： + +- `scalar`：标量结果，使用数字或布尔值；旧值 `return_value` 会兼容转换为 `scalar`。 +- `tensor`：向量或多维数组，必须同时声明 `output_dtype` 和 `output_shape`。 + +张量示例： + +```json +{ + "description": "2x2 matrix multiplication", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} +``` + +支持的 `output_dtype` 为 `bool`、`int32`、`int64`、`float32` 和 `float64`。加载元数据时，测试脚本会递归检查数组是否规则、实际形状是否与 `output_shape` 一致，以及每个元素是否符合声明的类型。 + +期望结果较大时，可以不使用内联的 `expected_return`，改为引用当前用例目录下的 JSON 文件： + +```json +{ + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "expected_output_file": "matmul_2x2.expected.json" +} +``` + +`expected_return` 和 `expected_output_file` 必须且只能存在一个。外部 JSON 可以直接保存标量或数组，也可以保存包含 `expected_return` 字段的对象。格式、shape 或 dtype 校验失败时，该用例报告 `metadata error` 并判定为 FAIL，不影响后续用例。 + +这一协议解决的是期望值的表达和校验。张量用例要在真实 TinyFive 路径下通过，还需要编译器后端约定输出内存地址并生成完整的数组计算及写回指令；这属于后端输入输出约定，不由元数据格式代替。 + +--- + +## 四、核心实现设计 + +### 4.1 编译阶段 + +函数：`run_compile(dsl_file: Path, timeout: float = 30.0)` + +作用： + +1. 接收一个 `.dsl` 文件路径。 +2. 调用 ScratchV 编译器： + +```text +python -m scratchv.main -o --optimize all --emit-register-map +``` + +3. 将生成的 RISC-V 汇编写入 `build/topic06/{case_name}.s`。 +4. 将寄存器分配结果写入 `build/topic06/{case_name}.registers.json`。 +5. 返回编译进程结果、汇编路径和寄存器映射路径。 + +这一阶段使用 `subprocess.run()` 调用编译器，能够捕获 `stdout`、`stderr` 和返回码。调用设置了默认 30 秒的 `timeout`：如果编译器在某个 DSL 上卡死，脚本会终止该编译子进程，返回错误 `compile timeout after 30s`，将当前用例标记为 FAIL，并继续运行后续用例。编译超时使用返回码 `124` 表示，不会进入 TinyFive 模拟阶段。 + +编译返回码非 0 时，`write_compile_failure_log()` 会生成： + +```text +benchmark_reports/topic06/failures/{category}-{case}.compile.log +``` + +日志保存以下现场信息： + +- 失败时间和 DSL 用例路径。 +- 完整编译命令。 +- 返回码和是否触发 30 秒 timeout。 +- 编译耗时、目标汇编路径以及汇编文件是否存在。 +- 完整 stdout 和 stderr，包括 Python traceback。 + +报告中的每条结果固定包含 `compile_returncode`、`compile_timed_out`、`compile_error` 和 `compile_log`。`compile_error` 保存最后一条非空错误信息，方便 CI 快速显示；`compile_log` 指向完整日志。成功用例的错误摘要和日志路径为 `null`。 + +### 4.2 模拟验证阶段 + +函数：`run_simulation(asm_file: Path, timeout: float = 5.0)` + +作用： + +1. 读取编译生成的 `.s` 汇编文件。 +2. 在独立 Python 子进程中调用： + +```python +from scratchv.simulator.tinyfive import verify_assembly +``` + +3. 获取 TinyFive 适配层返回的模拟结果。 +4. 如果模拟超过 5 秒，则终止该子进程，并返回： + +```python +{ + "success": False, + "backend": "timeout", + "error": "simulation timeout after 5s" +} +``` + +TinyFive 执行采用两层保护： + +1. `verify_assembly()` 内部调用 `m.run(instructions=100_000_000)`，最多执行一亿条指令，防止生成的汇编无限运行。 +2. `run_simulation()` 在外部使用独立子进程和 5 秒 timeout。即使 TinyFive 的指令上限执行过慢，或者模拟器内部其他步骤没有及时返回，主测试进程仍能终止该子进程。 + +如果真实 TinyFive 没有安装，`verify_assembly()` 会立即返回： + +```python +{ + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": "tinyfive not installed" +} +``` + +测试套件不会在 TinyFive 不可用时自动回退到 stub，因此报告中的 `backend: tinyfive` 表示实际调用了 TinyFive。TinyFive 未安装、执行异常或超过 5 秒都会使当前用例判定为 FAIL，但后续用例仍会继续执行。 + +这个设计解决了 branch 分支用例在真实 TinyFive 路径下可能长时间不返回的问题。当前 3 个 branch 用例触发 timeout，说明保护机制生效；它们的汇编跳转问题仍属于编译器后端问题，不属于 A2 保护机制本身。 + +### 4.3 TinyFive 直通验证 + +当前测试的输出正确性已经改为由 TinyFive 直通路径判断。流程是： + +```text +DSL + -> ScratchV 编译器 + -> RISC-V 汇编 + -> TinyFive 模拟执行 + -> 读取 a0/x10 作为 return_value + -> 与 .meta.json 中的 expected_return 对比 +``` + +寄存器分配器会直接产生虚拟寄存器到物理寄存器的映射，`CompileResult.stats["register_map"]` 保存该结构化数据。编译器 CLI 的 `--emit-register-map` 将其写为 JSON，例如： + +```json +{ + "schema_version": 1, + "register_map": { + "input": "t0", + "bias": "t1" + } +} +``` + +`run_topic06_benchmarks.py` 通过 `load_initial_registers()` 读取映射，并仅将 `.meta.json` 中可表示为整数寄存器值的标量输入注入 TinyFive。数组、矩阵和非整数浮点输入会明确标记为 `input_abi_unsupported`，即使模拟结果偶然等于期望值也不会判定编译路径通过。TinyFive 执行结束后，`verify_assembly()` 返回来自 `a0/x10` 的 `return_value`。 + +报告中的“TinyFive 输出”和“输出是否匹配”均来自真实编译产物的 TinyFive 执行结果。 + +### 4.4 PASS/FAIL 判断 + +一个用例通过需要同时满足： + +```python +ok = ( + result.returncode == 0 + and output_file.exists() + and sim_result["success"] + and matched + and not regression["regressed"] +) +``` + +也就是说，必须同时满足： + +- 编译成功。 +- 汇编文件生成成功。 +- TinyFive 模拟成功。 +- TinyFive 返回值与期望输出匹配。 +- Benchmark 模式下没有超过性能退化阈值。 +- Benchmark 重复没有因 timeout 或模拟失败提前停止。 + +如果 TinyFive 超时或执行失败，`sim_result["success"]` 为 `False`，该用例直接判定为 FAIL，不会使用其他执行结果替代。 + +### 4.5 指令数统计 + +模拟结果中包含 `instr_count` 字段。测试脚本会把该字段写入报告。 + +普通模式和 Benchmark 模式使用完全相同的报告字段。`instr_count` 始终记录首次 TinyFive 模拟的指令数；Benchmark 模式另外填写重复运行次数、平均值、最小值、最大值和 95% 置信区间。普通模式中的 Benchmark 专属字段统一写为 `null`，不再用 `0` 冒充未执行的统计结果。 + +相关函数： + +- `summarize_benchmark_runs(instr_counts)` +- `detect_regression(avg_instr_count, baseline_instr_count)` + +固定字段包括： + +| 字段 | 普通模式 | Benchmark 模式 | +|---|---|---| +| `mode` | `"normal"` | `"benchmark"` | +| `instr_count` | 单次模拟指令数 | 首次模拟指令数 | +| `compile_returncode` | 编译器返回码 | 编译器返回码 | +| `compile_timed_out` | 是否触发编译 timeout | 是否触发编译 timeout | +| `compile_error` / `compile_log` | 成功时为 `null`，失败时为摘要和日志路径 | 与普通模式相同 | +| `benchmark_runs` | `null` | 实际重复次数 | +| `benchmark_stopped_reason` | `null` | 正常完成时为 `null`，提前停止时记录原因 | +| `avg_instr_count` | `null` | 平均指令数 | +| `min_instr_count` / `max_instr_count` | `null` | 最小值/最大值 | +| `ci95_instr_count` | `null` | 95% 置信区间 | +| `baseline_instr_count` | `null` | 有基线时为基线值，否则为 `null` | +| `delta` / `delta_pct` | `null` | 有基线时为变化量和变化率，否则为 `null` | +| `threshold_pct` / `regressed` | `null` | 退化阈值和判断结果 | + +Markdown 和 HTML 报告固定展示相同列，不再根据模式增删列。CI 推荐读取 `benchmark_reports/topic06/report.json`，其顶层固定包含 `schema_version`、`mode`、`generated_at`、`summary`、`backend_matrix` 和 `results`，当前 schema 版本为 `2`。 + +### 4.6 耗时统计 + +测试脚本使用 `time.perf_counter()` 记录每个用例的耗时。 + +当前记录的字段包括： + +| 字段 | 说明 | +|---|---| +| `compile_time_sec` | 编译 DSL 到 RISC-V 汇编的耗时 | +| `simulation_time_sec` | TinyFive 模拟验证耗时 | +| `total_time_sec` | 单个用例完整处理耗时 | + +这些字段会输出到 Markdown 和 HTML 报告中，便于观察哪些用例编译慢、模拟慢或因为 timeout 导致耗时较长。 + +### 4.7 Benchmark 与性能基线 + +运行： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +表示每个用例重复模拟 3 次，统计平均指令数和 95% 置信区间。 + +运行： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --update-baseline +``` + +表示将当前 Benchmark 结果保存为性能基线： + +```text +benchmarks/topic06/baseline.json +``` + +后续再次运行： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +脚本会读取基线文件，并计算变化率： + +```text +变化率 = (当前平均指令数 - 基线指令数) / 基线指令数 * 100% +``` + +如果变化率超过配置阈值，则认为发生性能退化。默认阈值仍为 5%，可以使用命令行参数覆盖： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --regression-threshold 2 +``` + +上面的命令表示指令数相对基线增加超过 2% 就判定为退化。参数必须大于或等于 0；未指定时使用 `REGRESSION_THRESHOLD_PCT = 5.0`。实际阈值会传给 `detect_regression()`，并写入 Markdown、HTML 和 JSON 报告。这样日常开发、严格发布检查和宽松趋势监控可以使用不同标准。 + +Benchmark 模式会自动处理 timeout： + +1. 首次 TinyFive 模拟已经 timeout 时，直接跳过该用例的全部 Benchmark 重复。 +2. Benchmark 重复过程中一旦发生 timeout 或模拟失败，立即停止剩余次数。 +3. 只有成功运行的 `instr_count` 才进入平均值、最小值、最大值和置信区间计算，失败结果不会作为 `0` 混入统计。 +4. 报告中的 `benchmark_runs` 记录实际成功次数，`benchmark_stopped_reason` 记录跳过或提前停止原因。 +5. 没有成功 Benchmark 结果的用例不会写入性能基线，也不会进行变化率比较。 + +例如 branch 用例首次模拟等待 5 秒后 timeout，`--benchmark 3` 不会再额外等待 15 秒。 + +--- + +## 五、报告生成设计 + +### 5.1 Markdown 报告 + +输出文件： + +```text +benchmark_reports/topic06/report.md +``` + +内容包括： + +- 测试概览。 +- 用例总数、通过数量、失败数量、通过率。 +- 每个用例的状态、模拟后端、指令数、耗时、预期输出、实际输出。 +- Benchmark 模式下的平均指令数、置信区间、基线、变化率和是否退化。 +- Mermaid 指令数图表。 +- 每个用例的详细说明。 + +普通模式和 Benchmark 模式使用相同表头，未产生的 Benchmark 数据显示为 `null`。 + +Markdown 是默认轻量报告，不需要 `jinja2` 或 `matplotlib`。只有使用 `--full-report` 且图表生成成功时，Markdown 才会加入 PNG 图表链接，避免引用未生成或未刷新的图片。 + +### 5.2 JSON 报告 + +输出文件： + +```text +benchmark_reports/topic06/report.json +``` + +JSON 报告是提供给 CI 和其他程序的稳定解析接口。两种运行模式下字段集合保持一致，CI 只需要检查 `mode` 和字段值，不需要维护两套解析器。 + +顶层 `selection` 字段记录本次使用的 `category` 和 `filter`；未筛选时两个值均为 `null`，便于 CI 判断报告覆盖的是全量还是子集测试。 + +### 5.3 HTML 报告 + +输出文件： + +```text +benchmark_reports/topic06/report.html +``` + +HTML 报告主要用于浏览器查看，报告中会使用不同颜色标识 PASS 和 FAIL。它只在指定 `--full-report` 时生成，需要安装可选依赖 `jinja2`。 + +### 5.4 图表 + +输出文件： + +```text +benchmark_reports/topic06/course_report_instructions.png +``` + +图表由 `matplotlib` 生成，展示不同测试用例的指令数对比。它只在指定 `--full-report` 时生成；默认轻量模式不会导入 `matplotlib`，因此 CI 无需安装图表依赖。 + +### 5.5 报告模块重构 + +Commit Review 指出原 `run_tests.py` 曾保留多轮开发过程中产生的旧版中英文报告函数，存在多次重复定义。Python 实际只会使用最后一次定义，但这些不可达代码增加了文件体积和维护风险。 + +本次重构删除了全部未使用的历史报告生成器，只保留以下唯一实现： + +- `generate_unified_report_text_cn()`：生成 Markdown 报告。 +- `write_unified_html_report_cn()`：生成可选 HTML 报告。 +- `write_json_report()`：生成固定 schema 的 JSON 报告。 +- `write_report()`：统一协调 Markdown、JSON、HTML 和 PNG 输出。 + +`_markdown_cell()`、`_report_value()`、`_reported_instr_count()` 和 `write_chart()` 作为公共辅助函数继续保留。重构删除了约 850 行旧版代码；加入后续功能并迁移为 `scripts/run_topic06_benchmarks.py` 后仍约为 1000 行，且仅保留一套报告实现。报告字段和测试判定逻辑未因重构改变。 + +### 5.6 报告路径可移植性 + +测试运行时仍使用绝对路径调用编译器、读取用例和写入文件，避免工作目录变化影响执行。写入报告前，`_report_path()` 会将仓库内路径转换为相对于 ScratchV 仓库根目录的 POSIX 格式。 + +例如，原报告中的： + +```text +D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_only.dsl +``` + +现在写为： + +```text +tests/topic06/cases/activation/relu_only.dsl +``` + +该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖本机盘符与工作区位置，可以在 Windows、Linux、macOS 和 CI artifact 中查看。 + +--- + +## 六、运行方式 + +### 6.1 安装依赖 + +基础测试： + +```powershell +pip install -e ".[topic06]" +``` + +完整报告： + +```powershell +pip install -e ".[topic06-report]" +``` + +### 6.2 普通测试 + +```powershell +python scripts/run_topic06_benchmarks.py +``` + +默认只生成 `report.md` 和 `report.json`。生成 HTML 和 PNG 完整报告： + +```powershell +python scripts/run_topic06_benchmarks.py --full-report +``` + +### 6.3 子集测试 + +按测试目录类别筛选： + +```powershell +python scripts/run_topic06_benchmarks.py --category activation +``` + +按 DSL 文件名进行大小写不敏感的包含匹配： + +```powershell +python scripts/run_topic06_benchmarks.py --filter matmul +``` + +两个条件可以组合，组合时必须同时满足： + +```powershell +python scripts/run_topic06_benchmarks.py --category tensor --filter relu +``` + +筛选结果按路径排序，保证本地和 CI 的执行顺序稳定。没有匹配用例时返回退出码 2，不生成或覆盖报告。子集模式结合 `--update-baseline` 时会加载现有基线并只更新本次成功完成 Benchmark 的用例；全量模式仍会重建整份基线。 + +适合快速查看所有用例的 PASS/FAIL 和基本报告。 + +### 6.4 Benchmark 测试 + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +适合统计平均指令数和置信区间。 + +### 6.5 更新性能基线 + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --update-baseline +``` + +适合在修改编译器前或稳定版本上保存基线，供后续性能回归对比使用。 + +--- + +## 七、Mentor Review 对应状态 + +mentor review 中提出的问题分为架构、API、实现和测试覆盖四类。当前文档按真实代码状态整理如下： + +| 编号 | 问题 | 当前处理状态 | 说明 | +|---|---|---|---| +| P0 | 报告生成函数严重冗余 | 已处理 | 删除 5 组旧版/重复 Markdown 与 HTML 生成函数和约 850 行不可达代码；当前仅保留一套 Markdown、HTML、JSON 生成器和统一入口。 | +| P1a | 预生成报告包含 Windows 绝对路径 | 已处理 | `_report_path()` 在报告阶段统一输出相对仓库根目录的 POSIX 路径；生成报告迁入被忽略的 `benchmark_reports/topic06/`。 | +| P1b | 独立交付目录与项目测试/CI 脱节 | 已处理 | 核心脚本、用例、基线和文档已迁入仓库正式目录；`tests/topic06/test_integration.py` 和根目录 workflow 负责项目级集成。 | +| P1c | `infer_initial_registers()` 依赖 IR 文本格式 | 已处理 | 删除 IR 正则推断；寄存器分配器通过 `CompileResult.stats` 和 `--emit-register-map` 输出 JSON，测试套件按变量名读取真实物理寄存器。 | +| P2 | SPEC 中的 20 个通过、3 个失败与真实报告不一致 | 已处理 | 最新能力矩阵明确记录 DSLInterpreter 17 PASS/6 UNSUPPORTED，以及 TinyFive 12 PASS/3 TIMEOUT/8 INPUT_ABI_UNSUPPORTED；不再把数组输入未注入产生的偶然匹配计为通过。 | +| S1 | 参考解释器与 ScratchV 编译器语义脱节 | 已处理并演进 | 使用仓库正式合入的 `DSLInterpreter` 作为独立语义后端，同时保留真实 TinyFive 编译路径。两者分别记录状态，解释器不作为 TinyFive 的 fallback；循环和分支在解释器中明确标记为不支持。 | +| S2 | 编译阶段缺 timeout | 已处理 | `run_compile()` 默认设置 30 秒 timeout；超时后返回编译失败结果并继续执行后续用例。 | +| S3 | Benchmark/普通模式报告格式冲突 | 已处理 | Markdown、HTML 和 JSON 使用固定字段；普通模式的 Benchmark 专属字段为 `null`，CI 使用 `report.json` 统一解析。 | +| A1 | `.meta.json` 期望值表达能力有限 | 已处理 | 支持 `scalar`/`tensor`、嵌套数组、dtype/shape 校验和 `expected_output_file`；非法元数据只使当前用例失败。 | +| A2 | `verify_assembly` 调用风险 | 已处理 | TinyFive 未安装时明确失败且不回退 stub；内部设置一亿条指令上限，外层使用独立子进程和 5 秒 timeout，异常只影响当前用例。 | +| I1 | 报告依赖过重 | 已处理 | 默认只生成 Markdown 和 JSON，且不加载 `jinja2`/`matplotlib`；`--full-report` 才生成 HTML 和 PNG，可选依赖由 `pyproject.toml` 的 `topic06-report` 组管理。 | +| I2 | 5% 退化阈值过于刚性 | 已处理 | 新增 `--regression-threshold`，默认 5%，支持非负自定义值；实际阈值参与退化判断并写入 Markdown、HTML 和 JSON 报告。 | +| I3 | Benchmark 重复运行 timeout 用例 | 已处理 | 首次模拟 timeout 时跳过全部重复；重复过程中 timeout/失败则立即停止，失败结果不计入统计，报告记录实际次数和停止原因。 | +| T1 | 编译器 crash 无上下文保存 | 已处理 | 编译失败或超时时保存命令、返回码、耗时、stdout、stderr 和汇编状态到独立日志，并在 Markdown、HTML、JSON 和 CI artifact 中关联日志路径。 | +| T2 | 不支持增量/子集测试 | 已处理 | 新增大小写不敏感的 `--category` 和 `--filter`，支持组合筛选、稳定排序和无匹配退出码 2；子集更新基线采用合并策略。 | + +--- + +## 八、当前测试现状 + +最近一次使用真实 TinyFive 执行全部 23 个用例的报告结果如下（运行后可在 `benchmark_reports/topic06/` 查看）： + +```text +Total: 23 +Passed: 12 +Failed: 11 +Pass rate: 56.5% +``` + +早期 SPEC 中的 `Passed: 20, Failed: 3` 和后续 13/10 都包含过期或偶然匹配数据。当前 TinyFive 的真实状态是 12/23：3 个分支超时，8 个数组/矩阵输入 ABI 不支持。DSLInterpreter 独立状态为 17 PASS、6 个控制流不支持。 + +失败用例分为两类。 + +第一类是 `branch/` 目录下 3 个 if 分支用例： + +```text +branch/if_else.dsl +branch/if_relu.dsl +branch/if_then.dsl +``` + +失败原因是 TinyFive 模拟阶段 5 秒超时： + +```text +simulation timeout after 5s +``` + +第二类是以下 7 个数组或矩阵用例： + +```text +reduction/dot_4.dsl +reduction/dot_8.dsl +reduction/dot_relu_8.dsl +tensor/matmul_2x2.dsl +tensor/matmul_4x4.dsl +tensor/matmul_add_2x2.dsl +tensor/matmul_relu_2x2.dsl +``` + +这些用例失败有两层原因： + +1. 测试套件目前根据编译器输出的结构化寄存器映射初始化标量输入，但仍会跳过 `.meta.json` 中的数组和矩阵输入，也没有把数组写入 TinyFive 内存、传递首地址或从结果内存读取矩阵。 +2. 更根本的原因是 RISC-V 指令选择器当前把 `dot` 和 `matmul` 都降低成单条 `mul`。生成汇编没有数组元素读取指令、循环累加或矩阵结果写回，也没有使用 IR 中保存的 `length`、`m`、`n`、`k` 属性。因此只完善测试套件的内存初始化仍不能得到正确结果，后续需要先明确数组调用约定并实现后端降低逻辑。 + +`reduction/dot_relu_4.dsl` 的期望值和 TinyFive 未注入数组时的返回值都恰好为 0。新版能力判定会将它标记为 `input_abi_unsupported`，不再计入 PASS，也不写入性能基线。 + +当前报告中的 12 个通过用例可以完成编译、TinyFive 模拟和真实返回值对比。 + +--- + +## 九、CI 集成 + +仓库根目录包含正式 GitHub Actions workflow： + +```text +.github/workflows/topic06-benchmark.yml +``` + +CI 在 push 或 pull request 时执行三层检查： + +1. `tests/topic06/test_integration.py` 验证 23 组用例元数据、编译器结构化寄存器映射、Runner 映射读取和跨平台报告路径。 +2. `activation`、`elementwise`、`loop` 三个当前可靠类别使用 `--fail-on-test-failure` 作为阻塞门禁；任一用例失败时 workflow 失败。 +3. 全量 23 例运行 Benchmark 并生成诊断报告。已知未实现的 branch/reduction/tensor 后端能力不会隐藏，但在修复前不阻塞其他提交。 + +CI 使用 `pip install -e ".[topic06]"` 安装基础依赖，并通过 `if: always()` 上传 `benchmark_reports/topic06/` 中的报告与失败日志，同时保留 `benchmarks/topic06/baseline.json`。CI 不安装 `jinja2`、`matplotlib`，也不生成 HTML、PNG；完整可视化报告仍在本地通过 `--full-report` 生成。 + +--- + +## 十、已完成工作对照 + +| 阶段 | 完成情况 | +|---|---| +| W1 学习编译命令和 TinyFive 用法 | 已完成 | +| W2 使用 subprocess 自动调用编译器和模拟器 | 已完成 | +| W3 设计测试用例格式 | 已完成，采用 `.dsl + .meta.json` | +| W4 遍历用例并输出 PASS/FAIL | 已完成 | +| W5 提取指令数并加入报告 | 已完成 | +| W6 matplotlib 图表和 HTML 报告 | 已完成 | +| W7 扩充到 15 个以上用例并覆盖分支循环 | 已完成，共 23 个 | +| W8 使用 `time.perf_counter()` 统计耗时 | 已完成 | +| W9 回归测试模式和 5% 退化阈值 | 已完成 | +| W10 `--benchmark` 重复运行和置信区间 | 已完成 | +| W11 GitHub Actions CI 示例 | 已完成 | +| W12 完整文档 | 已完成 | + +--- + +## 十一、双后端与 Cost Model 整改（P0-P2） + +### P0：双后端独立验证 + +新增 `--verification-backend both|tinyfive|interpreter`，默认使用 `both`。DSLInterpreter 直接执行 DSL，用于验证语言语义和期望数据；TinyFive 执行 ScratchV 编译产生的 RISC-V，用于验证编译路径。报告分别保存 `interpreter_status` 和 `tinyfive_status`，任何一条路径失败都不会调用另一条路径伪造其结果。 + +解释器当前支持算术、激活、dot 和 matmul，但没有正确实现循环与条件分支语义。Runner 会在执行前识别 `for/if/while` 等控制流并标记 `UNSUPPORTED`，避免解释器静默跳过控制语句后产生错误结果。 + +### P1：能力矩阵与单用例报告 + +总报告新增 `backend_matrix`，每个用例记录后端状态、失败类型、实际输出、与期望值是否匹配以及两后端结果是否一致。只有 TinyFive 输入 ABI 有效且两边都成功执行时，才会进行跨后端结果比较。 + +`benchmark_reports/topic06/cases/` 为每个用例生成独立的 Markdown 和 JSON 报告，内容包括 DSL 元信息、双后端状态、失败分类、编译命令、寄存器映射、完整汇编、指令数、耗时和基线差异。当前失败类型包括编译错误、编译超时、解释器控制流不支持、TinyFive 超时、汇编编码错误、输入 ABI 不支持和结果不匹配。 + +### P2：Cost Model 性能看护 + +`baseline.json` 升级到 schema 2，主指标为 TinyFive 动态执行指令数，同时保存： + +- 静态汇编指令数； +- 编码后机器指令数； +- 代码大小（字节）； +- TinyFive 动态执行指令数； +- load、store、mul、add、madd、branch 动态分类计数； +- Benchmark 次数及生成环境。 + +各指标均按照 `--regression-threshold` 与基线独立比较，任一可比较指标超过阈值都会标记 cost-model 退化。只有输入 ABI 有效、真实 TinyFive 执行成功且输出正确的 Benchmark 才能写入基线。目前基线包含 activation、elementwise 和 loop 共 12 个稳定用例，避免使用 branch 超时和 reduction/tensor 未接通输入产生的不可信性能数字。 + +--- diff --git a/docs/topics/INDEX.md b/docs/topics/INDEX.md index c8ebe44..a0eb344 100644 --- a/docs/topics/INDEX.md +++ b/docs/topics/INDEX.md @@ -39,7 +39,7 @@ | 编号 | 课题 | 类型 | 状态 | |------|------|------|------| -| [06](06-性能基准套件.md) | 编译器性能测试套件 | 项目实战 | ✅ | +| [06](06-性能测试套件使用说明.md) | 编译器性能测试套件 | 项目实战 | ✅ | | [12](12-指令计数统计器.md) | RISC-V 后端指令计数统计器 | 项目实战 | ✅ | | [23](23-Cache模型.md) | Cache 行为分析（组相联 + LRU） | 参考分析 | ✅ | | [24](24-Spike仿真.md) | Spike 仿真集成 | 参考分析 | ✅ | diff --git a/pyproject.toml b/pyproject.toml index ce12d92..9fbc77f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ riscv = ["tinyfive"] llvm = ["llvmlite"] # optional: LLVM IR JIT execution verify = ["onnxruntime"] # optional: ONNX Runtime comparison all = ["tinyfive", "llvmlite", "onnxruntime"] +topic06 = ["tinyfive", "pytest"] +topic06-report = ["tinyfive", "pytest", "jinja2", "matplotlib"] [project.urls] Source = "https://github.com/kinsomwang/ScratchV" diff --git a/scratchv/backend/register_alloc.py b/scratchv/backend/register_alloc.py index 15d3e1b..134e214 100644 --- a/scratchv/backend/register_alloc.py +++ b/scratchv/backend/register_alloc.py @@ -64,6 +64,11 @@ def run(self) -> list[MachineInstr]: else: return self._allocate_greedy() + @property + def register_map(self) -> dict[str, str]: + """Return the current virtual-to-physical register mapping.""" + return dict(self._vreg_map) + def _allocate_naive(self) -> list[MachineInstr]: """Spill every virtual register to the stack.""" self._output = [] diff --git a/scratchv/compiler.py b/scratchv/compiler.py index fa5459e..633fb3d 100644 --- a/scratchv/compiler.py +++ b/scratchv/compiler.py @@ -221,6 +221,7 @@ class CompilerDriver: def __init__(self, config: CompilerConfig | None = None): self.config = config or CompilerConfig() + self._last_register_map: dict[str, str] = {} # ── Public API ────────────────────────────────────────────────────────── @@ -238,6 +239,7 @@ def compile(self, input_path: str, output_path: str | None = None, """ errors: list[str] = [] warnings: list[str] = [] + self._last_register_map = {} # Resolve output path if output_path is None: @@ -353,7 +355,11 @@ def compile(self, input_path: str, output_path: str | None = None, output_text=asm_text, output_path=output_path, ir_dump=ir_dump, - stats={"opt_message": opt_message, "cycle_report": cycle_report}, + stats={ + "opt_message": opt_message, + "cycle_report": cycle_report, + "register_map": dict(self._last_register_map), + }, warnings=warnings, ) @@ -444,10 +450,13 @@ def _generate_riscv_linear(self, program) -> str: ) ls_insts = block_from_machine_instrs(machine_instrs) lsa = LinearScanAllocator() - return lsa.emit(ls_insts) + asm_text = lsa.emit(ls_insts) + self._last_register_map = dict(lsa.alloc_map) + return asm_text alloc = RegisterAllocator(machine_instrs, mode=self.config.reg_alloc) allocated = alloc.run() + self._last_register_map = alloc.register_map emitter = AsmEmitter(allocated) return emitter.emit() @@ -468,6 +477,7 @@ def _generate_riscv_dag(self, program) -> str: alloc = RegisterAllocator(machine_instrs, mode=self.config.reg_alloc) allocated = alloc.run() + self._last_register_map = alloc.register_map emitter = AsmEmitter(allocated) return emitter.emit() diff --git a/scratchv/main.py b/scratchv/main.py index 52feff5..067c682 100644 --- a/scratchv/main.py +++ b/scratchv/main.py @@ -12,7 +12,9 @@ from __future__ import annotations import argparse +import json import sys +from pathlib import Path from scratchv.compiler import CompilerConfig, CompilerDriver, CompileResult @@ -55,6 +57,11 @@ def build_arg_parser() -> argparse.ArgumentParser: "--dump-ir", action="store_true", help="Dump IR before and after optimization", ) + parser.add_argument( + "--emit-register-map", + metavar="PATH", + help="Write the virtual-to-physical register mapping as JSON", + ) # ── Verification ──────────────────────────────────────────────────── parser.add_argument( @@ -256,6 +263,24 @@ def main(argv: list[str] | None = None) -> int: print(result.ir_dump, file=sys.stderr) if result.success: + if args.emit_register_map: + register_map_path = Path(args.emit_register_map) + try: + register_map_path.parent.mkdir(parents=True, exist_ok=True) + register_map_path.write_text( + json.dumps( + { + "schema_version": 1, + "register_map": result.stats.get("register_map", {}), + }, + indent=2, + ), + encoding="utf-8", + ) + except OSError as exc: + print(f"Error: failed to write register map: {exc}", file=sys.stderr) + return 1 + print(f"OK {args.backend.upper()} output written to {result.output_path}", file=sys.stderr) for w in result.warnings: diff --git a/scratchv/simulator/tinyfive.py b/scratchv/simulator/tinyfive.py index 310d47e..951fd6d 100644 --- a/scratchv/simulator/tinyfive.py +++ b/scratchv/simulator/tinyfive.py @@ -173,6 +173,43 @@ def run( after = int(self._m.ops.get('total', 0)) self.instr_count = after - before + def run_until( + self, + end: int, + max_instructions: int = 100_000_000, + start: Optional[int] = None, + *, + strict: bool = False, + ): + """Execute until the program counter reaches *end* or the limit.""" + if not self._available: + return + start_pc = self.pc if start is None else start + before = int(self._m.ops.get('total', 0)) + self.last_error = None + try: + # TinyFive's exe() also normalizes pc to a scalar before decoding. + # Keeping its initial one-element NumPy array breaks branch updates. + self._m.pc = int(start_pc) + with np.errstate(over="ignore"): + for _ in range(max(0, max_instructions)): + if self.pc == end: + break + inst = self._m.read_i32(self.pc) + self._m.dec(np.binary_repr(self._m.u(inst), 32)) + else: + raise RuntimeError( + f"instruction limit exceeded: {max_instructions}" + ) + except Exception as exc: + self.last_error = f"{type(exc).__name__}: {exc}" + if strict: + raise RuntimeError(self.last_error) from exc + finally: + self._m.x[0] = 0 + after = int(self._m.ops.get('total', 0)) + self.instr_count = after - before + # ── Register access ───────────────────────────────────────────────── def get_reg(self, idx: int) -> int: @@ -251,7 +288,7 @@ def _write_u32(self, addr: int, value): def _set_pc(self, val: int): if not self._available: return - if hasattr(self._m.pc, '__getitem__'): + if isinstance(self._m.pc, np.ndarray): self._m.pc[0] = np.uint32(val) # type: ignore[index] else: self._m.pc = int(val) @@ -261,7 +298,7 @@ def pc(self) -> int: if not self._available: return 0 pc_val = self._m.pc - if hasattr(pc_val, '__getitem__'): + if isinstance(pc_val, np.ndarray): return int(pc_val[0]) # type: ignore[index] return int(pc_val) @@ -343,7 +380,47 @@ def pc(self, val: int): self._pc = val -def verify_assembly(asm_code: str, verbose: bool = False) -> dict: +_REG_NUMS: dict[str, int] = { + "zero": 0, "x0": 0, + "ra": 1, "x1": 1, + "sp": 2, "x2": 2, + "gp": 3, "x3": 3, + "tp": 4, "x4": 4, + "t0": 5, "x5": 5, + "t1": 6, "x6": 6, + "t2": 7, "x7": 7, + "s0": 8, "fp": 8, "x8": 8, + "s1": 9, "x9": 9, + "a0": 10, "x10": 10, + "a1": 11, "x11": 11, + "a2": 12, "x12": 12, + "a3": 13, "x13": 13, + "a4": 14, "x14": 14, + "a5": 15, "x15": 15, + "a6": 16, "x16": 16, + "a7": 17, "x17": 17, + "s2": 18, "x18": 18, + "s3": 19, "x19": 19, + "s4": 20, "x20": 20, + "s5": 21, "x21": 21, + "s6": 22, "x22": 22, + "s7": 23, "x23": 23, + "s8": 24, "x24": 24, + "s9": 25, "x25": 25, + "s10": 26, "x26": 26, + "s11": 27, "x27": 27, + "t3": 28, "x28": 28, + "t4": 29, "x29": 29, + "t5": 30, "x30": 30, + "t6": 31, "x31": 31, +} + + +def verify_assembly( + asm_code: str, + verbose: bool = False, + initial_registers: Optional[dict[str, int]] = None, +) -> dict: """Verify generated assembly by running it in TinyFive. Uses ``RISCVAEncoder`` to assemble text → binary, then executes each @@ -353,16 +430,24 @@ def verify_assembly(asm_code: str, verbose: bool = False) -> dict: Args: asm_code: RISC-V assembly text. verbose: Print performance info. + initial_registers: Optional ABI register values to set before execution. Returns: - dict with keys: success, instr_count, error + dict with keys: success, instr_count, return_value, error """ m = ProfiledMachine(mem_size=128 * 1024 * 1024) + machine_code_instructions = None + code_size_bytes = None if not m.available: return { "success": False, "instr_count": 0, + "return_value": None, + "backend": "tinyfive", "error": "tinyfive not installed", + "machine_code_instructions": None, + "code_size_bytes": None, + "perf_counters": {}, } try: @@ -371,19 +456,48 @@ def verify_assembly(asm_code: str, verbose: bool = False) -> dict: binary = assemble_to_binary(asm_code) if not binary: raise ValueError("assembler produced empty binary") + code_size_bytes = len(binary) words = [ int.from_bytes(binary[i:i + 4], "little") for i in range(0, len(binary), 4) ] + machine_code_instructions = len(words) m.load_binary(words, origin=0) - m.run(instructions=len(words), start=0, strict=True) + + for reg_name, value in (initial_registers or {}).items(): + reg_num = _REG_NUMS.get(reg_name) + if reg_num is not None: + m.set_reg(reg_num, int(value)) + + return_address = len(binary) + m.set_reg(1, return_address) + m.run_until( + end=return_address, + max_instructions=100_000_000, + start=0, + strict=True, + ) except Exception as e: return { "success": False, "instr_count": m.instr_count, + "return_value": None, + "backend": "tinyfive", "error": str(e), + "machine_code_instructions": machine_code_instructions, + "code_size_bytes": code_size_bytes, + "perf_counters": m.get_perf(), } if verbose: m.print_perf() - return {"success": True, "instr_count": m.instr_count, "error": None} + return { + "success": True, + "instr_count": m.instr_count, + "return_value": m.get_reg(10), + "backend": "tinyfive", + "error": None, + "machine_code_instructions": machine_code_instructions, + "code_size_bytes": code_size_bytes, + "perf_counters": m.get_perf(), + } diff --git a/scripts/run_topic06_benchmarks.py b/scripts/run_topic06_benchmarks.py new file mode 100644 index 0000000..ecbc860 --- /dev/null +++ b/scripts/run_topic06_benchmarks.py @@ -0,0 +1,1521 @@ +import argparse +import json +import math +import os +import platform +import re +import shlex +import subprocess +import sys +import tempfile +import time +from datetime import datetime +from pathlib import Path + +from scratchv.simulator.tinyfive import verify_assembly + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +TEST_DIR = PROJECT_ROOT / "tests" / "topic06" / "cases" +BUILD_DIR = PROJECT_ROOT / "build" / "topic06" +REPORT_DIR = PROJECT_ROOT / "benchmark_reports" / "topic06" +REPORT_FILE = REPORT_DIR / "report.md" +HTML_REPORT_FILE = REPORT_DIR / "report.html" +JSON_REPORT_FILE = REPORT_DIR / "report.json" +CHART_FILE = REPORT_DIR / "course_report_instructions.png" +BASELINE_FILE = PROJECT_ROOT / "benchmarks" / "topic06" / "baseline.json" +FAILURE_DIR = REPORT_DIR / "failures" +CASE_REPORT_DIR = REPORT_DIR / "cases" +REGRESSION_THRESHOLD_PCT = 5.0 +COMPILE_TIMEOUT_SEC = 30.0 +SIMULATION_TIMEOUT_SEC = 5.0 +INTERPRETER_TIMEOUT_SEC = 5.0 + +INTERPRETER_SUPPORTED_OPS = { + "add", "sub", "mul", "div", "neg", "exp", "relu", "gelu", + "matmul", "dot", "softmax", "maxpool", +} + + +def run_compile(dsl_file: Path, timeout: float = COMPILE_TIMEOUT_SEC): + output_file = BUILD_DIR / (dsl_file.stem + ".s") + register_map_file = BUILD_DIR / (dsl_file.stem + ".registers.json") + register_map_file.unlink(missing_ok=True) + + cmd = [ + sys.executable, + "-m", + "scratchv.main", + str(dsl_file), + "-o", + str(output_file), + "--optimize", + "all", + "--emit-register-map", + str(register_map_file), + ] + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout.decode("utf-8", errors="ignore") if isinstance(exc.stdout, bytes) else (exc.stdout or "") + stderr = exc.stderr.decode("utf-8", errors="ignore") if isinstance(exc.stderr, bytes) else (exc.stderr or "") + timeout_message = f"compile timeout after {timeout:.0f}s" + stderr = f"{stderr.rstrip()}\n{timeout_message}" if stderr else timeout_message + result = subprocess.CompletedProcess( + args=cmd, + returncode=124, + stdout=stdout, + stderr=stderr, + ) + + return result, output_file, register_map_file + + +def _last_nonempty_line(text): + lines = [line.strip() for line in (text or "").splitlines() if line.strip()] + return lines[-1] if lines else None + + +def write_compile_failure_log(dsl_file, result, output_file, compile_time_sec): + FAILURE_DIR.mkdir(parents=True, exist_ok=True) + safe_name = re.sub( + r"[^A-Za-z0-9_.-]+", + "_", + f"{dsl_file.parent.name}-{dsl_file.stem}", + ) + log_file = FAILURE_DIR / f"{safe_name}.compile.log" + args = result.args + if isinstance(args, (list, tuple)): + command = shlex.join(str(arg) for arg in args) + else: + command = str(args or "(compiler was not started)") + stdout = result.stdout or "" + stderr = result.stderr or "" + log_file.write_text( + "\n".join([ + f"Timestamp: {datetime.now().isoformat(timespec='seconds')}", + f"Case: {dsl_file}", + f"Command: {command}", + f"Return code: {result.returncode}", + f"Timed out: {result.returncode == 124}", + f"Compile time (s): {compile_time_sec:.4f}", + f"Assembly output: {output_file}", + f"Assembly exists: {output_file.exists()}", + "", + "--- stdout ---", + stdout, + "", + "--- stderr ---", + stderr, + "", + ]), + encoding="utf-8", + ) + return log_file + + +SUPPORTED_OUTPUT_DTYPES = {"bool", "int32", "int64", "float32", "float64"} + + +def _value_shape(value): + if not isinstance(value, list): + return () + if not value: + return (0,) + child_shapes = [_value_shape(item) for item in value] + if any(shape != child_shapes[0] for shape in child_shapes[1:]): + raise ValueError("expected output must be a rectangular tensor") + return (len(value),) + child_shapes[0] + + +def _validate_output_dtype(value, dtype): + values = value if isinstance(value, list) else [value] + for item in values: + if isinstance(item, list): + _validate_output_dtype(item, dtype) + elif dtype == "bool" and not isinstance(item, bool): + raise ValueError(f"expected output contains non-bool value: {item!r}") + elif dtype.startswith("int") and (isinstance(item, bool) or not isinstance(item, int)): + raise ValueError(f"expected output contains non-integer value: {item!r}") + elif dtype.startswith("float") and (isinstance(item, bool) or not isinstance(item, (int, float))): + raise ValueError(f"expected output contains non-numeric value: {item!r}") + + +def _load_expected_output(metadata, meta_file): + has_inline = "expected_return" in metadata + has_file = "expected_output_file" in metadata + if has_inline == has_file: + raise ValueError("define exactly one of expected_return or expected_output_file") + + if has_inline: + return metadata["expected_return"] + + expected_file = meta_file.parent / metadata["expected_output_file"] + payload = json.loads(expected_file.read_text(encoding="utf-8")) + if isinstance(payload, dict) and "expected_return" in payload: + return payload["expected_return"] + return payload + + +def _validate_metadata(metadata, meta_file): + expected = _load_expected_output(metadata, meta_file) + output_type = metadata.get("expected_output_type", "scalar") + if output_type == "return_value": + output_type = "scalar" + if output_type not in {"scalar", "tensor"}: + raise ValueError(f"unsupported expected_output_type: {output_type}") + + actual_shape = _value_shape(expected) + if output_type == "scalar" and actual_shape: + raise ValueError("scalar expected output cannot contain a list") + if output_type == "tensor" and not actual_shape: + raise ValueError("tensor expected output must contain a nested JSON array") + + declared_shape = metadata.get("output_shape") + if output_type == "tensor" and declared_shape is None: + raise ValueError("tensor expected output requires output_shape") + if declared_shape is not None: + if not isinstance(declared_shape, list) or any( + isinstance(size, bool) or not isinstance(size, int) or size < 0 + for size in declared_shape + ): + raise ValueError("output_shape must be a list of non-negative integers") + if tuple(declared_shape) != actual_shape: + raise ValueError( + f"output_shape {declared_shape} does not match expected output shape {list(actual_shape)}" + ) + + dtype = metadata.get("output_dtype") + if output_type == "tensor" and dtype is None: + raise ValueError("tensor expected output requires output_dtype") + if dtype is not None: + if dtype not in SUPPORTED_OUTPUT_DTYPES: + raise ValueError(f"unsupported output_dtype: {dtype}") + _validate_output_dtype(expected, dtype) + + metadata["expected_output_type"] = output_type + metadata["expected_return"] = expected + return metadata + + +def load_metadata(dsl_file: Path): + meta_file = dsl_file.with_suffix(".meta.json") + if not meta_file.exists(): + return { + "description": "", + "expected_output_type": "scalar", + "expected_return": None, + "_metadata_error": f"metadata file not found: {meta_file}", + } + try: + metadata = json.loads(meta_file.read_text(encoding="utf-8")) + return _validate_metadata(metadata, meta_file) + except (OSError, ValueError, TypeError, json.JSONDecodeError) as exc: + return { + "description": "", + "expected_output_type": "invalid", + "expected_return": None, + "_metadata_error": str(exc), + } + + +def load_initial_registers(register_map_file: Path, inputs: dict) -> dict[str, int]: + """Load compiler-emitted register assignments for scalar DSL inputs.""" + try: + payload = json.loads(register_map_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid compiler register map: {exc}") from exc + + register_map = payload.get("register_map") + if not isinstance(register_map, dict): + raise ValueError("invalid compiler register map: missing register_map object") + + result: dict[str, int] = {} + for name, value in inputs.items(): + register = register_map.get(name) + if not isinstance(register, str): + continue + if isinstance(value, bool): + result[register] = int(value) + elif isinstance(value, int): + result[register] = value + elif isinstance(value, float) and value.is_integer(): + result[register] = int(value) + return result + + +def run_simulation( + asm_file: Path, + initial_registers: dict[str, int] | None = None, + timeout: float = SIMULATION_TIMEOUT_SEC, +): + if not asm_file.exists(): + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": "assembly file not found", + } + + code = "\n".join([ + "import json, sys", + "from pathlib import Path", + "from scratchv.simulator.tinyfive import verify_assembly", + "asm = Path(sys.argv[1]).read_text(encoding='utf-8')", + "initial_registers = json.loads(sys.argv[2])", + "try:", + " result = verify_assembly(asm, initial_registers=initial_registers)", + "except Exception as exc:", + " result = {'success': False, 'instr_count': 0, " + "'return_value': None, 'backend': 'tinyfive', 'error': str(exc)}", + "print(json.dumps(result, ensure_ascii=False))", + ]) + + try: + completed = subprocess.run( + [ + sys.executable, + "-c", + code, + str(asm_file), + json.dumps(initial_registers or {}), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "timeout", + "error": f"simulation timeout after {timeout:.0f}s", + } + + if completed.returncode != 0: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": (completed.stderr or completed.stdout or "simulation failed").strip(), + } + + try: + return json.loads(completed.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as exc: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": f"invalid simulation output: {exc}", + } + + +def interpreter_support_for_source(source: str) -> tuple[bool, str | None]: + """Reject syntax that DSLInterpreter currently accepts without executing correctly.""" + control_flow = re.search( + r"^\s*(for|endfor|if|else|endif|while|endwhile)\b", + source, + flags=re.MULTILINE | re.IGNORECASE, + ) + if control_flow: + return False, f"unsupported control flow: {control_flow.group(1).lower()}" + + operations = { + match.group(1).lower() + for match in re.finditer(r"^\s*\w+\s*=\s*(\w+)\s*\(", source, re.MULTILINE) + } + unsupported = sorted(operations - INTERPRETER_SUPPORTED_OPS) + if unsupported: + return False, f"unsupported interpreter operation(s): {', '.join(unsupported)}" + return True, None + + +def run_interpreter( + dsl_file: Path, + inputs: dict, + timeout: float = INTERPRETER_TIMEOUT_SEC, +): + """Run the DSL reference interpreter in an isolated process.""" + source = dsl_file.read_text(encoding="utf-8") + supported, reason = interpreter_support_for_source(source) + if not supported: + return { + "success": False, + "return_value": None, + "backend": "interpreter", + "status": "UNSUPPORTED", + "error": reason, + } + + code = "\n".join([ + "import json, sys", + "from pathlib import Path", + "import numpy as np", + "from scratchv.verification.verifier import DSLInterpreter", + "source = Path(sys.argv[1]).read_text(encoding='utf-8')", + "raw_inputs = json.loads(sys.argv[2])", + "inputs = {name: np.asarray(value) for name, value in raw_inputs.items()}", + "try:", + " value = DSLInterpreter().run(source, inputs)", + " if isinstance(value, np.ndarray):", + " value = value.item() if value.ndim == 0 else value.tolist()", + " elif isinstance(value, np.generic):", + " value = value.item()", + " result = {'success': True, 'return_value': value, " + "'backend': 'interpreter', 'status': 'EXECUTED', 'error': None}", + "except Exception as exc:", + " result = {'success': False, 'return_value': None, " + "'backend': 'interpreter', 'status': 'ERROR', 'error': str(exc)}", + "print(json.dumps(result, ensure_ascii=False))", + ]) + + try: + completed = subprocess.run( + [sys.executable, "-c", code, str(dsl_file), json.dumps(inputs)], + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return { + "success": False, + "return_value": None, + "backend": "interpreter", + "status": "TIMEOUT", + "error": f"interpreter timeout after {timeout:.0f}s", + } + + if completed.returncode != 0: + return { + "success": False, + "return_value": None, + "backend": "interpreter", + "status": "ERROR", + "error": (completed.stderr or completed.stdout or "interpreter failed").strip(), + } + try: + return json.loads(completed.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as exc: + return { + "success": False, + "return_value": None, + "backend": "interpreter", + "status": "ERROR", + "error": f"invalid interpreter output: {exc}", + } + + +def values_equal(lhs, rhs): + if isinstance(lhs, list) and isinstance(rhs, list): + if len(lhs) != len(rhs): + return False + return all(values_equal(a, b) for a, b in zip(lhs, rhs)) + if isinstance(lhs, float) or isinstance(rhs, float): + return math.isclose(lhs, rhs, rel_tol=1e-7, abs_tol=1e-7) + return lhs == rhs + + +def tinyfive_input_abi_supported(inputs: dict) -> tuple[bool, str | None]: + """Return whether every input can be injected directly into an integer register.""" + unsupported = [] + for name, value in inputs.items(): + if isinstance(value, bool) or isinstance(value, int): + continue + if isinstance(value, float) and value.is_integer(): + continue + unsupported.append(name) + if unsupported: + return False, "non-scalar TinyFive input ABI is unavailable: " + ", ".join(unsupported) + return True, None + + +def classify_tinyfive_result( + compile_result, + sim_result, + matched, + input_abi_supported, + metadata_error=None, + register_map_error=None, +): + if metadata_error: + return "SKIPPED", "metadata_error" + if compile_result.returncode == 124: + return "SKIPPED", "compile_timeout" + if compile_result.returncode != 0: + return "SKIPPED", "compile_error" + if register_map_error: + return "SKIPPED", "register_map_error" + if not input_abi_supported: + return "UNSUPPORTED", "input_abi_unsupported" + if sim_result.get("backend") == "skipped": + return "SKIPPED", "backend_not_selected" + if sim_result.get("backend") == "timeout": + return "TIMEOUT", "simulation_timeout" + if not sim_result.get("success"): + error = str(sim_result.get("error") or "").lower() + if "not installed" in error: + return "UNAVAILABLE", "tinyfive_unavailable" + if "unsupported" in error or "unknown" in error: + return "ERROR", "unsupported_assembly_instruction" + if "assembl" in error or "encode" in error: + return "ERROR", "assembly_encoding_error" + return "ERROR", "simulation_error" + if not matched: + return "MISMATCH", "result_mismatch" + return "PASS", None + + +def count_assembly_instructions(asm_file: Path) -> int | None: + if not asm_file.exists(): + return None + try: + from scratchv.backend._asm_parser import parse_asm + + return sum( + line.opcode is not None and not line.is_directive + for line in parse_asm(asm_file.read_text(encoding="utf-8")) + ) + except (OSError, ValueError): + return None + + +def summarize_backend_matrix(results): + def counts(field): + summary = {} + for result in results: + value = result.get(field) or "UNKNOWN" + summary[value] = summary.get(value, 0) + 1 + return summary + + comparable = [r for r in results if r.get("backend_outputs_match") is not None] + return { + "interpreter": counts("interpreter_status"), + "tinyfive": counts("tinyfive_status"), + "cross_backend": { + "comparable": len(comparable), + "matched": sum(r["backend_outputs_match"] is True for r in comparable), + "mismatched": sum(r["backend_outputs_match"] is False for r in comparable), + }, + } + + +def summarize_benchmark_runs(instr_counts): + if not instr_counts: + return { + "runs": 0, + "avg_instr_count": None, + "min_instr_count": None, + "max_instr_count": None, + "ci95_instr_count": None, + } + avg = sum(instr_counts) / len(instr_counts) + if len(instr_counts) > 1: + variance = sum((value - avg) ** 2 for value in instr_counts) / (len(instr_counts) - 1) + ci95 = 1.96 * math.sqrt(variance) / math.sqrt(len(instr_counts)) + else: + ci95 = 0.0 + return { + "runs": len(instr_counts), + "avg_instr_count": avg, + "min_instr_count": min(instr_counts), + "max_instr_count": max(instr_counts), + "ci95_instr_count": ci95, + } + + +def detect_regression( + avg_instr_count, + baseline_instr_count, + threshold_pct=REGRESSION_THRESHOLD_PCT, +): + delta = avg_instr_count - baseline_instr_count + delta_pct = 0.0 if baseline_instr_count == 0 else (delta / baseline_instr_count) * 100.0 + return { + "baseline_instr_count": baseline_instr_count, + "delta": round(delta, 4), + "delta_pct": round(delta_pct, 4), + "threshold_pct": threshold_pct, + "regressed": delta_pct > threshold_pct, + } + + +def build_cost_model_metrics( + static_asm_instruction_count, + sim_result, + avg_instr_count=None, + avg_perf_counters=None, +): + dynamic_count = avg_instr_count + if dynamic_count is None and sim_result.get("success"): + dynamic_count = sim_result.get("instr_count") + perf_counters = avg_perf_counters or sim_result.get("perf_counters") or {} + metrics = { + "static_asm_instructions": static_asm_instruction_count, + "machine_instructions": sim_result.get("machine_code_instructions"), + "code_size_bytes": sim_result.get("code_size_bytes"), + "dynamic_instructions": dynamic_count, + } + for name in ("load", "store", "mul", "add", "madd", "branch"): + metrics[f"dynamic_{name}"] = perf_counters.get(name) + return metrics + + +def summarize_perf_counters(samples): + if not samples: + return {} + keys = set().union(*(sample.keys() for sample in samples)) + return { + key: sum(float(sample.get(key, 0)) for sample in samples) / len(samples) + for key in sorted(keys) + } + + +def detect_cost_model_regressions(current, baseline, threshold_pct): + comparisons = {} + for metric, current_value in current.items(): + baseline_value = baseline.get(metric) + if current_value is None or baseline_value is None: + comparisons[metric] = { + "current": current_value, + "baseline": baseline_value, + "delta": None, + "delta_pct": None, + "regressed": None, + } + continue + delta = current_value - baseline_value + if baseline_value == 0: + delta_pct = 0.0 if current_value == 0 else None + regressed = current_value > 0 + else: + delta_pct = delta / baseline_value * 100.0 + regressed = delta_pct > threshold_pct + comparisons[metric] = { + "current": current_value, + "baseline": baseline_value, + "delta": round(delta, 4), + "delta_pct": None if delta_pct is None else round(delta_pct, 4), + "regressed": regressed, + } + return comparisons + + +def load_baseline(): + if not BASELINE_FILE.exists(): + return {} + payload = json.loads(BASELINE_FILE.read_text(encoding="utf-8")) + if isinstance(payload, dict) and isinstance(payload.get("cases"), dict): + return payload["cases"] + return payload + + +def save_baseline(results, preserve_existing=False): + BASELINE_FILE.parent.mkdir(parents=True, exist_ok=True) + cases = load_baseline() if preserve_existing else {} + for r in results: + if ( + r["avg_instr_count"] is None + or r.get("tinyfive_status") != "PASS" + or r.get("backend_outputs_match") is False + ): + continue + cases[r["name"]] = { + "category": r["category"], + "avg_instr_count": r["avg_instr_count"], + "runs": r["benchmark_runs"], + "cost_model": r["cost_model"], + } + payload = { + "schema_version": 2, + "primary_metric": "dynamic_instructions", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + }, + "cases": cases, + } + BASELINE_FILE.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _markdown_cell(value): + return str(value).replace("\n", " ").replace("|", "\\|") + + +def _report_path(path): + path = Path(path) + try: + return path.resolve().relative_to(PROJECT_ROOT).as_posix() + except ValueError: + return path.as_posix() + + +def write_chart(results): + try: + mpl_config_dir = Path(tempfile.gettempdir()) / "scratchv-matplotlib" + mpl_config_dir.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("MPLCONFIGDIR", str(mpl_config_dir)) + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + return None + + names = [result["name"] for result in results] + values = [_reported_instr_count(result) for result in results] + width = max(10, len(names) * 0.45) + fig, ax = plt.subplots(figsize=(width, 5)) + ax.bar(range(len(names)), values, color="#2563eb") + ax.set_title("ScratchV Course Benchmark Instruction Counts") + ax.set_ylabel("Instructions") + ax.set_xticks(range(len(names))) + ax.set_xticklabels(names, rotation=60, ha="right", fontsize=8) + ax.grid(axis="y", linestyle="--", alpha=0.35) + fig.tight_layout() + fig.savefig(CHART_FILE, dpi=160) + plt.close(fig) + return CHART_FILE + + +def _reported_instr_count(result): + average = result.get("avg_instr_count") + return result["instr_count"] if average is None else average + + +def _report_value(value, precision=None, prefix=""): + if value is None: + return "null" + if precision is not None: + return f"{prefix}{value:.{precision}f}" + return f"{prefix}{value}" + + +def generate_unified_report_text_cn( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + include_chart=False, +): + mode = results[0]["mode"] if results else "normal" + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- Schema 版本: 2\n", + f"- 运行模式: {mode}\n", + f"- 类别筛选: {_report_value(selection_category)}\n", + f"- 名称筛选: {_report_value(selection_filter)}\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{_report_path(TEST_DIR)}`\n", + f"- 汇编输出目录: `{_report_path(BUILD_DIR)}`\n", + f"- 性能基线文件: `{_report_path(BASELINE_FILE)}`\n", + f"- 性能退化阈值: {regression_threshold_pct:.2f}%\n", + f"- 单次编译超时: {COMPILE_TIMEOUT_SEC:.0f}s\n", + f"- 单次模拟超时: {SIMULATION_TIMEOUT_SEC:.0f}s\n\n", + "## 测试结果\n\n", + "| 用例 | 类别 | 状态 | 编译返回码 | 编译日志 | 模拟后端 | 指令数 | Benchmark 次数 | Benchmark 停止原因 | 平均指令数 | 95% 置信区间 | 最小 | 最大 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 基线 | 变化量 | 变化率(%) | 退化阈值(%) | 是否退化 | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ] + + for result in results: + lines.append( + f"| {result['name']} | {result['category']} | {result['status']} | " + f"{result['compile_returncode']} | {_report_value(result['compile_log'])} | " + f"{result['backend']} | {result['instr_count']} | " + f"{_report_value(result['benchmark_runs'])} | " + f"{_report_value(result['benchmark_stopped_reason'])} | " + f"{_report_value(result['avg_instr_count'], 2)} | " + f"{_report_value(result['ci95_instr_count'], 2, '±')} | " + f"{_report_value(result['min_instr_count'])} | " + f"{_report_value(result['max_instr_count'])} | " + f"{result['compile_time_sec']:.4f} | {result['simulation_time_sec']:.4f} | " + f"{result['total_time_sec']:.4f} | " + f"{_report_value(result['baseline_instr_count'], 2)} | " + f"{_report_value(result['delta'], 2)} | " + f"{_report_value(result['delta_pct'], 2)} | " + f"{_report_value(result['threshold_pct'], 2)} | " + f"{_report_value(result['regressed'])} | {_markdown_cell(result['expected'])} | " + f"{_markdown_cell(result['actual'])} | {result['matched']} | {result['asm']} |\n" + ) + + lines.extend([ + "\n## 双后端能力矩阵\n\n", + "| 用例 | 类别 | DSLInterpreter | TinyFive | TinyFive 失败类型 | 两后端输出一致 | 单用例报告 |\n", + "|---|---|---|---|---|---|---|\n", + ]) + for result in results: + lines.append( + f"| {result['name']} | {result['category']} | " + f"{result['interpreter_status']} | {result['tinyfive_status']} | " + f"{_report_value(result['tinyfive_failure_kind'])} | " + f"{_report_value(result['backend_outputs_match'])} | " + f"{result['case_report_md']} |\n" + ) + + if include_chart: + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + ]) + lines.append("\n## 用例详情\n\n") + for result in results: + lines.extend([ + f"### {result['name']}\n\n", + f"- 类别: {result['category']}\n", + f"- 描述: {result['description']}\n", + f"- 预期输出 ({result['expected_type']}): {result['expected']}\n", + f"- TinyFive 输出: {result['actual']}\n", + f"- 输出是否匹配: {result['matched']}\n", + f"- DSLInterpreter 状态: {result['interpreter_status']}\n", + f"- DSLInterpreter 输出: {result['interpreter_actual']}\n", + f"- DSLInterpreter 错误: {_report_value(result['interpreter_error'])}\n", + f"- TinyFive 状态: {result['tinyfive_status']}\n", + f"- TinyFive 失败类型: {_report_value(result['tinyfive_failure_kind'])}\n", + f"- TinyFive 输入 ABI 可用: {result['tinyfive_input_abi_supported']}\n", + f"- 两后端输出一致: {_report_value(result['backend_outputs_match'])}\n", + f"- TinyFive 初始寄存器: {result['initial_registers']}\n", + f"- 模拟后端: {result['backend']}\n", + f"- 指令数: {result['instr_count']}\n", + f"- 静态汇编指令数: {_report_value(result['static_asm_instruction_count'])}\n", + f"- 编码后机器指令数: {_report_value(result['machine_code_instruction_count'])}\n", + f"- 代码大小(bytes): {_report_value(result['code_size_bytes'])}\n", + f"- TinyFive 分类计数: {result['perf_counters']}\n", + f"- 编译返回码: {result['compile_returncode']}\n", + f"- 编译是否超时: {result['compile_timed_out']}\n", + f"- 编译错误摘要: {_report_value(result['compile_error'])}\n", + f"- 编译失败日志: {_report_value(result['compile_log'])}\n", + f"- Benchmark 重复次数: {_report_value(result['benchmark_runs'])}\n", + f"- Benchmark 停止原因: {_report_value(result['benchmark_stopped_reason'])}\n", + f"- 平均指令数: {_report_value(result['avg_instr_count'], 2)}\n", + f"- 95% 置信区间: {_report_value(result['ci95_instr_count'], 2, '±')}\n", + f"- 最小指令数: {_report_value(result['min_instr_count'])}\n", + f"- 最大指令数: {_report_value(result['max_instr_count'])}\n", + f"- 编译耗时(s): {result['compile_time_sec']:.4f}\n", + f"- 模拟耗时(s): {result['simulation_time_sec']:.4f}\n", + f"- 总耗时(s): {result['total_time_sec']:.4f}\n", + f"- 基线指令数: {_report_value(result['baseline_instr_count'], 2)}\n", + f"- 性能变化量: {_report_value(result['delta'], 2)}\n", + f"- 性能变化率(%): {_report_value(result['delta_pct'], 2)}\n", + f"- 性能退化阈值(%): {_report_value(result['threshold_pct'], 2)}\n", + f"- 是否性能退化: {_report_value(result['regressed'])}\n", + f"- Cost model 是否退化: {_report_value(result['cost_model_regressed'])}\n", + f"- 汇编文件: {result['asm']}\n\n", + ]) + return "".join(lines) + + +def write_unified_html_report_cn( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, +): + try: + from jinja2 import Template + except ImportError: + return None + + template = Template(""" + + + + ScratchV 测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

Schema 版本：2，运行模式：{{ mode }}

+

类别筛选：{{ fmt(selection_category) }}，名称筛选：{{ fmt(selection_filter) }}

+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}

+

性能退化阈值：{{ fmt(regression_threshold_pct, 2) }}%

+
+ 课程版指令数图表 +
+ + + + + + + + {% for r in results %} + + + + + + + + + + + + {% endfor %} +
用例类别状态解释器状态TinyFive 状态失败类型编译返回码编译日志模拟后端指令数Benchmark 次数Benchmark 停止原因平均指令数95% 置信区间最小最大编译耗时(s)模拟耗时(s)总耗时(s)基线变化量变化率(%)退化阈值(%)是否退化预期输出TinyFive 输出输出匹配汇编文件
{{ r.name }}{{ r.category }}{{ r.status }}{{ r.interpreter_status }}{{ r.tinyfive_status }}{{ fmt(r.tinyfive_failure_kind) }}{{ r.compile_returncode }}{{ fmt(r.compile_log) }}{{ r.backend }}{{ r.instr_count }}{{ fmt(r.benchmark_runs) }}{{ fmt(r.benchmark_stopped_reason) }}{{ fmt(r.avg_instr_count, 2) }}{{ fmt(r.ci95_instr_count, 2, '±') }}{{ fmt(r.min_instr_count) }}{{ fmt(r.max_instr_count) }}{{ fmt(r.compile_time_sec, 4) }}{{ fmt(r.simulation_time_sec, 4) }}{{ fmt(r.total_time_sec, 4) }}{{ fmt(r.baseline_instr_count, 2) }}{{ fmt(r.delta, 2) }}{{ fmt(r.delta_pct, 2) }}{{ fmt(r.threshold_pct, 2) }}{{ fmt(r.regressed) }}{{ r.expected }}{{ r.actual }}{{ r.matched }}{{ r.asm }}
+ + +""") + mode = results[0]["mode"] if results else "normal" + HTML_REPORT_FILE.write_text( + template.render( + mode=mode, + total=len(results), + passed=passed, + failed=failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + chart_name=CHART_FILE.name, + results=results, + fmt=_report_value, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def write_json_report( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + full_report=False, +): + mode = results[0]["mode"] if results else "normal" + payload = { + "schema_version": 2, + "mode": mode, + "report_level": "full" if full_report else "light", + "regression_threshold_pct": regression_threshold_pct, + "selection": { + "category": selection_category, + "filter": selection_filter, + }, + "generated_at": datetime.now().isoformat(timespec="seconds"), + "summary": { + "total": len(results), + "passed": passed, + "failed": failed, + }, + "backend_matrix": summarize_backend_matrix(results), + "results": results, + } + JSON_REPORT_FILE.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return JSON_REPORT_FILE + + +def write_case_reports(results): + CASE_REPORT_DIR.mkdir(parents=True, exist_ok=True) + for result in results: + report_stem = f"{result['category']}-{result['name']}" + markdown_path = CASE_REPORT_DIR / f"{report_stem}.md" + json_path = CASE_REPORT_DIR / f"{report_stem}.json" + result["case_report_md"] = _report_path(markdown_path) + result["case_report_json"] = _report_path(json_path) + + asm_text = "" + asm_path = PROJECT_ROOT / result["asm"] + if asm_path.exists(): + asm_text = asm_path.read_text(encoding="utf-8") + + markdown_path.write_text("".join([ + f"# {result['name']} 测试详情\n\n", + "## 基本信息\n\n", + f"- 类别: {result['category']}\n", + f"- DSL: `{result['path']}`\n", + f"- 描述: {result['description']}\n", + f"- 总体状态: {result['status']}\n", + f"- 验证模式: {result['verification_backend']}\n\n", + "## 后端能力矩阵\n\n", + "| 后端 | 状态 | 实际输出 | 与期望匹配 | 失败类型 | 错误 |\n", + "|---|---|---|---|---|---|\n", + f"| DSLInterpreter | {result['interpreter_status']} | " + f"{_markdown_cell(result['interpreter_actual'])} | {result['interpreter_matched']} | " + f"{_report_value(result['interpreter_failure_kind'])} | " + f"{_markdown_cell(_report_value(result['interpreter_error']))} |\n", + f"| TinyFive | {result['tinyfive_status']} | " + f"{_markdown_cell(result['actual'])} | {result['matched']} | " + f"{_report_value(result['tinyfive_failure_kind'])} | " + f"{_markdown_cell(_report_value(result['simulation_error']))} |\n\n", + f"- 期望输出: {result['expected']}\n", + f"- 两后端输出一致: {_report_value(result['backend_outputs_match'])}\n", + f"- TinyFive 输入 ABI 可用: {result['tinyfive_input_abi_supported']}\n", + f"- TinyFive 输入 ABI 说明: {_report_value(result['tinyfive_input_abi_reason'])}\n\n", + "## 性能指标\n\n", + f"- 静态汇编指令数: {_report_value(result['static_asm_instruction_count'])}\n", + f"- 编码后机器指令数: {_report_value(result['machine_code_instruction_count'])}\n", + f"- 代码大小(bytes): {_report_value(result['code_size_bytes'])}\n", + f"- TinyFive 动态执行指令数: {result['instr_count']}\n", + f"- TinyFive 分类计数: {result['perf_counters']}\n", + f"- 编译耗时(s): {result['compile_time_sec']:.6f}\n", + f"- 解释器耗时(s): {result['interpreter_time_sec']:.6f}\n", + f"- TinyFive 模拟耗时(s): {result['simulation_time_sec']:.6f}\n", + f"- 总耗时(s): {result['total_time_sec']:.6f}\n", + f"- 基线动态指令数: {_report_value(result['baseline_instr_count'])}\n", + f"- 动态指令变化率(%): {_report_value(result['delta_pct'])}\n", + f"- 是否退化: {_report_value(result['regressed'])}\n\n", + f"- Cost model 指标: {result['cost_model']}\n", + f"- Cost model 对比: {result['cost_model_comparison']}\n", + f"- Cost model 是否退化: {_report_value(result['cost_model_regressed'])}\n\n", + "## 编译信息\n\n", + f"- 命令: `{result['compile_command']}`\n", + f"- 返回码: {result['compile_returncode']}\n", + f"- 编译错误: {_report_value(result['compile_error'])}\n", + f"- 失败日志: {_report_value(result['compile_log'])}\n", + f"- 寄存器映射: `{result['register_map']}`\n", + f"- 汇编文件: `{result['asm']}`\n\n", + "## 生成汇编\n\n", + "```asm\n", + asm_text.rstrip(), + "\n```\n", + ]), encoding="utf-8") + json_path.write_text( + json.dumps(result, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def write_report( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + full_report=False, +): + REPORT_DIR.mkdir(parents=True, exist_ok=True) + write_case_reports(results) + chart_path = write_chart(results) if full_report else None + REPORT_FILE.write_text( + generate_unified_report_text_cn( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + include_chart=chart_path is not None, + ), + encoding="utf-8", + ) + html_path = ( + write_unified_html_report_cn( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + ) + if full_report else None + ) + json_path = write_json_report( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + full_report=full_report, + ) + print(f"\nMarkdown report written to {REPORT_FILE}") + print(f"JSON report written to {json_path}") + if full_report: + if html_path: + print(f"HTML report written to {html_path}") + else: + print("HTML report skipped: jinja2 is not installed") + if chart_path: + print(f"Chart written to {chart_path}") + else: + print("Chart skipped: matplotlib is not installed") + + +def non_negative_float(value): + parsed = float(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be greater than or equal to 0") + return parsed + + +def select_dsl_files(test_dir, category=None, name_filter=None): + dsl_files = sorted(test_dir.rglob("*.dsl"), key=lambda path: str(path).lower()) + if category: + expected_category = category.lower() + dsl_files = [ + path for path in dsl_files + if path.parent.name.lower() == expected_category + ] + if name_filter: + expected_name = name_filter.lower() + dsl_files = [ + path for path in dsl_files + if expected_name in path.stem.lower() + ] + return dsl_files + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description="Run ScratchV DSL benchmark suite.") + parser.add_argument("--benchmark", type=int, default=0, metavar="N", + help="Run each case N times and report average instruction count.") + parser.add_argument("--update-baseline", action="store_true", + help="Write current benchmark averages to the baseline file.") + parser.add_argument("--full-report", action="store_true", + help="Also generate the optional HTML report and PNG chart.") + parser.add_argument( + "--regression-threshold", + type=non_negative_float, + default=REGRESSION_THRESHOLD_PCT, + metavar="PERCENT", + help="Mark instruction-count increases above this percentage as regressions (default: 5).", + ) + parser.add_argument( + "--category", + help="Only run cases in this test category (for example: activation or tensor).", + ) + parser.add_argument( + "--filter", + dest="name_filter", + help="Only run cases whose file name contains this text.", + ) + parser.add_argument( + "--verification-backend", + choices=("both", "tinyfive", "interpreter"), + default="both", + help="Select result verification backend(s); both keeps verdicts independent.", + ) + parser.add_argument( + "--fail-on-test-failure", + action="store_true", + help="Return exit code 1 when one or more selected cases fail.", + ) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + if args.update_baseline and args.verification_backend == "interpreter": + print("Cannot update a TinyFive cost-model baseline in interpreter-only mode.") + return 2 + BUILD_DIR.mkdir(parents=True, exist_ok=True) + baseline = load_baseline() if args.benchmark else {} + + all_dsl_files = list(TEST_DIR.rglob("*.dsl")) + + if not all_dsl_files: + print("No DSL test files found.") + return 2 + + dsl_files = select_dsl_files( + TEST_DIR, + category=args.category, + name_filter=args.name_filter, + ) + if not dsl_files: + print("No DSL test cases matched the selected filters.") + return 2 + + passed = 0 + failed = 0 + results = [] + + print("Running DSL compiler tests...") + print("=" * 50) + + for dsl_file in dsl_files: + print(f"\n[TEST] {dsl_file}") + + case_start = time.perf_counter() + meta = load_metadata(dsl_file) + interpreter_start = time.perf_counter() + if args.verification_backend in {"both", "interpreter"} and not meta.get("_metadata_error"): + interpreter_result = run_interpreter(dsl_file, meta.get("inputs", {})) + else: + interpreter_result = { + "success": False, + "return_value": None, + "backend": "interpreter", + "status": "SKIPPED", + "error": ( + "interpreter backend not selected" + if not meta.get("_metadata_error") + else "invalid metadata" + ), + } + interpreter_time_sec = time.perf_counter() - interpreter_start + compile_start = time.perf_counter() + metadata_error = meta.get("_metadata_error") + if metadata_error: + output_file = BUILD_DIR / (dsl_file.stem + ".s") + register_map_file = BUILD_DIR / (dsl_file.stem + ".registers.json") + result = subprocess.CompletedProcess( + args=[], + returncode=2, + stdout="", + stderr=f"metadata error: {metadata_error}", + ) + else: + result, output_file, register_map_file = run_compile(dsl_file) + compile_time_sec = time.perf_counter() - compile_start + compile_log = None + compile_error = None + if result.returncode != 0: + compile_log = write_compile_failure_log( + dsl_file, + result, + output_file, + compile_time_sec, + ) + compile_error = _last_nonempty_line(result.stderr or result.stdout) + register_map_error = None + initial_registers = {} + input_abi_supported, input_abi_reason = tinyfive_input_abi_supported( + meta.get("inputs", {}), + ) + if result.returncode == 0: + try: + initial_registers = load_initial_registers( + register_map_file, + meta.get("inputs", {}), + ) + except ValueError as exc: + register_map_error = str(exc) + simulation_start = time.perf_counter() + if result.returncode != 0: + sim_result = { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": (result.stderr or result.stdout or "compile failed").strip(), + } + elif register_map_error: + sim_result = { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": register_map_error, + } + elif args.verification_backend == "interpreter": + sim_result = { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "skipped", + "error": "TinyFive backend not selected", + } + else: + sim_result = run_simulation(output_file, initial_registers) + simulation_time_sec = time.perf_counter() - simulation_start + expected_value = meta.get("expected_return") + actual_value = sim_result.get("return_value") + matched = bool(sim_result.get("success")) and values_equal(actual_value, expected_value) + interpreter_actual = interpreter_result.get("return_value") + interpreter_matched = bool(interpreter_result.get("success")) and values_equal( + interpreter_actual, + expected_value, + ) + if interpreter_result.get("status") == "EXECUTED": + interpreter_result["status"] = "PASS" if interpreter_matched else "MISMATCH" + interpreter_failure_kind = None + if interpreter_result.get("status") == "UNSUPPORTED": + interpreter_failure_kind = "interpreter_control_flow_unsupported" + elif interpreter_result.get("status") == "TIMEOUT": + interpreter_failure_kind = "interpreter_timeout" + elif interpreter_result.get("status") == "ERROR": + interpreter_failure_kind = "interpreter_error" + elif interpreter_result.get("status") == "MISMATCH": + interpreter_failure_kind = "interpreter_result_mismatch" + backend_outputs_match = ( + values_equal(interpreter_actual, actual_value) + if ( + interpreter_result.get("success") + and sim_result.get("success") + and input_abi_supported + ) + else None + ) + static_asm_instruction_count = count_assembly_instructions(output_file) + benchmark_counts = [] + benchmark_perf_samples = [] + benchmark_summary = { + "runs": None, + "avg_instr_count": None, + "min_instr_count": None, + "max_instr_count": None, + "ci95_instr_count": None, + } + regression = { + "baseline_instr_count": None, + "delta": None, + "delta_pct": None, + "threshold_pct": None, + "regressed": None, + } + benchmark_stopped_reason = None + + tinyfive_status, tinyfive_failure_kind = classify_tinyfive_result( + result, + sim_result, + matched, + input_abi_supported, + metadata_error=metadata_error, + register_map_error=register_map_error, + ) + + if args.benchmark > 0: + regression["threshold_pct"] = args.regression_threshold + if args.verification_backend == "interpreter": + benchmark_stopped_reason = "benchmark skipped: TinyFive backend not selected" + benchmark_summary = summarize_benchmark_runs([]) + elif not input_abi_supported: + benchmark_stopped_reason = "benchmark skipped: TinyFive input ABI unsupported" + benchmark_summary = summarize_benchmark_runs([]) + elif result.returncode != 0 or not output_file.exists(): + benchmark_stopped_reason = "benchmark skipped: compile failed" + benchmark_summary = summarize_benchmark_runs([]) + elif sim_result.get("backend") == "timeout": + benchmark_stopped_reason = "benchmark skipped: initial simulation timeout" + benchmark_summary = summarize_benchmark_runs([]) + elif not sim_result.get("success"): + benchmark_stopped_reason = "benchmark skipped: initial simulation failed" + benchmark_summary = summarize_benchmark_runs([]) + else: + for run_index in range(1, args.benchmark + 1): + benchmark_result = run_simulation(output_file, initial_registers) + if benchmark_result.get("backend") == "timeout": + benchmark_stopped_reason = ( + f"benchmark stopped: timeout on run {run_index}" + ) + break + if not benchmark_result.get("success"): + benchmark_stopped_reason = ( + f"benchmark stopped: simulation failed on run {run_index}" + ) + break + benchmark_counts.append(benchmark_result.get("instr_count", 0)) + benchmark_perf_samples.append( + benchmark_result.get("perf_counters") or {}, + ) + benchmark_summary = summarize_benchmark_runs(benchmark_counts) + + if benchmark_summary["avg_instr_count"] is not None and benchmark_stopped_reason is None: + regression["regressed"] = False + + baseline_entry = baseline.get(dsl_file.stem) + if ( + baseline_entry + and benchmark_summary["avg_instr_count"] is not None + and benchmark_stopped_reason is None + ): + regression = detect_regression( + avg_instr_count=benchmark_summary["avg_instr_count"], + baseline_instr_count=baseline_entry.get("avg_instr_count", 0.0), + threshold_pct=args.regression_threshold, + ) + + average_perf_counters = summarize_perf_counters(benchmark_perf_samples) + cost_model = build_cost_model_metrics( + static_asm_instruction_count, + sim_result, + avg_instr_count=benchmark_summary["avg_instr_count"], + avg_perf_counters=average_perf_counters, + ) + cost_model_comparison = {} + cost_model_regressed = None + baseline_entry = baseline.get(dsl_file.stem) + if ( + args.benchmark > 0 + and baseline_entry + and benchmark_summary["avg_instr_count"] is not None + and benchmark_stopped_reason is None + ): + baseline_cost_model = baseline_entry.get("cost_model") or { + "dynamic_instructions": baseline_entry.get("avg_instr_count"), + } + cost_model_comparison = detect_cost_model_regressions( + cost_model, + baseline_cost_model, + args.regression_threshold, + ) + comparable_regressions = [ + item["regressed"] + for item in cost_model_comparison.values() + if item["regressed"] is not None + ] + cost_model_regressed = any(comparable_regressions) + regression["regressed"] = bool(regression["regressed"] or cost_model_regressed) + total_time_sec = time.perf_counter() - case_start + + tinyfive_ok = ( + tinyfive_status == "PASS" + and regression["regressed"] is not True + and benchmark_stopped_reason is None + ) + interpreter_ok = ( + result.returncode == 0 + and interpreter_result.get("status") == "PASS" + ) + if args.verification_backend == "interpreter": + ok = interpreter_ok + elif args.verification_backend == "tinyfive": + ok = tinyfive_ok + else: + interpreter_required = interpreter_result.get("status") != "UNSUPPORTED" + ok = tinyfive_ok and (interpreter_ok or not interpreter_required) + + if ok: + print("PASS") + passed += 1 + status = "PASS" + else: + print("FAIL") + failed += 1 + status = "FAIL" + if tinyfive_failure_kind == "input_abi_unsupported": + print(input_abi_reason) + elif sim_result.get("error"): + print(sim_result["error"]) + else: + print( + "output mismatch: " + f"expected={expected_value}, " + f"tinyfive={actual_value}" + ) + + results.append({ + "mode": "benchmark" if args.benchmark > 0 else "normal", + "name": dsl_file.stem, + "category": dsl_file.parent.name, + "path": _report_path(dsl_file), + "status": status, + "description": meta.get("description", ""), + "expected_type": meta.get("expected_output_type", "scalar"), + "output_dtype": meta.get("output_dtype"), + "output_shape": meta.get("output_shape"), + "expected": expected_value, + "actual": actual_value, + "matched": matched, + "verification_backend": args.verification_backend, + "interpreter_status": interpreter_result.get("status"), + "interpreter_actual": interpreter_actual, + "interpreter_matched": interpreter_matched, + "interpreter_error": interpreter_result.get("error"), + "interpreter_failure_kind": interpreter_failure_kind, + "interpreter_time_sec": interpreter_time_sec, + "backend_outputs_match": backend_outputs_match, + "tinyfive_status": tinyfive_status, + "tinyfive_failure_kind": tinyfive_failure_kind, + "tinyfive_input_abi_supported": input_abi_supported, + "tinyfive_input_abi_reason": input_abi_reason, + "simulation_error": sim_result.get("error"), + "initial_registers": initial_registers, + "register_map": _report_path(register_map_file), + "backend": sim_result.get("backend", "none"), + "instr_count": sim_result.get("instr_count", 0), + "static_asm_instruction_count": static_asm_instruction_count, + "machine_code_instruction_count": sim_result.get("machine_code_instructions"), + "code_size_bytes": sim_result.get("code_size_bytes"), + "perf_counters": sim_result.get("perf_counters") or {}, + "cost_model": cost_model, + "cost_model_comparison": cost_model_comparison, + "cost_model_regressed": cost_model_regressed, + "compile_returncode": result.returncode, + "compile_timed_out": result.returncode == 124, + "compile_error": compile_error, + "compile_command": ( + shlex.join(str(arg) for arg in result.args) + if isinstance(result.args, (list, tuple)) and result.args + else "(compiler was not started)" + ), + "compile_log": _report_path(compile_log) if compile_log else None, + "compile_time_sec": compile_time_sec, + "simulation_time_sec": simulation_time_sec, + "total_time_sec": total_time_sec, + "benchmark_runs": benchmark_summary["runs"], + "benchmark_stopped_reason": benchmark_stopped_reason, + "avg_instr_count": benchmark_summary["avg_instr_count"], + "min_instr_count": benchmark_summary["min_instr_count"], + "max_instr_count": benchmark_summary["max_instr_count"], + "ci95_instr_count": benchmark_summary["ci95_instr_count"], + "baseline_instr_count": regression["baseline_instr_count"], + "delta": regression["delta"], + "delta_pct": regression["delta_pct"], + "threshold_pct": regression["threshold_pct"], + "regressed": regression["regressed"], + "asm": _report_path(output_file), + }) + + print("\n" + "=" * 50) + print(f"Total: {len(dsl_files)}") + print(f"Passed: {passed}") + print(f"Failed: {failed}") + + if args.benchmark and args.update_baseline: + save_baseline( + results, + preserve_existing=bool(args.category or args.name_filter), + ) + print(f"Baseline written to {BASELINE_FILE}") + + write_report( + results, + passed, + failed, + regression_threshold_pct=args.regression_threshold, + selection_category=args.category, + selection_filter=args.name_filter, + full_report=args.full_report, + ) + return 1 if args.fail_on_test_failure and failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_simulator.py b/tests/test_simulator.py index f32c708..3717ae9 100644 --- a/tests/test_simulator.py +++ b/tests/test_simulator.py @@ -122,6 +122,43 @@ def test_large_li_expands_and_simulates_equivalently(self): outputs.append(machine.get_reg(5)) assert outputs == [4098, 4098] + def test_run_until_executes_loop_and_stops_at_return_address(self): + binary = assemble_to_binary( + "li x5, 0\n" + "li x6, 4\n" + "loop:\n" + "addi x5, x5, 1\n" + "blt x5, x6, loop\n" + "jalr x0, x1, 0\n" + ) + words = [ + int.from_bytes(binary[i:i + 4], "little") + for i in range(0, len(binary), 4) + ] + machine = ProfiledMachine(mem_size=4096) + machine.load_binary(words, origin=0) + machine.set_reg(1, len(binary)) + + machine.run_until(end=len(binary), start=0, strict=True) + + assert machine.get_reg(5) == 4 + assert machine.pc == len(binary) + assert machine.instr_count == 11 + + def test_verify_assembly_injects_inputs_and_returns_a0(self): + result = verify_assembly( + "add x10, x5, x6\njalr x0, x1, 0\n", + initial_registers={"t0": -2, "t1": 7}, + ) + + assert result["success"] is True + assert result["backend"] == "tinyfive" + assert result["return_value"] == 5 + assert result["instr_count"] == 2 + assert result["machine_code_instructions"] == 2 + assert result["code_size_bytes"] == 8 + assert result["perf_counters"]["total"] == 2 + @pytest.mark.parametrize("value", [ -(1 << 31), -4097, -2049, -2048, -1, 0, 2047, 2048, 4096, 4098, (1 << 31) - 1, diff --git a/tests/topic06/cases/activation/add_relu_relu.dsl b/tests/topic06/cases/activation/add_relu_relu.dsl new file mode 100644 index 0000000..52c1325 --- /dev/null +++ b/tests/topic06/cases/activation/add_relu_relu.dsl @@ -0,0 +1,5 @@ +# Add followed by two ReLU stages +x = add(input, bias) +y = relu(x) +result = relu(y) +return result diff --git a/tests/topic06/cases/activation/add_relu_relu.meta.json b/tests/topic06/cases/activation/add_relu_relu.meta.json new file mode 100644 index 0000000..e587d8a --- /dev/null +++ b/tests/topic06/cases/activation/add_relu_relu.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply ReLU twice.", + "expected_output_type": "return_value", + "inputs": { + "input": -3, + "bias": 10 + }, + "expected_return": 7 +} diff --git a/tests/topic06/cases/activation/relu_add.dsl b/tests/topic06/cases/activation/relu_add.dsl new file mode 100644 index 0000000..7190772 --- /dev/null +++ b/tests/topic06/cases/activation/relu_add.dsl @@ -0,0 +1,4 @@ +# Add + ReLU activation +x = add(input, bias) +y = relu(x) +return y \ No newline at end of file diff --git a/tests/topic06/cases/activation/relu_add.meta.json b/tests/topic06/cases/activation/relu_add.meta.json new file mode 100644 index 0000000..f1b7f84 --- /dev/null +++ b/tests/topic06/cases/activation/relu_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply one ReLU.", + "expected_output_type": "return_value", + "inputs": { + "input": -2, + "bias": 5 + }, + "expected_return": 3 +} diff --git a/tests/topic06/cases/activation/relu_only.dsl b/tests/topic06/cases/activation/relu_only.dsl new file mode 100644 index 0000000..404a911 --- /dev/null +++ b/tests/topic06/cases/activation/relu_only.dsl @@ -0,0 +1,3 @@ +# Single ReLU activation +result = relu(x) +return result diff --git a/tests/topic06/cases/activation/relu_only.meta.json b/tests/topic06/cases/activation/relu_only.meta.json new file mode 100644 index 0000000..b08af80 --- /dev/null +++ b/tests/topic06/cases/activation/relu_only.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU directly to a single input value.", + "expected_output_type": "return_value", + "inputs": { + "x": -5 + }, + "expected_return": 0 +} diff --git a/tests/topic06/cases/activation/relu_twice.dsl b/tests/topic06/cases/activation/relu_twice.dsl new file mode 100644 index 0000000..7d2defe --- /dev/null +++ b/tests/topic06/cases/activation/relu_twice.dsl @@ -0,0 +1,4 @@ +# Two-stage ReLU activation +x = relu(input) +result = relu(x) +return result diff --git a/tests/topic06/cases/activation/relu_twice.meta.json b/tests/topic06/cases/activation/relu_twice.meta.json new file mode 100644 index 0000000..fabbb38 --- /dev/null +++ b/tests/topic06/cases/activation/relu_twice.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU twice to the same activation path.", + "expected_output_type": "return_value", + "inputs": { + "input": 4 + }, + "expected_return": 4 +} diff --git a/tests/topic06/cases/branch/if_else.dsl b/tests/topic06/cases/branch/if_else.dsl new file mode 100644 index 0000000..6541768 --- /dev/null +++ b/tests/topic06/cases/branch/if_else.dsl @@ -0,0 +1,8 @@ +# Branch takes the else path when flag is zero. +if (flag != 0): +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/tests/topic06/cases/branch/if_else.meta.json b/tests/topic06/cases/branch/if_else.meta.json new file mode 100644 index 0000000..fdacf6a --- /dev/null +++ b/tests/topic06/cases/branch/if_else.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 0, + "a": 9, + "b": 4 + }, + "expected_return": 5 +} diff --git a/tests/topic06/cases/branch/if_relu.dsl b/tests/topic06/cases/branch/if_relu.dsl new file mode 100644 index 0000000..1346060 --- /dev/null +++ b/tests/topic06/cases/branch/if_relu.dsl @@ -0,0 +1,8 @@ +# Branch selects whether to apply ReLU after an add. +sum = add(a, b) +if (use_relu != 0): +result = relu(sum) +return result +else +return sum +endif diff --git a/tests/topic06/cases/branch/if_relu.meta.json b/tests/topic06/cases/branch/if_relu.meta.json new file mode 100644 index 0000000..1925fb5 --- /dev/null +++ b/tests/topic06/cases/branch/if_relu.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch combined with add and relu.", + "expected_output_type": "return_value", + "inputs": { + "use_relu": 1, + "a": -8, + "b": 3 + }, + "expected_return": 0 +} diff --git a/tests/topic06/cases/branch/if_then.dsl b/tests/topic06/cases/branch/if_then.dsl new file mode 100644 index 0000000..c305c3d --- /dev/null +++ b/tests/topic06/cases/branch/if_then.dsl @@ -0,0 +1,8 @@ +# Branch takes the then path when flag is non-zero. +if (flag != 0): +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/tests/topic06/cases/branch/if_then.meta.json b/tests/topic06/cases/branch/if_then.meta.json new file mode 100644 index 0000000..04748d1 --- /dev/null +++ b/tests/topic06/cases/branch/if_then.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns add result when flag is non-zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 1, + "a": 9, + "b": 4 + }, + "expected_return": 13 +} diff --git a/tests/topic06/cases/elementwise/add_chain.dsl b/tests/topic06/cases/elementwise/add_chain.dsl new file mode 100644 index 0000000..d3917c9 --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain.dsl @@ -0,0 +1,4 @@ +# Chain of two vector adds +x = add(a, b) +y = add(x, c) +return y \ No newline at end of file diff --git a/tests/topic06/cases/elementwise/add_chain.meta.json b/tests/topic06/cases/elementwise/add_chain.meta.json new file mode 100644 index 0000000..b07adaf --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Add a and b, then add c to the intermediate result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/tests/topic06/cases/elementwise/add_chain_3.dsl b/tests/topic06/cases/elementwise/add_chain_3.dsl new file mode 100644 index 0000000..aeca4df --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain_3.dsl @@ -0,0 +1,5 @@ +# Chain of three add operations +x = add(a, b) +y = add(x, c) +z = add(y, d) +return z diff --git a/tests/topic06/cases/elementwise/add_chain_3.meta.json b/tests/topic06/cases/elementwise/add_chain_3.meta.json new file mode 100644 index 0000000..368b3aa --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain_3.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Chain three add operations across four symbolic inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4, + "d": 5 + }, + "expected_return": 14 +} diff --git a/tests/topic06/cases/elementwise/add_fan_in_4.dsl b/tests/topic06/cases/elementwise/add_fan_in_4.dsl new file mode 100644 index 0000000..0e4449e --- /dev/null +++ b/tests/topic06/cases/elementwise/add_fan_in_4.dsl @@ -0,0 +1,5 @@ +# Fan-in add over four inputs +x = add(a, b) +y = add(c, d) +result = add(x, y) +return result diff --git a/tests/topic06/cases/elementwise/add_fan_in_4.meta.json b/tests/topic06/cases/elementwise/add_fan_in_4.meta.json new file mode 100644 index 0000000..3eb1f4a --- /dev/null +++ b/tests/topic06/cases/elementwise/add_fan_in_4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute two independent adds and then merge them with a final add.", + "expected_output_type": "return_value", + "inputs": { + "a": 1, + "b": 2, + "c": 3, + "d": 4 + }, + "expected_return": 10 +} diff --git a/tests/topic06/cases/elementwise/add_reuse.dsl b/tests/topic06/cases/elementwise/add_reuse.dsl new file mode 100644 index 0000000..4618948 --- /dev/null +++ b/tests/topic06/cases/elementwise/add_reuse.dsl @@ -0,0 +1,4 @@ +# Reuse intermediate add result +x = add(a, b) +result = add(x, x) +return result diff --git a/tests/topic06/cases/elementwise/add_reuse.meta.json b/tests/topic06/cases/elementwise/add_reuse.meta.json new file mode 100644 index 0000000..2d2c396 --- /dev/null +++ b/tests/topic06/cases/elementwise/add_reuse.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 10 +} diff --git a/tests/topic06/cases/elementwise/vector_add.dsl b/tests/topic06/cases/elementwise/vector_add.dsl new file mode 100644 index 0000000..36ac648 --- /dev/null +++ b/tests/topic06/cases/elementwise/vector_add.dsl @@ -0,0 +1,3 @@ +# Vector add +result = add(a, b) +return result \ No newline at end of file diff --git a/tests/topic06/cases/elementwise/vector_add.meta.json b/tests/topic06/cases/elementwise/vector_add.meta.json new file mode 100644 index 0000000..7347fff --- /dev/null +++ b/tests/topic06/cases/elementwise/vector_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Single add over two symbolic vector inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/tests/topic06/cases/loop/loop_add_4.dsl b/tests/topic06/cases/loop/loop_add_4.dsl new file mode 100644 index 0000000..381f1c0 --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_4.dsl @@ -0,0 +1,5 @@ +# Loop with repeated add body over 4 iterations +for i = 0, 4 +x = add(a, b) +endfor +return x diff --git a/tests/topic06/cases/loop/loop_add_4.meta.json b/tests/topic06/cases/loop/loop_add_4.meta.json new file mode 100644 index 0000000..fc31fae --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/tests/topic06/cases/loop/loop_add_chain_4.dsl b/tests/topic06/cases/loop/loop_add_chain_4.dsl new file mode 100644 index 0000000..eca595b --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_chain_4.dsl @@ -0,0 +1,6 @@ +# Loop with chained add inside the body +for i = 0, 4 +x = add(a, b) +y = add(x, c) +endfor +return y diff --git a/tests/topic06/cases/loop/loop_add_chain_4.meta.json b/tests/topic06/cases/loop/loop_add_chain_4.meta.json new file mode 100644 index 0000000..154d91d --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_chain_4.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/tests/topic06/cases/loop/loop_relu_add_4.dsl b/tests/topic06/cases/loop/loop_relu_add_4.dsl new file mode 100644 index 0000000..44f3080 --- /dev/null +++ b/tests/topic06/cases/loop/loop_relu_add_4.dsl @@ -0,0 +1,6 @@ +# Loop with add followed by ReLU in the body +for i = 0, 4 +x = add(input, bias) +y = relu(x) +endfor +return y diff --git a/tests/topic06/cases/loop/loop_relu_add_4.meta.json b/tests/topic06/cases/loop/loop_relu_add_4.meta.json new file mode 100644 index 0000000..8baca28 --- /dev/null +++ b/tests/topic06/cases/loop/loop_relu_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "input": -4, + "bias": 6 + }, + "expected_return": 2 +} diff --git a/tests/topic06/cases/reduction/dot_4.dsl b/tests/topic06/cases/reduction/dot_4.dsl new file mode 100644 index 0000000..2118c72 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_4.dsl @@ -0,0 +1,3 @@ +# Dot product of two 4-element vectors +result = dot(a, b, len:4) +return result \ No newline at end of file diff --git a/tests/topic06/cases/reduction/dot_4.meta.json b/tests/topic06/cases/reduction/dot_4.meta.json new file mode 100644 index 0000000..748f260 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4], + "b": [5, 6, 7, 8] + }, + "expected_return": 70 +} diff --git a/tests/topic06/cases/reduction/dot_8.dsl b/tests/topic06/cases/reduction/dot_8.dsl new file mode 100644 index 0000000..e1b71fa --- /dev/null +++ b/tests/topic06/cases/reduction/dot_8.dsl @@ -0,0 +1,3 @@ +# Dot product of two 8-element vectors +result = dot(a, b, len:8) +return result diff --git a/tests/topic06/cases/reduction/dot_8.meta.json b/tests/topic06/cases/reduction/dot_8.meta.json new file mode 100644 index 0000000..d3172a2 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4, 5, 6, 7, 8], + "b": [1, 1, 1, 1, 1, 1, 1, 1] + }, + "expected_return": 36 +} diff --git a/tests/topic06/cases/reduction/dot_relu_4.dsl b/tests/topic06/cases/reduction/dot_relu_4.dsl new file mode 100644 index 0000000..279fc19 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_4.dsl @@ -0,0 +1,4 @@ +# Dot product followed by ReLU +x = dot(a, b, len:4) +result = relu(x) +return result diff --git a/tests/topic06/cases/reduction/dot_relu_4.meta.json b/tests/topic06/cases/reduction/dot_relu_4.meta.json new file mode 100644 index 0000000..24910bc --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, -2, 3, -4], + "b": [2, 3, 4, 5] + }, + "expected_return": 0 +} diff --git a/tests/topic06/cases/reduction/dot_relu_8.dsl b/tests/topic06/cases/reduction/dot_relu_8.dsl new file mode 100644 index 0000000..d2b906d --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_8.dsl @@ -0,0 +1,4 @@ +# Dot product of length 8 followed by ReLU +x = dot(a, b, len:8) +result = relu(x) +return result diff --git a/tests/topic06/cases/reduction/dot_relu_8.meta.json b/tests/topic06/cases/reduction/dot_relu_8.meta.json new file mode 100644 index 0000000..776fab5 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 0, 1, 0, 1, 0, 1, 0], + "b": [2, 2, 2, 2, 2, 2, 2, 2] + }, + "expected_return": 8 +} diff --git a/tests/topic06/cases/tensor/matmul_2x2.dsl b/tests/topic06/cases/tensor/matmul_2x2.dsl new file mode 100644 index 0000000..f0a80d9 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_2x2.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 2x2 * 2x2 +result = matmul(A, B, m:2, n:2, k:2) +return result \ No newline at end of file diff --git a/tests/topic06/cases/tensor/matmul_2x2.meta.json b/tests/topic06/cases/tensor/matmul_2x2.meta.json new file mode 100644 index 0000000..31a0848 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_2x2.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} diff --git a/tests/topic06/cases/tensor/matmul_4x4.dsl b/tests/topic06/cases/tensor/matmul_4x4.dsl new file mode 100644 index 0000000..ab5a865 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_4x4.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 4x4 * 4x4 +result = matmul(A, B, m:4, n:4, k:4) +return result diff --git a/tests/topic06/cases/tensor/matmul_4x4.meta.json b/tests/topic06/cases/tensor/matmul_4x4.meta.json new file mode 100644 index 0000000..2adb159 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_4x4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [4, 4], + "inputs": { + "A": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], + "B": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] + }, + "expected_return": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +} diff --git a/tests/topic06/cases/tensor/matmul_add_2x2.dsl b/tests/topic06/cases/tensor/matmul_add_2x2.dsl new file mode 100644 index 0000000..d41a2cb --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_add_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by add +x = matmul(A, B, m:2, n:2, k:2) +result = add(x, bias) +return result diff --git a/tests/topic06/cases/tensor/matmul_add_2x2.meta.json b/tests/topic06/cases/tensor/matmul_add_2x2.meta.json new file mode 100644 index 0000000..b336c80 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_add_2x2.meta.json @@ -0,0 +1,12 @@ +{ + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]], + "bias": [[1, 1], [1, 1]] + }, + "expected_return": [[20, 23], [44, 51]] +} diff --git a/tests/topic06/cases/tensor/matmul_relu_2x2.dsl b/tests/topic06/cases/tensor/matmul_relu_2x2.dsl new file mode 100644 index 0000000..28c7c61 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_relu_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by ReLU +x = matmul(A, B, m:2, n:2, k:2) +result = relu(x) +return result diff --git a/tests/topic06/cases/tensor/matmul_relu_2x2.meta.json b/tests/topic06/cases/tensor/matmul_relu_2x2.meta.json new file mode 100644 index 0000000..e2efb5e --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_relu_2x2.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[-1, 2], [-3, 4]], + "B": [[1, 0], [0, 1]] + }, + "expected_return": [[0, 2], [0, 4]] +} diff --git a/tests/topic06/test_integration.py b/tests/topic06/test_integration.py new file mode 100644 index 0000000..1af7eec --- /dev/null +++ b/tests/topic06/test_integration.py @@ -0,0 +1,196 @@ +"""Integration contracts for the Topic 06 benchmark suite.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CASE_DIR = ROOT / "tests" / "topic06" / "cases" +RUNNER_PATH = ROOT / "scripts" / "run_topic06_benchmarks.py" + + +def _load_runner(): + spec = importlib.util.spec_from_file_location("topic06_runner", RUNNER_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_topic06_case_manifest_is_complete(): + dsl_files = sorted(CASE_DIR.rglob("*.dsl")) + meta_files = sorted(CASE_DIR.rglob("*.meta.json")) + + assert len(dsl_files) == 23 + assert len(meta_files) == 23 + assert {path.with_suffix("") for path in dsl_files} == { + path.with_suffix("").with_suffix("") for path in meta_files + } + + +def test_compiler_emits_register_map_for_topic06_case(tmp_path): + from scratchv.compiler import CompilerConfig, CompilerDriver + + driver = CompilerDriver(CompilerConfig( + backend="riscv", + optimize_level="all", + reg_alloc="greedy", + )) + source = CASE_DIR / "activation" / "relu_add.dsl" + result = driver.compile(str(source), str(tmp_path / "relu_add.s")) + + assert result.success + assert result.stats["register_map"]["input"] == "t0" + assert result.stats["register_map"]["bias"] == "t1" + + +def test_runner_loads_compiler_register_map(tmp_path): + runner = _load_runner() + register_map_file = tmp_path / "registers.json" + register_map_file.write_text( + json.dumps({ + "schema_version": 1, + "register_map": {"input": "t0", "bias": "t1"}, + }), + encoding="utf-8", + ) + + initial = runner.load_initial_registers( + register_map_file, + {"input": -3, "bias": 7, "tensor": [1, 2]}, + ) + + assert initial == {"t0": -3, "t1": 7} + + +def test_topic06_report_paths_are_portable(): + runner = _load_runner() + case = runner.TEST_DIR / "activation" / "relu_only.dsl" + + assert runner._report_path(case) == "tests/topic06/cases/activation/relu_only.dsl" + + +def test_runner_uses_project_layout(): + runner = _load_runner() + + assert runner.PROJECT_ROOT == ROOT + assert runner.TEST_DIR == ROOT / "tests" / "topic06" / "cases" + assert runner.BUILD_DIR == ROOT / "build" / "topic06" + assert runner.REPORT_DIR == ROOT / "benchmark_reports" / "topic06" + assert runner.BASELINE_FILE == ROOT / "benchmarks" / "topic06" / "baseline.json" + + +def test_interpreter_verifies_scalar_and_tensor_cases(): + runner = _load_runner() + + for relative_path in ( + Path("activation/relu_add.dsl"), + Path("tensor/matmul_2x2.dsl"), + ): + dsl_file = CASE_DIR / relative_path + metadata = runner.load_metadata(dsl_file) + result = runner.run_interpreter(dsl_file, metadata["inputs"]) + + assert result["success"] + assert runner.values_equal( + result["return_value"], + metadata["expected_return"], + ) + + +def test_interpreter_marks_control_flow_as_unsupported(): + runner = _load_runner() + + for relative_path in ( + Path("loop/loop_add_4.dsl"), + Path("branch/if_else.dsl"), + ): + dsl_file = CASE_DIR / relative_path + metadata = runner.load_metadata(dsl_file) + result = runner.run_interpreter(dsl_file, metadata["inputs"]) + + assert result["status"] == "UNSUPPORTED" + assert not result["success"] + + +def test_runner_writes_independent_backend_case_report(tmp_path, monkeypatch): + runner = _load_runner() + report_dir = tmp_path / "reports" + monkeypatch.setattr(runner, "BUILD_DIR", tmp_path / "build") + monkeypatch.setattr(runner, "REPORT_DIR", report_dir) + monkeypatch.setattr(runner, "REPORT_FILE", report_dir / "report.md") + monkeypatch.setattr(runner, "JSON_REPORT_FILE", report_dir / "report.json") + monkeypatch.setattr(runner, "HTML_REPORT_FILE", report_dir / "report.html") + monkeypatch.setattr(runner, "CHART_FILE", report_dir / "chart.png") + monkeypatch.setattr(runner, "FAILURE_DIR", report_dir / "failures") + monkeypatch.setattr(runner, "CASE_REPORT_DIR", report_dir / "cases") + + exit_code = runner.main([ + "--filter", "relu_only", + "--verification-backend", "both", + "--fail-on-test-failure", + ]) + + assert exit_code == 0 + payload = json.loads((report_dir / "report.json").read_text(encoding="utf-8")) + result = payload["results"][0] + assert payload["schema_version"] == 2 + assert result["interpreter_status"] == "PASS" + assert result["tinyfive_status"] == "PASS" + assert result["backend_outputs_match"] is True + assert result["static_asm_instruction_count"] > 0 + assert (report_dir / "cases/activation-relu_only.md").exists() + assert (report_dir / "cases/activation-relu_only.json").exists() + + +def test_tinyfive_input_abi_marks_tensor_inputs_unsupported(): + runner = _load_runner() + + supported, reason = runner.tinyfive_input_abi_supported({"A": [[1, 2], [3, 4]]}) + + assert not supported + assert "A" in reason + + +def test_cost_model_baseline_uses_versioned_schema(tmp_path, monkeypatch): + runner = _load_runner() + baseline_file = tmp_path / "baseline.json" + monkeypatch.setattr(runner, "BASELINE_FILE", baseline_file) + + runner.save_baseline([{ + "name": "relu_only", + "category": "activation", + "avg_instr_count": 5.0, + "benchmark_runs": 3, + "tinyfive_status": "PASS", + "cost_model": { + "static_asm_instructions": 3, + "machine_instructions": 6, + "code_size_bytes": 24, + "dynamic_instructions": 5.0, + }, + }]) + + payload = json.loads(baseline_file.read_text(encoding="utf-8")) + assert payload["schema_version"] == 2 + assert payload["primary_metric"] == "dynamic_instructions" + assert payload["cases"]["relu_only"]["cost_model"]["code_size_bytes"] == 24 + assert runner.load_baseline() == payload["cases"] + + +def test_interpreter_only_mode_cannot_replace_cost_model_baseline(tmp_path, monkeypatch): + runner = _load_runner() + baseline_file = tmp_path / "baseline.json" + monkeypatch.setattr(runner, "BASELINE_FILE", baseline_file) + + exit_code = runner.main([ + "--benchmark", "1", + "--update-baseline", + "--verification-backend", "interpreter", + ]) + + assert exit_code == 2 + assert not baseline_file.exists()