From 82feb03e757bc0b1eff140608e82e9af6077c9fc Mon Sep 17 00:00:00 2001 From: LinxISA Automation Date: Wed, 1 Jul 2026 12:57:37 +0800 Subject: [PATCH] Port SuperNPUBench to PTO v0.57 block flow The LinxISA and PTO ISA benchmark trees now use the v0.57 tile descriptor syntax, updated TileOP API surface, synced kernel wrappers, and stricter compile wrappers that fail when an operator fails. Constraint: Issue #14 requires the hardware-visible memory, reduction, and elementwise TileOP API surface to compile without encoding sync, comm, pipe lifecycle, or IR-only PTO operations Rejected: Keep compile_all.sh exit-zero on failed operators | hidden failures made prior flow evidence unreliable Confidence: high Scope-risk: broad Directive: Keep benchmark-linxisa and benchmark-ptoisa wrapper APIs aligned when adding v0.57 TileOP forms Tested: COMPILER_DIR=/Users/zhoubot/linx-isa/compiler/llvm/build-linxisa-clang/bin bash workloads/SuperNPUBench/compile_all.sh all returned RC:0 Tested: 26 successful operator groups, 49 LinxISA ELFs, 56 PTO ISA ELFs, zero failure markers --- benchmark-linxisa/compile_all.sh | 21 +- .../kernels/broadcast/broadcast.hpp | 407 ++++-------- .../kernels/broadcast/broadcast_019.hpp | 2 +- .../kernels/broadcast/broadcast_039.hpp | 2 +- .../kernels/broadcast/broadcast_07.hpp | 2 +- .../kernels/broadcast/broadcast_07_simple.hpp | 2 +- .../kernels/broadcast/broadcast_Hunyuan.hpp | 2 +- .../kernels/broadcast/broadcast_mscatter.hpp | 2 +- .../kernels/broadcast/broadcast_nocopyout.hpp | 2 +- .../kernels/broadcast/broadcast_nomg.hpp | 2 +- .../kernels/broadcast/broadcast_pto.hpp | 248 ++++++++ .../kernels/broadcast/broadcast_simple.hpp | 2 +- .../kernels/broadcast/broadcast_vec_019.hpp | 218 +++---- .../broadcast/broadcast_vec_019_pto.hpp | 154 +++++ .../kernels/broadcast/broadcast_vec_039.hpp | 137 +--- .../broadcast/broadcast_vec_039_pto.hpp | 155 +++++ .../kernels/broadcast/broadcast_vec_07.hpp | 152 ++--- .../broadcast/broadcast_vec_07_handwrite.hpp | 2 +- .../broadcast/broadcast_vec_07_pto.hpp | 132 ++++ .../kernels/concat/concat_gather.hpp | 354 +++++------ .../kernels/concat/concat_scatter.hpp | 317 ++++------ .../kernels/control/hashtable_lookup_simd.hpp | 1 + .../kernels/element_wise/gelu.hpp | 177 ++---- .../kernels/element_wise/gelu_pto.hpp | 277 ++++++++ benchmark-linxisa/kernels/fa/fa_2d_unroll.hpp | 95 ++- benchmark-linxisa/kernels/fa/fa_fp4_utils.h | 5 + benchmark-linxisa/kernels/fa/fa_utils.h | 4 + benchmark-linxisa/kernels/fa/linx_blkc.h | 121 ++++ benchmark-linxisa/kernels/gather/gather.hpp | 283 +++------ .../kernels/gather/gather_pto.hpp | 199 ++++++ benchmark-linxisa/kernels/matmul/matmul.hpp | 9 +- .../kernels/matmul/matmul_mx.hpp | 18 +- .../kernels/reduction/cumsum_colvec.hpp | 177 +----- .../kernels/reduction/cumsum_rowvec.hpp | 170 +---- .../kernels/reduction/reducemax_colvec.hpp | 167 ++--- .../reducemax_colvec_unalign_120_8.hpp | 157 +---- .../kernels/reduction/reducemax_rowvec.hpp | 189 ++---- .../reducemax_rowvec_single_tree.hpp | 227 +------ .../kernels/reduction/reduceprod_colvec.hpp | 141 ++--- .../kernels/reduction/reduceprod_rowvec.hpp | 162 ++--- .../kernels/reduction/reducesum_colvec.hpp | 169 ++--- .../reducesum_colvec_single_tree.hpp | 235 ++----- .../reducesum_colvec_unalign_120_8.hpp | 162 +---- .../kernels/reduction/reducesum_rowvec.hpp | 184 ++---- .../reducesum_rowvec_single_tree.hpp | 227 +------ benchmark-linxisa/kernels/sort/topk.hpp | 14 +- .../kernels/transpose/transpose.hpp | 589 ++++++++++++------ .../transpose/transpose_vector_007.hpp | 114 ++-- .../transpose/transpose_vector_050.hpp | 136 ++-- .../kernels/utils/layout_transform.hpp | 24 +- benchmark-linxisa/test/common/Makefile.common | 28 +- benchmark-linxisa/test/common/_start.s | 4 +- .../test/common/src/benchmark_boot_linx.s | 4 +- benchmark-linxisa/test/common/template_asm.h | 221 +++---- .../test/kernel/control/compile.all | 2 +- .../data_obj/build_data_obj.sh | 6 +- .../control/hkv/data_obj/build_data_obj.sh | 4 +- .../test/kernel/element_wise/gelu/compile.all | 2 +- benchmark-linxisa/test/kernel/fa/Makefile | 10 +- benchmark-linxisa/test/kernel/fa/compile.all | 19 +- .../test/kernel/fa/src/fa_2d_unroll.cpp | 9 + .../test/kernel/gather/compile.all | 2 +- .../test/kernel/matmul/compile.all | 32 +- .../test/kernel/matmul/src/A16W4.cpp | 4 +- .../reduction/reducemax_col/compile.all | 4 +- .../reducemax_col/src/reducemax_col.cpp | 25 +- .../reduction/reducemax_row/compile.all | 4 +- .../reduction/reducesum_col/compile.all | 4 +- .../reduction/reducesum_row/compile.all | 4 +- .../sort/topk/data_obj/build_data_obj.sh | 6 +- .../test/kernel/sort/topk/topk.cpp | 42 +- benchmark-ptoisa/compile_all.sh | 21 +- .../kernels/broadcast/broadcast.hpp | 112 ++++ .../kernels/broadcast/broadcast_pto.hpp | 2 +- .../kernels/broadcast/broadcast_vec_019.hpp | 99 +++ .../broadcast/broadcast_vec_019_pto.hpp | 8 +- .../kernels/broadcast/broadcast_vec_039.hpp | 6 + .../broadcast/broadcast_vec_039_pto.hpp | 8 +- .../kernels/broadcast/broadcast_vec_07.hpp | 69 ++ .../broadcast/broadcast_vec_07_pto.hpp | 2 +- .../kernels/concat/concat_gather.hpp | 3 +- .../kernels/concat/concat_scatter.hpp | 3 +- .../kernels/control/hashtable_lookup_simd.hpp | 1 + .../kernels/element_wise/gelu.hpp | 77 +++ benchmark-ptoisa/kernels/fa/fa_2d_unroll.hpp | 95 ++- benchmark-ptoisa/kernels/fa/fa_fp4_utils.h | 5 + benchmark-ptoisa/kernels/fa/fa_utils.h | 4 + benchmark-ptoisa/kernels/fa/linx_blkc.h | 121 ++++ benchmark-ptoisa/kernels/gather/gather.hpp | 92 +++ benchmark-ptoisa/kernels/matmul/matmul.hpp | 9 +- benchmark-ptoisa/kernels/matmul/matmul_mx.hpp | 18 +- .../kernels/reduction/cumsum_colvec.hpp | 1 - .../kernels/reduction/cumsum_rowvec.hpp | 1 - .../kernels/reduction/reducemax_colvec.hpp | 1 - .../reducemax_colvec_unalign_120_8.hpp | 1 - .../kernels/reduction/reducemax_rowvec.hpp | 1 - .../reducemax_rowvec_single_tree.hpp | 1 - .../kernels/reduction/reduceprod_colvec.hpp | 1 - .../kernels/reduction/reduceprod_rowvec.hpp | 1 - .../kernels/reduction/reducesum_colvec.hpp | 3 +- .../reducesum_colvec_single_tree.hpp | 1 - .../reducesum_colvec_unalign_120_8.hpp | 1 - .../kernels/reduction/reducesum_rowvec.hpp | 1 - .../reducesum_rowvec_single_tree.hpp | 1 - benchmark-ptoisa/kernels/sort/topk.hpp | 14 +- .../kernels/transpose/transpose.hpp | 14 + .../transpose/transpose_vector_007.hpp | 1 - .../transpose/transpose_vector_050.hpp | 1 - .../kernels/utils/layout_transform.hpp | 24 +- benchmark-ptoisa/test/common/Makefile.common | 28 +- benchmark-ptoisa/test/common/_start.s | 4 +- .../test/common/src/benchmark_boot_linx.s | 4 +- benchmark-ptoisa/test/common/template_asm.h | 221 +++---- .../test/kernel/control/compile.all | 2 +- .../data_obj/build_data_obj.sh | 6 +- .../control/hkv/data_obj/build_data_obj.sh | 4 +- .../test/kernel/element_wise/gelu/compile.all | 2 +- benchmark-ptoisa/test/kernel/fa/Makefile | 10 +- benchmark-ptoisa/test/kernel/fa/compile.all | 19 +- .../test/kernel/fa/src/fa_2d_unroll.cpp | 9 + .../test/kernel/gather/compile.all | 2 +- .../test/kernel/matmul/compile.all | 32 +- .../test/kernel/matmul/src/A16W4.cpp | 4 +- .../reduction/reducemax_col/compile.all | 4 +- .../reducemax_col/src/reducemax_col.cpp | 25 +- .../reduction/reducemax_row/compile.all | 4 +- .../reduction/reducesum_col/compile.all | 4 +- .../reduction/reducesum_row/compile.all | 4 +- .../sort/topk/data_obj/build_data_obj.sh | 6 +- .../test/kernel/sort/topk/topk.cpp | 42 +- compile_all.sh | 24 +- 131 files changed, 4397 insertions(+), 4071 deletions(-) create mode 100644 benchmark-linxisa/kernels/broadcast/broadcast_pto.hpp create mode 100644 benchmark-linxisa/kernels/broadcast/broadcast_vec_019_pto.hpp create mode 100644 benchmark-linxisa/kernels/broadcast/broadcast_vec_039_pto.hpp create mode 100644 benchmark-linxisa/kernels/broadcast/broadcast_vec_07_pto.hpp create mode 100644 benchmark-linxisa/kernels/element_wise/gelu_pto.hpp create mode 100644 benchmark-linxisa/kernels/fa/linx_blkc.h create mode 100644 benchmark-linxisa/kernels/gather/gather_pto.hpp create mode 100644 benchmark-ptoisa/kernels/broadcast/broadcast.hpp create mode 100644 benchmark-ptoisa/kernels/broadcast/broadcast_vec_019.hpp create mode 100644 benchmark-ptoisa/kernels/broadcast/broadcast_vec_039.hpp create mode 100644 benchmark-ptoisa/kernels/broadcast/broadcast_vec_07.hpp create mode 100644 benchmark-ptoisa/kernels/element_wise/gelu.hpp create mode 100644 benchmark-ptoisa/kernels/fa/linx_blkc.h create mode 100644 benchmark-ptoisa/kernels/gather/gather.hpp diff --git a/benchmark-linxisa/compile_all.sh b/benchmark-linxisa/compile_all.sh index 172590a..a586d36 100755 --- a/benchmark-linxisa/compile_all.sh +++ b/benchmark-linxisa/compile_all.sh @@ -4,15 +4,19 @@ # Don't use set -e as some operators may fail to compile SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -export COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/GitHub/linx-toolchain-build/output/linx_blockisa_llvm_musl/bin} +LINX_ISA_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +export COMPILER_DIR=${COMPILER_DIR:-$LINX_ISA_ROOT/compiler/llvm/build-linxisa-clang/bin} REPO_ROOT=${REPO_ROOT:-$SCRIPT_DIR} echo "==========================================" echo "[LinxISA] Starting full compilation" echo "REPO_ROOT: $REPO_ROOT" +echo "COMPILER_DIR: $COMPILER_DIR" echo "==========================================" # Function to compile an operator +FAILURES=0 + compile_operator() { local operator_path=$1 local operator_name=$2 @@ -25,10 +29,15 @@ compile_operator() { if [ ! -d "$operator_path" ]; then echo "Warning: Directory not found: $operator_path" + FAILURES=$((FAILURES + 1)) return 1 fi - cd "$operator_path" + if ! cd "$operator_path"; then + echo "✗ $operator_name compilation failed: cannot enter directory" + FAILURES=$((FAILURES + 1)) + return 1 + fi if [ -f "compile.all" ]; then echo "Running compile.all with baremetal=${baremetal:-off}..." @@ -37,9 +46,12 @@ compile_operator() { echo "✓ $operator_name compilation completed" else echo "✗ $operator_name compilation failed" + FAILURES=$((FAILURES + 1)) + return 1 fi else echo "Warning: No compile.all found in $operator_path" + FAILURES=$((FAILURES + 1)) return 1 fi } @@ -67,3 +79,8 @@ echo "" echo "Generated ELF files:" find "$REPO_ROOT/output" -name "*.elf" -type f | wc -l echo "ELF files are located in: $REPO_ROOT/output/" + +if [ "$FAILURES" -ne 0 ]; then + echo "Compilation failures: $FAILURES" + exit 1 +fi diff --git a/benchmark-linxisa/kernels/broadcast/broadcast.hpp b/benchmark-linxisa/kernels/broadcast/broadcast.hpp index 5276b2b..75082ee 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast.hpp @@ -1,305 +1,112 @@ +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_HPP + #include -#include "template_asm.h" +#include #include -#include - - -#define DUMP_TILE(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%12lld ", (long long)DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) - -// ============================================== -// 维度规则：从后面对齐，前面自动补 1，维度=1 可广播 -// ============================================== -template -void __vec__ gen_offset( - typename tile_shape::TileDType __out__ out, - #if MAX_DIMs >= 1 - size_t in0, size_t out0, - #endif - #if MAX_DIMs >= 2 - size_t in1, size_t out1, - #endif - #if MAX_DIMs >= 3 - size_t in2, size_t out2, - #endif - #if MAX_DIMs >= 4 - size_t in3, size_t out3, - #endif - #if MAX_DIMs >= 5 - size_t in4, size_t out4, - #endif - #if MAX_DIMs >= 6 - size_t in5, size_t out5, - #endif - #if MAX_DIMs >= 7 - size_t in6, size_t out6, - #endif - #if MAX_DIMs >= 8 - size_t in7, size_t out7, - #endif - const size_t base, - const size_t total_elements -) { - size_t index = blkv_get_index_x(); - // if (index >= total_elements) return; - size_t idx = index + base; - - size_t out_coord[MAX_DIM] = {0}; - size_t tmp = idx; - - // ====== 输出坐标计算 ====== - for (int d = OUT_DIM - 1; d >= 0; d--) { - if (d == 0) { out_coord[d] = tmp % out0; tmp /= out0; } - #if MAX_DIMs >=2 - else if (d == 1) { out_coord[d] = tmp % out1; tmp /= out1; } - #endif - #if MAX_DIMs >=3 - else if (d == 2) { out_coord[d] = tmp % out2; tmp /= out2; } - #endif - #if MAX_DIMs >=4 - else if (d == 3) { out_coord[d] = tmp % out3; tmp /= out3; } - #endif - #if MAX_DIMs >=5 - else if (d == 4) { out_coord[d] = tmp % out4; tmp /= out4; } - #endif - #if MAX_DIMs >=6 - else if (d == 5) { out_coord[d] = tmp % out5; tmp /= out5; } - #endif - #if MAX_DIMs >=7 - else if (d == 6) { out_coord[d] = tmp % out6; tmp /= out6; } - #endif - #if MAX_DIMs >=8 - else if (d == 7) { out_coord[d] = tmp % out7; tmp /= out7; } - #endif - } - - // ====== 输入坐标计算 ====== - size_t in_coord[MAX_DIM] = {0}; - for (int i = OUT_DIM - 1; i >= 0; i--) { - int o = OUT_DIM - IN_DIM + i; - size_t in_dim; - - if (i == 0) in_dim = in0; - #if MAX_DIMs >=2 - else if (i == 1) in_dim = in1; - #endif - #if MAX_DIMs >=3 - else if (i == 2) in_dim = in2; - #endif - #if MAX_DIMs >=4 - else if (i == 3) in_dim = in3; - #endif - #if MAX_DIMs >=5 - else if (i == 4) in_dim = in4; - #endif - #if MAX_DIMs >=6 - else if (i == 5) in_dim = in5; - #endif - #if MAX_DIMs >=7 - else if (i == 6) in_dim = in6; - #endif - #if MAX_DIMs >=8 - else if (i == 7) in_dim = in7; - #endif - - if (in_dim == 1) - in_coord[i] = 0; - else - in_coord[i] = out_coord[o]; - } - - // ====== 计算偏移 ====== - size_t in_offset = 0; - size_t data_width = sizeof(dtype); - for (int i = 0; i < IN_DIM; i++) { - if (i == 0) in_offset = in_offset * in0 + in_coord[0]; - #if MAX_DIMs >=2 - else if (i == 1) in_offset = in_offset * in1 + in_coord[1]; - #endif - #if MAX_DIMs >=3 - else if (i == 2) in_offset = in_offset * in2 + in_coord[2]; - #endif - #if MAX_DIMs >=4 - else if (i == 3) in_offset = in_offset * in3 + in_coord[3]; - #endif - #if MAX_DIMs >=5 - else if (i == 4) in_offset = in_offset * in4 + in_coord[4]; - #endif - #if MAX_DIMs >=6 - else if (i == 5) in_offset = in_offset * in5 + in_coord[5]; - #endif - #if MAX_DIMs >=7 - else if (i == 6) in_offset = in_offset * in6 + in_coord[6]; - #endif - #if MAX_DIMs >=8 - else if (i == 7) in_offset = in_offset * in7 + in_coord[7]; - #endif - } - in_offset *= data_width; - - blkv_get_tile_ptr(out)[index] = in_offset; -} -template -void gen_offset_impl( - tile_shapeOffset &out, - const size_t *in_shape, - const size_t *out_shape, - const size_t base, - const size_t total_elements) { - static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, - "Only static shape supported"); - - #if MAX_DIMs >= 1 - size_t in_shape0 = in_shape[0]; - size_t out_shape0 = out_shape[0]; - #endif - #if MAX_DIMs >= 2 - size_t in_shape1 = in_shape[1]; - size_t out_shape1 = out_shape[1]; - #endif - #if MAX_DIMs >= 3 - size_t in_shape2 = in_shape[2]; - size_t out_shape2 = out_shape[2]; - #endif - #if MAX_DIMs >= 4 - size_t in_shape3 = in_shape[3]; - size_t out_shape3 = out_shape[3]; - #endif - #if MAX_DIMs >= 5 - size_t in_shape4 = in_shape[4]; - size_t out_shape4 = out_shape[4]; - #endif - #if MAX_DIMs >= 6 - size_t in_shape5 = in_shape[5]; - size_t out_shape5 = out_shape[5]; - #endif - #if MAX_DIMs >= 7 - size_t in_shape6 = in_shape[6]; - size_t out_shape6 = out_shape[6]; - #endif - #if MAX_DIMs >= 8 - size_t in_shape7 = in_shape[7]; - size_t out_shape7 = out_shape[7]; - #endif - - gen_offset<<>>( - out.data(), - #if MAX_DIMs >= 1 - in_shape0, out_shape0, - #endif - #if MAX_DIMs >= 2 - in_shape1, out_shape1, - #endif - #if MAX_DIMs >= 3 - in_shape2, out_shape2, - #endif - #if MAX_DIMs >= 4 - in_shape3, out_shape3, - #endif - #if MAX_DIMs >= 5 - in_shape4, out_shape4, - #endif - #if MAX_DIMs >= 6 - in_shape5, out_shape5, - #endif - #if MAX_DIMs >= 7 - in_shape6, out_shape6, - #endif - #if MAX_DIMs >= 8 - in_shape7, out_shape7, - #endif - base, - total_elements); -} - - - -template -void broadcast( - dtype *in_ptr, - dtype *out_ptr, - const size_t *in_shape, - const size_t *out_shape - ) { - const size_t Mb = gOM / tM; - const size_t rmd_M = gOM % tM; - - using gm_shapeIn = global_tensor>; - using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; - using tile_shapeOffset = Tile; - using tile_shapeData_rmd = Tile; - using tile_shapeOffset_rmd = Tile; - - gm_shapeIn inGm(in_ptr); - // gm_shapeOut outGm(out_ptr); - tile_shapeData outTile; - tile_shapeOffset offsetTile; - tile_shapeData_rmd outTile_rmd; - tile_shapeOffset_rmd offsetTile_rmd; - size_t base = 0; - size_t all_num = gOM; // 总元素数量 - - using itOut = global_iterator; - - // tile_shapeData inTile; - // using itIn = global_iterator; - - itOut gOIter(out_ptr); - // itIn gIIter(in_ptr); - // for test /////////////////////////////////////// - // alignas(256) static uint32_t g_dump[tM]; - // alignas(256) static dtype g_dump_outTile[tM]; - // alignas(256) static dtype g_dump_inTile[tM]; - // /////////////////////////////////////// - - size_t total_elements = tM; - for (int i = 0; i < Mb; ++i) { - auto gO = gOIter(0, i); - // auto gI = gIIter(0, i); - // printf("iter = %d\n", i); - // printf("base = %d\n", base); - // printf("total_elements = %d\n", total_elements); - // printf("in_shape[0] = %d\n", in_shape[0]); - // printf("inGm = %ld\n", inGm); - - // TCOPYIN(inTile, gI); - // DUMP_TILE("inTile", inTile, g_dump_inTile, 1, tM); - gen_offset_impl(offsetTile, in_shape, out_shape, base, total_elements); - base += total_elements; - - // DUMP_TILE("offsetTile", offsetTile, g_dump, 1, tM); - - MGATHER(outTile, inGm, offsetTile); - - // DUMP_TILE("outTile", outTile, g_dump_outTile, 1, tM); - TCOPYOUT(gO, outTile); - } - if constexpr (rmd_M) { - // printf("rmd_M = %d\n", rmd_M); - auto gO = gOIter(0, Mb); - total_elements = rmd_M; - gen_offset_impl(offsetTile_rmd, in_shape, out_shape, base, total_elements); - base += total_elements; - MGATHER(outTile_rmd, inGm, offsetTile_rmd); - TCOPYOUT(gO, outTile_rmd); - } - +#include + +using namespace pto; + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t *in_shape, + const size_t *out_shape) { + constexpr size_t kBCast = gOM / gIM; + constexpr size_t kElementsPerTile = 512; + constexpr size_t kFullTiles = gOM / kElementsPerTile; + constexpr size_t kTail = gOM % kElementsPerTile; + + static_assert(MAX_DIM >= IN_DIM && MAX_DIM >= OUT_DIM, + "MAX_DIM must cover input and output ranks"); + static_assert(gOM % gIM == 0, + "broadcast output element count must be a multiple of input"); + static_assert(kBCast > 0, "broadcast factor must be positive"); + (void)kTileHint; + + size_t inner = 1; + size_t in_dim = IN_DIM; + size_t out_dim = OUT_DIM; + while (in_dim > 0 && out_dim > 0 && + in_shape[in_dim - 1] == out_shape[out_dim - 1]) { + inner *= out_shape[out_dim - 1]; + --in_dim; + --out_dim; + } + + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + using DataTile = Tile; + using OffsetTile = Tile; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputIterator output_iter(out_ptr); + + auto emit_tile = [&](auto &data_tile, auto &linear, auto &batch, + auto &batch_base, auto &inner_q, auto &inner_base, + auto &inner_idx, auto &inner_bytes, auto &offset, + std::uint32_t base, auto output_addr) { + using LinearTile = std::remove_reference_t; + const auto inner_u32 = static_cast(inner); + const auto group_u32 = static_cast(kBCast * inner); + TCI(linear, base); + + TDIVS(batch, linear, group_u32); + TMULS(batch_base, batch, + static_cast(inner * sizeof(dtype))); + + TDIVS(inner_q, linear, inner_u32); + TMULS(inner_base, inner_q, inner_u32); + TSUB(inner_idx, linear, inner_base); + TMULS(inner_bytes, inner_idx, static_cast(sizeof(dtype))); + + TADD(offset, batch_base, inner_bytes); + MGATHER(data_tile, input_global, offset); + TSTORE(output_addr, data_tile); + }; + + for (size_t tile = 0; tile < kFullTiles; ++tile) { + DataTile data_tile; + OffsetTile linear; + OffsetTile batch; + OffsetTile batch_base; + OffsetTile inner_q; + OffsetTile inner_base; + OffsetTile inner_idx; + OffsetTile inner_bytes; + OffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, + inner_idx, inner_bytes, offset, + static_cast(tile * kElementsPerTile), + output_iter(0, static_cast(tile))); + } + + if constexpr (kTail != 0) { + using TailDataTile = Tile; + using TailOffsetTile = + Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailDataTile data_tile; + TailOffsetTile linear; + TailOffsetTile batch; + TailOffsetTile batch_base; + TailOffsetTile inner_q; + TailOffsetTile inner_base; + TailOffsetTile inner_idx; + TailOffsetTile inner_bytes; + TailOffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, + inner_idx, inner_bytes, offset, + static_cast(kFullTiles * kElementsPerTile), + tail_output_iter(0, static_cast(kFullTiles))); + } } - - +#endif // SUPERNPUBENCH_PTOISA_BROADCAST_HPP diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_019.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_019.hpp index b6916f2..1a5f750 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_019.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_019.hpp @@ -260,7 +260,7 @@ void gen_offset_impl( -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_039.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_039.hpp index b3fc5e5..fe3bc66 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_039.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_039.hpp @@ -309,7 +309,7 @@ void gen_offset_impl( -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_07.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_07.hpp index 4b3b7b1..28a0649 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_07.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_07.hpp @@ -260,7 +260,7 @@ void gen_offset_impl( -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_07_simple.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_07_simple.hpp index 75e03d0..ef429b7 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_07_simple.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_07_simple.hpp @@ -32,7 +32,7 @@ void gen_offset_impl( } } -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_Hunyuan.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_Hunyuan.hpp index 80b5b1c..5ec65ef 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_Hunyuan.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_Hunyuan.hpp @@ -265,7 +265,7 @@ void gen_offset_impl( -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_mscatter.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_mscatter.hpp index f38161b..509e498 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_mscatter.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_mscatter.hpp @@ -86,7 +86,7 @@ void gen_scatter_offset_impl( // 通用MSCATTER版 Broadcast // 自动识别任意广播维度 // ============================================== -template +template void broadcast_mscatter( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_nocopyout.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_nocopyout.hpp index 5340d09..825261d 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_nocopyout.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_nocopyout.hpp @@ -228,7 +228,7 @@ void gen_offset_impl( -template +template void broadcast_nocopyout( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_nomg.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_nomg.hpp index 16c5332..391ccf9 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_nomg.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_nomg.hpp @@ -228,7 +228,7 @@ void gen_offset_impl( -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_pto.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_pto.hpp new file mode 100644 index 0000000..cfaf699 --- /dev/null +++ b/benchmark-linxisa/kernels/broadcast/broadcast_pto.hpp @@ -0,0 +1,248 @@ +// ============================================================================ +// Broadcast 算子 — PTO 一层编程模型 +// +// 以 Tile_ISA_Reference 文档为准，使用 PTO ISA 定义的 C++ Intrinsic 接口。 +// 当前编译器 (Linx-TileOP-API) 尚未完全实现 PTO ISA，本文件按 ISA 文档书写， +// 暂不保证可编译通过。 +// +// ┌─────────────────────────────────────────────────────────────────────────┐ +// │ 当前编译器不支持 / 不完整的指令汇总 │ +// ├──────────┬──────────────────┬──────────────────────────────────────────┤ +// │ Pto ISA │ 当前编译器状态 │ 说明 │ +// │ 指令 │ │ │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TCI │ API 有，二层实现 │ pto_tileop.hpp 有声明； │ +// │ │ │ jcore/TCI.hpp 用 __vec__ + blkv_get_* │ +// │ │ │ 实现，非真正一层 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TEXPANDS │ API 有(名不同)， │ Pto ISA 名 TEXPANDS； │ +// │ │ 二层实现 │ 当前编译器名 TEXPANDSCALAR； │ +// │ │ │ jcore 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TREMS │ 完全缺失 │ pto_tileop.hpp 中无此 API； │ +// │ │ │ 当前编译器无任何标量取余实现 │ +// │ │ │ (仅有 tile-tile 的 TREM，且仅 int32) │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TDIVS │ API 有，二层实现 │ jcore/TDivs.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TMULS │ API 有，二层实现 │ jcore/TMuls.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TADD │ API 有，二层实现 │ jcore/TAdd.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ MGATHER │ 已支持(一层) │ template_asm.h 用 asm volatile │ +// │ │ │ (BSTART.TMA) 实现； │ +// │ │ │ 但缺少 Coalesce::Elem 模板参数支持 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TSTORE │ API 有(名不同)， │ Pto ISA 名 TSTORE； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYOUT； │ +// │ │ │ jcore/TCopyOut.hpp 用 __vec__ 实现 │ +// └──────────┴──────────────────┴──────────────────────────────────────────┘ +// +// PTO ISA 文档签名 (Declared in include/pto/pto_instr.hpp): +// +// TCI: +// template +// PTO_INST RecordEvent TCI(TileData &dst, T start, WaitEvents &... events); +// +// TEXPANDS: +// template +// PTO_INST RecordEvent TEXPANDS(TileData &dst, typename TileData::DType scalar, +// WaitEvents &... events); +// +// TREMS: +// template +// PTO_INST RecordEvent TREMS(TileDataDst &dst, TileDataSrc &src, +// typename TileDataSrc::DType scalar, +// TileDataTmp &tmp, WaitEvents &... events); +// +// TDIVS: +// template +// PTO_INST RecordEvent TDIVS(TileDataDst &dst, TileDataSrc &src0, +// typename TileDataSrc::DType scalar, +// WaitEvents &... events); +// +// TMULS: +// template +// PTO_INST RecordEvent TMULS(TileDataDst &dst, TileDataSrc &src0, +// typename TileDataSrc::DType scalar, +// WaitEvents &... events); +// +// TADD: +// template +// PTO_INST RecordEvent TADD(TileDataDst &dst, TileDataSrc0 &src0, +// TileDataSrc1 &src1, WaitEvents &... events); +// +// MGATHER: +// template +// PTO_INST RecordEvent MGATHER(TileDst &dst, GlobalData &src, +// TileInd &indexes, WaitEvents &... events); +// +// TSTORE: +// template +// PTO_INST RecordEvent TSTORE(GlobalData &dst, TileData &src, +// WaitEvents &... events); +// +// ============================================================================ + +#include // Tile, GlobalTensor 等类型 (当前编译器已有) +#include // global_iterator 工具类型 (当前编译器已有) +// #include // [!] PTO ISA C++ Intrinsic — 当前编译器未提供 + +#include +#include + +// ============================================================================ +// 维度规则：从后面对齐，前面自动补 1，维度=1 可广播 +// ============================================================================ + +// ---------------------------------------------------------------------------- +// gen_offset_pto: 用 PTO ISA Tile 指令计算广播偏移 (一层编程，无 __vec__ 块) +// +// 算法 (stride 累加法，从最内维到最外维): +// 1. TCI 生成索引序列 base, base+1, ..., base+N-1 +// 2. TEXPANDS 将输出偏移 tile 初始化为 0 +// 3. 对每个输出维 d (从 OUT_DIM-1 到 0): +// TREMS coord = idx % out_shape[d] (标量取余) +// TDIVS idx = idx / out_shape[d] (标量整除) +// 若 d 对应输入维 i = d-(OUT_DIM-IN_DIM) >= 0: +// 非广播维 (in_shape[i]!=1): +// TMULS tmp = coord * stride +// TADD out += tmp +// stride *= in_shape[i] +// +// 注意: PTO ISA MGATHER 使用元素索引 (非字节偏移), +// 所以这里不再乘 sizeof(dtype)。 +// 原始 broadcast.hpp 的 __vec__ 版本需要乘 sizeof(dtype) 是因为 +// template_asm.h 里的旧 MGATHER 按字节偏移取数。 +// ---------------------------------------------------------------------------- +template +void gen_offset_pto( + tile_shapeOffset &out, + const size_t *in_shape, + const size_t *out_shape, + const size_t base, + const size_t total_elements) +{ + static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, + "Only static shape supported"); + + using off_t = typename tile_shapeOffset::DType; // uint32_t + + tile_shapeOffset idxTile; // 当前正在分解的线性索引 + tile_shapeOffset coordTile; // 当前输出维坐标 + tile_shapeOffset tmpTile; // TREMS 需要 (A2A3 要求 tmp >= 1 行, 列数 >= dst) + + // ---- Step 1: TCI 生成索引序列 [base, base+1, ..., base+N-1] ---- + // [当前编译器] TCI 在 pto_tileop.hpp 有声明，但 jcore 实现为 __vec__ (二层) + TCI(idxTile, (off_t)base); + + // ---- Step 2: TEXPANDS 初始化 out = 0 ---- + // [当前编译器] PTO ISA 名 TEXPANDS，当前编译器名 TEXPANDSCALAR；jcore 为 __vec__ + TEXPANDS(out, (off_t)0); + + size_t stride = 1; + + // ---- Step 3: 从最内维到最外维逐维 divmod + stride 累加 ---- + #pragma clang loop unroll(full) + for (int d = (int)OUT_DIM - 1; d >= 0; d--) { + off_t out_d = (off_t)out_shape[d]; + + // TREMS: coord = idx % out_d + // [当前编译器] 完全缺失！pto_tileop.hpp 无 TREMS API。 + // 退路: 用 TDIVS+TMULS+TSUB 三条拼出取余。 + TREMS(coordTile, idxTile, out_d, tmpTile); + + // TDIVS: idx = idx / out_d (推进到下一维) + // [当前编译器] API 有，jcore 为 __vec__ + TDIVS(idxTile, idxTile, out_d); + + // 该输出维是否对应输入维 + int i = d - (int)(OUT_DIM - IN_DIM); + if (i >= 0) { + if (in_shape[i] != 1) { // 非广播维才累加 + // TMULS: tmp = coord * stride + // [当前编译器] API 有，jcore 为 __vec__ + TMULS(tmpTile, coordTile, (off_t)stride); + // TADD: out += tmp + // [当前编译器] API 有，jcore 为 __vec__ + TADD(out, out, tmpTile); + } + stride *= in_shape[i]; // stride 更新 (广播维 ==1 不变) + } + } + + // 不再乘 sizeof(dtype): + // PTO ISA MGATHER 按"元素索引"取数 (dst[i,j] = src[idx[i,j]])， + // 而非旧 MGATHER 的字节偏移。 +} + + +// ---------------------------------------------------------------------------- +// broadcast: 接口与原 broadcast.hpp 一致 +// ---------------------------------------------------------------------------- +template +void broadcast( + dtype *in_ptr, + dtype *out_ptr, + const size_t *in_shape, + const size_t *out_shape + ) { + const size_t Mb = gOM / tM; + const size_t rmd_M = gOM % tM; + + using gm_shapeIn = global_tensor>; + using gm_shapeOut = global_tensor>; + using tile_shapeData = Tile; + using tile_shapeOffset = Tile; + using tile_shapeData_rmd = Tile; + using tile_shapeOffset_rmd= Tile; + + gm_shapeIn inGm(in_ptr); + tile_shapeData outTile; + tile_shapeOffset offsetTile; + tile_shapeData_rmd outTile_rmd; + tile_shapeOffset_rmd offsetTile_rmd; + size_t base = 0; + + using itOut = global_iterator; + itOut gOIter(out_ptr); + + size_t total_elements = tM; + for (int i = 0; i < Mb; ++i) { + auto gO = gOIter(0, i); + + // 计算偏移 tile (元素索引) + gen_offset_pto( + offsetTile, in_shape, out_shape, base, total_elements); + base += total_elements; + + // MGATHER: 按 offsetTile 中的元素索引从 inGm 取数 + // [当前编译器] template_asm.h 的 MGATHER 已用 asm volatile (一层)， + // 但不支持 Coalesce::Elem 模板参数； + // 且旧实现按字节偏移取数，与 PTO ISA 的元素索引语义不同。 + MGATHER(outTile, inGm, offsetTile); + + // TSTORE: 将 outTile 写回 global memory + // [当前编译器] PTO ISA 名 TSTORE，当前编译器名 TCOPYOUT；jcore 为 __vec__ + TSTORE(gO, outTile); + } + if constexpr (rmd_M) { + auto gO = gOIter(0, Mb); + total_elements = rmd_M; + gen_offset_pto( + offsetTile_rmd, in_shape, out_shape, base, total_elements); + base += total_elements; + MGATHER(outTile_rmd, inGm, offsetTile_rmd); + TSTORE(gO, outTile_rmd); + } +} diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_simple.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_simple.hpp index c8ce29f..1e1b926 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_simple.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_simple.hpp @@ -151,7 +151,7 @@ void gen_offset_impl( -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_019.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_019.hpp index e58991b..e6332b6 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_vec_019.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_019.hpp @@ -1,133 +1,99 @@ +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_VEC_019_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_VEC_019_HPP + #include + +#include #include -#include +#include using namespace pto; -// ===================================================================== -// Broadcast (B,1,K) -> (B,N,K) via TCOPYIN + __vec__ broadcast + TCOPYOUT -// -// Optimized for: (1280,1,49) -> (1280,8,49), dtype=half -// -// Data layout (row-major): -// Input: [B][1][K] -> flat = B*K, batch b data at offset b*K -// Output: [B][N][K] -> flat = B*N*K, batch b data at offset b*N*K -// Broadcast along dim1: 1 -> N, dim0 & dim2 preserved. -// -// Processing strategy: -// Divide B batches into tiles of kTileBatch batches each. -// Per tile: -// 1. TCOPYIN (kTileBatch, K) from GlobalMem -> TileReg -// Reads kTileBatch * K contiguous elements. -// 2. __vec__ broadcast within TileReg: -// Launch <<>> threads: -// x = inner column index (0..K-1, direct use, no modulo) -// y = broadcast_copy * kTileBatch + batch_idx (0..N*kTileBatch-1) -// copy = y / kTileBatch (0..N-1, power-of-2 safe) -// batch_idx = y & (kTileBatch - 1) (0..kTileBatch-1, bitwise) -// Read src[batch_idx * RowStride + x] -// Write dst[batch_idx * RowStride + copy * K + x] -// 3. TCOPYOUT (kTileBatch, N*K) from TileReg -> GlobalMem -// -// TileReg layout: -// Physical tile cols = 512 (padded for 512B alignment). -// Constraint: kTileBatch must be power-of-2, N must be power-of-2. -// -// Note: kInner (K=49) is NOT power-of-2, so we avoid modulo on K by -// making x the direct column index (0..K-1). All division on -// y uses power-of-2 kTileBatch (bitwise ops only). -// -// Alignment notes (dtype=half): -// kTileBatch=16, Cols=512 -> 16*512*2 = 16384 = 512*2^5 (aligned) -// For (1280,1,49)->(1280,8,49) with kTileBatch=16: -// 1280/16 = 80 tiles, no remainder. -// -// Template params: -// dtype - data type (__half, float, etc.) -// MAX_DIM - max dimensions (kept for compat) -// IN_DIM - input dim count (kept for compat) -// OUT_DIM - output dim count (kept for compat) -// gIM - total input elements = B * K -// gOM - total output elements = B * N * K -// kTileBatch - batches per tile, power-of-2 -// kInner - inner dimension K (e.g. 49, need not be power-of-2) -// ===================================================================== - -template -void __vec__ -vec_broadcast_3d(typename tile_shape_out::TileDType __out__ dst, - const typename tile_shape_in::TileDType __in__ src) { - size_t x = blkv_get_index_x(); - size_t y = blkv_get_index_y(); - - size_t batch_idx = y & (kTileBatch - 1); - size_t copy = y >> 4; - - size_t in_index = batch_idx * tile_shape_in::RowStride + x; - size_t out_index = batch_idx * tile_shape_out::RowStride + copy * kInnerCols + x; - - blkv_get_tile_ptr(dst)[out_index] = blkv_get_tile_ptr(src)[in_index]; +template +void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t *, const size_t *) { + constexpr size_t kBCast = gOM / gIM; + constexpr size_t kElementsPerTile = 512; + constexpr size_t kFullTiles = gOM / kElementsPerTile; + constexpr size_t kTail = gOM % kElementsPerTile; + + static_assert(MAX_DIM >= IN_DIM && MAX_DIM >= OUT_DIM, + "MAX_DIM must cover input and output ranks"); + static_assert(gOM % gIM == 0, + "broadcast output element count must be a multiple of input"); + static_assert(gIM % kInner == 0, + "input element count must be divisible by kInner"); + static_assert(kBCast > 0, "broadcast factor must be positive"); + static_assert(kTileBatch > 0, "kTileBatch must be positive"); + + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + using DataTile = Tile; + using OffsetTile = Tile; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputIterator output_iter(out_ptr); + + auto emit_tile = [&](auto &data_tile, auto &linear, auto &batch, + auto &batch_base, auto &inner_q, auto &inner_base, + auto &inner, auto &inner_bytes, auto &offset, + std::uint32_t base, auto output_addr) { + using LinearTile = std::remove_reference_t; + TCI(linear, base); + + TDIVS(batch, linear, static_cast(kBCast * kInner)); + TMULS(batch_base, batch, + static_cast(kInner * sizeof(dtype))); + + TDIVS(inner_q, linear, static_cast(kInner)); + TMULS(inner_base, inner_q, static_cast(kInner)); + TSUB(inner, linear, inner_base); + TMULS(inner_bytes, inner, static_cast(sizeof(dtype))); + + TADD(offset, batch_base, inner_bytes); + MGATHER(data_tile, input_global, offset); + TSTORE(output_addr, data_tile); + }; + + for (size_t tile = 0; tile < kFullTiles; ++tile) { + DataTile data_tile; + OffsetTile linear; + OffsetTile batch; + OffsetTile batch_base; + OffsetTile inner_q; + OffsetTile inner_base; + OffsetTile inner; + OffsetTile inner_bytes; + OffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, inner, + inner_bytes, offset, + static_cast(tile * kElementsPerTile), + output_iter(0, static_cast(tile))); + } + + if constexpr (kTail != 0) { + using TailDataTile = Tile; + using TailOffsetTile = + Tile; + TailDataTile data_tile; + TailOffsetTile linear; + TailOffsetTile batch; + TailOffsetTile batch_base; + TailOffsetTile inner_q; + TailOffsetTile inner_base; + TailOffsetTile inner; + TailOffsetTile inner_bytes; + TailOffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, inner, + inner_bytes, offset, + static_cast(kFullTiles * kElementsPerTile), + output_iter(0, static_cast(kFullTiles))); + } } -template -void broadcast(dtype *in_ptr, dtype *out_ptr, - const size_t * /*in_shape*/, const size_t * /*out_shape*/) { - constexpr size_t kBCast = gOM / gIM; - constexpr size_t kBatch = gIM / kInner; - constexpr size_t tileCols = 512; - - static_assert(gOM % gIM == 0, - "gOM must be divisible by gIM for (B,1,K)->(B,N,K) broadcast"); - static_assert(gIM % kInner == 0, - "gIM must be divisible by kInner (B = gIM/kInner must be integer)"); - static_assert((kTileBatch & (kTileBatch - 1)) == 0, - "kTileBatch must be power of 2 for 512B tile alignment"); - static_assert((kBCast & (kBCast - 1)) == 0, - "kBCast (N) must be power of 2 for bitwise division in SIMT"); - static_assert(tileCols >= kBCast * kInner, - "padded tileCols (512) must >= broadcast target width (N*K)"); - - using tile_in = Tile; - using tile_out = Tile; - using gm_in = global_tensor>; - using gm_out = global_tensor>; - - constexpr size_t Nb = kBatch / kTileBatch; - constexpr size_t rmd = kBatch % kTileBatch; - - tile_in inTile; - tile_out outTile; - - using tile_in_r = Tile; - using tile_out_r = Tile; - tile_in_r inTile_rmd; - tile_out_r outTile_rmd; - - for (size_t i = 0; i < Nb; i++) { - gm_in gsrc(in_ptr + i * kTileBatch * kInner); - TCOPYIN(inTile, gsrc); - - vec_broadcast_3d - <<>>(outTile.data(), inTile.data()); - - gm_out gdst(out_ptr + i * kTileBatch * kBCast * kInner); - TCOPYOUT(gdst, outTile); - } - - if constexpr (rmd > 0) { - gm_in gsrc(in_ptr + Nb * kTileBatch * kInner); - TCOPYIN(inTile_rmd, gsrc); - - vec_broadcast_3d - <<>>(outTile_rmd.data(), inTile_rmd.data()); - - gm_out gdst(out_ptr + Nb * kTileBatch * kBCast * kInner); - TCOPYOUT(gdst, outTile_rmd); - } -} \ No newline at end of file +#endif diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_019_pto.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_019_pto.hpp new file mode 100644 index 0000000..d1aa674 --- /dev/null +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_019_pto.hpp @@ -0,0 +1,154 @@ +// ============================================================================ +// Broadcast (B,1,K) -> (B,N,K) — PTO 一层编程模型 +// +// 原始 broadcast_vec_019.hpp 策略: +// TCOPYIN (kTileBatch,K) -> __vec__ 行复制 (K 个元素复制 N 次) -> TCOPYOUT (kTileBatch,N*K) +// __vec__ 块: dst[batch*RowStride + copy*K + x] = src[batch*RowStride + x] +// +// PTO 一层策略: +// TLOAD (kTileBatch,K) -> TINSERT × N (将输入 tile 插入输出 tile 的 N 个列偏移) -> TSTORE (kTileBatch,N*K) +// TINSERT 语义: dst[indexRow+i, indexCol+j] = src[i,j] +// 对 copy c = 0..N-1: TINSERT(outTile, inTile, 0, c*K) +// N 次插入互不重叠, 合起来恰好填满输出 tile 的 valid 区域 +// +// ┌─────────────────────────────────────────────────────────────────────────┐ +// │ 当前编译器不支持 / 不完整的指令汇总 │ +// ├──────────┬──────────────────┬──────────────────────────────────────────┤ +// │ Pto ISA │ 当前编译器状态 │ 说明 │ +// │ 指令 │ │ │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TLOAD │ API 有(名不同)， │ PTO ISA 名 TLOAD； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYIN； │ +// │ │ │ jcore/TCopyIn.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TINSERT │ v0.57 supported │ TileOP API lowers to BSTART.FIXP │ +// │ │ │ with FIXP.Function.TINSERT │ +// │ │ │ (仅有反向操作 TEXTRACT) │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYOUT； │ +// │ │ │ jcore/TCopyOut.hpp 用 __vec__ 实现 │ +// └──────────┴──────────────────┴──────────────────────────────────────────┘ +// +// PTO ISA 文档签名 (Declared in include/pto/pto_instr.hpp): +// +// TLOAD: +// template +// PTO_INST RecordEvent TLOAD(TileData &dst, GlobalData &src, +// WaitEvents &... events); +// +// TINSERT: +// template +// PTO_INST RecordEvent TINSERT(DstTileData &dst, SrcTileData &src, +// uint16_t indexRow, uint16_t indexCol, +// WaitEvents &... events); +// 语义: dst[indexRow+i, indexCol+j] = src[i,j] +// for 0 <= i < src.ValidRow, 0 <= j < src.ValidCol +// +// TSTORE: +// template +// PTO_INST RecordEvent TSTORE(GlobalData &dst, TileData &src, +// WaitEvents &... events); +// ============================================================================ + +#include +#include +// #include // [!] PTO ISA C++ Intrinsic — 当前编译器未提供 + +#include +#include + +// ===================================================================== +// Broadcast (B,1,K) -> (B,N,K) via TLOAD + TINSERT×N + TSTORE +// +// Data layout (row-major): +// Input: [B][1][K] -> flat = B*K, batch b data at offset b*K +// Output: [B][N][K] -> flat = B*N*K, batch b data at offset b*N*K +// Broadcast along dim1: 1 -> N, dim0 & dim2 preserved. +// +// Template params (与原 broadcast_vec_019 一致): +// dtype - data type (__half, float, etc.) +// MAX_DIM - max dimensions (kept for compat) +// IN_DIM - input dim count (kept for compat) +// OUT_DIM - output dim count (kept for compat) +// gIM - total input elements = B * K +// gOM - total output elements = B * N * K +// kTileBatch - batches per tile, power-of-2 +// kInner - inner dimension K (e.g. 49, need not be power-of-2) +// ===================================================================== + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, + const size_t * /*in_shape*/, const size_t * /*out_shape*/) { + constexpr size_t kBCast = gOM / gIM; + constexpr size_t kBatch = gIM / kInner; + constexpr size_t tileCols = 512; + + static_assert(gOM % gIM == 0, + "gOM must be divisible by gIM for (B,1,K)->(B,N,K) broadcast"); + static_assert(gIM % kInner == 0, + "gIM must be divisible by kInner (B = gIM/kInner must be integer)"); + static_assert((kTileBatch & (kTileBatch - 1)) == 0, + "kTileBatch must be power of 2 for 512B tile alignment"); + static_assert((kBCast & (kBCast - 1)) == 0, + "kBCast (N) must be power of 2 for bitwise division in SIMT"); + static_assert(tileCols >= kBCast * kInner, + "padded tileCols (512) must >= broadcast target width (N*K)"); + + using tile_in = Tile; + using tile_out = Tile; + using gm_in = global_tensor>; + using gm_out = global_tensor>; + + constexpr size_t Nb = kBatch / kTileBatch; + constexpr size_t rmd = kBatch % kTileBatch; + + tile_in inTile; + tile_out outTile; + + for (size_t i = 0; i < Nb; i++) { + gm_in gsrc(in_ptr + i * kTileBatch * kInner); + gm_out gdst(out_ptr + i * kTileBatch * kBCast * kInner); + + // TLOAD: GM -> UB, 加载 (kTileBatch, kInner) 输入 tile + // [当前编译器] 名为 TCOPYIN, jcore 为 __vec__ + TLOAD(inTile, gsrc); + + // TINSERT × kBCast: 将输入 tile 插入输出 tile 的 N 个列偏移 + // 每次 TINSERT 写入 kInner 列, N 次互不重叠, 合起来填满 N*kInner 列 + // v0.57 lowers TINSERT through the FIXP tile block. + #pragma clang loop unroll(full) + for (size_t c = 0; c < kBCast; c++) { + TINSERT(outTile, inTile, /*indexRow=*/0, /*indexCol=*/(uint16_t)(c * kInner)); + } + + // TSTORE: UB -> GM, 写回 (kTileBatch, kBCast*kInner) 输出 tile + // [当前编译器] 名为 TCOPYOUT, jcore 为 __vec__ + TSTORE(gdst, outTile); + } + + using tile_in_r = Tile; + using tile_out_r = Tile; + tile_in_r inTile_rmd; + tile_out_r outTile_rmd; + if constexpr (rmd > 0) { + gm_in gsrc(in_ptr + Nb * kTileBatch * kInner); + gm_out gdst(out_ptr + Nb * kTileBatch * kBCast * kInner); + + TLOAD(inTile_rmd, gsrc); + + #pragma clang loop unroll(full) + for (size_t c = 0; c < kBCast; c++) { + TINSERT(outTile_rmd, inTile_rmd, 0, (uint16_t)(c * kInner)); + } + + TSTORE(gdst, outTile_rmd); + } +} diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_039.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_039.hpp index 72cccbd..a918659 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_vec_039.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_039.hpp @@ -1,135 +1,6 @@ -#include -#include -#include +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_VEC_039_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_VEC_039_HPP -using namespace pto; +#include "broadcast_vec_019.hpp" -// ===================================================================== -// Broadcast (B,1,K) -> (B,N,K) via TCOPYIN + __vec__ broadcast + TCOPYOUT -// -// Optimized for: (8192,1,16) -> (8192,8,16), dtype=half -// -// Data layout (row-major): -// Input: [B][1][K] -> flat = B*K, batch b data at offset b*K -// Output: [B][N][K] -> flat = B*N*K, batch b data at offset b*N*K -// Broadcast along dim1: 1 -> N, dim0 & dim2 preserved. -// -// Processing strategy: -// Divide B batches into tiles of kTileBatch batches each. -// Per tile: -// 1. TCOPYIN (kTileBatch, K) from GlobalMem -> TileReg -// Reads kTileBatch * K contiguous elements. -// 2. __vec__ broadcast within TileReg: -// For each batch, replicate its K elements N times (row-wise). -// Launch <<>> threads: -// x = output position within one batch (0..N*K-1) -// y = batch index within tile (0..kTileBatch-1) -// copy = x / K (broadcast copy 0..N-1) -// col = x % K (inner column 0..K-1) -// Read src[y * RowStride + col] -// Write dst[y * RowStride + x] -// 3. TCOPYOUT (kTileBatch, N*K) from TileReg -> GlobalMem -// -// TileReg layout: -// Physical tile cols = 256 (padded for 512B alignment). -// Tile bytes = kTileBatch * 256 * sizeof(dtype) = 512 * kTileBatch -// Constraint: kTileBatch must be power-of-2. -// -// Alignment notes: -// K and N must be powers of 2 for bit-op index decomposition. -// For 512B-aligned global tile strides (dtype=half): -// Input stride = kTileBatch * K * sizeof(half) = kTileBatch * K * 2 -// 512-aligned when kTileBatch * K * 2 >= 512 and is 512 * 2^n -// e.g. kTileBatch=16, K=16 -> 512B per tile (512-aligned) -// Output stride = kTileBatch * N * K * sizeof(half) -// e.g. kTileBatch=16, N=8, K=16 -> 4096B per tile (512-aligned) -// For (8192,1,16)->(8192,8,16) with kTileBatch=16: -// 8192/16 = 512 tiles, no remainder, all aligned. -// -// Template params: -// dtype - data type (__half, float, etc.) -// MAX_DIM - max dimensions (kept for compat) -// IN_DIM - input dim count (kept for compat) -// OUT_DIM - output dim count (kept for compat) -// gIM - total input elements = B * K (e.g. 8192*16 = 131072) -// gOM - total output elements = B * N * K (e.g. 8192*8*16 = 1048576) -// kTileBatch - batches per tile, power-of-2 (e.g. 1,2,4,8,16,32,64) -// kInner - inner dimension K, power-of-2 (e.g. 16) -// ===================================================================== - -template -void __vec__ -vec_broadcast_3d(typename tile_shape_out::TileDType __out__ dst, - const typename tile_shape_in::TileDType __in__ src) { - size_t x = blkv_get_index_x(); - size_t y = blkv_get_index_y(); - - size_t in_index = y * tile_shape_in::RowStride + (x & (kInnerCols - 1)); - size_t out_index = y * tile_shape_out::RowStride + x; - - blkv_get_tile_ptr(dst)[out_index] = blkv_get_tile_ptr(src)[in_index]; -} - -template -void broadcast(dtype *in_ptr, dtype *out_ptr, - const size_t * /*in_shape*/, const size_t * /*out_shape*/) { - constexpr size_t kBCast = gOM / gIM; // 8 - constexpr size_t kBCast_in = 64 / kInner; // 2 - constexpr size_t kBCast_out = kBCast / kBCast_in; // 4 - constexpr size_t kBatch = gIM / kInner; - constexpr size_t tileCols = 256; - - static_assert(gOM % gIM == 0, - "gOM must be divisible by gIM for (B,1,K)->(B,N,K) broadcast"); - static_assert(gIM % kInner == 0, - "gIM must be divisible by kInner (B = gIM/kInner must be integer)"); - static_assert((kInner & (kInner - 1)) == 0, - "kInner must be power of 2 for bit-op index decomposition"); - static_assert((kTileBatch & (kTileBatch - 1)) == 0, - "kTileBatch must be power of 2 for 512B tile alignment"); - static_assert(tileCols >= kBCast * kInner, - "padded tileCols (256) must >= broadcast target width (N*K)"); - - using tile_in = Tile; - using tile_out = Tile; - using gm_in = global_tensor>; - using gm_out = global_tensor>; - - constexpr size_t Nb = kBatch / kTileBatch; - constexpr size_t rmd = kBatch % kTileBatch; - - tile_in inTile; - tile_out outTile; - - using tile_in_r = Tile; - using tile_out_r = Tile; - tile_in_r inTile_rmd; - tile_out_r outTile_rmd; - - for (size_t i = 0; i < Nb; i++) { - gm_in gsrc(in_ptr + i * kTileBatch * kInner); - TCOPYIN(inTile, gsrc); - - vec_broadcast_3d - <<>>(outTile.data(), inTile.data()); - - gm_out gdst(out_ptr + i * kTileBatch * kBCast * kInner); - TCOPYOUT(gdst, outTile); - } - - if constexpr (rmd > 0) { - gm_in gsrc(in_ptr + Nb * kTileBatch * kInner); - TCOPYIN(inTile_rmd, gsrc); - - vec_broadcast_3d - <<>>(outTile_rmd.data(), inTile_rmd.data()); - - gm_out gdst(out_ptr + Nb * kTileBatch * kBCast * kInner); - TCOPYOUT(gdst, outTile_rmd); - } -} \ No newline at end of file +#endif diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_039_pto.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_039_pto.hpp new file mode 100644 index 0000000..38894d8 --- /dev/null +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_039_pto.hpp @@ -0,0 +1,155 @@ +// ============================================================================ +// Broadcast (B,1,K) -> (B,N,K) — PTO 一层编程模型 +// +// 原始 broadcast_vec_039.hpp 策略: +// TCOPYIN (kTileBatch,K) -> __vec__ 行复制 (K 个元素复制 N 次) -> TCOPYOUT (kTileBatch,N*K) +// __vec__ 块: dst[y*RowStride + x] = src[y*RowStride + (x & (K-1))] +// (K 为 2 的幂, 用位与代替取余) +// +// PTO 一层策略: +// TLOAD (kTileBatch,K) -> TINSERT × N (将输入 tile 插入输出 tile 的 N 个列偏移) -> TSTORE (kTileBatch,N*K) +// TINSERT 语义: dst[indexRow+i, indexCol+j] = src[i,j] +// 对 copy c = 0..N-1: TINSERT(outTile, inTile, 0, c*K) +// N 次插入互不重叠, 合起来恰好填满输出 tile 的 valid 区域 +// +// ┌─────────────────────────────────────────────────────────────────────────┐ +// │ 当前编译器不支持 / 不完整的指令汇总 │ +// ├──────────┬──────────────────┬──────────────────────────────────────────┤ +// │ Pto ISA │ 当前编译器状态 │ 说明 │ +// │ 指令 │ │ │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TLOAD │ API 有(名不同)， │ PTO ISA 名 TLOAD； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYIN； │ +// │ │ │ jcore/TCopyIn.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TINSERT │ v0.57 supported │ TileOP API lowers to BSTART.FIXP │ +// │ │ │ with FIXP.Function.TINSERT │ +// │ │ │ (仅有反向操作 TEXTRACT) │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYOUT； │ +// │ │ │ jcore/TCopyOut.hpp 用 __vec__ 实现 │ +// └──────────┴──────────────────┴──────────────────────────────────────────┘ +// +// PTO ISA 文档签名 (Declared in include/pto/pto_instr.hpp): +// +// TLOAD: +// template +// PTO_INST RecordEvent TLOAD(TileData &dst, GlobalData &src, +// WaitEvents &... events); +// +// TINSERT: +// template +// PTO_INST RecordEvent TINSERT(DstTileData &dst, SrcTileData &src, +// uint16_t indexRow, uint16_t indexCol, +// WaitEvents &... events); +// 语义: dst[indexRow+i, indexCol+j] = src[i,j] +// for 0 <= i < src.ValidRow, 0 <= j < src.ValidCol +// +// TSTORE: +// template +// PTO_INST RecordEvent TSTORE(GlobalData &dst, TileData &src, +// WaitEvents &... events); +// ============================================================================ + +#include +#include +// #include // [!] PTO ISA C++ Intrinsic — 当前编译器未提供 + +#include +#include + +// ===================================================================== +// Broadcast (B,1,K) -> (B,N,K) via TLOAD + TINSERT×N + TSTORE +// +// Data layout (row-major): +// Input: [B][1][K] -> flat = B*K, batch b data at offset b*K +// Output: [B][N][K] -> flat = B*N*K, batch b data at offset b*N*K +// Broadcast along dim1: 1 -> N, dim0 & dim2 preserved. +// +// Template params (与原 broadcast_vec_039 一致): +// dtype - data type (__half, float, etc.) +// MAX_DIM - max dimensions (kept for compat) +// IN_DIM - input dim count (kept for compat) +// OUT_DIM - output dim count (kept for compat) +// gIM - total input elements = B * K (e.g. 8192*16 = 131072) +// gOM - total output elements = B * N * K (e.g. 8192*8*16 = 1048576) +// kTileBatch - batches per tile, power-of-2 (e.g. 1,2,4,8,16,32,64) +// kInner - inner dimension K, power-of-2 (e.g. 16) +// ===================================================================== + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, + const size_t * /*in_shape*/, const size_t * /*out_shape*/) { + constexpr size_t kBCast = gOM / gIM; + constexpr size_t kBatch = gIM / kInner; + constexpr size_t tileCols = 256; + + static_assert(gOM % gIM == 0, + "gOM must be divisible by gIM for (B,1,K)->(B,N,K) broadcast"); + static_assert(gIM % kInner == 0, + "gIM must be divisible by kInner (B = gIM/kInner must be integer)"); + static_assert((kInner & (kInner - 1)) == 0, + "kInner must be power of 2 for bit-op index decomposition"); + static_assert((kTileBatch & (kTileBatch - 1)) == 0, + "kTileBatch must be power of 2 for 512B tile alignment"); + static_assert(tileCols >= kBCast * kInner, + "padded tileCols (256) must >= broadcast target width (N*K)"); + + using tile_in = Tile; + using tile_out = Tile; + using gm_in = global_tensor>; + using gm_out = global_tensor>; + + constexpr size_t Nb = kBatch / kTileBatch; + constexpr size_t rmd = kBatch % kTileBatch; + + tile_in inTile; + tile_out outTile; + + for (size_t i = 0; i < Nb; i++) { + gm_in gsrc(in_ptr + i * kTileBatch * kInner); + gm_out gdst(out_ptr + i * kTileBatch * kBCast * kInner); + + // TLOAD: GM -> UB, 加载 (kTileBatch, kInner) 输入 tile + // [当前编译器] 名为 TCOPYIN, jcore 为 __vec__ + TLOAD(inTile, gsrc); + + // TINSERT × kBCast: 将输入 tile 插入输出 tile 的 N 个列偏移 + // 每次 TINSERT 写入 kInner 列, N 次互不重叠, 合起来填满 N*kInner 列 + // v0.57 lowers TINSERT through the FIXP tile block. + #pragma clang loop unroll(full) + for (size_t c = 0; c < kBCast; c++) { + TINSERT(outTile, inTile, /*indexRow=*/0, /*indexCol=*/(uint16_t)(c * kInner)); + } + + // TSTORE: UB -> GM, 写回 (kTileBatch, kBCast*kInner) 输出 tile + // [当前编译器] 名为 TCOPYOUT, jcore 为 __vec__ + TSTORE(gdst, outTile); + } + + using tile_in_r = Tile; + using tile_out_r = Tile; + tile_in_r inTile_rmd; + tile_out_r outTile_rmd; + if constexpr (rmd > 0) { + gm_in gsrc(in_ptr + Nb * kTileBatch * kInner); + gm_out gdst(out_ptr + Nb * kTileBatch * kBCast * kInner); + + TLOAD(inTile_rmd, gsrc); + + #pragma clang loop unroll(full) + for (size_t c = 0; c < kBCast; c++) { + TINSERT(outTile_rmd, inTile_rmd, 0, (uint16_t)(c * kInner)); + } + + TSTORE(gdst, outTile_rmd); + } +} diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_07.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_07.hpp index 5934187..e4b8cf6 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_vec_07.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_07.hpp @@ -1,105 +1,69 @@ -#include -#include -#include - -using namespace pto; - -// ===================================================================== -// Broadcast (N,1) -> (N,C) via TCOPYIN + __vec__ broadcast + TCOPYOUT -// -// Optimized for: (1443,1) -> (1443,129), dtype=half -// -// Processing strategy: -// Divide N rows into tiles of kTileRows rows each. -// Per tile: -// 1. TCOPYIN (kTileRows, 1) from GlobalMem -> TileReg -// 2. __vec__ broadcast (kTileRows, 1) -> (kTileRows, C) within TileReg -// Launch <<>> threads: -// x = column index (0..kC-1), y = row index (0..kTileRows-1) -// Each thread reads src[j*src_RowStride] (col 0 of row j) -// and writes to dst[i + j*dst_RowStride] (col i of row j) -// 3. TCOPYOUT (kTileRows, C) from TileReg -> GlobalMem -// -// TileReg layout: -// Physical tile cols padded to 256 for 512B alignment. -// Tile bytes = kTileRows * 256 * sizeof(dtype) = 512 * kTileRows -// Constraint: kTileRows must be power-of-2 (512 * 2^n bytes). -// -// Template params (backward compatible with original broadcast_07): -// dtype - data type (__half, float, etc.) -// MAX_DIM - max dimensions (unused, kept for compat) -// IN_DIM - input dim count (unused) -// OUT_DIM - output dim count (unused) -// gIM - total input elements = N*1 = N (e.g. 1443) -// gOM - total output elements = N*C (e.g. 1443*129) -// kTileRows - rows per tile, must be power-of-2 (e.g. 1,2,4,..,64) -// ===================================================================== - -template -void __vec__ -vec_broadcast_rowmajor(typename tile_shape_out::TileDType __out__ dst, - const typename tile_shape_in::TileDType __in__ src) { - size_t i = blkv_get_index_x(); - size_t j = blkv_get_index_y(); - size_t out_index = i + j * tile_shape_out::RowStride; - blkv_get_tile_ptr(dst)[out_index] = - blkv_get_tile_ptr(src)[j * tile_shape_in::RowStride]; -} - -template -void broadcast(dtype *in_ptr, dtype *out_ptr, - const size_t * /*in_shape*/, const size_t * /*out_shape*/) { - constexpr size_t kN = gIM; - constexpr size_t kC = gOM / gIM; - constexpr size_t tileCols = 256; +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_VEC_07_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_VEC_07_HPP - static_assert(gOM % gIM == 0, - "gOM must be divisible by gIM for (N,1)->(N,C) broadcast"); - static_assert(tileCols >= kC, - "padded tileCols (256) must >= broadcast target columns"); - static_assert((kTileRows & (kTileRows - 1)) == 0, - "kTileRows must be power of 2 for 512B tile alignment"); +#include - using tile_in = Tile; - using tile_out = Tile; - using gm_in = global_tensor>; - using gm_out = global_tensor>; +#include - constexpr size_t Nb = kN / kTileRows; - constexpr size_t rmd = kN % kTileRows; +using namespace pto; - tile_in inTile; - tile_out outTile; +template +void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t *, const size_t *) { + constexpr size_t kN = gIM; + constexpr size_t kC = gOM / gIM; + constexpr size_t kTileCols = kC; + constexpr size_t kMaxRowsByBytes = + (4096 / (kTileCols * sizeof(dtype))) == 0 + ? 1 + : (4096 / (kTileCols * sizeof(dtype))); + constexpr size_t kRowsPerTile = + kTileRows < kMaxRowsByBytes ? kTileRows : kMaxRowsByBytes; + static_assert(MAX_DIM >= IN_DIM && MAX_DIM >= OUT_DIM, + "MAX_DIM must cover input and output ranks"); + static_assert(gOM % gIM == 0, + "broadcast output element count must be a multiple of input"); + static_assert(kRowsPerTile > 0, "broadcast tile must contain at least one row"); - for (size_t i = 0; i < Nb; i++) { - gm_in gsrc(in_ptr + i * kTileRows); - TCOPYIN(inTile, gsrc); + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + using InputTile = Tile; + using OutputTile = Tile; + using InputIterator = global_iterator; + using OutputIterator = global_iterator; - vec_broadcast_rowmajor - <<>>(outTile.data(), inTile.data()); + InputIterator input_iter(in_ptr); + OutputIterator output_iter(out_ptr); - gm_out gdst(out_ptr + i * kTileRows * kC); - TCOPYOUT(gdst, outTile); - } + constexpr size_t kFullTiles = kN / kRowsPerTile; + constexpr size_t kTail = kN % kRowsPerTile; - using tile_in_r = Tile; - using tile_out_r = Tile; - tile_in_r inTile_rmd; - tile_out_r outTile_rmd; - if constexpr (rmd > 0) { - gm_in gsrc(in_ptr + Nb * kTileRows); - TCOPYIN(inTile_rmd, gsrc); + for (size_t tile = 0; tile < kFullTiles; ++tile) { + InputTile input_tile; + OutputTile output_tile; + TLOAD(input_tile, input_iter(static_cast(tile), 0)); + TROWEXPAND(output_tile, input_tile); + TSTORE(output_iter(static_cast(tile), 0), output_tile); + } - vec_broadcast_rowmajor - <<>>(outTile_rmd.data(), inTile_rmd.data()); + if constexpr (kTail != 0) { + using TailInputTile = Tile; + using TailOutputTile = Tile; + using TailInputIterator = global_iterator; + using TailOutputIterator = global_iterator; + TailInputIterator tail_input_iter(in_ptr); + TailOutputIterator tail_output_iter(out_ptr); + TailInputTile input_tile; + TailOutputTile output_tile; + TLOAD(input_tile, tail_input_iter(static_cast(kFullTiles), 0)); + TROWEXPAND(output_tile, input_tile); + TSTORE(tail_output_iter(static_cast(kFullTiles), 0), output_tile); + } +} - gm_out gdst(out_ptr + Nb * kTileRows * kC); - TCOPYOUT(gdst, outTile_rmd); - } -} \ No newline at end of file +#endif // SUPERNPUBENCH_PTOISA_BROADCAST_VEC_07_HPP diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_handwrite.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_handwrite.hpp index 88cccf2..c90e364 100644 --- a/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_handwrite.hpp +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_handwrite.hpp @@ -39,7 +39,7 @@ void __vec__ vec_broadcast( // (N,1)->(N,129) -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_pto.hpp b/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_pto.hpp new file mode 100644 index 0000000..550a9f4 --- /dev/null +++ b/benchmark-linxisa/kernels/broadcast/broadcast_vec_07_pto.hpp @@ -0,0 +1,132 @@ +// ============================================================================ +// Broadcast (N,1) -> (N,C) — PTO 一层编程模型 +// +// 原始 broadcast_vec_07.hpp 策略: +// TCOPYIN (kTileRows,1) -> __vec__ 行广播 -> TCOPYOUT (kTileRows,C) +// __vec__ 块: dst[x + y*RowStride] = src[y*RowStride] (col 0 -> all cols) +// +// PTO 一层策略: +// TLOAD (kTileRows,1) -> TROWEXPAND (col 0 广播到全部 C 列) -> TSTORE (kTileRows,C) +// TROWEXPAND 语义: dst[i,j] = src[i,0], 恰好是行广播 +// +// ┌─────────────────────────────────────────────────────────────────────────┐ +// │ 当前编译器不支持 / 不完整的指令汇总 │ +// ├──────────┬──────────────────┬──────────────────────────────────────────┤ +// │ Pto ISA │ 当前编译器状态 │ 说明 │ +// │ 指令 │ │ │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TLOAD │ API 有(名不同)， │ PTO ISA 名 TLOAD； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYIN； │ +// │ │ │ jcore/TCopyIn.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │TROWEXPAND│ API 有(名不同)， │ PTO ISA 名 TROWEXPAND； │ +// │ │ 二层实现 │ 当前编译器名 TEXPANDROW； │ +// │ │ │ jcore/TExpandRow.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE； │ +// │ │ 二层实现 │ 当前编译器名 TCOPYOUT； │ +// │ │ │ jcore/TCopyOut.hpp 用 __vec__ 实现 │ +// └──────────┴──────────────────┴──────────────────────────────────────────┘ +// +// PTO ISA 文档签名 (Declared in include/pto/pto_instr.hpp): +// +// TLOAD: +// template +// PTO_INST RecordEvent TLOAD(TileData &dst, GlobalData &src, +// WaitEvents &... events); +// +// TROWEXPAND: +// template +// PTO_INST RecordEvent TROWEXPAND(TileDataDst &dst, TileDataSrc &src, +// WaitEvents &... events); +// 约束: dst[i,j] = src[i,0]; dst 与 src 必须为 Vec tile, ND layout; +// A2A3: srcValidRow == dstValidRow; srcValidCol >= 1 +// +// TSTORE: +// template +// PTO_INST RecordEvent TSTORE(GlobalData &dst, TileData &src, +// WaitEvents &... events); +// ============================================================================ + +#include +#include +// #include // [!] PTO ISA C++ Intrinsic — 当前编译器未提供 + +#include +#include + +// ===================================================================== +// Broadcast (N,1) -> (N,C) via TLOAD + TROWEXPAND + TSTORE +// +// Template params (与原 broadcast_vec_07 一致): +// dtype - data type (__half, float, etc.) +// MAX_DIM - max dimensions (unused, kept for compat) +// IN_DIM - input dim count (unused) +// OUT_DIM - output dim count (unused) +// gIM - total input elements = N*1 = N (e.g. 1443) +// gOM - total output elements = N*C (e.g. 1443*129) +// kTileRows - rows per tile, must be power-of-2 (e.g. 1,2,4,..,64) +// ===================================================================== + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, + const size_t * /*in_shape*/, const size_t * /*out_shape*/) { + constexpr size_t kN = gIM; + constexpr size_t kC = gOM / gIM; + constexpr size_t tileCols = 256; + + static_assert(gOM % gIM == 0, + "gOM must be divisible by gIM for (N,1)->(N,C) broadcast"); + static_assert(tileCols >= kC, + "padded tileCols (256) must >= broadcast target columns"); + static_assert((kTileRows & (kTileRows - 1)) == 0, + "kTileRows must be power of 2 for 512B tile alignment"); + + using tile_in = Tile; + using tile_out = Tile; + using gm_in = global_tensor>; + using gm_out = global_tensor>; + + constexpr size_t Nb = kN / kTileRows; + constexpr size_t rmd = kN % kTileRows; + + tile_in inTile; + tile_out outTile; + + for (size_t i = 0; i < Nb; i++) { + gm_in gsrc(in_ptr + i * kTileRows); + gm_out gdst(out_ptr + i * kTileRows * kC); + + // TLOAD: GM -> UB, 加载 (kTileRows, 1) 输入 tile + // [当前编译器] 名为 TCOPYIN, jcore 为 __vec__ + TLOAD(inTile, gsrc); + + // TROWEXPAND: 将每行 col 0 广播到全部 kC 列 -> (kTileRows, kC) + // [当前编译器] 名为 TEXPANDROW, jcore 为 __vec__ + TROWEXPAND(outTile, inTile); + + // TSTORE: UB -> GM, 写回 (kTileRows, kC) 输出 tile + // [当前编译器] 名为 TCOPYOUT, jcore 为 __vec__ + TSTORE(gdst, outTile); + } + + using tile_in_r = Tile; + using tile_out_r = Tile; + tile_in_r inTile_rmd; + tile_out_r outTile_rmd; + if constexpr (rmd > 0) { + gm_in gsrc(in_ptr + Nb * kTileRows); + gm_out gdst(out_ptr + Nb * kTileRows * kC); + + TLOAD(inTile_rmd, gsrc); + TROWEXPAND(outTile_rmd, inTile_rmd); + TSTORE(gdst, outTile_rmd); + } +} diff --git a/benchmark-linxisa/kernels/concat/concat_gather.hpp b/benchmark-linxisa/kernels/concat/concat_gather.hpp index 85a69fd..94c738a 100644 --- a/benchmark-linxisa/kernels/concat/concat_gather.hpp +++ b/benchmark-linxisa/kernels/concat/concat_gather.hpp @@ -1,199 +1,189 @@ -#ifndef TRANSPOSE_KERNEL_HPP -#define TRANSPOSE_KERNEL_HPP +#ifndef CONCAT_GATHER_KERNEL_HPP +#define CONCAT_GATHER_KERNEL_HPP #include -#include "template_asm.h" - -using namespace pto; - -#pragma once #include #include -/* -#define DUMP_TILE(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%12lld ", (long long)DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) -*/ - -// ============================================== -// 维度规则：交换transpose_dim0和transpose_dim1 -// ============================================== -template -void __vec__ gen_offset_concat( - typename tile_shape::TileDType __out__ out, - typename tile_Inshape::TileDType __in__ in_shape, - typename tile_Outshape::TileDType __in__ out_shape, -// const size_t in_dim, - const size_t base, - const size_t total_elements -) { - size_t index = blkv_get_index_x(); - size_t idx = blkv_get_index_x(); - - __vbuf__ typename tile_Inshape::DType *in_shape_ptr = blkv_get_tile_ptr(in_shape); - __vbuf__ typename tile_Outshape::DType *out_shape_ptr = blkv_get_tile_ptr(out_shape); - - if (index >= total_elements) return; - idx = idx + base; // todo idx是个向量，base是个标量，获得所有的基地址或者说基offset - - //转置维度交换stride。 -// uint16_t stride[IN_DIM]; - uint32_t stride[DATA_DIM]; - stride[DATA_DIM-1] = 1; - - #pragma clang loop unroll(full) - for(int i = DATA_DIM-2 ; i >= 0; --i) - { - stride[i] = stride[i+1] * in_shape_ptr[i+1]; - } - - - // 输出一维索引 → 输出坐标 - size_t out_coord[MAX_DIM] = {0}; // - size_t tmp = idx; // - - #pragma clang loop unroll(full) - for (int d = DATA_DIM - 1; d >= 0; d--) { - out_coord[d] = tmp % out_shape_ptr[d]; - tmp /= out_shape_ptr[d]; - } - - size_t n = out_coord[CONCAT_DIM] / in_shape_ptr[CONCAT_DIM]; - size_t offset = out_coord[CONCAT_DIM] % in_shape_ptr[CONCAT_DIM]; - - out_coord[CONCAT_DIM] = offset; - -/* - // 输出坐标 → 输入坐标 - size_t in_coord[MAX_DIM] = {0}; - for (size_t i = 0; i < in_dim; i++) { - size_t o = out_dim - in_dim + i; // 从后面对齐 - if (in_shape[i] == 1) { - in_coord[i] = 0; - } else { - in_coord[i] = out_coord[o]; - } - } -*/ -// uint16_t in_offset = 0; - uint32_t in_offset = 0; - - #pragma clang loop unroll(full) - for (int i = 0; i < DATA_DIM; i++) { - in_offset += out_coord[i] * stride[i] * sizeof(dtype); - } - in_offset += n * (stride[0]*in_shape_ptr[0]) * sizeof(dtype); - - // 赋值 - blkv_get_tile_ptr(out)[index] = in_offset; -} - -template -void gen_offset_Impl( - tile_shapeOffset &out, - tile_Inshape &in_shape, - tile_Outshape &out_shape, -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0, - const size_t base, - const size_t total_elements - ) -{ - static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, - "Only static shape supported"); - gen_offset_concat<<>>(out.data(), in_shape.data(), out_shape.data(), base, total_elements); // todo 这部分的tile shape是怎么传递的？ -} - - +using namespace pto; -template +/** + * @brief 通用 N 维 concat gather（Tile ISA 实现） + * + * 算法：对输出张量的每个元素，计算其多维坐标，确定来自哪个输入张量（n） + * 以及在输入张量内的偏移，用 MGATHER 从输入中 gather 数据。 + * + * @tparam DType 数据类型 + * @tparam MAX_DIM 最大维度数 + * @tparam gIM 输入总元素数 + * @tparam gOM 输出总元素数 + * @tparam tM 每个 tile 处理的元素数 + * @tparam DATA_DIM 数据维度数（编译期常量） + * @tparam CONCAT_DIM 拼接维度索引（编译期常量） + */ +template void concat_gather( - dtype *in_ptr, - dtype *out_ptr, + DType *in_ptr, + DType *out_ptr, size_t *in_shape, size_t *out_shape -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0 -) +) { + constexpr int kFullTiles = gOM / tM; + constexpr int kTailElements = gOM % tM; + + using InputGlobal = pto::global_tensor>; + using OutputGlobal = pto::global_tensor>; + using DataTile = pto::Tile; + using OffsetTile = pto::Tile; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputIterator output_iter(out_ptr); + + uint32_t output_base = 0; + for (int tile_index = 0; tile_index < kFullTiles; ++tile_index) { + auto output_global = output_iter(0, tile_index); + DataTile output_tile; + OffsetTile offset_tile; + + OffsetTile linear_index; + OffsetTile quotient; + OffsetTile cycle; + OffsetTile cycle_base; + OffsetTile coordinate; + OffsetTile contribution; + + TCI(linear_index, output_base); + TEXPANDS(offset_tile, static_cast(0)); + + // 计算输入 stride（标量核心，运行时计算） + uint32_t input_stride[DATA_DIM]; + input_stride[DATA_DIM - 1] = 1; + for (int i = static_cast(DATA_DIM) - 2; i >= 0; --i) { + input_stride[i] = input_stride[i + 1] * static_cast(in_shape[i + 1]); + } + uint32_t input_size = input_stride[0] * static_cast(in_shape[0]); + + // 对每个维度提取坐标并计算 byte offset + for (int d = static_cast(DATA_DIM) - 1; d >= 0; --d) { + uint32_t output_stride = 1; + for (int dd = d + 1; dd < static_cast(DATA_DIM); ++dd) { + output_stride *= static_cast(out_shape[dd]); + } + + if (output_stride == 1) { + TMOV(quotient, linear_index); + } else { + TDIVS(quotient, linear_index, output_stride); + } + + uint32_t out_dim_size = static_cast(out_shape[d]); + TDIVS(cycle, quotient, out_dim_size); + TMULS(cycle_base, cycle, out_dim_size); + TSUB(coordinate, quotient, cycle_base); + + if (static_cast(d) == CONCAT_DIM) { + uint32_t in_dim_size = static_cast(in_shape[d]); + OffsetTile n_tile; + OffsetTile offset_in; + OffsetTile n_times_shape; + + TDIVS(n_tile, coordinate, in_dim_size); + TMULS(n_times_shape, n_tile, in_dim_size); + TSUB(offset_in, coordinate, n_times_shape); + + TMULS(contribution, offset_in, + input_stride[d] * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + + TMULS(contribution, n_tile, + input_size * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + } else { + TMULS(contribution, coordinate, + input_stride[d] * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + } + } - const int Mb = gOM / tM; - - const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_shapeOut = global_tensor>; - - using gm_InDataShape = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_OutDataShape = global_tensor>; - - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - using tile_Inshape = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - using tile_Outshape = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec -// using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - gm_shapeIn inGm(in_ptr); - - gm_InDataShape InShapeGm(in_shape); - gm_OutDataShape OutShapeGm(out_shape); - - tile_shapeData dataTile; - tile_shapeOffset offsetTile; - - tile_Inshape InshapeTile; - tile_Outshape OutshapeTile; - - int base = 0;// todo 生成一个标量 - int all_num = gOM; // 总元素数量 - - using itOut = global_iterator; - itOut gOIter(out_ptr); - -// alignas(256) static uint32_t g_dump[tM]; + MGATHER(output_tile, input_global, offset_tile); + TSTORE(output_global, output_tile); - int total_elements = tM; + output_base += tM; + } + if constexpr (kTailElements != 0) { + using TailDataTile = pto::Tile; + using TailOffsetTile = pto::Tile; + + auto output_global = output_iter(0, kFullTiles); + TailDataTile output_tile; + TailOffsetTile offset_tile; + + TailOffsetTile linear_index; + TailOffsetTile quotient; + TailOffsetTile cycle; + TailOffsetTile cycle_base; + TailOffsetTile coordinate; + TailOffsetTile contribution; + + TCI(linear_index, output_base); + TEXPANDS(offset_tile, static_cast(0)); + + uint32_t input_stride[DATA_DIM]; + input_stride[DATA_DIM - 1] = 1; + for (int i = static_cast(DATA_DIM) - 2; i >= 0; --i) { + input_stride[i] = input_stride[i + 1] * static_cast(in_shape[i + 1]); + } + uint32_t input_size = input_stride[0] * static_cast(in_shape[0]); + + for (int d = static_cast(DATA_DIM) - 1; d >= 0; --d) { + uint32_t output_stride = 1; + for (int dd = d + 1; dd < static_cast(DATA_DIM); ++dd) { + output_stride *= static_cast(out_shape[dd]); + } + + if (output_stride == 1) { + TMOV(quotient, linear_index); + } else { + TDIVS(quotient, linear_index, output_stride); + } + + uint32_t out_dim_size = static_cast(out_shape[d]); + TDIVS(cycle, quotient, out_dim_size); + TMULS(cycle_base, cycle, out_dim_size); + TSUB(coordinate, quotient, cycle_base); + + if (static_cast(d) == CONCAT_DIM) { + uint32_t in_dim_size = static_cast(in_shape[d]); + TailOffsetTile n_tile; + TailOffsetTile offset_in; + TailOffsetTile n_times_shape; + + TDIVS(n_tile, coordinate, in_dim_size); + TMULS(n_times_shape, n_tile, in_dim_size); + TSUB(offset_in, coordinate, n_times_shape); + + TMULS(contribution, offset_in, + input_stride[d] * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + + TMULS(contribution, n_tile, + input_size * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + } else { + TMULS(contribution, coordinate, + input_stride[d] * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + } + } - for (int i = 0; i < Mb; ++i) { - auto gO = gOIter(0, i); - TCOPYIN(InshapeTile, InShapeGm); - TCOPYIN(OutshapeTile, OutShapeGm); - gen_offset_Impl(offsetTile, InshapeTile, OutshapeTile, base, total_elements); -// printf("end genoffset\n"); - base += total_elements; -// DUMP_TILE("offsetTile", offsetTile, g_dump, 1, tM); - MGATHER(dataTile, inGm, offsetTile); -// printf("end mgather\n"); - TCOPYOUT(gO, dataTile); - } - if constexpr (rmd_M) { - auto gO = gOIter(0, Mb); - TCOPYIN(InshapeTile, InShapeGm); - TCOPYIN(OutshapeTile, OutShapeGm); - total_elements = rmd_M;//尾片的大小。 - gen_offset_Impl(offsetTile, InshapeTile, OutshapeTile, base, total_elements); - base += total_elements; - MGATHER(dataTile, inGm, offsetTile); - TCOPYOUT(gO, dataTile); + MGATHER(output_tile, input_global, offset_tile); + TSTORE(output_global, output_tile); } } diff --git a/benchmark-linxisa/kernels/concat/concat_scatter.hpp b/benchmark-linxisa/kernels/concat/concat_scatter.hpp index e0f14b8..2846f33 100644 --- a/benchmark-linxisa/kernels/concat/concat_scatter.hpp +++ b/benchmark-linxisa/kernels/concat/concat_scatter.hpp @@ -2,206 +2,153 @@ #define CONCAT_SCATTER_KERNEL_HPP #include -#include "template_asm.h" - -using namespace pto; - -#pragma once #include #include +using namespace pto; -#define DUMP_TILE(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%12lld ", (long long)DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) - - -// ============================================== -// 维度规则：交换transpose_dim0和transpose_dim1 -// ============================================== -template -void __vec__ gen_offset_concat( - typename tile_shape::TileDType __out__ out, - typename tile_Inshape::TileDType __in__ in_shape, - typename tile_Outshape::TileDType __in__ out_shape, -// const size_t in_dim, - const size_t base, - const size_t total_elements -) { - size_t index = blkv_get_index_x(); - size_t idx = blkv_get_index_x(); - - __vbuf__ typename tile_Inshape::DType *in_shape_ptr = blkv_get_tile_ptr(in_shape); - __vbuf__ typename tile_Outshape::DType *out_shape_ptr = blkv_get_tile_ptr(out_shape); - - if (index >= total_elements) return; - idx = idx + base; // todo idx是个向量，base是个标量，获得所有的基地址或者说基offset - - //转置维度交换stride。 -// uint16_t stride[IN_DIM]; - uint16_t stride[DATA_DIM]; - stride[DATA_DIM-1] = 1; - - #pragma clang loop unroll(full) - for(int i = DATA_DIM-2 ; i >= 0; --i) - { - stride[i] = stride[i+1] * out_shape_ptr[i+1]; - } - - - // 输出一维索引 → 输出坐标 - size_t in_coord[MAX_DIM] = {0}; // - size_t tmp = idx; // - - #pragma clang loop unroll(full) - for (int d = DATA_DIM - 1; d >= 0; d--) { - in_coord[d] = tmp % in_shape_ptr[d]; - tmp /= in_shape_ptr[d]; - } - size_t n = tmp; - in_coord[CONCAT_DIM] = n * in_shape_ptr[CONCAT_DIM] + in_coord[CONCAT_DIM]; - -// size_t n = out_coord[CONCAT_DIM] / in_shape_ptr[CONCAT_DIM]; -// size_t offset = out_coord[CONCAT_DIM] % in_shape_ptr[CONCAT_DIM]; - -// out_coord[CONCAT_DIM] = offset; - -/* - // 输出坐标 → 输入坐标 - size_t in_coord[MAX_DIM] = {0}; - for (size_t i = 0; i < in_dim; i++) { - size_t o = out_dim - in_dim + i; // 从后面对齐 - if (in_shape[i] == 1) { - in_coord[i] = 0; - } else { - in_coord[i] = out_coord[o]; - } - } -*/ -// uint16_t in_offset = 0; -// uint32_t out_offset = 0; - uint16_t out_offset = 0; - - #pragma clang loop unroll(full) - for (int i = 0; i < DATA_DIM; i++) { - out_offset += in_coord[i] * stride[i] * sizeof(dtype); - } - - // 赋值 - blkv_get_tile_ptr(out)[index] = out_offset; -} - -template -void gen_offset_Impl( - tile_shapeOffset &out, - tile_Inshape &in_shape, - tile_Outshape &out_shape, -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0, - const size_t base, - const size_t total_elements - ) -{ - static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, - "Only static shape supported"); - gen_offset_concat<<>>(out.data(), in_shape.data(), out_shape.data(), base, total_elements); // todo 这部分的tile shape是怎么传递的？ -} - - - -template +/** + * @brief 通用 N 维 concat scatter（Tile ISA 实现） + * + * 算法：对输入张量的每个元素，计算其多维坐标，用输出 stride 计算输出 byte offset， + * 用 MSCATTER 将数据写入输出。 + * + * 注意：当前实现中 n（来自哪个输入张量）始终为 0，即输入被放置在输出的起始位置。 + * 这与原始 __vec__ 版本行为一致。 + * + * @tparam DType 数据类型 + * @tparam MAX_DIM 最大维度数 + * @tparam gIM 输入总元素数 + * @tparam gOM 输出总元素数 + * @tparam tM 每个 tile 处理的元素数 + * @tparam DATA_DIM 数据维度数（编译期常量） + * @tparam CONCAT_DIM 拼接维度索引（编译期常量） + */ +template void concat_scatter( - dtype *in_ptr, - dtype *out_ptr, + DType *in_ptr, + DType *out_ptr, size_t *in_shape, size_t *out_shape -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0 -) +) { + constexpr int kFullTiles = gIM / tM; + constexpr int kTailElements = gIM % tM; + + using InputGlobal = pto::global_tensor>; + using OutputGlobal = pto::global_tensor>; + using DataTile = pto::Tile; + using OffsetTile = pto::Tile; + using InputIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputGlobal output_global(out_ptr); + InputIterator input_iter(in_ptr); + + uint32_t input_base = 0; + for (int tile_index = 0; tile_index < kFullTiles; ++tile_index) { + auto input_tile_global = input_iter(0, tile_index); + DataTile data_tile; + OffsetTile offset_tile; + + OffsetTile linear_index; + OffsetTile quotient; + OffsetTile cycle; + OffsetTile cycle_base; + OffsetTile coordinate; + OffsetTile contribution; + + TCI(linear_index, static_cast(input_base)); + TEXPANDS(offset_tile, static_cast(0)); + + // 计算输出 stride（标量核心，运行时计算） + uint16_t output_stride[DATA_DIM]; + output_stride[DATA_DIM - 1] = 1; + for (int i = static_cast(DATA_DIM) - 2; i >= 0; --i) { + output_stride[i] = output_stride[i + 1] * static_cast(out_shape[i + 1]); + } - const int Mb = gOM / tM; - - const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_shapeOut = global_tensor>; - - using gm_InDataShape = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_OutDataShape = global_tensor>; - - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec -// using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - using tile_Inshape = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - using tile_Outshape = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec -// using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - -// gm_shapeIn inGm(in_ptr); - gm_shapeOut outGm(out_ptr); - - gm_InDataShape InShapeGm(in_shape); - gm_OutDataShape OutShapeGm(out_shape); - - tile_shapeData dataTile; - tile_shapeOffset offsetTile; - - tile_Inshape InshapeTile; - tile_Outshape OutshapeTile; - - int base = 0;// todo 生成一个标量 - int all_num = gIM; // 总元素数量 - -// using itOut = global_iterator; - using itIn = global_iterator; + // 对每个维度提取输入坐标并计算输出 byte offset + for (int d = static_cast(DATA_DIM) - 1; d >= 0; --d) { + uint32_t input_stride_for_dim = 1; + for (int dd = d + 1; dd < static_cast(DATA_DIM); ++dd) { + input_stride_for_dim *= static_cast(in_shape[dd]); + } + + if (input_stride_for_dim == 1) { + TMOV(quotient, linear_index); + } else { + TDIVS(quotient, linear_index, static_cast(input_stride_for_dim)); + } + + uint16_t in_dim_size = static_cast(in_shape[d]); + TDIVS(cycle, quotient, in_dim_size); + TMULS(cycle_base, cycle, in_dim_size); + TSUB(coordinate, quotient, cycle_base); + + TMULS(contribution, coordinate, + static_cast(output_stride[d] * sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + } - itIn gIIter(in_ptr); -// itOut gOIter(out_ptr); + TLOAD(data_tile, input_tile_global); + MSCATTER(output_global, data_tile, offset_tile); - alignas(256) static uint16_t g_dump[tM]; + input_base += tM; + } - int total_elements = tM; + if constexpr (kTailElements != 0) { + using TailDataTile = pto::Tile; + using TailOffsetTile = pto::Tile; + + auto input_tile_global = input_iter(0, kFullTiles); + TailDataTile data_tile; + TailOffsetTile offset_tile; + + TailOffsetTile linear_index; + TailOffsetTile quotient; + TailOffsetTile cycle; + TailOffsetTile cycle_base; + TailOffsetTile coordinate; + TailOffsetTile contribution; + + TCI(linear_index, static_cast(input_base)); + TEXPANDS(offset_tile, static_cast(0)); + + uint16_t output_stride[DATA_DIM]; + output_stride[DATA_DIM - 1] = 1; + for (int i = static_cast(DATA_DIM) - 2; i >= 0; --i) { + output_stride[i] = output_stride[i + 1] * static_cast(out_shape[i + 1]); + } + for (int d = static_cast(DATA_DIM) - 1; d >= 0; --d) { + uint32_t input_stride_for_dim = 1; + for (int dd = d + 1; dd < static_cast(DATA_DIM); ++dd) { + input_stride_for_dim *= static_cast(in_shape[dd]); + } + + if (input_stride_for_dim == 1) { + TMOV(quotient, linear_index); + } else { + TDIVS(quotient, linear_index, static_cast(input_stride_for_dim)); + } + + uint16_t in_dim_size = static_cast(in_shape[d]); + TDIVS(cycle, quotient, in_dim_size); + TMULS(cycle_base, cycle, in_dim_size); + TSUB(coordinate, quotient, cycle_base); + + TMULS(contribution, coordinate, + static_cast(output_stride[d] * sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + } - for (int i = 0; i < Mb; ++i) { - auto gI = gIIter(0, i); - TCOPYIN(InshapeTile, InShapeGm); - TCOPYIN(OutshapeTile, OutShapeGm); - gen_offset_Impl(offsetTile, InshapeTile, OutshapeTile, base, total_elements); -// printf("end genoffset\n"); - base += total_elements; -// DUMP_TILE("offsetTile", offsetTile, g_dump, 1, tM); - TCOPYIN(dataTile, gI); - MSCATTER(outGm, dataTile, offsetTile); - } - if constexpr (rmd_M) { - auto gI = gIIter(0, Mb); - TCOPYIN(InshapeTile, InShapeGm); - TCOPYIN(OutshapeTile, OutShapeGm); - total_elements = rmd_M;//尾片的大小。 - gen_offset_Impl(offsetTile, InshapeTile, OutshapeTile, base, total_elements); - base += total_elements; - TCOPYIN(dataTile, gI); - MSCATTER(outGm, dataTile, offsetTile); + TLOAD(data_tile, input_tile_global); + MSCATTER(output_global, data_tile, offset_tile); } } #endif - diff --git a/benchmark-linxisa/kernels/control/hashtable_lookup_simd.hpp b/benchmark-linxisa/kernels/control/hashtable_lookup_simd.hpp index 19659f3..6f273bc 100644 --- a/benchmark-linxisa/kernels/control/hashtable_lookup_simd.hpp +++ b/benchmark-linxisa/kernels/control/hashtable_lookup_simd.hpp @@ -1,6 +1,7 @@ #ifndef HASHTABLE_LOOKUP_SIMD_HPP #define HASHTABLE_LOOKUP_SIMD_HPP +#include #include #include "template_asm.h" // MGATHER #include diff --git a/benchmark-linxisa/kernels/element_wise/gelu.hpp b/benchmark-linxisa/kernels/element_wise/gelu.hpp index b2e3043..76cd3e3 100644 --- a/benchmark-linxisa/kernels/element_wise/gelu.hpp +++ b/benchmark-linxisa/kernels/element_wise/gelu.hpp @@ -1,128 +1,77 @@ -#include +#ifndef SUPERNPUBENCH_PTOISA_GELU_HPP +#define SUPERNPUBENCH_PTOISA_GELU_HPP -#include "template_asm.h" -#include -#include -// 海思解决方案 新版多项式拟合 +#include -// ============================================== -// ============================================== +#include -template -void __vec__ gelu_simd( - typename tile_shape::TileDType __in__ in, - typename tile_shape::TileDType __out__ out - // bool approximate = false // false:none, true:tanh -) { - size_t index = blkv_get_index_x(); - typename tile_shape::DType indata; - indata.data = blkv_get_tile_ptr(in)[index].data; // 直接拷贝 short +using namespace pto; - // 数据格式转换 V.FCVT - float x = static_cast(indata); - - constexpr uint32_t TOTAL_COUNT = 24*8*1024; - constexpr float SCALAR_A5 = -3.5123395303315874e-09f; - constexpr float SCALAR_A4 = 2.6452661927578447e-07f; - constexpr float SCALAR_A3 = -7.9294877650681883e-06f; - constexpr float SCALAR_A2 = 1.1061238183174282e-04f; - constexpr float SCALAR_A1 = 6.5189960878342390e-05f; - constexpr float SCALAR_A0 = -7.2666168212890625e-02f; - constexpr float SCALAR_AM1 = -1.5957698822021484e+00f; - constexpr float FP32_MAX = 5.75f; - - float t = blkv_max(x, -FP32_MAX); - t = blkv_min(t, FP32_MAX); - float t2 = t * t; +template +inline void gelu_tile(TileT &dst, TileT &src) { + TileT t; + TileT t2; + TileT p; + TileT tmp; + TileT denom; + TileT recip; - float p = SCALAR_A5 * t2 + SCALAR_A4; - p = p * t2 + SCALAR_A3; - p = p * t2 + SCALAR_A2; - p = p * t2 + SCALAR_A1; - p = p * t2 + SCALAR_A0; - p = p * t2 + SCALAR_AM1; + TMAXS(t, src, -5.75f); + TMINS(t, t, 5.75f); + TMUL(t2, t, t); - float exp_val = blkv_fexp(t * p); - float y = x / (1.0f + exp_val); - - BLKC_ASSIGN_CAST(out, index, y); - // blkv_get_tile_ptr(out)[index] = static_cast(result); -} + TMULS(p, t2, -3.5123395303315874e-09f); + TADDS(p, p, 2.6452661927578447e-07f); + TMUL(p, p, t2); + TADDS(p, p, -7.9294877650681883e-06f); + TMUL(p, p, t2); + TADDS(p, p, 1.1061238183174282e-04f); + TMUL(p, p, t2); + TADDS(p, p, 6.5189960878342390e-05f); + TMUL(p, p, t2); + TADDS(p, p, -7.2666168212890625e-02f); + TMUL(p, p, t2); + TADDS(p, p, -1.5957698822021484e+00f); -template -void gelu_impl( - tile_shapeData &in, - tile_shapeData &out -){ - static_assert(tile_shapeData::ValidRow != -1 && tile_shapeData::ValidCol != -1, - "Only static shape supported"); - gelu_simd<<>>(in.data(), out.data()); + TMUL(tmp, t, p); + TEXP(tmp, tmp); + TADDS(denom, tmp, 1.0f); + TRECIP(recip, denom); + TMUL(dst, src, recip); } -#define DUMP_TILE(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%.4f ", DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) +template +void gelu(dtype *in_ptr, dtype *out_ptr, bool = false) { + constexpr int kFullTiles = gM / tM; + constexpr int kTail = gM % tM; -template -void gelu( - dtype *in_ptr, - dtype *out_ptr, - bool approximate = false // false:none, true:tanh - ) { - const int Mb = gM / tM; - - const int rmd_M = gM % tM; + using Global = global_tensor>; + using TileT = Tile; + using Iterator = global_iterator; - using gm_shape = global_tensor>; - using tile_shapeData = Tile; - using tile_shapeData_rmd = Tile; + Iterator input_iter(in_ptr); + Iterator output_iter(out_ptr); - gm_shape inGm(in_ptr); - gm_shape outGm(out_ptr); - tile_shapeData inTile; - tile_shapeData outTile; - tile_shapeData_rmd inTile_rmd; - tile_shapeData_rmd outTile_rmd; + for (int tile = 0; tile < kFullTiles; ++tile) { + TileT input_tile; + TileT output_tile; + TLOAD(input_tile, input_iter(0, tile)); + gelu_tile(output_tile, input_tile); + TSTORE(output_iter(0, tile), output_tile); + } - using itIn = global_iterator; - using itOut = global_iterator; - - itIn gIIter(in_ptr); - itOut gOIter(out_ptr); - // for test /////////////////////////////////////// - alignas(256) static dtype g_dump_intTile[tM]; - alignas(256) static dtype g_dump_outTile[tM]; - // /////////////////////////////////////// - - // printf("MB = %d\n", Mb); - for (int i = 0; i < Mb; ++i) { - // printf("iter i %d\n",i); - auto gI = gIIter(0, i); - auto gO = gOIter(0, i); - TCOPYIN(inTile, gI); - // DUMP_TILE("inTile", inTile, g_dump_intTile, 1, tM); - gelu_impl(inTile, outTile); - // DUMP_TILE("outTile", outTile, g_dump_outTile, 1, tM); - TCOPYOUT(gO, outTile); - } - if constexpr (rmd_M) { - auto gI = gIIter(0, Mb); - auto gO = gOIter(0, Mb); - TCOPYIN(inTile_rmd, gI); - gelu_impl(inTile_rmd, outTile_rmd); - // DUMP_TILE("offsetTile", offsetTile, g_dump, 1, tM); - TCOPYOUT(gO, outTile_rmd); - } + if constexpr (kTail != 0) { + using TailTile = + Tile; + using TailIterator = global_iterator; + TailIterator tail_input_iter(in_ptr); + TailIterator tail_output_iter(out_ptr); + TailTile input_tile; + TailTile output_tile; + TLOAD(input_tile, tail_input_iter(0, kFullTiles)); + gelu_tile(output_tile, input_tile); + TSTORE(tail_output_iter(0, kFullTiles), output_tile); + } } + +#endif // SUPERNPUBENCH_PTOISA_GELU_HPP diff --git a/benchmark-linxisa/kernels/element_wise/gelu_pto.hpp b/benchmark-linxisa/kernels/element_wise/gelu_pto.hpp new file mode 100644 index 0000000..f292a3e --- /dev/null +++ b/benchmark-linxisa/kernels/element_wise/gelu_pto.hpp @@ -0,0 +1,277 @@ +// ============================================================================ +// GELU 算子 — PTO 一层编程模型 +// +// 原始 gelu.hpp 策略: +// TCOPYIN (half) -> __vec__ gelu_simd (多项式拟合) -> TCOPYOUT (half) +// __vec__ 块逐元素: fp16→fp32, clamp, Horner 多项式, exp, 除法, fp32→fp16 +// +// PTO 一层策略: +// TLOAD (half) -> TCVT(fp16→fp32) -> tile 指令链计算 GELU -> TCVT(fp32→fp16) -> TSTORE (half) +// 全部用 Tile 级内联函数, 无 __vec__ 块 +// +// ┌─────────────────────────────────────────────────────────────────────────┐ +// │ 当前编译器不支持 / 不完整的指令汇总 │ +// ├──────────┬──────────────────┬──────────────────────────────────────────┤ +// │ Pto ISA │ 当前编译器状态 │ 说明 │ +// │ 指令 │ │ │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TLOAD │ API 有(名不同)， │ PTO ISA 名 TLOAD；当前编译器名 TCOPYIN； │ +// │ │ 二层实现 │ jcore/TCopyIn.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TCVT │ API 有(签名不同)，│ PTO ISA: TCVT(dst,src,tmp,mode,satMode) │ +// │ │ 二层实现 │ 当前编译器: TCVT(dst,src) 无 tmp/mode； │ +// │ │ │ jcore/TCvt.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TMAXS │ API 有，二层实现 │ jcore/TMaxs.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TMINS │ API 有，二层实现 │ jcore/TMins.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TMUL │ API 有，二层实现 │ jcore/TMul.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TMULS │ API 有，二层实现 │ jcore/TMuls.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TADDS │ API 有，二层实现 │ jcore/TAdds.hpp 用 __vec__ 实现 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TEXP │ API 有，二层实现 │ jcore/TExp.hpp 用 __vec__ 实现 │ +// │ │ │ (template_asm.h 有 TEXP_TEPL 内联汇编, │ +// │ │ │ 但不在 pto_tileop.hpp API 中) │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TRECIP │ API 有，二层实现 │ jcore/TRecip.hpp 用 __vec__ 实现 │ +// │ │ │ (template_asm.h 有 TRECIP_TEPL 内联汇编, │ +// │ │ │ 但不在 pto_tileop.hpp API 中) │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE；当前编译器名 TCOPYOUT；│ +// │ │ 二层实现 │ jcore/TCopyOut.hpp 用 __vec__ 实现 │ +// └──────────┴──────────────────┴──────────────────────────────────────────┘ +// +// PTO ISA 文档签名 (Declared in include/pto/pto_instr.hpp): +// +// TLOAD: +// template +// PTO_INST RecordEvent TLOAD(TileData &dst, GlobalData &src, WaitEvents &... events); +// +// TCVT: +// template +// PTO_INST RecordEvent TCVT(TileDataD &dst, TileDataS &src, TmpTileData &tmp, +// RoundMode mode, SaturationMode satMode, +// WaitEvents &... events); +// +// TMAXS: +// template +// PTO_INST RecordEvent TMAXS(TileDataDst &dst, TileDataSrc &src, +// typename TileDataSrc::DType scalar, +// WaitEvents &... events); +// +// TMINS: +// template +// PTO_INST RecordEvent TMINS(TileDataDst &dst, TileDataSrc &src, +// typename TileDataSrc::DType scalar, +// WaitEvents &... events); +// +// TMUL: +// template +// PTO_INST RecordEvent TMUL(TileDataDst &dst, TileDataSrc0 &src0, +// TileDataSrc1 &src1, WaitEvents &... events); +// +// TMULS: +// template +// PTO_INST RecordEvent TMULS(TileDataDst &dst, TileDataSrc &src0, +// typename TileDataSrc::DType scalar, +// WaitEvents &... events); +// +// TADDS: +// template +// PTO_INST RecordEvent TADDS(TileDataDst &dst, TileDataSrc &src0, +// typename TileDataSrc::DType scalar, +// WaitEvents &... events); +// +// TEXP: +// template +// PTO_INST RecordEvent TEXP(TileDataDst &dst, TileDataSrc &src, +// WaitEvents &... events); +// +// TRECIP: +// template +// PTO_INST RecordEvent TRECIP(TileDataDst &dst, TileDataSrc &src, +// WaitEvents &... events); +// +// TSTORE: +// template +// PTO_INST RecordEvent TSTORE(GlobalData &dst, TileData &src, +// WaitEvents &... events); +// ============================================================================ + +#include +#include +// #include // [!] PTO ISA C++ Intrinsic — 当前编译器未提供 + +#include +#include + +// ============================================================================ +// GELU 多项式拟合系数 (与原始 gelu.hpp 一致) +// GELU(x) = x / (1 + exp(t * P(t²))) +// P(t²) = A5*t²⁵ + A4*t²⁴ + A3*t²³ + A2*t²² + A1*t² + A0 + AM1/t² +// (Horner: p = ((((A5*t2 + A4)*t2 + A3)*t2 + A2)*t2 + A1)*t2 + A0)*t2 + AM1) +// ============================================================================ +namespace gelu_coeffs { + constexpr float A5 = -3.5123395303315874e-09f; + constexpr float A4 = 2.6452661927578447e-07f; + constexpr float A3 = -7.9294877650681883e-06f; + constexpr float A2 = 1.1061238183174282e-04f; + constexpr float A1 = 6.5189960878342390e-05f; + constexpr float A0 = -7.2666168212890625e-02f; + constexpr float AM1 = -1.5957698822021484e+00f; + constexpr float CLAMP_MAX = 5.75f; +} + +// ---------------------------------------------------------------------------- +// gelu_impl: 用 PTO ISA Tile 指令计算 GELU (一层编程, 无 __vec__ 块) +// +// 输入: inTile — fp16 tile, shape (1, tM) +// 输出: outTile — fp16 tile, shape (1, tM) +// 中间: 全部在 fp32 tile 上计算 +// +// 算法: +// x = (float)in +// t = clamp(x, -5.75, 5.75) +// t2 = t * t +// p = Horner(t2, [A5,A4,A3,A2,A1,A0,AM1]) +// e = exp(t * p) +// y = x * (1 / (1 + e)) // 用 TRECIP + TMUL 代替除法 +// out = (half)y +// ---------------------------------------------------------------------------- +template +void gelu_impl( + tile_shapeData &inTile, + tile_shapeData &outTile, + tile_shapeFP32 &tmpCvt // TCVT 需要的临时 tile +) { + using fp_t = typename tile_shapeFP32::DType; // float + + tile_shapeFP32 xTile; // x = (float)input + tile_shapeFP32 tTile; // t = clamp(x) + tile_shapeFP32 t2Tile; // t² + tile_shapeFP32 pTile; // 多项式值 + tile_shapeFP32 scratchTile; // 复用: tp -> exp -> denom -> recip -> y + + // ---- Step 1: fp16 -> fp32 ---- + // [当前编译器] TCVT(dst, src) 无 tmp/mode/satMode 参数; jcore 为 __vec__ + TCVT(xTile, inTile, tmpCvt, RoundMode::Nearest, SaturationMode::Default); + + // ---- Step 2: clamp x to [-5.75, 5.75] ---- + // [当前编译器] TMAXS/TMINS API 有, jcore 为 __vec__ + TMAXS(tTile, xTile, (fp_t)(-gelu_coeffs::CLAMP_MAX)); // t = max(x, -5.75) + TMINS(tTile, tTile, (fp_t)gelu_coeffs::CLAMP_MAX); // t = min(t, 5.75) + + // ---- Step 3: t² = t * t ---- + // [当前编译器] TMUL API 有, jcore 为 __vec__ + TMUL(t2Tile, tTile, tTile); + + // ---- Step 4: 多项式 Horner 法 ---- + // p = A5*t2 + A4 + // [当前编译器] TMULS/TADDS API 有, jcore 为 __vec__ + TMULS(pTile, t2Tile, gelu_coeffs::A5); + TADDS(pTile, pTile, gelu_coeffs::A4); + + // p = p*t2 + A3 + TMUL(pTile, pTile, t2Tile); + TADDS(pTile, pTile, gelu_coeffs::A3); + + // p = p*t2 + A2 + TMUL(pTile, pTile, t2Tile); + TADDS(pTile, pTile, gelu_coeffs::A2); + + // p = p*t2 + A1 + TMUL(pTile, pTile, t2Tile); + TADDS(pTile, pTile, gelu_coeffs::A1); + + // p = p*t2 + A0 + TMUL(pTile, pTile, t2Tile); + TADDS(pTile, pTile, gelu_coeffs::A0); + + // p = p*t2 + AM1 + TMUL(pTile, pTile, t2Tile); + TADDS(pTile, pTile, gelu_coeffs::AM1); + + // ---- Step 5: exp_val = exp(t * p) ---- + // scratch = t * p + TMUL(scratchTile, tTile, pTile); + // exp_val = exp(scratch) + // [当前编译器] TEXP API 有, jcore 为 __vec__ + // (template_asm.h 有 TEXP_TEPL 内联汇编, 但不在 pto_tileop.hpp 中) + TEXP(scratchTile, scratchTile); // scratch = exp(t*p) + + // ---- Step 6: y = x / (1 + exp_val) ---- + // denom = 1 + exp_val + TADDS(scratchTile, scratchTile, (fp_t)1.0f); // scratch = 1 + exp + // recip = 1 / denom + // [当前编译器] TRECIP API 有, jcore 为 __vec__ + // (template_asm.h 有 TRECIP_TEPL 内联汇编, 但不在 pto_tileop.hpp 中) + TRECIP(scratchTile, scratchTile); // scratch = 1 / (1+exp) + // y = x * recip + TMUL(scratchTile, xTile, scratchTile); // scratch = x * recip = y + + // ---- Step 7: fp32 -> fp16 ---- + TCVT(outTile, scratchTile, tmpCvt, RoundMode::Nearest, SaturationMode::Default); +} + + +// ---------------------------------------------------------------------------- +// gelu: 主入口, 接口与原 gelu.hpp 一致 +// ---------------------------------------------------------------------------- +template +void gelu( + dtype *in_ptr, + dtype *out_ptr, + bool approximate = false + ) { + using gm_shape = global_tensor>; + using tile_shapeData = Tile; + using tile_shapeFP32 = Tile; + using tile_shapeData_rmd = Tile; + using tile_shapeFP32_rmd = Tile; + + const int Mb = gM / tM; + const int rmd_M = gM % tM; + + using itIn = global_iterator; + using itOut = global_iterator; + + itIn gIIter(in_ptr); + itOut gOIter(out_ptr); + + tile_shapeData inTile, outTile; + tile_shapeFP32 tmpCvt; // TCVT 临时 tile + tile_shapeData_rmd inTile_rmd, outTile_rmd; + tile_shapeFP32_rmd tmpCvt_rmd; + + for (int i = 0; i < Mb; ++i) { + auto gI = gIIter(0, i); + auto gO = gOIter(0, i); + + // TLOAD: GM -> UB + // [当前编译器] 名为 TCOPYIN, jcore 为 __vec__ + TLOAD(inTile, gI); + + gelu_impl(inTile, outTile, tmpCvt); + + // TSTORE: UB -> GM + // [当前编译器] 名为 TCOPYOUT, jcore 为 __vec__ + TSTORE(gO, outTile); + } + if constexpr (rmd_M) { + auto gI = gIIter(0, Mb); + auto gO = gOIter(0, Mb); + + TLOAD(inTile_rmd, gI); + gelu_impl(inTile_rmd, outTile_rmd, tmpCvt_rmd); + TSTORE(gO, outTile_rmd); + } +} diff --git a/benchmark-linxisa/kernels/fa/fa_2d_unroll.hpp b/benchmark-linxisa/kernels/fa/fa_2d_unroll.hpp index 296e6e9..709e203 100644 --- a/benchmark-linxisa/kernels/fa/fa_2d_unroll.hpp +++ b/benchmark-linxisa/kernels/fa/fa_2d_unroll.hpp @@ -1,6 +1,10 @@ #include "fa_utils.h" #include "fa_fp4_utils.h" +using namespace pto; +using namespace pto::blkv; +using pto::type_traits; + #ifndef Xdim #define Xdim 2 #endif @@ -13,6 +17,11 @@ #define __vbuf__ #endif +#ifndef BLKC_ASSIGN_CAST +#define BLKC_ASSIGN_CAST(tile, idx, value) \ + (pto::blkv::blkv_get_tile_ptr(tile)[(idx)] = (value)) +#endif + template void __vec__ new_max_1src( typename tileMax::TileDType __out__ scale, @@ -797,7 +806,6 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype TEXPANDSCALAR(tSum[x], 0); } - tileO_out tPV_out; tileO tO[Xdim], tPV[Xdim]; tileScale tScale[Xdim]; @@ -840,7 +848,8 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype #if Ydim == 1 #pragma clang loop unroll(full) for(int x=0;x<<>>( + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + flashsoftmax_dn_mout_cast_kernel( tScale[x].data(), tNewMax[x].data(), tNewSum[x].data(), @@ -849,16 +858,19 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype tMax[x].data(), tSum[x].data(), scale); + }); } #elif Ydim == 2 #pragma clang loop unroll(full) for(int x=0;x<<>>( + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_2src( tScale[x].data(), tNewMax[x].data(), tW[x][0].data(), tW[x][1].data(), tMax[x].data(), scale); + }); // src_exp_2src<<>>( // tExpW[x][0].data(), tExpW[x][1].data(), // tW[x][0].data(), tW[x][1].data(), @@ -870,39 +882,49 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype // tSum[x].data(), // tScale[x].data() // ); - src_exp_2src_with_new_sum<<>>( + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_new_sum( tNewSum[x].data(), tExpW[x][0].data(), tExpW[x][1].data(), tW[x][0].data(), tW[x][1].data(), tNewMax[x].data(), tSum[x].data(), tScale[x].data(), scale); + }); } #elif Ydim == 4 tileSum tLocalSum[Xdim][2]; #pragma clang loop unroll(full) for(int x=0;x<<>>( + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_4src( tScale[x].data(), tNewMax[x].data(), tW[x][0].data(), tW[x][1].data(), tW[x][2].data(), tW[x][3].data(), tMax[x].data(), scale); + }); // src_exp_4src<<>>( // tExpW[x][0].data(), tExpW[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), // tW[x][0].data(), tW[x][1].data(), tW[x][2].data(), tW[x][3].data(), // tNewMax[x].data(), // scale); - src_exp_2src_with_local_sum<<>>(tLocalSum[x][0].data(), tExpW[x][0].data(), tExpW[x][1].data(), + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_local_sum(tLocalSum[x][0].data(), tExpW[x][0].data(), tExpW[x][1].data(), tW[x][0].data(), tW[x][1].data(), tNewMax[x].data(), scale); - src_exp_2src_with_local_sum<<>>(tLocalSum[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), + }); + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_local_sum(tLocalSum[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), tW[x][2].data(), tW[x][3].data(), tNewMax[x].data(), scale); + }); // new_sum_4src<<>>( // tNewSum[x].data(), // tExpW[x][0].data(), tExpW[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), // tSum[x].data(), // tScale[x].data() // ); - new_sum_of_2_loc_sum<<>>(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tSum[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + new_sum_of_2_loc_sum(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tSum[x].data(), tScale[x].data()); + }); } #elif Ydim == 8 tileMax tLocalMax[Xdim][2]; @@ -912,17 +934,25 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype for(int x=0;x<<>>(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + local_max_4src(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + }); } - new_max_of_2_loc_max<<>>(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tMax[x].data()); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_of_2_loc_max(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tMax[x].data()); + }); #pragma clang loop unroll(full) for(int k=0;k<4;k++){ - src_exp_2src_with_local_sum<<>>(tLocalSum[x][k].data(), tExpW[x][2*k].data(), tExpW[x][2*k+1].data(), + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_local_sum(tLocalSum[x][k].data(), tExpW[x][2*k].data(), tExpW[x][2*k+1].data(), tW[x][2*k].data(), tW[x][2*k+1].data(), tNewMax[x].data(), scale); + }); } - new_sum_of_4_loc_sum<<>>(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + new_sum_of_4_loc_sum(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + }); } #elif Ydim == 16 tileMax tLocalMax[Xdim][4]; @@ -931,25 +961,35 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype #pragma clang loop unroll(full) for(int x=0;x<<>>(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + local_max_4src(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + }); } - new_max_of_4_loc_max<<>>(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tLocalMax[x][2].data(), tLocalMax[x][3].data(), tMax[x].data()); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_of_4_loc_max(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tLocalMax[x][2].data(), tLocalMax[x][3].data(), tMax[x].data()); + }); #pragma clang loop unroll(full) for(int k=0;k<4;k++){ - src_exp_4src<<>>( + pto::blkv::blkv_for_2d(tileW::ValidRow, tileW::ValidCol, [&] { + src_exp_4src( tExpW[x][4*k].data(), tExpW[x][4*k+1].data(), tExpW[x][4*k+2].data(), tExpW[x][4*k+3].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), tNewMax[x].data(), scale); + }); } #pragma clang loop unroll(full) for(int k=0;k<4;k++){ - local_sum_4src<<>>(tLocalSum[x][k].data(), tExpW[x][4*k].data(), tExpW[x][4*k+1].data(), tExpW[x][4*k+2].data(), tExpW[x][4*k+3].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + local_sum_4src(tLocalSum[x][k].data(), tExpW[x][4*k].data(), tExpW[x][4*k+1].data(), tExpW[x][4*k+2].data(), tExpW[x][4*k+3].data()); + }); } - new_sum_of_4_loc_sum<<>>(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + new_sum_of_4_loc_sum(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + }); } #else #ifdef _2D_UNROLL @@ -978,6 +1018,7 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype tileW_left tW_left[Xdim][Ydim]; #pragma clang loop unroll(full) for(int x=0;x::bits == 4) { @@ -1001,19 +1042,21 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype if(j==0){ #pragma clang loop unroll(full) for(int x=0;x<<>>(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileO::ValidRow, [&] { + global_update(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data()); + }); } } // 更新最大值状态 #pragma clang loop unroll(full) for(int x=0;x::bits == 4) { - normalize_with_last_update_nocast<<>>(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + pto::blkv::blkv_for_2d(tileO::ValidRow, tileO::ValidCol, [&] { + normalize_with_last_update_nocast(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + }); TMOV_NORM(tO_cast[x], tO[x]); } else { - normalize_with_last_update<<>>(tO_cast[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + pto::blkv::blkv_for_2d(tileO::ValidRow, tileO::ValidCol, [&] { + normalize_with_last_update(tO_cast[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + }); } } // 写回全局内存 @@ -1046,4 +1093,6 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype } } +#ifdef _UNALIGN_2D_UNROLL #include "fa_unalign_2d_unroll.hpp" +#endif diff --git a/benchmark-linxisa/kernels/fa/fa_fp4_utils.h b/benchmark-linxisa/kernels/fa/fa_fp4_utils.h index d2f776b..ecdbd51 100644 --- a/benchmark-linxisa/kernels/fa/fa_fp4_utils.h +++ b/benchmark-linxisa/kernels/fa/fa_fp4_utils.h @@ -1,4 +1,9 @@ +#include + +using namespace pto::blkv; +using pto::type_traits; + template concept is_fp4_tile = requires (T tile) { type_traits::bits == 4; diff --git a/benchmark-linxisa/kernels/fa/fa_utils.h b/benchmark-linxisa/kernels/fa/fa_utils.h index 5744a1d..59ee045 100644 --- a/benchmark-linxisa/kernels/fa/fa_utils.h +++ b/benchmark-linxisa/kernels/fa/fa_utils.h @@ -1,5 +1,9 @@ // FA utility functions shared across different implementations +#include + +using namespace pto::blkv; + template void __vec__ flashsoftmax_dn_mout_cast_kernel( typename tileScale::TileDType __out__ rescale, diff --git a/benchmark-linxisa/kernels/fa/linx_blkc.h b/benchmark-linxisa/kernels/fa/linx_blkc.h new file mode 100644 index 0000000..df68bc9 --- /dev/null +++ b/benchmark-linxisa/kernels/fa/linx_blkc.h @@ -0,0 +1,121 @@ +#ifndef PTO_FA_LINX_BLKC_H +#define PTO_FA_LINX_BLKC_H + +#include + +#ifndef __bf16 +#define __bf16 pto_bf16_t +#endif + +using pto::blkv::blkv_fexp; +using pto::blkv::blkv_fsqrt; +using pto::blkv::blkv_get_index_x; +using pto::blkv::blkv_get_index_y; +using pto::blkv::blkv_get_index_z; +using pto::blkv::blkv_get_tile_ptr; +using pto::blkv::blkv_max; + +using __fp4_e1m2x2 = pto::fp4_e2m1_t; +using __bf16x2 = uint32_t; + +template +inline To linx_cvt_as(From value) { + return static_cast(value); +} + +template +inline void linx_cvt(To &dst, From value) { + dst = linx_cvt_as(value); +} + +template +inline void linx_cvt_package(To &dst, A a, B b) { + const uint16_t hi = + static_cast(pto::lowp_word_from_bf16(pto::bf16_t{static_cast(a)}) & + 0xffffu); + const uint16_t lo = + static_cast(pto::lowp_word_from_bf16(pto::bf16_t{static_cast(b)}) & + 0xffffu); + if constexpr (sizeof(To) >= 4) { + dst = static_cast((static_cast(hi) << 16) | lo); + } else { + dst = static_cast(pto::float_to_fp4_e2m1(static_cast(a))); + } +} + +inline void blkv_bf16_fmax(__bf16 &dst, __bf16 a, __bf16 b) { + dst = a < b ? b : a; +} + +inline __bf16 blkv_bf16_max(__bf16 a, __bf16 b) { return a < b ? b : a; } +inline __bf16 blkv_bf16_mul(__bf16 a, __bf16 b) { return a * b; } +inline __bf16 blkv_bf16_div(__bf16 a, __bf16 b) { return a / b; } + +inline void blkv_bf16_fadd(__bf16 &dst, __bf16 a, __bf16 b) { dst = a + b; } +inline void blkv_bf16_fsub(__bf16 &dst, __bf16 a, __bf16 b) { dst = a - b; } +inline void blkv_bf16_fmul(__bf16 &dst, __bf16 a, __bf16 b) { dst = a * b; } +inline void blkv_bf16_fdiv(__bf16 &dst, __bf16 a, __bf16 b) { dst = a / b; } +inline void blkv_bf16_fexp(__bf16 &dst, __bf16 a) { + dst = pto::blkv::blkv_fexp(static_cast(a)); +} + +inline float linx_bf16x2_hi(__bf16x2 value) { + return static_cast(pto::bf16_from_lowp_word((value >> 16) & 0xffffu)); +} + +inline float linx_bf16x2_lo(__bf16x2 value) { + return static_cast(pto::bf16_from_lowp_word(value & 0xffffu)); +} + +inline __bf16x2 linx_pack_bf16x2(float hi, float lo) { + return (pto::lowp_word_from_bf16(pto::bf16_t{hi}) << 16) | + pto::lowp_word_from_bf16(pto::bf16_t{lo}); +} + +inline void blkv_bf16x2_fmax(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(pto::blkv::blkv_max(linx_bf16x2_hi(a), linx_bf16x2_hi(b)), + pto::blkv::blkv_max(linx_bf16x2_lo(a), linx_bf16x2_lo(b))); +} + +inline __bf16x2 blkv_bf16x2_mul(__bf16x2 a, __bf16x2 b) { + return linx_pack_bf16x2(linx_bf16x2_hi(a) * linx_bf16x2_hi(b), + linx_bf16x2_lo(a) * linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_mul(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = blkv_bf16x2_mul(a, b); +} + +inline void blkv_bf16x2_fadd(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(linx_bf16x2_hi(a) + linx_bf16x2_hi(b), + linx_bf16x2_lo(a) + linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_fsub(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(linx_bf16x2_hi(a) - linx_bf16x2_hi(b), + linx_bf16x2_lo(a) - linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_fmul(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = blkv_bf16x2_mul(a, b); +} + +inline void blkv_bf16x2_fdiv(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(linx_bf16x2_hi(a) / linx_bf16x2_hi(b), + linx_bf16x2_lo(a) / linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_fmsub(__bf16x2 &dst, __bf16x2 a, __bf16x2 b, + __bf16x2 c) { + dst = linx_pack_bf16x2((linx_bf16x2_hi(a) * linx_bf16x2_hi(b)) - + linx_bf16x2_hi(c), + (linx_bf16x2_lo(a) * linx_bf16x2_lo(b)) - + linx_bf16x2_lo(c)); +} + +inline void blkv_bf16x2_fexp(__bf16x2 &dst, __bf16x2 a) { + dst = linx_pack_bf16x2(pto::blkv::blkv_fexp(linx_bf16x2_hi(a)), + pto::blkv::blkv_fexp(linx_bf16x2_lo(a))); +} + +#endif // PTO_FA_LINX_BLKC_H diff --git a/benchmark-linxisa/kernels/gather/gather.hpp b/benchmark-linxisa/kernels/gather/gather.hpp index 0222852..a19ec69 100644 --- a/benchmark-linxisa/kernels/gather/gather.hpp +++ b/benchmark-linxisa/kernels/gather/gather.hpp @@ -1,207 +1,92 @@ +#ifndef SUPERNPUBENCH_PTOISA_GATHER_HPP +#define SUPERNPUBENCH_PTOISA_GATHER_HPP + #include -#include "template_asm.h" +#include #include -#include - - -#define DUMP_TILE(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%d ", DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) - - -#define DUMP_TILE_FLOAT(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%f ", DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) -template -void __vec__ gen_offset( - typename tile_shapeInOffset::TileDType __in__ in, // inOffset - typename tile_shapeOffset::TileDType __out__ out, // - const size_t n_base -) { - size_t data_width = sizeof(dtype); - size_t index_x = blkv_get_index_x(); - size_t index_y = blkv_get_index_y(); - size_t index = index_y * tile_shapeOffset::RowStride + index_x; - - size_t n_offset = (index_x + n_base); - size_t index_m = blkv_get_tile_ptr(in)[index_y] * gN; // 这里不需要m_base，因为传入的inOffset已经被切分为tile，起始元素就是m_base - size_t out_offset = index_m + n_offset; - out_offset *= data_width; - - blkv_get_tile_ptr(out)[index] = out_offset; -} - -template -void gen_offset_impl( - tile_shapeInOffset &in_offset, - tile_shapeOffset &out_offset, - const size_t n_base - ) { - static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, - "Only static shape supported"); - - - gen_offset<<>>( - in_offset.data(), - out_offset.data(), - n_base); -} - -template -void gather( - dtype *in_data_ptr, - otype *in_offset_ptr, - dtype *out_ptr - ) { - const size_t Mb = gM / tM; - const size_t Nb = gN / tN; - const size_t rmd_M = gM % tM; - const size_t rmd_N = gN % tN; - - using gm_shapeInOffset = global_tensor>; - using gm_shapeIn = global_tensor>; - using gm_shapeOut = global_tensor>; - - using tile_shapeInData = Tile; - using itIn = global_iterator; - tile_shapeInData inTile; - itIn gInIter(in_data_ptr); - - using tile_shapeInOffset = Tile; - using tile_shapeData = Tile; - using tile_shapeOffset = Tile; - using tile_shapeInOffset_rmd_n = Tile; - using tile_shapeData_rmd_n = Tile; - using tile_shapeOffset_rmd_n = Tile; - - using tile_shapeInOffset_rmd_mn = Tile; - using tile_shapeData_rmd_mn = Tile; - using tile_shapeOffset_rmd_mn = Tile; - - using tile_shapeInOffset_rmd_m = Tile; - using tile_shapeData_rmd_m = Tile; - using tile_shapeOffset_rmd_m = Tile; - - gm_shapeIn inGm(in_data_ptr); - - tile_shapeInOffset inOffsetTile; - tile_shapeData outTile; - tile_shapeOffset offsetTile; - - tile_shapeInOffset_rmd_n inOffsetTile_rmd_n; - tile_shapeData_rmd_n outTile_rmd_n; - tile_shapeOffset_rmd_n offsetTile_rmd_n; - - tile_shapeInOffset_rmd_mn inOffsetTile_rmd_mn; - tile_shapeData_rmd_mn outTile_rmd_mn; - tile_shapeOffset_rmd_mn offsetTile_rmd_mn; - - tile_shapeInOffset_rmd_m inOffsetTile_rmd_m; - tile_shapeData_rmd_m outTile_rmd_m; - tile_shapeOffset_rmd_m offsetTile_rmd_m; - - using itInOffset = global_iterator; - using itOut = global_iterator; - - itInOffset gInOffsetIter(in_offset_ptr); - itOut gOIter(out_ptr); - // for test /////////////////////////////////////// - // alignas(256) static uint32_t g_dump[tM * rmd_N]; - alignas(256) static uint32_t g_dump[tM * rmd_N]; - // // alignas(256) static uint32_t g_dump_inoffset[rmd_N]; - // alignas(256) static uint32_t g_dump_inoffset[rmd_N]; - // alignas(256) static dtype g_dump_outdata[tM * rmd_N]; - // /////////////////////////////////////// - - size_t n_base = 0; - - // #pragma clang loop unroll(full) - for (int j = 0; j < Mb; ++j) { - printf("j = %d\n", j); - for (int i = 0; i < Nb; ++i) { - auto gInOffset = gInOffsetIter(0, j); - auto gO = gOIter(j, i); - TCOPYIN(inOffsetTile, gInOffset); - // test - // auto gIn = gInIter(j, i); - // TCOPYIN(inTile, gIn); - n_base = i * tN; - // printf("j = %d\n", j); - // printf("i = %d\n", i); - // printf("base = %d\n", base); - // printf("in_shape[0] = %d\n", in_shape[0]); - gen_offset_impl(inOffsetTile, offsetTile, n_base); - - MGATHER(outTile, inGm, offsetTile); - - // printf("inGm = %d\n", inGm); - // DUMP_TILE("inOffsetTile", inOffsetTile, g_dump_inoffset, 1, tN); - // DUMP_TILE("offsetTile", offsetTile, g_dump, 1, tN); - // DUMP_TILE_FLOAT("inTile", inTile, g_dump_outdata, 1, tN); - // DUMP_TILE_FLOAT("outTile", outTile, g_dump_outdata, 1, tN); - TCOPYOUT(gO, outTile); - } - if constexpr (rmd_N) { - auto gInOffset = gInOffsetIter(0, j); - auto gO = gOIter(j, Nb); - n_base = Nb * tN; - TCOPYIN(inOffsetTile_rmd_n, gInOffset); - gen_offset_impl(inOffsetTile_rmd_n, offsetTile_rmd_n, n_base); - MGATHER(outTile_rmd_n, inGm, offsetTile_rmd_n); - // DUMP_TILE("inOffsetTile_rmd_n", inOffsetTile_rmd_n, g_dump_inoffset, 1, rmd_N); - // DUMP_TILE_FLOAT("outTile_rmd_n", outTile_rmd_n, g_dump_outdata, 20, rmd_N); - // DUMP_TILE("offsetTile_rmd_n", offsetTile_rmd_n, g_dump, 20, rmd_N); - TCOPYOUT(gO, outTile_rmd_n); - } +using namespace pto; + +template +void gather(dtype *in_data_ptr, otype *in_offset_ptr, dtype *out_ptr) { + constexpr size_t kFullRows = gM / tM; + constexpr size_t kTailRows = gM % tM; + constexpr size_t kFullCols = gN / tN; + constexpr size_t kTailCols = gN % tN; + + using InputGlobal = global_tensor>; + using OffsetGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + + using OffsetTile = + Tile; + using DataTile = Tile; + using OffsetIterator = global_iterator; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_data_ptr); + OffsetIterator offset_iter(in_offset_ptr); + OutputIterator output_iter(out_ptr); + + for (size_t row_tile = 0; row_tile < kFullRows; ++row_tile) { + OffsetTile row_index; + TLOAD(row_index, offset_iter(0, static_cast(row_tile))); + for (size_t col_tile = 0; col_tile < kFullCols; ++col_tile) { + DataTile out_tile; + InputGlobal adjusted_input(in_data_ptr + col_tile * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(output_iter(static_cast(row_tile), + static_cast(col_tile)), + out_tile); } - if constexpr (rmd_M) { - for (int i = 0; i < Nb; ++i) { - auto gInOffset = gInOffsetIter(0, Mb); - auto gO = gOIter(Mb, i); - n_base = i * tN; - TCOPYIN(inOffsetTile_rmd_m, gInOffset); - gen_offset_impl(inOffsetTile_rmd_m, offsetTile_rmd_m, n_base); - MGATHER(outTile_rmd_m, inGm, offsetTile_rmd_m); - TCOPYOUT(gO, outTile_rmd_m); - } - if constexpr (rmd_N) { - auto gInOffset = gInOffsetIter(0, Mb); - auto gO = gOIter(Mb, Nb); - n_base = Nb * tN; - TCOPYIN(inOffsetTile_rmd_mn, gInOffset); - gen_offset_impl(inOffsetTile_rmd_mn, offsetTile_rmd_mn, n_base); - MGATHER(outTile_rmd_mn, inGm, offsetTile_rmd_mn); - TCOPYOUT(gO, outTile_rmd_mn); - } + if constexpr (kTailCols != 0) { + using TailColTile = Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailColTile out_tile; + InputGlobal adjusted_input(in_data_ptr + kFullCols * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(tail_output_iter(static_cast(row_tile), + static_cast(kFullCols)), + out_tile); } - + } + + if constexpr (kTailRows != 0) { + using TailOffsetTile = + Tile; + using TailOffsetIterator = global_iterator; + TailOffsetIterator tail_offset_iter(in_offset_ptr); + TailOffsetTile row_index; + TLOAD(row_index, tail_offset_iter(0, static_cast(kFullRows))); + for (size_t col_tile = 0; col_tile < kFullCols; ++col_tile) { + using TailRowTile = Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailRowTile out_tile; + InputGlobal adjusted_input(in_data_ptr + col_tile * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(tail_output_iter(static_cast(kFullRows), + static_cast(col_tile)), + out_tile); + } + if constexpr (kTailCols != 0) { + using TailTile = Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailTile out_tile; + InputGlobal adjusted_input(in_data_ptr + kFullCols * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(tail_output_iter(static_cast(kFullRows), + static_cast(kFullCols)), + out_tile); + } + } } - - +#endif // SUPERNPUBENCH_PTOISA_GATHER_HPP diff --git a/benchmark-linxisa/kernels/gather/gather_pto.hpp b/benchmark-linxisa/kernels/gather/gather_pto.hpp new file mode 100644 index 0000000..8835c43 --- /dev/null +++ b/benchmark-linxisa/kernels/gather/gather_pto.hpp @@ -0,0 +1,199 @@ +// ============================================================================ +// Gather 算子 — PTO 一层编程模型 +// +// 原始 gather.hpp 策略: +// 对每个输出 tile (tM, tN): +// 1. TCOPYIN 加载 offset tile (行索引, 1×tM) from GM +// 2. __vec__ gen_offset 计算字节偏移: +// offset[row,col] = (in_offset[row] * gN + n_base + col) * sizeof(dtype) +// 3. MGATHER 按字节偏移从数据表取数 (旧 MGATHER, 字节偏移语义) +// 4. TCOPYOUT 写回输出 +// +// PTO 一层策略: +// 对每个输出 tile (tM, tN): +// 1. TLOAD 加载 offset tile (行索引, 1×tM) from GM +// 2. MGATHER 按行索引直接取数: +// dst[r,:] = table[idx[r], :] +// 通过调整 table 指针 (+n_base) 处理列偏移 +// (PTO ISA MGATHER 使用行索引, 非 字节偏移, 无需 gen_offset) +// 3. TSTORE 写回输出 +// +// gen_offset __vec__ 块完全消除: MGATHER 内部完成 +// 行索引 → 行地址 的转换 (tablePtr + idx * tableRowStride)。 +// +// ┌─────────────────────────────────────────────────────────────────────────┐ +// │ 当前编译器不支持 / 不完整的指令汇总 │ +// ├──────────┬──────────────────┬──────────────────────────────────────────┤ +// │ Pto ISA │ 当前编译器状态 │ 说明 │ +// │ 指令 │ │ │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TLOAD │ API 有(名不同)， │ PTO ISA 名 TLOAD；当前编译器名 TCOPYIN； │ +// │ │ │ 核心参数 (dst, src) 和行为一致 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ MGATHER │ 部分支持 │ template_asm.h 有 MGATHER (asm volatile);│ +// │ │ │ 但缺 Coalesce 模板参数 (无法选 Row/Elem);│ +// │ │ │ 且旧实现按字节偏移取数, │ +// │ │ │ PTO ISA Coalesce::Row 按行索引取数 │ +// ├──────────┼──────────────────┼──────────────────────────────────────────┤ +// │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE；当前编译器名 TCOPYOUT；│ +// │ │ │ 核心参数 (dst, src) 和行为一致 │ +// └──────────┴──────────────────┴──────────────────────────────────────────┘ +// +// PTO ISA 文档签名 (Declared in include/pto/pto_instr.hpp): +// +// TLOAD: +// template +// PTO_INST RecordEvent TLOAD(TileData &dst, GlobalData &src, +// WaitEvents &... events); +// +// MGATHER (Coalesce::Row 模式): +// template +// PTO_INST RecordEvent MGATHER(TileDst &dst, GlobalData &src, +// TileInd &indexes, WaitEvents &... events); +// +// Row 模式语义: dst[r,:] = table[idx[r], :] +// - idx 为行索引 (非字节偏移) +// - tablePtr + idx * tableRowStride 定位到行起始 +// - 读取 validCol 个元素到 dst 对应行 +// - 约束 (A2A3): TileIdx::ValidRow==1, TileIdx::ValidCol==TileDst::ValidRow +// - 约束 (A5): 同上, 或 [R,1] ColMajor; 且 staticShape[4]==ValidCol +// +// TSTORE: +// template +// PTO_INST RecordEvent TSTORE(GlobalData &dst, TileData &src, +// WaitEvents &... events); +// ============================================================================ + +#include +#include +// #include // [!] PTO ISA C++ Intrinsic — 当前编译器未提供 + +#include +#include + +// ---------------------------------------------------------------------------- +// gather: 按行索引从数据表中 gather 行 +// +// 输入: +// in_data_ptr — 数据表, shape (gK, gN), row-major +// in_offset_ptr — 行索引数组, shape (gM,), 每个元素是数据表的行号 +// out_ptr — 输出, shape (gM, gN), out[j,:] = in_data[in_offset[j],:] +// +// 模板参数 (与原 gather.hpp 一致): +// dtype — 数据类型 (float, half, ...) +// otype — 索引类型 (uint32_t, int32_t) +// gK — 数据表行数 +// gM — 输出行数 (= 索引数组长度) +// gN — 数据表列数 (= 每行元素数) +// tM — tile 行数 +// tN — tile 列数 +// ---------------------------------------------------------------------------- +template +void gather( + dtype *in_data_ptr, + otype *in_offset_ptr, + dtype *out_ptr + ) { + const size_t Mb = gM / tM; + const size_t Nb = gN / tN; + const size_t rmd_M = gM % tM; + const size_t rmd_N = gN % tN; + + using gm_shapeInOffset = global_tensor>; + using gm_shapeIn = global_tensor>; + using gm_shapeOut = global_tensor>; + + using tile_shapeInOffset = Tile; + using tile_shapeData = Tile; + using tile_shapeInOffset_rmd_n = Tile; + using tile_shapeData_rmd_n = Tile; + using tile_shapeInOffset_rmd_mn = Tile; + using tile_shapeData_rmd_mn = Tile; + using tile_shapeInOffset_rmd_m = Tile; + using tile_shapeData_rmd_m = Tile; + + tile_shapeInOffset inOffsetTile; + tile_shapeData outTile; + tile_shapeInOffset_rmd_n inOffsetTile_rmd_n; + tile_shapeData_rmd_n outTile_rmd_n; + tile_shapeInOffset_rmd_mn inOffsetTile_rmd_mn; + tile_shapeData_rmd_mn outTile_rmd_mn; + tile_shapeInOffset_rmd_m inOffsetTile_rmd_m; + tile_shapeData_rmd_m outTile_rmd_m; + + using itInOffset = global_iterator; + using itOut = global_iterator; + + itInOffset gInOffsetIter(in_offset_ptr); + itOut gOIter(out_ptr); + + // ---- 主循环: Mb × Nb 个完整 tile ---- + for (int j = 0; j < Mb; ++j) { + for (int i = 0; i < Nb; ++i) { + auto gInOffset = gInOffsetIter(0, j); + auto gO = gOIter(j, i); + size_t n_base = i * tN; + + // TLOAD: 加载行索引 tile (1, tM) from GM + // [当前编译器] 名为 TCOPYIN + TLOAD(inOffsetTile, gInOffset); + + // MGATHER: 按行索引从数据表取数 + // dst[r,:] = table[idx[r], :] + // table 指针偏移 n_base 个元素, 使取数起始列为 n_base + // (tablePtr + idx * gN + n_base 定位到 row idx, col n_base) + // [当前编译器] template_asm.h 的 MGATHER 无 Coalesce 模板参数; + // 且按字节偏移取数, 非行索引 + gm_shapeIn adjustedGm(in_data_ptr + n_base); + MGATHER(outTile, adjustedGm, inOffsetTile); + + // TSTORE: 写回输出 tile (tM, tN) to GM + // [当前编译器] 名为 TCOPYOUT + TSTORE(gO, outTile); + } + + // ---- rmd_N: 最后一个列块不完整 ---- + if constexpr (rmd_N) { + auto gInOffset = gInOffsetIter(0, j); + auto gO = gOIter(j, Nb); + size_t n_base = Nb * tN; + + TLOAD(inOffsetTile_rmd_n, gInOffset); + gm_shapeIn adjustedGm(in_data_ptr + n_base); + MGATHER(outTile_rmd_n, adjustedGm, inOffsetTile_rmd_n); + TSTORE(gO, outTile_rmd_n); + } + } + + // ---- rmd_M: 最后一个行块不完整 ---- + if constexpr (rmd_M) { + for (int i = 0; i < Nb; ++i) { + auto gInOffset = gInOffsetIter(0, Mb); + auto gO = gOIter(Mb, i); + size_t n_base = i * tN; + + TLOAD(inOffsetTile_rmd_m, gInOffset); + gm_shapeIn adjustedGm(in_data_ptr + n_base); + MGATHER(outTile_rmd_m, adjustedGm, inOffsetTile_rmd_m); + TSTORE(gO, outTile_rmd_m); + } + + // ---- rmd_M + rmd_N: 右下角不完整 ---- + if constexpr (rmd_N) { + auto gInOffset = gInOffsetIter(0, Mb); + auto gO = gOIter(Mb, Nb); + size_t n_base = Nb * tN; + + TLOAD(inOffsetTile_rmd_mn, gInOffset); + gm_shapeIn adjustedGm(in_data_ptr + n_base); + MGATHER(outTile_rmd_mn, adjustedGm, inOffsetTile_rmd_mn); + TSTORE(gO, outTile_rmd_mn); + } + } +} diff --git a/benchmark-linxisa/kernels/matmul/matmul.hpp b/benchmark-linxisa/kernels/matmul/matmul.hpp index a3671c7..14f574e 100644 --- a/benchmark-linxisa/kernels/matmul/matmul.hpp +++ b/benchmark-linxisa/kernels/matmul/matmul.hpp @@ -9,7 +9,12 @@ using namespace pto; -template +enum class PadValue { Null }; +enum class LayoutCvtEnum { ND2ZZ, ND2NN }; +template +void blk_tload(Args...); + +template void TCOPYOUT_ACC(GmOut &Gout, TileAcc &tAcc){ using TileAccOut = Tile; TileAccOut tAccOut; @@ -17,7 +22,7 @@ void TCOPYOUT_ACC(GmOut &Gout, TileAcc &tAcc){ TCOPYOUT(Gout, tAccOut); } -template +template void TCOPYOUT_ACC_DYNAMIC(GmOut &Gout, TileAcc &tAcc, size_t valid_row, size_t valid_col){ using TileAccOut = Tile; TileAccOut tAccOut(valid_row, valid_col); diff --git a/benchmark-linxisa/kernels/matmul/matmul_mx.hpp b/benchmark-linxisa/kernels/matmul/matmul_mx.hpp index b26e87f..d45b12b 100644 --- a/benchmark-linxisa/kernels/matmul/matmul_mx.hpp +++ b/benchmark-linxisa/kernels/matmul/matmul_mx.hpp @@ -1399,7 +1399,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { MATMUL(tACC, tA, tB); TCVT(tACCin, tACC);//[tM, tN] 256->1 , 256 -> 2 scaling factor // static_assert(tile_shapeB::ValidCol % (width_factor*128) == 0); // TODO, 暂不考虑padding，假设形状是规整的, 方便处理, taccin*ts_adder=tc_dequant - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_shapeACC::ValidRow, tile_shapeACC::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } k++; @@ -1419,7 +1421,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { MATMUL(tACC, tA, tB); TCVT(tACCin, tACC); - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_shapeACC::ValidRow, tile_shapeACC::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } TCOPYOUT(gACC, tAdder[(k+1)%2]); @@ -1449,7 +1453,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { TCOPYIN(ts, gS); MATMUL(tACC, tA, tB); TCVT(tACCin, tACC); - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_ACCin_tcols::ValidRow, tile_ACCin_tcols::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } k++; @@ -1467,7 +1473,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { TCOPYIN(ts, gS); MATMUL(tACC, tA, tB); TCVT(tACCin, tACC); - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_shapeACC::ValidRow, tile_shapeACC::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } TCOPYOUT(gACC, tAdder[(k+1)%2]); @@ -1518,4 +1526,4 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { } -#endif \ No newline at end of file +#endif diff --git a/benchmark-linxisa/kernels/reduction/cumsum_colvec.hpp b/benchmark-linxisa/kernels/reduction/cumsum_colvec.hpp index e7ca443..a8219aa 100644 --- a/benchmark-linxisa/kernels/reduction/cumsum_colvec.hpp +++ b/benchmark-linxisa/kernels/reduction/cumsum_colvec.hpp @@ -1,180 +1,41 @@ #ifndef CUMSUMCOL_KERNEL_HPP #define CUMSUMCOL_KERNEL_HPP - #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== - - - -template -void __vec__ cumsum_col_kernel( - typename tileSum::TileDType __out__ new_sum, - typename tileData::TileDType __out__ out, - const typename tileData::TileDType __in__ src, - const typename tileSum::TileDType __in__ old_sum -) -{ - size_t i = blkv_get_index_x(); - size_t sum_idx = i * tileSum::RowStride; - - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileData::DType *out_ptr = blkv_get_tile_ptr(out); - __vbuf__ typename tileData::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - - typename tileSum::DType upd_sum = old_sum_ptr[i]; -// printf("upd_sum = %d",upd_sum); -// typename tileData::DType upd_out = old_sum_ptr[i]; -/* - for(size_t j=0;j(sum_out); - } - new_sum_ptr[i] = upd_sum; -} - - - +using namespace pto; template void cumsum_col_rand( dtype *in_ptr, -// dtype *inzero_ptr, dtype *out_ptr -) +) { + using ScalarTile = Tile; + using gm_shape = global_tensor>; + using itType = global_iterator; - const int Mb = gIM / tM; - const int Nb = gIN / tN; - - const int rmd_M = gIM % tM; - const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeSum = global_tensor>; - using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // - using tile_shapeData_col = Tile; // - using tile_shapeSum = Tile; // - - using tile_shapeData_row = Tile; // - using tile_shapeData_cor = Tile; // - using tile_shapeSum_row = Tile; // - //need tM = 1; - - - gm_shapeIn inGm(in_ptr); -// gm_shapeOut ZeroGm(inzero_ptr); - gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); - - tile_shapeData dataTile; - tile_shapeData OutTile; - tile_shapeData_col dataTile_col; - tile_shapeData_col OutTile_col; - tile_shapeSum SumTile; - tile_shapeSum oldSumTile; - - tile_shapeData_row dataTile_row; - tile_shapeData_row OutTile_row; - tile_shapeData_cor dataTile_cor; - tile_shapeData_cor OutTile_cor; - - tile_shapeSum_row SumTile_row; - tile_shapeSum_row oldSumTile_row; - -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 + itType gIIter(in_ptr); + itType gOIter(out_ptr); - using itIn = global_iterator; -// using itZero = global_iterator; - using itOut = global_iterator; -// using itSum = global_iterator; + // 对每一列独立计算累积和 + for (int j = 0; j < gIN; ++j) { + ScalarTile running_sum; + TEXPANDS(running_sum, static_cast(0)); - itIn gIIter(in_ptr); - itOut gOIter(out_ptr); + for (int i = 0; i < gIM; ++i) { + auto gI = gIIter(i, j); + ScalarTile input_elem; + TLOAD(input_elem, gI); -// dtype zero = 0; + TADD(running_sum, running_sum, input_elem); - for (int j = 0; j < Nb; ++j) { - TEXPANDSCALAR(oldSumTile, 0);//初始化为0 - for (int i = 0; i < Mb; ++i) { - auto gI = gIIter(i, j); - auto gO = gOIter(i, j); - TCOPYIN(dataTile, gI); -// printf("in0 : %d, %d\n",in_ptr[i*tM], i*tM); - cumsum_col_kernel<<>>(SumTile.data(), OutTile.data(), dataTile.data(), oldSumTile.data()); - oldSumTile = SumTile; - TCOPYOUT(gO, OutTile); -// printf("out0 : %d,%d\n", out_ptr[i*tM],i*tM); - } - if constexpr (rmd_M > 0){ - auto gI = gIIter(Mb, j); - auto gO = gOIter(Mb, j); - TCOPYIN(dataTile_col, gI); - cumsum_col_kernel<<>>(SumTile.data(), OutTile_col.data(), dataTile_col.data(), oldSumTile.data()); - oldSumTile = SumTile; - TCOPYOUT(gO, OutTile_col); + auto gO = gOIter(i, j); + TSTORE(gO, running_sum); } -// TCOPYOUT(gO, SumTile); } - if constexpr (rmd_N > 0){ -// auto gZero = gZeroIter(0, Nb); -// auto gO = gOIter(0, Nb); - TEXPANDSCALAR(oldSumTile_row, 0);//初始化为0 -// TCOPYIN(oldSumTile_row, gZero);//初始化为0 - for (int i = 0; i < Mb; ++i) { - auto gI = gIIter(i, Nb); - auto gO = gOIter(i, Nb); - TCOPYIN(dataTile_row, gI); - cumsum_col_kernel<<>>(SumTile_row.data(), OutTile_row.data(), dataTile_row.data(), oldSumTile_row.data()); - oldSumTile_row = SumTile_row; - TCOPYOUT(gO, OutTile_row); - } - if constexpr (rmd_M > 0){ - auto gI = gIIter(Mb, Nb); - auto gO = gOIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - cumsum_col_kernel<<>>(SumTile_row.data(), OutTile_cor.data(), dataTile_cor.data(), oldSumTile_row.data()); - oldSumTile_row = SumTile_row; - TCOPYOUT(gO, OutTile_cor); - } - } -/* - for(int i = 0; i < gIN; i++){ - printf("out%d = %d\n", i, out_ptr[i]); - } -*/ } - - -#endif \ No newline at end of file +#endif diff --git a/benchmark-linxisa/kernels/reduction/cumsum_rowvec.hpp b/benchmark-linxisa/kernels/reduction/cumsum_rowvec.hpp index ec998c1..9be3b11 100644 --- a/benchmark-linxisa/kernels/reduction/cumsum_rowvec.hpp +++ b/benchmark-linxisa/kernels/reduction/cumsum_rowvec.hpp @@ -1,170 +1,42 @@ -#ifndef REDUCESUMTROWSUM_KERNEL_HPP -#define REDUCESUMTROWSUM_KERNEL_HPP - -#ifndef __vbuf__ -#define __vbuf__ -#endif - -#include - -using namespace pto; +#ifndef CUMSUMROW_KERNEL_HPP +#define CUMSUMROW_KERNEL_HPP #pragma once +#include #include #include -template -void __vec__ cumsum_row_kernel( - typename tileSum::TileDType __out__ new_sum, - const typename tileData::TileDType __out__ out, - const typename tileData::TileDType __in__ src, - const typename tileSum::TileDType __in__ old_sum -) -{ -// size_t i = blkv_get_index_x(); - size_t j = blkv_get_index_y(); - size_t sum_idx = j * tileSum::RowStride; - - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileData::DType *out_ptr = blkv_get_tile_ptr(out); - __vbuf__ typename tileData::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - - typename tileSum::DType upd_sum = old_sum_ptr[sum_idx]; -/* - for(size_t j=0;j(sum_out); - } - new_sum_ptr[sum_idx] = upd_sum; -} - - +using namespace pto; template void cumsum_row_rand( dtype *in_ptr, dtype *out_ptr -) +) { + using ScalarTile = Tile; + using gm_shape = global_tensor>; + using itType = global_iterator; - const int Mb = gIM / tM; - const int Nb = gIN / tN; - - const int rmd_M = gIM % tM; // todo 尾块怎么处理？ - const int rmd_N = gIN % tN; // todo 尾块怎么处理？ - - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeSum = global_tensor>; - using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_row = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeSum = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - - using tile_shapeData_col = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_cor = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeSum_col = Tile; - + itType gIIter(in_ptr); + itType gOIter(out_ptr); - gm_shapeIn inGm(in_ptr); - gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); + // 对每一行独立计算累积和 + for (int i = 0; i < gIM; ++i) { + ScalarTile running_sum; + TEXPANDS(running_sum, static_cast(0)); - tile_shapeData dataTile; - tile_shapeData_row dataTile_row; - tile_shapeData_col dataTile_col; - tile_shapeData_cor dataTile_cor; + for (int j = 0; j < gIN; ++j) { + auto gI = gIIter(i, j); + ScalarTile input_elem; + TLOAD(input_elem, gI); - tile_shapeData OutTile; - tile_shapeData_row OutTile_row; - tile_shapeData_col OutTile_col; - tile_shapeData_cor OutTile_cor; - - tile_shapeSum SumTile; - tile_shapeSum oldSumTile; - tile_shapeSum_col SumTile_col; - tile_shapeSum_col oldSumTile_col; + TADD(running_sum, running_sum, input_elem); -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; - using itOut = global_iterator; - - itIn gIIter(in_ptr); - itOut gOIter(out_ptr); - -// printf("tile_shapeSum::ValidCol = %d\n", tile_shapeSum::ValidCol); -// printf("tile_shapeSum::ValidRow = %d\n", tile_shapeSum::ValidRow); - - for (int j = 0; j < Mb; ++j) { -// auto gO = gOIter(j, 0); - TEXPANDSCALAR(oldSumTile, 0);//初始化为0 - //初始化old_sum的tile - for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(j, i); - auto gO = gOIter(j, i); - TCOPYIN(dataTile, gI); - cumsum_row_kernel<<<1, tile_shapeSum::ValidRow, 1>>>(SumTile.data(), OutTile.data(), dataTile.data(), oldSumTile.data()); -// reducesum_row_kernel<<>>(SumTile.data(), dataTile.data(), oldSumTile.data()); - oldSumTile = SumTile; - TCOPYOUT(gO, OutTile); + auto gO = gOIter(i, j); + TSTORE(gO, running_sum); } -// printf("end for%d\n",j); - //for row corner - if constexpr (rmd_N > 0){ - auto gI = gIIter(j, Nb); - auto gO = gOIter(j, Nb); - TCOPYIN(dataTile_row, gI); - cumsum_row_kernel<<<1, tile_shapeSum::ValidRow, 1>>>(SumTile.data(), OutTile_row.data(), dataTile_row.data(), oldSumTile.data()); -// reducesum_row_kernel<<>>(SumTile.data(), dataTile_row.data(), oldSumTile.data()); - oldSumTile = SumTile; - TCOPYOUT(gO, OutTile_row); - } - } - //for col cor - if constexpr (rmd_M > 0){ - TEXPANDSCALAR(oldSumTile_col, 0);//初始化为0 - //初始化old_sum的tile - for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(Mb, i); - auto gO = gOIter(Mb, i); - TCOPYIN(dataTile_col, gI); - cumsum_row_kernel<<<1, tile_shapeSum_col::ValidRow, 1>>>(SumTile_col.data(), OutTile_col.data(), dataTile_col.data(), oldSumTile_col.data()); - oldSumTile_col = SumTile_col; - TCOPYOUT(gO, OutTile_col); - } - if constexpr (rmd_N > 0){ - auto gI = gIIter(Mb, Nb); - auto gO = gOIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - cumsum_row_kernel<<<1, tile_shapeSum_col::ValidRow, 1>>>(SumTile_col.data(), OutTile_cor.data(), dataTile_cor.data(), oldSumTile_col.data()); - oldSumTile_col = SumTile_col; - TCOPYOUT(gO, OutTile_cor); - } - } -/* - for(int i = 0; i < gIM; i++){ - printf("out%d = %d\n", i, out_ptr[i]); } -*/ } #endif diff --git a/benchmark-linxisa/kernels/reduction/reducemax_colvec.hpp b/benchmark-linxisa/kernels/reduction/reducemax_colvec.hpp index 502db96..0c4dc7d 100644 --- a/benchmark-linxisa/kernels/reduction/reducemax_colvec.hpp +++ b/benchmark-linxisa/kernels/reduction/reducemax_colvec.hpp @@ -1,170 +1,85 @@ -#ifndef REDUCESUMCOLVEC_KERNEL_HPP -#define REDUCESUMCOLVEC_KERNEL_HPP - +#ifndef REDUCEMAXCOLVEC_KERNEL_HPP +#define REDUCEMAXCOLVEC_KERNEL_HPP #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== -template -void __vec__ reducemax_col_kernel( - typename tileMax::TileDType __out__ new_max, - const typename tileSrc::TileDType __in__ src, - const typename tileMax::TileDType __in__ old_max -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileMax::DType *new_max_ptr = blkv_get_tile_ptr(new_max); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileMax::DType *old_max_ptr = blkv_get_tile_ptr(old_max); - - - typename tileMax::DType upd_max = old_max_ptr[i]; -/* - for(size_t j=0;j void reducemax_col_rand( dtype *in_ptr, -// dtype *inzero_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - + const int Nb = gIN / tN; const int rmd_M = gIM % tM; const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - using gm_shapeIn = global_tensor>; // -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // - using tile_shapeData_col = Tile; // - using tile_shapeMax = Tile; // - - - - using tile_shapeData_row = Tile; // - using tile_shapeData_cor = Tile; // - using tile_shapeMax_row = Tile; // - //need tM = 1; - - - gm_shapeIn inGm(in_ptr); -// gm_shapeOut ZeroGm(inzero_ptr); + using tile_shapeData = Tile; + using tile_shapeData_col = Tile; + using tile_shapeMax = Tile; + using tile_shapeData_row = Tile; + using tile_shapeData_cor = Tile; + using tile_shapeMax_row = Tile; + + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); tile_shapeData dataTile; - tile_shapeData_col dataTile_col; + tile_shapeData_col dataTile_col; tile_shapeMax MaxTile; tile_shapeMax oldMaxTile; - tile_shapeData_row dataTile_row; - tile_shapeData_cor dataTile_cor; + tile_shapeData_cor dataTile_cor; tile_shapeMax_row MaxTile_row; - tile_shapeMax_row oldMaxTile_row; - -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 + tile_shapeMax_row oldMaxTile_row; - using itIn = global_iterator; - using itIn_row = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); - itIn_row gIIter_rmd_row(in_ptr); -// itZero gZeroIter(inzero_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// dtype zero = 0; - for (int j = 0; j < Nb; ++j) { -// auto gZero = gZeroIter(0, j); auto gO = gOIter(0, j); - TEXPANDSCALAR(oldMaxTile, 0);//初始化为0 -// TCOPYIN(oldSumTile, gZero);//初始化为0 - //初始化old_sum的tile - //need + TEXPANDS(oldMaxTile, static_cast(0)); + for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, j); - TCOPYIN(dataTile, gI); - reducemax_col_kernel<<>>(MaxTile.data(), dataTile.data(), oldMaxTile.data()); - oldMaxTile = MaxTile; + TLOAD(dataTile, gI); + TCOLMAX(MaxTile, dataTile); + TMAX(oldMaxTile, oldMaxTile, MaxTile); } - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gI = gIIter(Mb, j); - TCOPYIN(dataTile_col, gI); - reducemax_col_kernel<<>>(MaxTile.data(), dataTile_col.data(), oldMaxTile.data()); - oldMaxTile = MaxTile; + TLOAD(dataTile_col, gI); + TCOLMAX(MaxTile, dataTile_col); + TMAX(oldMaxTile, oldMaxTile, MaxTile); } - TCOPYOUT(gO, MaxTile); + TSTORE(gO, oldMaxTile); } - if constexpr (rmd_N > 0){ -// auto gZero = gZeroIter(0, Nb); + if constexpr (rmd_N > 0) { auto gO = gOIter(0, Nb); - TEXPANDSCALAR(oldMaxTile_row, 0);//初始化为0 -// TCOPYIN(oldSumTile_row, gZero);//初始化为0 - for (int i = 0; i < Mb; ++i) { + TEXPANDS(oldMaxTile_row, static_cast(0)); + + for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, Nb); - TCOPYIN(dataTile_row, gI); - reducemax_col_kernel<<>>(MaxTile_row.data(), dataTile_row.data(), oldMaxTile_row.data()); - oldMaxTile_row = MaxTile_row; + TLOAD(dataTile_row, gI); + TCOLMAX(MaxTile_row, dataTile_row); + TMAX(oldMaxTile_row, oldMaxTile_row, MaxTile_row); } - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gI = gIIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - reducemax_col_kernel<<>>(MaxTile_row.data(), dataTile_cor.data(), oldMaxTile_row.data()); - oldMaxTile_row = MaxTile_row; + TLOAD(dataTile_cor, gI); + TCOLMAX(MaxTile_row, dataTile_cor); + TMAX(oldMaxTile_row, oldMaxTile_row, MaxTile_row); } - TCOPYOUT(gO, MaxTile_row); + TSTORE(gO, oldMaxTile_row); } } diff --git a/benchmark-linxisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp b/benchmark-linxisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp index adab096..04fa543 100644 --- a/benchmark-linxisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp +++ b/benchmark-linxisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp @@ -1,167 +1,46 @@ -#ifndef REDUCESUMCOLVEC_KERNEL_HPP -#define REDUCESUMCOLVEC_KERNEL_HPP - +#ifndef REDUCEMAXCOLVEC_KERNEL_HPP +#define REDUCEMAXCOLVEC_KERNEL_HPP #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== -template -void __vec__ reducemax_col_tmp( - typename tileTmp::TileDType __out__ tmp_max, - const typename tileSrc::TileDType __in__ src -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileTmp::DType *tmp_max_ptr = blkv_get_tile_ptr(tmp_max); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); -// __vbuf__ typename tileMax::DType *old_max_ptr = blkv_get_tile_ptr(old_max); - - typename tileTmp::DType upd_tmp_max = 0; - - #pragma clang loop unroll(full) - for(size_t j=0;j -void __vec__ reducemax_col_final( - typename tileMax::TileDType __out__ new_max, - const typename tileTmp::TileDType __in__ src, - const typename tileMax::TileDType __in__ old_max -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileMax::DType *new_max_ptr = blkv_get_tile_ptr(new_max); - __vbuf__ typename tileTmp::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileMax::DType *old_max_ptr = blkv_get_tile_ptr(old_max); - - - typename tileMax::DType upd_max = old_max_ptr[i]; - - - size_t src_idx_0 = i * tileMax::ColStride + 0 * tileMax::ValidCol; - size_t src_idx_1 = i * tileMax::ColStride + 1 * tileMax::ValidCol; - size_t src_idx_2 = i * tileMax::ColStride + 2 * tileMax::ValidCol; - size_t src_idx_3 = i * tileMax::ColStride + 3 * tileMax::ValidCol; - size_t src_idx_4 = i * tileMax::ColStride + 4 * tileMax::ValidCol; - size_t src_idx_5 = i * tileMax::ColStride + 5 * tileMax::ValidCol; - size_t src_idx_6 = i * tileMax::ColStride + 6 * tileMax::ValidCol; - size_t src_idx_7 = i * tileMax::ColStride + 7 * tileMax::ValidCol; - typename tileMax::DType max_01 = blkv_max(src_ptr[src_idx_0], src_ptr[src_idx_1]); - typename tileMax::DType max_23 = blkv_max(src_ptr[src_idx_2], src_ptr[src_idx_3]); - typename tileMax::DType max_45 = blkv_max(src_ptr[src_idx_4], src_ptr[src_idx_5]); - typename tileMax::DType max_67 = blkv_max(src_ptr[src_idx_6], src_ptr[src_idx_7]); - typename tileMax::DType max_0123 = blkv_max(max_01, max_23); - typename tileMax::DType max_4567 = blkv_max(max_45, max_67); - typename tileMax::DType max_all = blkv_max(max_0123, max_4567); - -// upd_max = upd_max + max_tmp; - - - new_max_ptr[i] = blkv_max(max_all, upd_max); -} - - - - +using namespace pto; template void reducemax_col_rand( dtype *in_ptr, -// dtype *inzero_ptr, dtype *out_ptr -) +) { - -// const int Mb = (gIM/8) / tM; - - const int rmd_M = gIM % tM; - const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - - using gm_shapeIn = global_tensor>; // -// using gm_shapeMax = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // -// using tile_shapeData_col = Tile; // - using tile_shapeTmp = Tile; // - using tile_shapeMax = Tile; // - - - -// using tile_shapeData_row = Tile; // -// using tile_shapeData_cor = Tile; // -// using tile_shapeMax_row = Tile; // - //need tM = 1; + using tile_shapeData = Tile; + using tile_shapeTmp = Tile; + using tile_shapeMax = Tile; - - gm_shapeIn inGm(in_ptr); -// gm_shapeOut ZeroGm(inzero_ptr); + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeMax olcMaxGm(old_max_ptr); tile_shapeData dataTile; -// tile_shapeData_col dataTile_col; - tile_shapeTmp TmpTile; + tile_shapeTmp TmpTile; tile_shapeMax MaxTile; tile_shapeMax oldMaxTile; -// tile_shapeData_row dataTile_row; -// tile_shapeData_cor dataTile_cor; -// tile_shapeMax_row MaxTile_row; -// tile_shapeMax_row oldMaxTile_row; - -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; - using itIn_row = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); - itIn_row gIIter_rmd_row(in_ptr); -// itZero gZeroIter(inzero_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); - auto gO = gOIter(0, 0); - TEXPANDSCALAR(oldMaxTile, 0);//初始化为0 + TEXPANDS(oldMaxTile, static_cast(0)); auto gI = gIIter(0, 0); - TCOPYIN(dataTile, gI);//补0的TLOAD - reducemax_col_tmp<<>>(TmpTile.data(), dataTile.data()); - reducemax_col_final<<>>(MaxTile.data(), TmpTile.data(), oldMaxTile.data()); - oldMaxTile = MaxTile; - TCOPYOUT(gO, MaxTile); + TLOAD(dataTile, gI); + TCOLMAX(TmpTile, dataTile); + TCOLMAX(MaxTile, TmpTile); + TMAX(oldMaxTile, oldMaxTile, MaxTile); + TSTORE(gO, MaxTile); } #endif diff --git a/benchmark-linxisa/kernels/reduction/reducemax_rowvec.hpp b/benchmark-linxisa/kernels/reduction/reducemax_rowvec.hpp index 6b95d70..cb5b23e 100644 --- a/benchmark-linxisa/kernels/reduction/reducemax_rowvec.hpp +++ b/benchmark-linxisa/kernels/reduction/reducemax_rowvec.hpp @@ -1,180 +1,95 @@ -#ifndef REDUCESUMTROWSUM_KERNEL_HPP -#define REDUCESUMTROWSUM_KERNEL_HPP - -#ifndef __vbuf__ -#define __vbuf__ -#endif - -#include -#include "template_asm.h" - -using namespace pto; +#ifndef REDUCEMAXROWVEC_KERNEL_HPP +#define REDUCEMAXROWVEC_KERNEL_HPP #pragma once +#include #include #include -template -void __vec__ reducemax_row_kernel( - typename tileMax::TileDType __out__ new_max, - const typename tileSrc::TileDType __in__ src, - const typename tileMax::TileDType __in__ old_max -) -{ -// size_t i = blkv_get_index_x(); - size_t j = blkv_get_index_x(); -// size_t j = blkv_get_index_y(); - size_t idx = j * tileMax::RowStride; - - __vbuf__ typename tileMax::DType *new_max_ptr = blkv_get_tile_ptr(new_max); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileMax::DType *old_max_ptr = blkv_get_tile_ptr(old_max); - - - typename tileMax::DType upd_max = old_max_ptr[idx]; - - - #pragma clang loop unroll(full) - for(size_t i=0;i void reducemax_row_rand( dtype *in_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - - const int rmd_M = gIM % tM; // todo 尾块怎么处理？ - const int rmd_N = gIN % tN; // todo 尾块怎么处理？ + const int Nb = gIN / tN; + const int rmd_M = gIM % tM; + const int rmd_N = gIN % tN; - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_row = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeMax = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - - using tile_shapeData_col = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_cor = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeMax_col = Tile; - - - gm_shapeIn inGm(in_ptr); + using tile_shapeData = Tile; + using tile_shapeData_row = Tile; + using tile_shapeMax = Tile; + using tile_shapeData_col = Tile; + using tile_shapeData_cor = Tile; + using tile_shapeMax_col = Tile; + using tile_shapeTmp = Tile; + using tile_shapeTmp_row = Tile; + using tile_shapeTmp_col = Tile; + using tile_shapeTmp_cor = Tile; + + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); - tile_shapeData dataTile; + tile_shapeData dataTile; tile_shapeData_row dataTile_row; tile_shapeData_col dataTile_col; - tile_shapeData_cor dataTile_cor; - + tile_shapeData_cor dataTile_cor; tile_shapeMax MaxTile; tile_shapeMax oldMaxTile; tile_shapeMax_col MaxTile_col; - tile_shapeMax_col oldMaxTile_col; + tile_shapeMax_col oldMaxTile_col; + tile_shapeTmp tmpTile; + tile_shapeTmp_row tmpTile_row; + tile_shapeTmp_col tmpTile_col; + tile_shapeTmp_cor tmpTile_cor; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// printf("tile_shapeSum::ValidCol = %d\n", tile_shapeSum::ValidCol); -// printf("tile_shapeSum::ValidRow = %d\n", tile_shapeSum::ValidRow); -// printf("before for\n"); for (int j = 0; j < Mb; ++j) { auto gO = gOIter(j, 0); - TEXPANDSCALAR(oldMaxTile, 0);//初始化为0 - //初始化old_sum的tile + TEXPANDS(oldMaxTile, static_cast(0)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(j, i); -// printf("before copy in , %d\n", i); - TCOPYIN(dataTile, gI); - reducemax_row_kernel<<>>(MaxTile.data(), dataTile.data(), oldMaxTile.data()); -// reducesum_row_kernel<<<1, tile_shapeSum::ValidRow, 1>>>(SumTile.data(), dataTile.data(), oldSumTile.data()); -// printf("kernel , %d\n", i); - oldMaxTile = MaxTile; + auto gI = gIIter(j, i); + TLOAD(dataTile, gI); + TROWMAX(MaxTile, dataTile, tmpTile); + TMAX(oldMaxTile, oldMaxTile, MaxTile); } -// printf("end for%d\n",j); - //for row corner - if constexpr (rmd_N > 0){ + if constexpr (rmd_N > 0) { auto gI = gIIter(j, Nb); - TCOPYIN(dataTile_row, gI); - reducemax_row_kernel<<>>(MaxTile.data(), dataTile_row.data(), oldMaxTile.data()); -// reducesum_row_kernel<<>>(SumTile.data(), dataTile_row.data(), oldSumTile.data()); - oldMaxTile = MaxTile; + TLOAD(dataTile_row, gI); + TROWMAX(MaxTile, dataTile_row, tmpTile_row); + TMAX(oldMaxTile, oldMaxTile, MaxTile); } -// printf("before tcopyout\n"); - TCOPYOUT(gO, MaxTile); -// printf("end tcopyout\n"); + TSTORE(gO, oldMaxTile); } - //for col cor - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gO = gOIter(Mb, 0); - TEXPANDSCALAR(oldMaxTile_col, 0);//初始化为0 - //初始化old_sum的tile + TEXPANDS(oldMaxTile_col, static_cast(0)); + for (int i = 0; i < Nb; ++i) { auto gI = gIIter(Mb, i); - TCOPYIN(dataTile_col, gI); - reducemax_row_kernel<<>>(MaxTile_col.data(), dataTile_col.data(), oldMaxTile_col.data()); - oldMaxTile_col = MaxTile_col; + TLOAD(dataTile_col, gI); + TROWMAX(MaxTile_col, dataTile_col, tmpTile_col); + TMAX(oldMaxTile_col, oldMaxTile_col, MaxTile_col); } - if constexpr (rmd_N > 0){ + if constexpr (rmd_N > 0) { auto gI = gIIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - reducemax_row_kernel<<>>(MaxTile_col.data(), dataTile_cor.data(), oldMaxTile_col.data()); - oldMaxTile_col = MaxTile_col; + TLOAD(dataTile_cor, gI); + TROWMAX(MaxTile_col, dataTile_cor, tmpTile_cor); + TMAX(oldMaxTile_col, oldMaxTile_col, MaxTile_col); } - TCOPYOUT(gO, MaxTile_col); - } -/* - for(int i = 0; i < gIM; i++){ - printf("out%d = %d\n", i, out_ptr[i]); + TSTORE(gO, oldMaxTile_col); } -*/ -// printf("end program\n"); } #endif diff --git a/benchmark-linxisa/kernels/reduction/reducemax_rowvec_single_tree.hpp b/benchmark-linxisa/kernels/reduction/reducemax_rowvec_single_tree.hpp index 8f066d2..8eae796 100644 --- a/benchmark-linxisa/kernels/reduction/reducemax_rowvec_single_tree.hpp +++ b/benchmark-linxisa/kernels/reduction/reducemax_rowvec_single_tree.hpp @@ -1,233 +1,60 @@ #ifndef REDUCEMAXTROWMAX_KERNEL_HPP #define REDUCEMAXTROWMAX_KERNEL_HPP -#ifndef __vbuf__ -#define __vbuf__ -#endif - -#include - -using namespace pto; - #pragma once +#include #include #include -template -void __vec__ reducemax_row_kernel( - typename tileTmpMax::TileDType __out__ new_max, - const typename tileSrc::TileDType __in__ src, - const typename tileSrcCol::TileDType __in__ src_col, - const typename tileTmpMax::TileDType __in__ old_max, - const size_t tile_idx -) -{ - - size_t j = blkv_get_index_x(); - size_t z = blkv_get_index_y(); - size_t stride_src = z * (tileSrc::ValidCol/4) * tileSrc::ColStride; - size_t stride_src_col = z * (tileSrcCol::ValidCol/4) * tileSrcCol::ColStride; - - __vbuf__ typename tileTmpMax::DType *new_max_ptr = blkv_get_tile_ptr(new_max); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSrc::DType *src_col_ptr = blkv_get_tile_ptr(src_col); - __vbuf__ typename tileTmpMax::DType *old_max_ptr = blkv_get_tile_ptr(old_max); - -/* - #pragma clang loop unroll(full) - for(size_t i=0;i -void __vec__ reducemax_row_final_kernel( - typename tileMax::TileDType __out__ new_max, - const typename tileTmpMax::TileDType __in__ tmp_max -){ - size_t j = blkv_get_index_x(); - size_t idx = j * tileMax::RowStride; - - __vbuf__ typename tileMax::DType *new_max_ptr = blkv_get_tile_ptr(new_max); - __vbuf__ typename tileTmpMax::DType *tmp_max_ptr = blkv_get_tile_ptr(tmp_max); - - #pragma clang loop unroll(full) - for(size_t i=0;i void reducemax_row_rand( dtype *in_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - - const int rmd_M = gIM % tM; // todo 尾块怎么处理？ - const int rmd_N = gIN % tN; // todo 尾块怎么处理？ - + const int Nb = gIN / tN; + const int rmd_M = gIM % tM; + const int rmd_N = gIN % tN; - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeMax = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeDataCol = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeMax = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - using tile_shapeTmpMax = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec + using tile_shapeData = Tile; + using tile_shapeMax = Tile; + using tile_shapeTmpMax = Tile; + using tile_shapeTmp = Tile; - - gm_shapeIn inGm(in_ptr); + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeMax olcMaxGm(old_max_ptr); - tile_shapeData dataTile; - tile_shapeDataCol dataTile_col; + tile_shapeData dataTile; tile_shapeMax MaxTile; tile_shapeTmpMax oldtmpMaxTile; tile_shapeTmpMax tmpMaxTile; + tile_shapeTmp tmpTile; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); - auto gO = gOIter(0, 0); - TEXPANDSCALAR(oldtmpMaxTile, 0);//初始化为0 - TEXPANDSCALAR(dataTile_col, 0);//初始化为0 + TEXPANDS(oldtmpMaxTile, static_cast(0)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(0, i); - TCOPYIN(dataTile, gI); - reducemax_row_kernel<<>>(tmpMaxTile.data(), - dataTile.data(), - dataTile_col.data(), - oldtmpMaxTile.data(), - i); - oldtmpMaxTile = tmpMaxTile; + auto gI = gIIter(0, i); + TLOAD(dataTile, gI); + tile_shapeMax partialMax; + TROWMAX(partialMax, dataTile, tmpTile); + TMAX(oldtmpMaxTile, oldtmpMaxTile, partialMax); } - reducemax_row_final_kernel<<>>(MaxTile.data(), - tmpMaxTile.data()); - TCOPYOUT(gO, MaxTile); + TMOV(tmpMaxTile, oldtmpMaxTile); + + TROWMAX(MaxTile, tmpMaxTile, tmpTile); + TSTORE(gO, MaxTile); } #endif - diff --git a/benchmark-linxisa/kernels/reduction/reduceprod_colvec.hpp b/benchmark-linxisa/kernels/reduction/reduceprod_colvec.hpp index 15d12c1..a5b1d4b 100644 --- a/benchmark-linxisa/kernels/reduction/reduceprod_colvec.hpp +++ b/benchmark-linxisa/kernels/reduction/reduceprod_colvec.hpp @@ -1,144 +1,85 @@ -#ifndef REDUCESUMCOLVEC_KERNEL_HPP -#define REDUCESUMCOLVEC_KERNEL_HPP - +#ifndef REDUCEPRODCOLVEC_KERNEL_HPP +#define REDUCEPRODCOLVEC_KERNEL_HPP #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== -template -void __vec__ reduceprod_col_kernel( - typename timeProd::TileDType __out__ new_prod, - const typename tileSrc::TileDType __in__ src, - const typename timeProd::TileDType __in__ old_prod -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename timeProd::DType *new_prod_ptr = blkv_get_tile_ptr(new_prod); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename timeProd::DType *old_prod_ptr = blkv_get_tile_ptr(old_prod); - - - typename timeProd::DType upd_prod = old_prod_ptr[i]; - - #pragma clang loop unroll(full) - for(size_t j=0;j void reduceprod_col_rand( dtype *in_ptr, -// dtype *inzero_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - + const int Nb = gIN / tN; const int rmd_M = gIM % tM; const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - using gm_shapeIn = global_tensor>; // -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // - using tile_shapeData_col = Tile; // - using tile_shapeProd = Tile; // - - - - using tile_shapeData_row = Tile; // - using tile_shapeData_cor = Tile; // - using tile_shapeProd_row = Tile; // - //need tM = 1; - - - gm_shapeIn inGm(in_ptr); -// gm_shapeOut ZeroGm(inzero_ptr); + using tile_shapeData = Tile; + using tile_shapeData_col = Tile; + using tile_shapeProd = Tile; + using tile_shapeData_row = Tile; + using tile_shapeData_cor = Tile; + using tile_shapeProd_row = Tile; + + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); tile_shapeData dataTile; - tile_shapeData_col dataTile_col; + tile_shapeData_col dataTile_col; tile_shapeProd ProdTile; tile_shapeProd oldProdTile; - tile_shapeData_row dataTile_row; - tile_shapeData_cor dataTile_cor; + tile_shapeData_cor dataTile_cor; tile_shapeProd_row ProdTile_row; - tile_shapeProd_row oldProdTile_row; + tile_shapeProd_row oldProdTile_row; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; - using itIn_row = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); - itIn_row gIIter_rmd_row(in_ptr); -// itZero gZeroIter(inzero_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// dtype zero = 0; - for (int j = 0; j < Nb; ++j) { -// auto gZero = gZeroIter(0, j); auto gO = gOIter(0, j); - TEXPANDSCALAR(oldProdTile, 0);//初始化为0 -// TCOPYIN(oldSumTile, gZero);//初始化为0 - //初始化old_sum的tile - //need + TEXPANDS(oldProdTile, static_cast(1)); + for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, j); - TCOPYIN(dataTile, gI); - reduceprod_col_kernel<<>>(ProdTile.data(), dataTile.data(), oldProdTile.data()); - oldProdTile = ProdTile; + TLOAD(dataTile, gI); + TCOLPROD(ProdTile, dataTile); + TMUL(oldProdTile, oldProdTile, ProdTile); } - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gI = gIIter(Mb, j); - TCOPYIN(dataTile_col, gI); - reduceprod_col_kernel<<>>(ProdTile.data(), dataTile_col.data(), oldProdTile.data()); - oldProdTile = ProdTile; + TLOAD(dataTile_col, gI); + TCOLPROD(ProdTile, dataTile_col); + TMUL(oldProdTile, oldProdTile, ProdTile); } - TCOPYOUT(gO, ProdTile); + TSTORE(gO, oldProdTile); } - if constexpr (rmd_N > 0){ -// auto gZero = gZeroIter(0, Nb); + if constexpr (rmd_N > 0) { auto gO = gOIter(0, Nb); - TEXPANDSCALAR(oldProdTile_row, 0);//初始化为0 -// TCOPYIN(oldSumTile_row, gZero);//初始化为0 - for (int i = 0; i < Mb; ++i) { + TEXPANDS(oldProdTile_row, static_cast(1)); + + for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, Nb); - TCOPYIN(dataTile_row, gI); - reduceprod_col_kernel<<>>(ProdTile_row.data(), dataTile_row.data(), oldProdTile_row.data()); - oldProdTile_row = ProdTile_row; + TLOAD(dataTile_row, gI); + TCOLPROD(ProdTile_row, dataTile_row); + TMUL(oldProdTile_row, oldProdTile_row, ProdTile_row); } - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gI = gIIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - reduceprod_col_kernel<<>>(ProdTile_row.data(), dataTile_cor.data(), oldProdTile_row.data()); - oldProdTile_row = ProdTile_row; + TLOAD(dataTile_cor, gI); + TCOLPROD(ProdTile_row, dataTile_cor); + TMUL(oldProdTile_row, oldProdTile_row, ProdTile_row); } - TCOPYOUT(gO, ProdTile_row); + TSTORE(gO, oldProdTile_row); } } diff --git a/benchmark-linxisa/kernels/reduction/reduceprod_rowvec.hpp b/benchmark-linxisa/kernels/reduction/reduceprod_rowvec.hpp index 7494130..274021b 100644 --- a/benchmark-linxisa/kernels/reduction/reduceprod_rowvec.hpp +++ b/benchmark-linxisa/kernels/reduction/reduceprod_rowvec.hpp @@ -1,151 +1,95 @@ -#ifndef REDUCESUMTROWSUM_KERNEL_HPP -#define REDUCESUMTROWSUM_KERNEL_HPP - -#ifndef __vbuf__ -#define __vbuf__ -#endif - -#include - -using namespace pto; +#ifndef REDUCEPRODROWVEC_KERNEL_HPP +#define REDUCEPRODROWVEC_KERNEL_HPP #pragma once +#include #include #include -template -void __vec__ reduceprod_row_kernel( - typename tileProd::TileDType __out__ new_prod, - const typename tileSrc::TileDType __in__ src, - const typename tileProd::TileDType __in__ old_prod -) -{ -// size_t i = blkv_get_index_x(); - size_t j = blkv_get_index_x(); -// size_t j = blkv_get_index_y(); - size_t idx = j * tileProd::RowStride; - - __vbuf__ typename tileProd::DType *new_prod_ptr = blkv_get_tile_ptr(new_prod); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileProd::DType *old_prod_ptr = blkv_get_tile_ptr(old_prod); - - - typename tileProd::DType upd_prod = old_prod_ptr[idx]; - - #pragma clang loop unroll(full) - for(size_t i=0;i void reduceprod_row_rand( dtype *in_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; + const int Nb = gIN / tN; + const int rmd_M = gIM % tM; + const int rmd_N = gIN % tN; - const int rmd_M = gIM % tM; // todo 尾块怎么处理？ - const int rmd_N = gIN % tN; // todo 尾块怎么处理？ - - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_row = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeProd = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - - using tile_shapeData_col = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_cor = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeProd_col = Tile; - - - gm_shapeIn inGm(in_ptr); + using tile_shapeData = Tile; + using tile_shapeData_row = Tile; + using tile_shapeProd = Tile; + using tile_shapeData_col = Tile; + using tile_shapeData_cor = Tile; + using tile_shapeProd_col = Tile; + using tile_shapeTmp = Tile; + using tile_shapeTmp_row = Tile; + using tile_shapeTmp_col = Tile; + using tile_shapeTmp_cor = Tile; + + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); - tile_shapeData dataTile; + tile_shapeData dataTile; tile_shapeData_row dataTile_row; tile_shapeData_col dataTile_col; - tile_shapeData_cor dataTile_cor; - + tile_shapeData_cor dataTile_cor; tile_shapeProd ProdTile; tile_shapeProd oldProdTile; tile_shapeProd_col ProdTile_col; - tile_shapeProd_col oldProdTile_col; - -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 + tile_shapeProd_col oldProdTile_col; + tile_shapeTmp tmpTile; + tile_shapeTmp_row tmpTile_row; + tile_shapeTmp_col tmpTile_col; + tile_shapeTmp_cor tmpTile_cor; - using itIn = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// printf("tile_shapeSum::ValidCol = %d\n", tile_shapeSum::ValidCol); -// printf("tile_shapeSum::ValidRow = %d\n", tile_shapeSum::ValidRow); -// printf("before for\n"); for (int j = 0; j < Mb; ++j) { auto gO = gOIter(j, 0); - TEXPANDSCALAR(oldProdTile, 0);//初始化为0 - //初始化old_sum的tile + TEXPANDS(oldProdTile, static_cast(1)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(j, i); -// printf("before copy in , %d\n", i); - TCOPYIN(dataTile, gI); - reduceprod_row_kernel<<>>(ProdTile.data(), dataTile.data(), oldProdTile.data()); -// reducesum_row_kernel<<<1, tile_shapeSum::ValidRow, 1>>>(SumTile.data(), dataTile.data(), oldSumTile.data()); -// printf("kernel , %d\n", i); - oldProdTile = ProdTile; + auto gI = gIIter(j, i); + TLOAD(dataTile, gI); + TROWPROD(ProdTile, dataTile, tmpTile); + TMUL(oldProdTile, oldProdTile, ProdTile); } -// printf("end for%d\n",j); - //for row corner - if constexpr (rmd_N > 0){ + if constexpr (rmd_N > 0) { auto gI = gIIter(j, Nb); - TCOPYIN(dataTile_row, gI); - reduceprod_row_kernel<<>>(ProdTile.data(), dataTile_row.data(), oldProdTile.data()); -// reducesum_row_kernel<<>>(SumTile.data(), dataTile_row.data(), oldSumTile.data()); - oldProdTile = ProdTile; + TLOAD(dataTile_row, gI); + TROWPROD(ProdTile, dataTile_row, tmpTile_row); + TMUL(oldProdTile, oldProdTile, ProdTile); } -// printf("before tcopyout\n"); - TCOPYOUT(gO, ProdTile); -// printf("end tcopyout\n"); + TSTORE(gO, oldProdTile); } - //for col cor - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gO = gOIter(Mb, 0); - TEXPANDSCALAR(oldProdTile_col, 0);//初始化为0 - //初始化old_sum的tile + TEXPANDS(oldProdTile_col, static_cast(1)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(Mb, i); - TCOPYIN(dataTile_col, gI); - reduceprod_row_kernel<<>>(ProdTile_col.data(), dataTile_col.data(), oldProdTile_col.data()); - oldProdTile_col = ProdTile_col; + auto gI = gIIter(Mb, i); + TLOAD(dataTile_col, gI); + TROWPROD(ProdTile_col, dataTile_col, tmpTile_col); + TMUL(oldProdTile_col, oldProdTile_col, ProdTile_col); } - if constexpr (rmd_N > 0){ + if constexpr (rmd_N > 0) { auto gI = gIIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - reduceprod_row_kernel<<>>(ProdTile_col.data(), dataTile_cor.data(), oldProdTile_col.data()); - oldProdTile_col = ProdTile_col; + TLOAD(dataTile_cor, gI); + TROWPROD(ProdTile_col, dataTile_cor, tmpTile_cor); + TMUL(oldProdTile_col, oldProdTile_col, ProdTile_col); } - TCOPYOUT(gO, ProdTile_col); - } -/* - for(int i = 0; i < gIM; i++){ - printf("out%d = %d\n", i, out_ptr[i]); + TSTORE(gO, oldProdTile_col); } -*/ -// printf("end program\n"); } #endif diff --git a/benchmark-linxisa/kernels/reduction/reducesum_colvec.hpp b/benchmark-linxisa/kernels/reduction/reducesum_colvec.hpp index 431d8d4..31e54a0 100644 --- a/benchmark-linxisa/kernels/reduction/reducesum_colvec.hpp +++ b/benchmark-linxisa/kernels/reduction/reducesum_colvec.hpp @@ -1,168 +1,95 @@ #ifndef REDUCESUMCOLVEC_KERNEL_HPP #define REDUCESUMCOLVEC_KERNEL_HPP - #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== -template -void __vec__ reducesum_col_kernel( - typename tileSum::TileDType __out__ new_sum, - const typename tileSrc::TileDType __in__ src, - const typename tileSum::TileDType __in__ old_sum -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - -// typename tileSum::DType upd_sum = old_sum_ptr[i]; - typename tileSum::DType upd_sum = 0; - - #pragma clang loop unroll(full) - for(size_t j=0;j void reducesum_colsum_rand( dtype *in_ptr, -// dtype *inzero_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - + const int Nb = gIN / tN; const int rmd_M = gIM % tM; const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - using gm_shapeIn = global_tensor>; // -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // - using tile_shapeData_col = Tile; // - using tile_shapeSum = Tile; // - - - - using tile_shapeData_row = Tile; // - using tile_shapeData_cor = Tile; // - using tile_shapeSum_row = Tile; // - //need tM = 1; - - - gm_shapeIn inGm(in_ptr); -// gm_shapeOut ZeroGm(inzero_ptr); + using tile_shapeData = Tile; + using tile_shapeData_col = Tile; + using tile_shapeSum = Tile; + using tile_shapeData_row = Tile; + using tile_shapeData_cor = Tile; + using tile_shapeSum_row = Tile; + + using tile_shapeTmp = Tile; + using tile_shapeTmp_col = Tile; + using tile_shapeTmp_row = Tile; + using tile_shapeTmp_cor = Tile; + + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); tile_shapeData dataTile; - tile_shapeData_col dataTile_col; + tile_shapeData_col dataTile_col; tile_shapeSum SumTile; tile_shapeSum oldSumTile; - tile_shapeData_row dataTile_row; - tile_shapeData_cor dataTile_cor; + tile_shapeData_cor dataTile_cor; tile_shapeSum_row SumTile_row; - tile_shapeSum_row oldSumTile_row; + tile_shapeSum_row oldSumTile_row; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 + tile_shapeTmp tmpTile; + tile_shapeTmp_col tmpTile_col; + tile_shapeTmp_row tmpTile_row; + tile_shapeTmp_cor tmpTile_cor; - using itIn = global_iterator; - using itIn_row = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); - itIn_row gIIter_rmd_row(in_ptr); -// itZero gZeroIter(inzero_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// dtype zero = 0; - for (int j = 0; j < Nb; ++j) { -// auto gZero = gZeroIter(0, j); auto gO = gOIter(0, j); - TEXPANDSCALAR(oldSumTile, 0);//初始化为0 -// TCOPYIN(oldSumTile, gZero);//初始化为0 - //初始化old_sum的tile - //need + TEXPANDS(oldSumTile, static_cast(0.0f)); + for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, j); - TCOPYIN(dataTile, gI); - reducesum_col_kernel<<>>(SumTile.data(), dataTile.data(), oldSumTile.data()); - oldSumTile = SumTile; + TLOAD(dataTile, gI); + TCOLSUM(SumTile, dataTile, tmpTile, /*isBinary=*/true); + TADD(oldSumTile, oldSumTile, SumTile); } - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gI = gIIter(Mb, j); - TCOPYIN(dataTile_col, gI); - reducesum_col_kernel<<>>(SumTile.data(), dataTile_col.data(), oldSumTile.data()); - oldSumTile = SumTile; + TLOAD(dataTile_col, gI); + TCOLSUM(SumTile, dataTile_col, tmpTile_col, /*isBinary=*/true); + TADD(oldSumTile, oldSumTile, SumTile); } - TCOPYOUT(gO, SumTile); + TSTORE(gO, oldSumTile); } - if constexpr (rmd_N > 0){ -// auto gZero = gZeroIter(0, Nb); + if constexpr (rmd_N > 0) { auto gO = gOIter(0, Nb); - TEXPANDSCALAR(oldSumTile_row, 0);//初始化为0 -// TCOPYIN(oldSumTile_row, gZero);//初始化为0 - for (int i = 0; i < Mb; ++i) { + TEXPANDS(oldSumTile_row, static_cast(0)); + + for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, Nb); - TCOPYIN(dataTile_row, gI); - reducesum_col_kernel<<>>(SumTile_row.data(), dataTile_row.data(), oldSumTile_row.data()); - oldSumTile_row = SumTile_row; + TLOAD(dataTile_row, gI); + TCOLSUM(SumTile_row, dataTile_row, tmpTile_row, /*isBinary=*/true); + TADD(oldSumTile_row, oldSumTile_row, SumTile_row); } - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gI = gIIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - reducesum_col_kernel<<>>(SumTile_row.data(), dataTile_cor.data(), oldSumTile_row.data()); - oldSumTile_row = SumTile_row; + TLOAD(dataTile_cor, gI); + TCOLSUM(SumTile_row, dataTile_cor, tmpTile_cor, /*isBinary=*/true); + TADD(oldSumTile_row, oldSumTile_row, SumTile_row); } - TCOPYOUT(gO, SumTile_row); + TSTORE(gO, oldSumTile_row); } } diff --git a/benchmark-linxisa/kernels/reduction/reducesum_colvec_single_tree.hpp b/benchmark-linxisa/kernels/reduction/reducesum_colvec_single_tree.hpp index f24fb56..19dbd18 100644 --- a/benchmark-linxisa/kernels/reduction/reducesum_colvec_single_tree.hpp +++ b/benchmark-linxisa/kernels/reduction/reducesum_colvec_single_tree.hpp @@ -1,227 +1,78 @@ #ifndef REDUCESUMCOLVEC_KERNEL_HPP #define REDUCESUMCOLVEC_KERNEL_HPP - #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== -//tile内进行reduce，所有tile的reduce结果统一存到一个tile中。 -template -void __vec__ reducesum_col_kernel( - typename tileTmpSum::TileDType __out__ new_sum, - const typename tileSrc::TileDType __in__ src, - const typename tileTmpSum::TileDType __in__ old_sum, - const size_t tile_idx -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileTmpSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileTmpSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - #pragma clang loop unroll(full) - for(size_t j=0;j -void __vec__ reducesum_col_final_kernel( - typename tileSum::TileDType __out__ new_sum, - const typename tileTmpSum::TileDType __in__ tmp_sum -){ - size_t i = blkv_get_index_x(); - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileTmpSum::DType *tmp_sum_ptr = blkv_get_tile_ptr(tmp_sum); - - #pragma clang loop unroll(full) - for(size_t j=0;j void reducesum_colsum_rand( - dtype *in_ptr, + dtype *in_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - + const int Nb = gIN / tN; const int rmd_M = gIM % tM; const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - using gm_shapeIn = global_tensor>; // + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // - using tile_shapeData_col = Tile; // - using tile_shapeSum = Tile; // - using tile_shapeTmpSum = Tile; // -// using tile_shapeTmpSum_l2 = Tile; // + using tile_shapeData = Tile; + using tile_shapeData_col = Tile; + using tile_shapeSum = Tile; + using tile_shapeTmpSum = Tile; + using tile_shapeTmp = Tile; + using tile_shapeTmp_col = Tile; + using tile_shapeTmp_final = Tile; - -// using tile_shapeData_row = Tile; // -// using tile_shapeData_cor = Tile; // -// using tile_shapeSum_row = Tile; // - //need tM = 1; - - - gm_shapeIn inGm(in_ptr); - gm_shapeOut outGm(out_ptr); + gm_shapeIn inGm(in_ptr); + gm_shapeOut outGm(out_ptr); tile_shapeData dataTile; - tile_shapeData_col dataTile_col; + tile_shapeData_col dataTile_col; tile_shapeSum SumTile; - tile_shapeTmpSum oldtmpSumTile; tile_shapeTmpSum tmpSumTile; -// tile_shapeTmpSum_l2 tmpSumTile_l2; -// tile_shapeData_row dataTile_row; -// tile_shapeData_cor dataTile_cor; -// tile_shapeSum_row SumTile_row; -// tile_shapeSum_row oldSumTile_row; - -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 + tile_shapeTmp tmpTile; + tile_shapeTmp_col tmpTile_col; + tile_shapeTmp_final tmpTile_final; using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// dtype zero = 0; - -// for (int j = 0; j < Nb; ++j) { -// auto gZero = gZeroIter(0, j); auto gO = gOIter(0, 0); - TEXPANDSCALAR(oldtmpSumTile, 0);//初始化为0 -// TEXPANDSCALAR(tmpSumTile, 0);//初始化为0 -// TEXPANDSCALAR(tmpSumTile_l2, 0);//初始化为0 - for (size_t i = 0; i < Mb; ++i){ + + for (size_t i = 0; i < static_cast(Mb); ++i) { auto gI = gIIter(i, 0); - TCOPYIN(dataTile, gI); - reducesum_col_kernel<<>>(tmpSumTile.data(), - dataTile.data(), - oldtmpSumTile.data(), - i); - oldtmpSumTile = tmpSumTile; + TLOAD(dataTile, gI); + tile_shapeSum partialSum; + TCOLSUM(partialSum, dataTile, tmpTile, /*isBinary=*/true); + + using SingleRow = Tile; + SingleRow rowView; + TMOV(rowView, partialSum); + if (i == 0) { + TMOV(tmpSumTile, partialSum); + } else { + TADD(tmpSumTile, tmpSumTile, partialSum); + } + } + if constexpr (rmd_M > 0) { + auto gI = gIIter(Mb, 0); + TLOAD(dataTile_col, gI); + tile_shapeSum partialSum; + TCOLSUM(partialSum, dataTile_col, tmpTile_col, /*isBinary=*/true); + TADD(tmpSumTile, tmpSumTile, partialSum); } - reducesum_col_final_kernel<<>>(SumTile.data(), - tmpSumTile.data()); - TCOPYOUT(gO, SumTile); -} + TCOLSUM(SumTile, tmpSumTile, tmpTile_final, /*isBinary=*/true); + TSTORE(gO, SumTile); +} #endif diff --git a/benchmark-linxisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp b/benchmark-linxisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp index 45cb8f8..b7015c2 100644 --- a/benchmark-linxisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp +++ b/benchmark-linxisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp @@ -1,174 +1,52 @@ #ifndef REDUCESUMCOLVEC_KERNEL_HPP #define REDUCESUMCOLVEC_KERNEL_HPP - #include -#include "template_asm.h" - - -using namespace pto; - -#pragma once #include #include - -// ============================================== -// ============================================== -template -void __vec__ reducesum_col_tmp( - typename tileTmp::TileDType __out__ tmp_sum, - const typename tileSrc::TileDType __in__ src -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileTmp::DType *tmp_sum_ptr = blkv_get_tile_ptr(tmp_sum); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); -// __vbuf__ typename tileSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - typename tileTmp::DType upd_tmp_sum = 0; - - #pragma clang loop unroll(full) - for(size_t j=0;j -void __vec__ reducesum_col_final( - typename tileSum::TileDType __out__ new_sum, - const typename tileTmp::TileDType __in__ src, - const typename tileSum::TileDType __in__ old_sum -) -{ - size_t i = blkv_get_index_x(); - - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileTmp::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - - typename tileSum::DType upd_sum = old_sum_ptr[i]; - - - size_t src_idx_0 = i * tileSum::ColStride + 0 * tileSum::ValidCol; - size_t src_idx_1 = i * tileSum::ColStride + 1 * tileSum::ValidCol; - size_t src_idx_2 = i * tileSum::ColStride + 2 * tileSum::ValidCol; - size_t src_idx_3 = i * tileSum::ColStride + 3 * tileSum::ValidCol; - size_t src_idx_4 = i * tileSum::ColStride + 4 * tileSum::ValidCol; - size_t src_idx_5 = i * tileSum::ColStride + 5 * tileSum::ValidCol; - size_t src_idx_6 = i * tileSum::ColStride + 6 * tileSum::ValidCol; - size_t src_idx_7 = i * tileSum::ColStride + 7 * tileSum::ValidCol; - typename tileSum::DType sum_01 = src_ptr[src_idx_0] + src_ptr[src_idx_1]; - typename tileSum::DType sum_23 = src_ptr[src_idx_2] + src_ptr[src_idx_3]; - typename tileSum::DType sum_45 = src_ptr[src_idx_4] + src_ptr[src_idx_5]; - typename tileSum::DType sum_67 = src_ptr[src_idx_6] + src_ptr[src_idx_7]; - typename tileSum::DType sum_0123 = sum_01 + sum_23; - typename tileSum::DType sum_4567 = sum_45 + sum_67; - typename tileSum::DType sum_all = sum_0123 + sum_4567; - -// upd_sum = upd_sum + sum_tmp; - -/* - #pragma clang loop unroll(full) - for(size_t j=0;j void reducesum_colsum_rand( dtype *in_ptr, -// dtype *inzero_ptr, dtype *out_ptr -) +) { - -// const int Mb = (gIM/8) / tM; - - const int rmd_M = gIM % tM; - const int rmd_N = gIN % tN; -// const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - - using gm_shapeIn = global_tensor>; // -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // -// using tile_shapeData_col = Tile; // - using tile_shapeTmp = Tile; // - using tile_shapeSum = Tile; // - + using tile_shapeData = Tile; + using tile_shapeTmp = Tile; + using tile_shapeSum = Tile; + using tile_shapeTmpData = Tile; + using tile_shapeTmpTmp = Tile; -// using tile_shapeData_row = Tile; // -// using tile_shapeData_cor = Tile; // -// using tile_shapeSum_row = Tile; // - //need tM = 1; - - - gm_shapeIn inGm(in_ptr); -// gm_shapeOut ZeroGm(inzero_ptr); + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); tile_shapeData dataTile; -// tile_shapeData_col dataTile_col; - tile_shapeTmp TmpTile; + tile_shapeTmp TmpTile; tile_shapeSum SumTile; tile_shapeSum oldSumTile; -// tile_shapeData_row dataTile_row; -// tile_shapeData_cor dataTile_cor; -// tile_shapeSum_row SumTile_row; -// tile_shapeSum_row oldSumTile_row; + tile_shapeTmpData tmpTileData; + tile_shapeTmpTmp tmpTileTmp; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; - using itIn_row = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); - itIn_row gIIter_rmd_row(in_ptr); -// itZero gZeroIter(inzero_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); - auto gO = gOIter(0, 0); - TEXPANDSCALAR(oldSumTile, 0);//初始化为0 + TEXPANDS(oldSumTile, static_cast(0)); auto gI = gIIter(0, 0); - TCOPYIN(dataTile, gI);//TLOAD应补0，目前gfrun默认补0，需要接口去弄 - reducesum_col_tmp<<>>(TmpTile.data(), dataTile.data()); - reducesum_col_final<<>>(SumTile.data(), TmpTile.data(), oldSumTile.data()); - oldSumTile = SumTile; - TCOPYOUT(gO, SumTile); + TLOAD(dataTile, gI); + TCOLSUM(TmpTile, dataTile, tmpTileData, /*isBinary=*/true); + TCOLSUM(SumTile, TmpTile, tmpTileTmp, /*isBinary=*/true); + TADD(oldSumTile, oldSumTile, SumTile); + TSTORE(gO, SumTile); } #endif diff --git a/benchmark-linxisa/kernels/reduction/reducesum_rowvec.hpp b/benchmark-linxisa/kernels/reduction/reducesum_rowvec.hpp index be8b749..401f873 100644 --- a/benchmark-linxisa/kernels/reduction/reducesum_rowvec.hpp +++ b/benchmark-linxisa/kernels/reduction/reducesum_rowvec.hpp @@ -1,177 +1,95 @@ #ifndef REDUCESUMTROWSUM_KERNEL_HPP #define REDUCESUMTROWSUM_KERNEL_HPP -#ifndef __vbuf__ -#define __vbuf__ -#endif - -#include - -using namespace pto; - #pragma once +#include #include #include -template -void __vec__ reducesum_row_kernel( - typename tileSum::TileDType __out__ new_sum, - const typename tileSrc::TileDType __in__ src, - const typename tileSum::TileDType __in__ old_sum -) -{ -// size_t i = blkv_get_index_x(); - size_t j = blkv_get_index_x(); -// size_t j = blkv_get_index_y(); - size_t idx = j * tileSum::RowStride; - - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - - - typename tileSum::DType upd_sum = old_sum_ptr[idx]; - - #pragma clang loop unroll(full) - for(size_t i=0;i void reducesum_trowsum_rand( dtype *in_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - - const int rmd_M = gIM % tM; // todo 尾块怎么处理？ - const int rmd_N = gIN % tN; // todo 尾块怎么处理？ + const int Nb = gIN / tN; + const int rmd_M = gIM % tM; + const int rmd_N = gIN % tN; - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_row = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeSum = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - - using tile_shapeData_col = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeData_cor = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeSum_col = Tile; - - - gm_shapeIn inGm(in_ptr); + using tile_shapeData = Tile; + using tile_shapeData_row = Tile; + using tile_shapeSum = Tile; + using tile_shapeData_col = Tile; + using tile_shapeData_cor = Tile; + using tile_shapeSum_col = Tile; + using tile_shapeTmp = Tile; + using tile_shapeTmp_row = Tile; + using tile_shapeTmp_col = Tile; + using tile_shapeTmp_cor = Tile; + + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); - tile_shapeData dataTile; + tile_shapeData dataTile; tile_shapeData_row dataTile_row; tile_shapeData_col dataTile_col; - tile_shapeData_cor dataTile_cor; - + tile_shapeData_cor dataTile_cor; tile_shapeSum SumTile; tile_shapeSum oldSumTile; tile_shapeSum_col SumTile_col; - tile_shapeSum_col oldSumTile_col; + tile_shapeSum_col oldSumTile_col; + tile_shapeTmp tmpTile; + tile_shapeTmp_row tmpTile_row; + tile_shapeTmp_col tmpTile_col; + tile_shapeTmp_cor tmpTile_cor; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); -// printf("tile_shapeSum::ValidCol = %d\n", tile_shapeSum::ValidCol); -// printf("tile_shapeSum::ValidRow = %d\n", tile_shapeSum::ValidRow); -// printf("before for\n"); for (int j = 0; j < Mb; ++j) { auto gO = gOIter(j, 0); - TEXPANDSCALAR(oldSumTile, 0);//初始化为0 - //初始化old_sum的tile + TEXPANDS(oldSumTile, static_cast(0)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(j, i); -// printf("before copy in , %d\n", i); - TCOPYIN(dataTile, gI); - reducesum_row_kernel<<>>(SumTile.data(), dataTile.data(), oldSumTile.data()); -// reducesum_row_kernel<<<1, tile_shapeSum::ValidRow, 1>>>(SumTile.data(), dataTile.data(), oldSumTile.data()); -// printf("kernel , %d\n", i); - oldSumTile = SumTile; + auto gI = gIIter(j, i); + TLOAD(dataTile, gI); + TROWSUM(SumTile, dataTile, tmpTile); + TADD(oldSumTile, oldSumTile, SumTile); } -// printf("end for%d\n",j); - //for row corner - if constexpr (rmd_N > 0){ + if constexpr (rmd_N > 0) { auto gI = gIIter(j, Nb); - TCOPYIN(dataTile_row, gI); - reducesum_row_kernel<<>>(SumTile.data(), dataTile_row.data(), oldSumTile.data()); -// reducesum_row_kernel<<>>(SumTile.data(), dataTile_row.data(), oldSumTile.data()); - oldSumTile = SumTile; + TLOAD(dataTile_row, gI); + TROWSUM(SumTile, dataTile_row, tmpTile_row); + TADD(oldSumTile, oldSumTile, SumTile); } -// printf("before tcopyout\n"); - TCOPYOUT(gO, SumTile); -// printf("end tcopyout\n"); + TSTORE(gO, oldSumTile); } - //for col cor - if constexpr (rmd_M > 0){ + if constexpr (rmd_M > 0) { auto gO = gOIter(Mb, 0); - TEXPANDSCALAR(oldSumTile_col, 0);//初始化为0 - //初始化old_sum的tile + TEXPANDS(oldSumTile_col, static_cast(0)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(Mb, i); - TCOPYIN(dataTile_col, gI); - reducesum_row_kernel<<>>(SumTile_col.data(), dataTile_col.data(), oldSumTile_col.data()); - oldSumTile_col = SumTile_col; + auto gI = gIIter(Mb, i); + TLOAD(dataTile_col, gI); + TROWSUM(SumTile_col, dataTile_col, tmpTile_col); + TADD(oldSumTile_col, oldSumTile_col, SumTile_col); } - if constexpr (rmd_N > 0){ + if constexpr (rmd_N > 0) { auto gI = gIIter(Mb, Nb); - TCOPYIN(dataTile_cor, gI); - reducesum_row_kernel<<>>(SumTile_col.data(), dataTile_cor.data(), oldSumTile_col.data()); - oldSumTile_col = SumTile_col; + TLOAD(dataTile_cor, gI); + TROWSUM(SumTile_col, dataTile_cor, tmpTile_cor); + TADD(oldSumTile_col, oldSumTile_col, SumTile_col); } - TCOPYOUT(gO, SumTile_col); - } -/* - for(int i = 0; i < gIM; i++){ - printf("out%d = %d\n", i, out_ptr[i]); + TSTORE(gO, oldSumTile_col); } -*/ -// printf("end program\n"); } #endif diff --git a/benchmark-linxisa/kernels/reduction/reducesum_rowvec_single_tree.hpp b/benchmark-linxisa/kernels/reduction/reducesum_rowvec_single_tree.hpp index ddfcca7..60ddc02 100644 --- a/benchmark-linxisa/kernels/reduction/reducesum_rowvec_single_tree.hpp +++ b/benchmark-linxisa/kernels/reduction/reducesum_rowvec_single_tree.hpp @@ -1,233 +1,60 @@ #ifndef REDUCESUMTROWSUM_KERNEL_HPP #define REDUCESUMTROWSUM_KERNEL_HPP -#ifndef __vbuf__ -#define __vbuf__ -#endif - -#include - -using namespace pto; - #pragma once +#include #include #include -template -void __vec__ reducesum_row_kernel( - typename tileTmpSum::TileDType __out__ new_sum, - const typename tileSrc::TileDType __in__ src, - const typename tileSrcCol::TileDType __in__ src_col, - const typename tileTmpSum::TileDType __in__ old_sum, - const size_t tile_idx -) -{ - - size_t j = blkv_get_index_x(); - size_t z = blkv_get_index_y(); - size_t stride_src = z * (tileSrc::ValidCol/4) * tileSrc::ColStride; - size_t stride_src_col = z * (tileSrcCol::ValidCol/4) * tileSrcCol::ColStride; - - __vbuf__ typename tileTmpSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileSrc::DType *src_ptr = blkv_get_tile_ptr(src); - __vbuf__ typename tileSrc::DType *src_col_ptr = blkv_get_tile_ptr(src_col); - __vbuf__ typename tileTmpSum::DType *old_sum_ptr = blkv_get_tile_ptr(old_sum); - -/* - #pragma clang loop unroll(full) - for(size_t i=0;i -void __vec__ reducesum_row_final_kernel( - typename tileSum::TileDType __out__ new_sum, - const typename tileTmpSum::TileDType __in__ tmp_sum -){ - size_t j = blkv_get_index_x(); - size_t idx = j * tileSum::RowStride; - - __vbuf__ typename tileSum::DType *new_sum_ptr = blkv_get_tile_ptr(new_sum); - __vbuf__ typename tileTmpSum::DType *tmp_sum_ptr = blkv_get_tile_ptr(tmp_sum); - - #pragma clang loop unroll(full) - for(size_t i=0;i void reducesum_trowsum_rand( dtype *in_ptr, dtype *out_ptr -) +) { - const int Mb = gIM / tM; - const int Nb = gIN / tN; - - const int rmd_M = gIM % tM; // todo 尾块怎么处理？ - const int rmd_N = gIN % tN; // todo 尾块怎么处理？ - + const int Nb = gIN / tN; + const int rmd_M = gIM % tM; + const int rmd_N = gIN % tN; - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 -// using gm_shapeSum = global_tensor>; + using gm_shapeIn = global_tensor>; using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeDataCol = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeSum = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - using tile_shapeTmpSum = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec + using tile_shapeData = Tile; + using tile_shapeSum = Tile; + using tile_shapeTmpSum = Tile; + using tile_shapeTmp = Tile; - - gm_shapeIn inGm(in_ptr); + gm_shapeIn inGm(in_ptr); gm_shapeOut outGm(out_ptr); -// gm_shapeSum olcSumGm(old_sum_ptr); - tile_shapeData dataTile; - tile_shapeDataCol dataTile_col; + tile_shapeData dataTile; tile_shapeSum SumTile; tile_shapeTmpSum oldtmpSumTile; tile_shapeTmpSum tmpSumTile; + tile_shapeTmp tmpTile; -// int base = 0;// todo 生成一个标量 -// int all_num = gOM; // 总元素数量 - - using itIn = global_iterator; + using itIn = global_iterator; using itOut = global_iterator; - itIn gIIter(in_ptr); + itIn gIIter(in_ptr); itOut gOIter(out_ptr); - auto gO = gOIter(0, 0); - TEXPANDSCALAR(oldtmpSumTile, 0);//初始化为0 - TEXPANDSCALAR(dataTile_col, 0);//初始化为0 + TEXPANDS(oldtmpSumTile, static_cast(0)); + for (int i = 0; i < Nb; ++i) { - auto gI = gIIter(0, i); - TCOPYIN(dataTile, gI); - reducesum_row_kernel<<>>(tmpSumTile.data(), - dataTile.data(), - dataTile_col.data(), - oldtmpSumTile.data(), - i); - oldtmpSumTile = tmpSumTile; + auto gI = gIIter(0, i); + TLOAD(dataTile, gI); + tile_shapeSum partialSum; + TROWSUM(partialSum, dataTile, tmpTile); + TADD(oldtmpSumTile, oldtmpSumTile, partialSum); } - reducesum_row_final_kernel<<>>(SumTile.data(), - tmpSumTile.data()); - TCOPYOUT(gO, SumTile); + TMOV(tmpSumTile, oldtmpSumTile); + + TROWSUM(SumTile, tmpSumTile, tmpTile); + TSTORE(gO, SumTile); } #endif - diff --git a/benchmark-linxisa/kernels/sort/topk.hpp b/benchmark-linxisa/kernels/sort/topk.hpp index 3bba98b..d9ac6b9 100644 --- a/benchmark-linxisa/kernels/sort/topk.hpp +++ b/benchmark-linxisa/kernels/sort/topk.hpp @@ -1,9 +1,12 @@ #ifndef TOPK_HPP #define TOPK_HPP +#include #include #include +using namespace pto::blkv; + // ============================================================================ // Constants // ============================================================================ @@ -82,15 +85,18 @@ void __vec__ ExtractLow8HistForKthBin_Vec_RowMajor( template void ExtractHigh8Hist_Impl(tile_shape_out& dst, const uint16_t* src) { - ExtractHigh8Hist_Vec_RowMajor - <<<1, 256, 1>>>(dst.data(), src); + pto::blkv::blkv_for_2d(1, 256, [&] { + ExtractHigh8Hist_Vec_RowMajor(dst.data(), src); + }); } template void ExtractLow8HistForKthBin_Impl(tile_shape_out& dst, const uint16_t* src, uint16_t kth_bin) { - ExtractLow8HistForKthBin_Vec_RowMajor - <<<1, 256, 1>>>(dst.data(), src, kth_bin); + pto::blkv::blkv_for_2d(1, 256, [&] { + ExtractLow8HistForKthBin_Vec_RowMajor(dst.data(), src, + kth_bin); + }); } // ============================================================================ diff --git a/benchmark-linxisa/kernels/transpose/transpose.hpp b/benchmark-linxisa/kernels/transpose/transpose.hpp index 2747040..9db50d2 100644 --- a/benchmark-linxisa/kernels/transpose/transpose.hpp +++ b/benchmark-linxisa/kernels/transpose/transpose.hpp @@ -1,201 +1,428 @@ -#ifndef TRANSPOSE_KERNEL_HPP -#define TRANSPOSE_KERNEL_HPP +/** + * @file transpose_tile_isa.hpp + * @brief 基于纯 Tile ISA 的转置算子实现 + * + * 本文件提供两个转置算子： + * 1. tile_transpose_nd: 通用 N 维任意轴交换，使用 offset 计算 + MGATHER + * 2. tile_transpose_2d: 2D 矩阵转置，使用硬件原生 TTRANS 指令 + * + * 核心设计思想： + * - 将"标量计算"与"数据并行计算"分离到不同执行路径 + * - 维度循环在标量核心上运行（编译期模板展开） + * - 每个维度内的坐标提取用 tile 操作并行处理所有元素 + * - 避免使用 __vec__ 的 per-element 标量循环模式 + */ + +#ifndef SUPERNPU_TRANSPOSE_TILE_ISA_HPP +#define SUPERNPU_TRANSPOSE_TILE_ISA_HPP #include #include "template_asm.h" +#include +#include -using namespace pto; +// TTRANS 指令的 API 兼容性配置 +// 旧 API: TTRANS(dst, src) +// 新 API: TTRANS(dst, src, tmp) - 需要显式传入临时 tile +// 根据实际 PTO 版本设置此宏 +#ifndef SUPERNPU_PTO_TTRANS_NEEDS_TMP +#define SUPERNPU_PTO_TTRANS_NEEDS_TMP 0 +#endif -#pragma once -#include -#include - - -#define DUMP_TILE(label, TileVar, DumpBuf, Rows, Cols) \ - do { \ - GlobalTensor, \ - Stride<1,1,1,Cols,1>> _g(DumpBuf); \ - TCOPYOUT(_g, TileVar); \ - printf("[DUMP] %s (shape=%dx%d):\n", label, Rows, Cols); \ - for (int ri = 0; ri < Rows; ri++) { \ - printf(" row%2d: ", ri); \ - for (int ci = 0; ci < Cols; ci++) \ - printf("%12lld ", (long long)DumpBuf[ri * Cols + ci]); \ - printf("\n"); \ - } \ - fflush(stdout); \ - } while (0) - -// ============================================== -// 维度规则：交换transpose_dim0和transpose_dim1 -// ============================================== -template -void __vec__ gen_offset_trans( - typename tile_shape::TileDType __out__ out, - typename tile_Inshape::TileDType __in__ in_shape, - typename tile_Outshape::TileDType __in__ out_shape, -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0, - const size_t base, - const size_t total_elements -) { - size_t index = blkv_get_index_x(); - size_t idx = blkv_get_index_x(); - - __vbuf__ typename tile_Inshape::DType *in_shape_ptr = blkv_get_tile_ptr(in_shape); - __vbuf__ typename tile_Outshape::DType *out_shape_ptr = blkv_get_tile_ptr(out_shape); - -// if (index >= total_elements) return; - idx = idx + base; // todo idx是个向量，base是个标量，获得所有的基地址或者说基offset - - //转置维度交换stride。 -// uint16_t stride[IN_DIM]; - uint32_t stride[IN_DIM]; - stride[IN_DIM-1] = 1; - - #pragma clang loop unroll(full) - for(int i = IN_DIM-2 ; i >= 0; --i) - { - stride[i] = stride[i+1] * in_shape_ptr[i+1]; - } - std::swap(stride[TRANSPOSE_DIM1],stride[TRANSPOSE_DIM0]); - - // 输出一维索引 → 输出坐标 - size_t out_coord[MAX_DIM] = {0}; // - size_t tmp = idx; // - - #pragma clang loop unroll(full) - for (int d = OUT_DIM - 1; d >= 0; d--) { - out_coord[d] = tmp % out_shape_ptr[d]; - tmp /= out_shape_ptr[d]; - } +namespace supernpu::tile_isa { + +/** + * @brief 通用 N 维转置（任意轴交换） + * + * 算法核心： + * 1. 对输出张量的每个元素，计算其多维坐标 + * 2. 通过轴映射得到对应的输入坐标 + * 3. 用输入 stride 计算 byte offset + * 4. 用 MGATHER 按 offset 从输入中 gather 数据 + * + * 关键优化： + * - Rank、Axis0、Axis1 是模板参数 → 维度循环编译期完全展开 + * - 每个维度的坐标提取用 tile 操作并行处理 tile 内所有元素 + * - 不是"per-element 串行迭代"，而是"per-dimension 并行处理" + * + * @tparam DType 数据类型 + * @tparam Rank 张量维度数（编译期常量，> 1） + * @tparam Axis0 转置轴 0（编译期常量，[0, Rank)） + * @tparam Axis1 转置轴 1（编译期常量，[0, Rank) 且 != Axis0） + * @tparam Elements 总元素数（编译期常量，> 0） + * @tparam TileElements 每个 tile 处理的元素数（默认 512） + * + * @param input 输入数据指针（展平为一维） + * @param output 输出数据指针（展平为一维） + * @param input_shape 输入张量各维度尺寸（运行时数组） + * @param output_shape 输出张量各维度尺寸（运行时数组） + */ +template +void tile_transpose_nd(DType *input, DType *output, std::uint32_t *input_shape, + std::uint32_t *output_shape) { + // ========== 编译期安全检查 ========== + static_assert(Rank > 1, "转置至少需要 2 维"); + static_assert(Axis0 >= 0 && Axis0 < Rank, "Axis0 超出维度范围"); + static_assert(Axis1 >= 0 && Axis1 < Rank, "Axis1 超出维度范围"); + static_assert(Axis0 != Axis1, "转置的两个轴必须不同"); + static_assert(Elements > 0, "元素数必须为正"); + static_assert(TileElements > 0, "TileElements 必须为正"); + // Tile 行必须 32 字节对齐（硬件要求） + static_assert(TileElements * static_cast(sizeof(DType)) % 32 == 0, + "RowMajor Tile 的行必须 32 字节对齐"); + // MGATHER 的 offset tile 使用 uint32 byte offset，不能溢出 + static_assert(static_cast(Elements) * sizeof(DType) <= + 0x100000000ULL, + "MGATHER 的 byte offset 使用 uint32，不能溢出"); + + // ========== 分块参数 ========== + constexpr int kFullTiles = Elements / TileElements; // 完整 tile 数量 + constexpr int kTailElements = Elements % TileElements; // 尾部剩余元素数 + + // ========== 类型定义 ========== + // 输入/输出全局张量：展平为一维（转置不改变存储，只改变逻辑索引） + using InputGlobal = pto::global_tensor>; + using OutputGlobal = pto::global_tensor>; + + // 数据 tile：存储实际数据 + using DataTile = pto::Tile; + // Offset tile：存储每个元素的 byte offset（用于 MGATHER） + using OffsetTile = pto::Tile; + // 输出迭代器：用于遍历输出张量的 tile + using OutputIterator = global_iterator; + + // ========== 初始化全局张量和迭代器 ========== + InputGlobal input_global(input); + OutputIterator output_iter(output); + + // ========== 处理完整 tile ========== + std::uint32_t output_base = 0; // 当前 tile 在输出中的起始线性索引 + for (int tile_index = 0; tile_index < kFullTiles; ++tile_index) { + auto output_global = output_iter(0, tile_index); + DataTile output_tile; // 存储 gather 回来的数据 + OffsetTile offset_tile; // 存储每个元素的 byte offset + + // 临时 tile（用于坐标提取和 offset 计算） + OffsetTile linear_index; // 线性索引 [output_base, output_base+1, ...] + OffsetTile quotient; // 除法商 + OffsetTile cycle; // 周期数（用于取模） + OffsetTile cycle_base; // 周期基址 + OffsetTile coordinate; // 当前维度的坐标 + OffsetTile contribution; // 当前维度对 offset 的贡献 + + // Step 1: 生成线性索引序列 [output_base, output_base+1, ..., output_base+TileElements-1] + TCI(linear_index, output_base); + + // Step 2: 初始化 offset 累加器为 0 + TEXPANDS(offset_tile, static_cast(0)); + + // Step 3: 对每个维度计算坐标并累加 offset + // 关键：这个循环在标量核心上运行，由于 Rank 是模板参数，编译期完全展开 + // 每次迭代用 tile 操作并行处理 tile 内所有元素的当前维度 + std::uint32_t input_stride = 1; // 输入张量当前维度的 stride（从内层开始） + for (int input_dim = Rank - 1; input_dim >= 0; --input_dim) { + // 轴映射：交换 Axis0 和 Axis1 + // 例如：如果 input_dim == Axis0，则对应的 output_dim == Axis1 + int output_dim = input_dim; + if (input_dim == Axis0) { + output_dim = Axis1; + } else if (input_dim == Axis1) { + output_dim = Axis0; + } -/* - // 输出坐标 → 输入坐标 - size_t in_coord[MAX_DIM] = {0}; - for (size_t i = 0; i < in_dim; i++) { - size_t o = out_dim - in_dim + i; // 从后面对齐 - if (in_shape[i] == 1) { - in_coord[i] = 0; - } else { - in_coord[i] = out_coord[o]; - } + // 计算 output_dim 维度的 stride（标量计算，在标量核心上完成） + // output_stride = output_shape[output_dim+1] * output_shape[output_dim+2] * ... * output_shape[Rank-1] + std::uint32_t output_stride = 1; + for (int dim = output_dim + 1; dim < Rank; ++dim) { + output_stride *= output_shape[dim]; + } + + // 提取当前维度的坐标：coordinate = (linear_index / output_stride) % output_shape[output_dim] + // 用 tile 操作并行处理所有元素 + + // quotient = linear_index / output_stride + // 优化：如果 output_stride == 1，直接拷贝（避免除法） + if (output_stride == 1) { + TMOV(quotient, linear_index); + } else { + TDIVS(quotient, linear_index, output_stride); + } + + // 取模实现：a % b = a - (a / b) * b + // cycle = quotient / output_shape[output_dim] + TDIVS(cycle, quotient, output_shape[output_dim]); + // cycle_base = cycle * output_shape[output_dim] + TMULS(cycle_base, cycle, output_shape[output_dim]); + // coordinate = quotient - cycle_base （即 quotient % output_shape[output_dim]） + TSUB(coordinate, quotient, cycle_base); + + // 计算当前维度对 byte offset 的贡献 + // contribution = coordinate * input_stride * sizeof(DType) + // 注意：coordinate 是 output_coord[output_dim]，由于轴映射，它等于 input_coord[input_dim] + TMULS(contribution, coordinate, + input_stride * static_cast(sizeof(DType))); + + // 累加到总 offset + TADD(offset_tile, offset_tile, contribution); + + // 更新 input_stride（从内层向外层累积） + input_stride *= input_shape[input_dim]; } -*/ -// uint16_t in_offset = 0; - uint32_t in_offset = 0; - #pragma clang loop unroll(full) - for (int i = 0; i < IN_DIM; i++) { - in_offset += out_coord[i] * stride[i] * sizeof(dtype); + // Step 4: 按 byte offset 从输入中 gather 数据 + MGATHER(output_tile, input_global, offset_tile); + + // Step 5: 写回到输出全局内存 + TSTORE(output_global, output_tile); + + // 更新下一个 tile 的起始索引 + output_base += TileElements; + } + + // ========== 处理尾部元素 ========== + // 如果总元素数不是 TileElements 的整数倍，处理剩余元素 + if constexpr (kTailElements != 0) { + // 尾部 tile 的类型定义（物理尺寸相同，但 valid region 缩小） + using TailDataTile = pto::Tile; + using TailOffsetTile = + pto::Tile; + + auto output_global = output_iter(0, kFullTiles); + TailDataTile output_tile; + TailOffsetTile offset_tile; + + TailOffsetTile linear_index; + TailOffsetTile quotient; + TailOffsetTile cycle; + TailOffsetTile cycle_base; + TailOffsetTile coordinate; + TailOffsetTile contribution; + + // 尾部 tile 的处理逻辑与完整 tile 完全一致 + // 区别仅在于 valid region 更小（kTailElements 而非 TileElements） + // tile 操作以 valid region 为迭代域，因此自动只处理有效元素 + TCI(linear_index, output_base); + TEXPANDS(offset_tile, static_cast(0)); + + std::uint32_t input_stride = 1; + for (int input_dim = Rank - 1; input_dim >= 0; --input_dim) { + int output_dim = input_dim; + if (input_dim == Axis0) { + output_dim = Axis1; + } else if (input_dim == Axis1) { + output_dim = Axis0; + } + + std::uint32_t output_stride = 1; + for (int dim = output_dim + 1; dim < Rank; ++dim) { + output_stride *= output_shape[dim]; + } + + if (output_stride == 1) { + TMOV(quotient, linear_index); + } else { + TDIVS(quotient, linear_index, output_stride); + } + + TDIVS(cycle, quotient, output_shape[output_dim]); + TMULS(cycle_base, cycle, output_shape[output_dim]); + TSUB(coordinate, quotient, cycle_base); + TMULS(contribution, coordinate, + input_stride * static_cast(sizeof(DType))); + TADD(offset_tile, offset_tile, contribution); + + input_stride *= input_shape[input_dim]; } - // 赋值 - blkv_get_tile_ptr(out)[index] = in_offset; + MGATHER(output_tile, input_global, offset_tile); + TSTORE(output_global, output_tile); + } } -template -void gen_offset_Impl( - tile_shapeOffset &out, - tile_Inshape &in_shape, - tile_Outshape &out_shape, -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0, - const size_t base, - const size_t total_elements - ) -{ - static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, - "Only static shape supported"); - gen_offset_trans<<>>(out.data(), in_shape.data(), out_shape.data(), base, total_elements); // todo 这部分的tile shape是怎么传递的？ -} +/** + * @brief 2D 矩阵转置（高性能路径） + * + * 使用硬件原生 TTRANS 指令，性能最优。 + * 算法：标准分块转置 + * 1. 将 Rows × Cols 矩阵划分为 TileRows × TileCols 的块 + * 2. 对每个块：TLOAD → TTRANS → TSTORE + * 3. 输入块 TileRows × TileCols，转置后输出块 TileCols × TileRows + * 4. 输出位置为 (col_tile, row_tile)（行列互换） + * + * @tparam DType 数据类型 + * @tparam Rows 输入矩阵行数 + * @tparam Cols 输入矩阵列数 + * @tparam TileRows tile 块行数（默认 16） + * @tparam TileCols tile 块列数（默认 16） + * + * @param input 输入矩阵指针（Rows × Cols） + * @param output 输出矩阵指针（Cols × Rows） + */ +template +void tile_transpose_2d(DType *input, DType *output) { + // ========== 编译期安全检查 ========== + static_assert(Rows > 0 && Cols > 0, "矩阵维度必须为正"); + static_assert(TileRows > 0 && TileCols > 0, "tile 维度必须为正"); + // TTRANS 对行宽有 32 字节对齐要求 + static_assert(TileRows * static_cast(sizeof(DType)) % 32 == 0, + "TTRANS 输出行宽必须 32 字节对齐"); + static_assert(TileCols * static_cast(sizeof(DType)) % 32 == 0, + "TTRANS 输入行宽必须 32 字节对齐"); + + // ========== 分块参数 ========== + constexpr int kRowTiles = Rows / TileRows; // 行方向完整 tile 数 + constexpr int kColTiles = Cols / TileCols; // 列方向完整 tile 数 + constexpr int kTailRows = Rows % TileRows; // 行方向尾部元素数 + constexpr int kTailCols = Cols % TileCols; // 列方向尾部元素数 + + // ========== 类型定义 ========== + using InputGlobal = pto::global_tensor>; + using OutputGlobal = pto::global_tensor>; + + // 输入 tile：TileRows × TileCols + using SrcTile = pto::Tile; + // 输出 tile：TileCols × TileRows（转置后） + using DstTile = pto::Tile; + + using InputIterator = global_iterator; + using OutputIterator = global_iterator; + + InputIterator input_iter(input); + OutputIterator output_iter(output); + // ========== 处理完整 tile 块 ========== + for (int row_tile = 0; row_tile < kRowTiles; ++row_tile) { + for (int col_tile = 0; col_tile < kColTiles; ++col_tile) { + // 输入位置：(row_tile, col_tile) + auto input_global = input_iter(row_tile, col_tile); + // 输出位置：(col_tile, row_tile) - 行列互换 + auto output_global = output_iter(col_tile, row_tile); + SrcTile src_tile; + DstTile dst_tile; -template -void transpose( - dtype *in_ptr, - dtype *out_ptr, - uint32_t *in_shape, - uint32_t *out_shape -// const size_t in_dim, -// const size_t out_dim, -// const size_t transpose_dim1, -// const size_t transpose_dim0 -) - { - - const int Mb = gOM / tM; - - - const int rmd_M = gOM % tM; // todo 尾块怎么处理？ - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_shapeOut = global_tensor>; - - using gm_InMatShape = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_OutMatShape = global_tensor>; - - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec -// using tile_shapeOffset = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - - using tile_shapeData_rmd = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeOffset_rmd = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - using tile_Inshape = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_Outshape = Tile; // todo 这里的location，一定要是Vec吗？哪怕没有传入Vec - - gm_shapeIn inGm(in_ptr); - - gm_InMatShape InShapeGm(in_shape); - gm_OutMatShape OutShapeGm(out_shape); - - tile_shapeData dataTile; - tile_shapeOffset offsetTile; - - tile_shapeData_rmd dataTile_rmd; - tile_shapeOffset_rmd offsetTile_rmd; - - tile_Inshape InshapeTile; - tile_Outshape OutshapeTile; - - int base = 0;// todo 生成一个标量 - int all_num = gOM; // 总元素数量 - - using itOut = global_iterator; - itOut gOIter(out_ptr); - - TCOPYIN(InshapeTile, InShapeGm); - TCOPYIN(OutshapeTile, OutShapeGm); - - - int total_elements = tM; - for (int i = 0; i < Mb; ++i) { - auto gO = gOIter(0, i); - gen_offset_Impl(offsetTile, InshapeTile, OutshapeTile, base, total_elements); -// printf("end genoffset\n"); - base += total_elements; -// DUMP_TILE("offsetTile", offsetTile, g_dump, 1, tM); - MGATHER(dataTile, inGm, offsetTile); -// printf("end mgather\n"); - TCOPYOUT(gO, dataTile); -// TCOPYOUT(gO, dataTile); + // 从全局内存加载输入 tile + TLOAD(src_tile, input_global); + + // 硬件转置：TileRows × TileCols → TileCols × TileRows +#if SUPERNPU_PTO_TTRANS_NEEDS_TMP + SrcTile tmp_tile; + TTRANS(dst_tile, src_tile, tmp_tile); +#else + TTRANS(dst_tile, src_tile); +#endif + + // 写回到全局内存 + TSTORE(output_global, dst_tile); } - if constexpr (rmd_M) { - auto gO = gOIter(0, Mb); - total_elements = rmd_M;//尾片的大小。 - gen_offset_Impl(offsetTile_rmd, InshapeTile, OutshapeTile, base, total_elements); - base += total_elements; - MGATHER(dataTile_rmd, inGm, offsetTile_rmd); - TCOPYOUT(gO, dataTile_rmd); + + // ========== 处理右侧尾部 tile（列方向） ========== + if constexpr (kTailCols != 0) { + // 输入 tile：TileRows × kTailCols（valid region） + using SrcRightTile = + pto::Tile; + // 输出 tile：kTailCols × TileRows（转置后） + using DstBottomTile = + pto::Tile; + + auto input_global = input_iter(row_tile, kColTiles); + auto output_global = output_iter(kColTiles, row_tile); + + SrcRightTile src_tile; + DstBottomTile dst_tile; + + TLOAD(src_tile, input_global); +#if SUPERNPU_PTO_TTRANS_NEEDS_TMP + SrcRightTile tmp_tile; + TTRANS(dst_tile, src_tile, tmp_tile); +#else + TTRANS(dst_tile, src_tile); +#endif + TSTORE(output_global, dst_tile); } + } + + // ========== 处理底部尾部 tile（行方向） ========== + if constexpr (kTailRows != 0) { + for (int col_tile = 0; col_tile < kColTiles; ++col_tile) { + // 输入 tile：kTailRows × TileCols（valid region） + using SrcBottomTile = + pto::Tile; + // 输出 tile：TileCols × kTailRows（转置后） + using DstRightTile = + pto::Tile; + + auto input_global = input_iter(kRowTiles, col_tile); + auto output_global = output_iter(col_tile, kRowTiles); + + SrcBottomTile src_tile; + DstRightTile dst_tile; + + TLOAD(src_tile, input_global); +#if SUPERNPU_PTO_TTRANS_NEEDS_TMP + SrcBottomTile tmp_tile; + TTRANS(dst_tile, src_tile, tmp_tile); +#else + TTRANS(dst_tile, src_tile); +#endif + TSTORE(output_global, dst_tile); + } + + // ========== 处理右下角尾部 tile ========== + if constexpr (kTailCols != 0) { + // 输入 tile：kTailRows × kTailCols（valid region） + using SrcCornerTile = + pto::Tile; + // 输出 tile：kTailCols × kTailRows（转置后） + using DstCornerTile = + pto::Tile; + + auto input_global = input_iter(kRowTiles, kColTiles); + auto output_global = output_iter(kColTiles, kRowTiles); + + SrcCornerTile src_tile; + DstCornerTile dst_tile; + + TLOAD(src_tile, input_global); +#if SUPERNPU_PTO_TTRANS_NEEDS_TMP + SrcCornerTile tmp_tile; + TTRANS(dst_tile, src_tile, tmp_tile); +#else + TTRANS(dst_tile, src_tile); +#endif + TSTORE(output_global, dst_tile); + } + } +} + +} // namespace supernpu::tile_isa + +template +void transpose(DType *input, DType *output, std::uint32_t *input_shape, + std::uint32_t *output_shape) { + static_assert(gIM == gOM, "transpose preserves element count"); + static_assert(IN_DIM == OUT_DIM, "transpose rank must be preserved"); + (void)MAX_DIM; + supernpu::tile_isa::tile_transpose_nd< + DType, static_cast(IN_DIM), static_cast(TRANSPOSE_DIM0), + static_cast(TRANSPOSE_DIM1), gOM, tM>(input, output, input_shape, + output_shape); } #endif diff --git a/benchmark-linxisa/kernels/transpose/transpose_vector_007.hpp b/benchmark-linxisa/kernels/transpose/transpose_vector_007.hpp index c9fbe07..254ac86 100644 --- a/benchmark-linxisa/kernels/transpose/transpose_vector_007.hpp +++ b/benchmark-linxisa/kernels/transpose/transpose_vector_007.hpp @@ -1,59 +1,91 @@ +/** + * @file transpose_vector_007_tile.hpp + * @brief 4096×3 矩阵转置为 3×4096 矩阵（Tile ISA 实现，使用 TTRANS） + * + * 输入：4096×3 矩阵 + * 输出：3×4096 矩阵 + * + * 算法：分块 2D 转置 + * - 将 4096×3 分成多个 TileRows×TileCols 的小块 + * - 对每个小块用 TTRANS 转置 + * - 存储到输出的对应位置 + */ + +#ifndef TRANSPOSE_VECTOR_007_TILE_HPP +#define TRANSPOSE_VECTOR_007_TILE_HPP + #include -#include "gemm.hpp" +#include using namespace pto; -//AI/IA = A, placeholder -//like Ttrans -template -void __vec__ transpose_007_impl( - typename tileOutData::TileDType __out__ out, - const typename tileInData::TileDType __in__ in +/** + * @brief 4096×3 → 3×4096 转置（使用 TTRANS） + * + * 分块策略： + * - TileRows = 16, TileCols = 3 + * - 输入块：16×3 + * - 输出块：3×16（TTRANS 后） + * - 共 256 个块（4096/16 = 256） + * + * @tparam dtype 数据类型 + * @param out_ptr 输出指针（3×4096） + * @param in_ptr 输入指针（4096×3） + */ +template +void transpose_007_tile( + dtype *out_ptr, + dtype *in_ptr ) { - size_t i = blkv_get_index_x(); // 4096 - size_t j = blkv_get_index_y(); // 3 + constexpr int INPUT_ROWS = 4096; + constexpr int INPUT_COLS = 3; + constexpr int OUTPUT_ROWS = 3; + constexpr int OUTPUT_COLS = 4096; - size_t idx_src = i * tileOutData::ValidRow + j; - size_t idx_dst = i * tileOutData::ColStride + j * tileOutData::RowStride; - blkv_get_tile_ptr(out)[idx_dst] = blkv_get_tile_ptr(in)[idx_src]; -} + // 分块参数 + constexpr int TILE_ROWS = 16; // 输入块行数 + constexpr int TILE_COLS = 3; // 输入块列数 + constexpr int NUM_ROW_TILES = INPUT_ROWS / TILE_ROWS; // 256 + constexpr int NUM_COL_TILES = INPUT_COLS / TILE_COLS; // 1 + // 全局张量定义 + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; -// output = input * A^T + bias ;;; input = [batch_size, in_features] , A = [out_features, in_features] -// A should be row majo + // Tile 定义 + using SrcTile = Tile; + using DstTile = Tile; -// y = x1^T.A.x2 + b, where x1=1xd1, x2=1xd2, A=doxd1xd2 -// in1[DimIn1, 1] in2 [DimIn2, 1] bias[DimOut, 1] weight [DimOut, DimIn1, DimIn2] -template -void transpose_007( - dtype *out_ptr, - dtype *in_ptr -) -{ - const int Mb = 4096 / 4096; + // 迭代器 + using InputIterator = global_iterator; + using OutputIterator = global_iterator; + + InputIterator input_iter(in_ptr); + OutputIterator output_iter(out_ptr); - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_shapeOut = global_tensor>; - using tile_shapeInData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - using tile_shapeOutData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 + // 遍历所有块 + for (int row_tile = 0; row_tile < NUM_ROW_TILES; ++row_tile) { + for (int col_tile = 0; col_tile < NUM_COL_TILES; ++col_tile) { + // 输入位置：(row_tile, col_tile) -> 对应输入矩阵的 [row_tile*16 : (row_tile+1)*16, col_tile*3 : (col_tile+1)*3] + auto input_global = input_iter(row_tile, col_tile); - using itIn = global_iterator; - using itOut = global_iterator; + // 输出位置：(col_tile, row_tile) -> 对应输出矩阵的 [col_tile*3 : (col_tile+1)*3, row_tile*16 : (row_tile+1)*16] + auto output_global = output_iter(col_tile, row_tile); - tile_shapeInData InDataTile; - tile_shapeOutData OutDataTile; + SrcTile src_tile; + DstTile dst_tile; - itIn gIIter(in_ptr); - itOut gOIter(out_ptr); + // 加载输入块（16×3） + TLOAD(src_tile, input_global); - for (int i = 0; i < Mb; ++i) { - auto gI = gIIter(0, i); - auto gO = gOIter(0, i); - TCOPYIN(InDataTile, gI); - transpose_007_impl<<>>(OutDataTile.data(), InDataTile.data()); - TCOPYOUT(gO, OutDataTile); - } + // 转置：16×3 -> 3×16 + TTRANS(dst_tile, src_tile); + // 存储到输出 + TSTORE(output_global, dst_tile); + } + } } +#endif // TRANSPOSE_VECTOR_007_TILE_HPP diff --git a/benchmark-linxisa/kernels/transpose/transpose_vector_050.hpp b/benchmark-linxisa/kernels/transpose/transpose_vector_050.hpp index d280dd3..10b535f 100644 --- a/benchmark-linxisa/kernels/transpose/transpose_vector_050.hpp +++ b/benchmark-linxisa/kernels/transpose/transpose_vector_050.hpp @@ -1,58 +1,94 @@ +/** + * @file transpose_vector_050_tile.hpp + * @brief 8×64×64 张量转置为 64×8×64 张量（Tile ISA 实现，逐行处理） + * + * 输入：8×64×64 张量（展平为 512×64） + * 输出：64×8×64 张量（展平为 512×64） + * + * 转置关系：交换第 0 维和第 1 维 + * - input[a][b][c] → output[b][a][c] + * - 其中 a∈[0,8), b∈[0,64), c∈[0,64) + * + * 行置换关系： + * - 输入行号 = a*64 + b + * - 输出行号 = b*8 + a + * - 每行 64 个元素 + * + * 算法：逐行复制 + * - 对每个 (a, b) 对，复制 input 行 (a*64+b) 到 output 行 (b*8+a) + * - 共 8×64 = 512 行，每行用 TLOAD + TSTORE 处理 + */ + +#ifndef TRANSPOSE_VECTOR_050_TILE_HPP +#define TRANSPOSE_VECTOR_050_TILE_HPP + #include -#include "gemm.hpp" +#include using namespace pto; - -template -void __vec__ transpose_050_impl( - typename tileData::TileDType __out__ out, - const typename tileData::TileDType __in__ in -) -{ - size_t i = blkv_get_index_x(); // y - size_t j = blkv_get_index_y(); // x 4096*3 - - size_t row_out = j / 8; - size_t col_out = j % 8; - - size_t idx_in = col_out * 64 + row_out; - - size_t idx_src = i * tileData::ColStride + idx_in * tileData::RowStride; - size_t idx_dst = i * tileData::ColStride + j * tileData::RowStride; - blkv_get_tile_ptr(out)[idx_dst] = blkv_get_tile_ptr(in)[idx_src]; -} - - +/** + * @brief 8×64×64 → 64×8×64 转置（逐行处理） + * + * 实现方式： + * - 外层循环遍历 a ∈ [0, 8) + * - 内层循环遍历 b ∈ [0, 64) + * - 对每个 (a, b)，将 input 行 (a*64+b) 复制到 output 行 (b*8+a) + * + * 每行处理： + * - 用 1×64 的 tile 加载一行 + * - 用 TSTORE 存储到目标位置 + * + * @tparam dtype 数据类型 + * @param out_ptr 输出指针（64×8×64 = 512×64） + * @param in_ptr 输入指针（8×64×64 = 512×64） + */ template -void transpose_050( - dtype *out_ptr, - dtype *in_ptr +void transpose_050_tile( + dtype *out_ptr, + dtype *in_ptr ) -{ - - - using gm_shapeIn = global_tensor>; //将gm中的Tensor先声明为一维数据 - using gm_shapeOut = global_tensor>; - using tile_shapeData = Tile; // todo 尾块怎么处理？是否要作为参数写在这 - - using itIn = global_iterator; - using itOut = global_iterator; - - tile_shapeData InDataTile; - tile_shapeData OutDataTile; - - itIn gIIter(in_ptr); - itOut gOIter(out_ptr); - - - auto gI = gIIter(0, 0); - auto gO = gOIter(0, 0); - TCOPYIN(InDataTile, gI); - transpose_050_impl<<>>(OutDataTile.data(), InDataTile.data()); - TCOPYOUT(gO, OutDataTile); - - +{ + constexpr int DIM0 = 8; // 第 0 维大小 + constexpr int DIM1 = 64; // 第 1 维大小 + constexpr int DIM2 = 64; // 第 2 维大小（每行元素数） + constexpr int TOTAL_ROWS = DIM0 * DIM1; // 512 + + // 全局张量定义 + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + + // 行 tile：1×64 + using RowTile = Tile; + + // 行迭代器 + using RowIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputGlobal output_global(out_ptr); + RowIterator input_row_iter(in_ptr); + RowIterator output_row_iter(out_ptr); + + RowTile row_tile; + + // 逐行处理 + for (int a = 0; a < DIM0; ++a) { + for (int b = 0; b < DIM1; ++b) { + // 源行号：a*64 + b + int src_row = a * DIM1 + b; + + // 目标行号：b*8 + a + int dst_row = b * DIM0 + a; + + // 加载源行 + auto src_global = input_row_iter(src_row, 0); + TLOAD(row_tile, src_global); + + // 存储到目标行 + auto dst_global = output_row_iter(dst_row, 0); + TSTORE(dst_global, row_tile); + } + } } - +#endif // TRANSPOSE_VECTOR_050_TILE_HPP diff --git a/benchmark-linxisa/kernels/utils/layout_transform.hpp b/benchmark-linxisa/kernels/utils/layout_transform.hpp index 3e07196..62b40f8 100644 --- a/benchmark-linxisa/kernels/utils/layout_transform.hpp +++ b/benchmark-linxisa/kernels/utils/layout_transform.hpp @@ -1,3 +1,9 @@ +#include + +using namespace pto::blkv; +using pto::is_global_data_v; +using pto::is_tile_data_v; + template void __vec__ gen_offset_ND2ZZ( typename tile_shape::TileDType __out__ out, @@ -98,8 +104,12 @@ void gen_ND2ZZ_offset_Impl( const uint32_t j) { static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, "Only static shape supported"); - // gen_offset_ND2ZZ<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); - gen_offset_ND2ZZ<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); + pto::blkv::blkv_for_1d(tl_tensor::ValidCol, [&] { + gen_offset_ND2ZZ(offset.data(), glb_tensor::ColStride, + glb_tensor::RowStride, + tl_tensor::ValidRow, + tl_tensor::ValidCol, i, j); + }); } template @@ -111,7 +121,9 @@ void gen_ND2NN_offset_Impl( const uint32_t j) { static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, "Only static shape supported"); - // gen_offset_ND2NN<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); - // gen_offset_ND2NN<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); - gen_offset_ND2NN_new<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); -} \ No newline at end of file + pto::blkv::blkv_for_1d(tl_tensor::ValidRow, [&] { + gen_offset_ND2NN_new( + offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, + tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); + }); +} diff --git a/benchmark-linxisa/test/common/Makefile.common b/benchmark-linxisa/test/common/Makefile.common index 35caf4d..6b97f6b 100644 --- a/benchmark-linxisa/test/common/Makefile.common +++ b/benchmark-linxisa/test/common/Makefile.common @@ -1,8 +1,11 @@ ROOT := $(shell echo $(CURDIR) | sed -E 's@(.*)/(benchmark-linxisa|benchmark-ptoisa)/.*@\1/\2@') +LINX_ISA_ROOT ?= $(shell cd $(ROOT)/../../.. && pwd) +LINX_SYSROOT ?= $(LINX_ISA_ROOT)/out/libc/musl/install/phase-b +PTO_KERNELS_INCLUDE ?= $(LINX_ISA_ROOT)/workloads/pto_kernels/include TEST_ROOT := $(shell echo $(CURDIR) | sed -E 's@(.*)/(benchmark-linxisa|benchmark-ptoisa)/test/.*@\1/\2/test@') CATEGORY := $(shell echo $(CURDIR) | sed -E 's@.*/(benchmark-linxisa|benchmark-ptoisa)/test/(.*)@\2@') CATEGORY_NAME := $(shell echo $(CATEGORY) | sed -e 's/\//_/g') -OBJ_ROOT := $(shell realpath $(TEST_ROOT)/../output) +OBJ_ROOT := $(abspath $(TEST_ROOT)/../output) CASE_SRC_DIR := $(CATEGORY)/src ELF_DIR := $(OBJ_ROOT)/$(CATEGORY)/elf SRC_DIR := $(shell dirname $(SRC_FILE)) @@ -51,7 +54,7 @@ CXX_VER ?= -std=c++20 else ifeq ($(PLAT), linx) DEFINES += -D__linx ifndef COMPILER_DIR -$(error COMPILER_DIR is not set. Export COMPILER_DIR pointing to the linx_blockisa_llvm_musl toolchain bin directory, e.g. export COMPILER_DIR=/path/to/linx_blockisa_llvm_musl/bin) +$(error COMPILER_DIR is not set. Export COMPILER_DIR pointing to the Linx Clang bin directory) endif AS = $(COMPILER_DIR)/clang CC = $(COMPILER_DIR)/clang @@ -59,17 +62,16 @@ CXX = $(COMPILER_DIR)/clang++ LINK = $(COMPILER_DIR)/clang++ DUMP = $(COMPILER_DIR)/llvm-objdump COPY = $(COMPILER_DIR)/llvm-objcopy -CC_O = -c -mlxbc -fenable-matrix -O2 -mllvm -enable-all-vector-as-tilereg=true $(CFLAGS) +LINX_TARGET ?= linx64-unknown-linux-musl +LINX_CFLAGS ?= -mlxbc --target=$(LINX_TARGET) -fenable-matrix -O2 +LINX_LDFLAGS ?= -nostdlib++ -unwindlib=none +ifneq ($(wildcard $(LINX_SYSROOT)/usr/include/bits/alltypes.h),) +LINX_CFLAGS += --sysroot=$(LINX_SYSROOT) -idirafter $(LINX_SYSROOT)/usr/include -D_GNU_SOURCE +CC_LINK += --sysroot=$(LINX_SYSROOT) +endif +CC_O = -c $(LINX_CFLAGS) $(CFLAGS) +CC_LINK += $(LINX_CFLAGS) $(LINX_LDFLAGS) CXX_VER ?= -std=c++20 - ifneq ($(CC_OPT), default) - CC_O += -mllvm -linxv5-enable-HL-Inst-Opt=true \ - -mllvm -linxv5-enable-dim-opt=true \ - -mllvm -linxv5-enable-ldst-bridge=false \ - -mllvm -linxv5-enable-continuous-mem-opt=true \ - -mllvm -linxv5-enable-tile-clock-hand=false \ - -mllvm -linxv5-enable-simt-clock-hand=true \ - -mllvm -enable-misched=false - endif ifeq ($(baremetal), on) CC_LINK += -static -lm -nostartfiles -L $(OBJ_ROOT)/$(COMM_SRC_DIR) -T $(LINK_SCRIPT) @@ -92,7 +94,7 @@ CC_O += -fPIC CC_LINK += -shared endif -INCLUDE += -I$(ROOT)/include -I$(ROOT)/test/common -I$(ROOT)/test/common/src -I$(ROOT)/kernels -I$(ROOT)/models +INCLUDE += -I$(ROOT)/include -I$(ROOT)/test -I$(ROOT)/test/common -I$(ROOT)/test/common/src -I$(ROOT)/kernels -I$(ROOT)/models -I$(PTO_KERNELS_INCLUDE) QEMU ?= /remote/lms60/c00622284/qemu/LinxBlockModel/build/qemu-linx CC_O_ALL = $(CC_O) $(CC_OPTS) diff --git a/benchmark-linxisa/test/common/_start.s b/benchmark-linxisa/test/common/_start.s index 150e10c..654d9bb 100644 --- a/benchmark-linxisa/test/common/_start.s +++ b/benchmark-linxisa/test/common/_start.s @@ -5,7 +5,7 @@ _start: bstart.std call main c.setret 2, ->ra _end: - bstart.aux fall + bstart.sys fall addi zero, 0x5e, ->x1 acrc 1 - c.bstop \ No newline at end of file + c.bstop diff --git a/benchmark-linxisa/test/common/src/benchmark_boot_linx.s b/benchmark-linxisa/test/common/src/benchmark_boot_linx.s index 4cae8da..146c0f5 100644 --- a/benchmark-linxisa/test/common/src/benchmark_boot_linx.s +++ b/benchmark-linxisa/test/common/src/benchmark_boot_linx.s @@ -7,7 +7,7 @@ _start: bstart.std call _linx_start c.setret 2, ->ra _end: - bstart.aux fall + bstart.sys fall addi zero, 0x5e, ->x1 acrc 1 - c.bstop \ No newline at end of file + c.bstop diff --git a/benchmark-linxisa/test/common/template_asm.h b/benchmark-linxisa/test/common/template_asm.h index 2f60707..8f1e0d6 100644 --- a/benchmark-linxisa/test/common/template_asm.h +++ b/benchmark-linxisa/test/common/template_asm.h @@ -3,78 +3,66 @@ #include +using namespace pto; + +template +struct pto_v057_tile_alloc { + static constexpr unsigned SizeCode = tile_type_traits::TilesizeCode; + static_assert(SizeCode >= 3, "v0.57 B.OTA allocation must be at least one 128-byte CELL"); + static constexpr unsigned CellCountM1 = (1u << (SizeCode - 3u)) - 1u; +}; + template void MGATHER(tile_shape_out &dst, gm_shape &src, tile_shape_offset &offset) { - asm volatile( - "BSTART.TMA 4, %c[DataType]\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT [%[s1]], last, ->%[d0]<%c[TileSize]>\n" - "B.IOR [%[s0]], []\n" - : [d0]"=Tr"(dst.data()) - : [s0]"r"(src.data()), - [s1]"Tr"(offset.data()), - [DataType]"i"(type_traits::TypeCode), - [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(tile_shape_offset::ValidCol), [VROW]"i"(tile_shape_offset::ValidRow) - ); + pto::MGATHER(dst, src, offset); } template void MSCATTER(gm_shape &dst, tile_shape_in &src, tile_shape_offset &offset) { - asm volatile( - "BSTART.TMA 5, %c[SrcType]\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT [%[s0], %[s1]], last\n" - "B.IOR [%[d0]], []\n" - : - : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), - [s1]"Tr"(offset.data()), - [SrcType]"i"(type_traits::TypeCode), - [VCOL]"i"(tile_shape_offset::ValidCol), [VROW]"i"(tile_shape_offset::ValidRow) - ); + pto::MSCATTER(dst, src, offset); } template void TMAX_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 11, %c1\n" + "BSTART.TEPL 0x25, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TMULS_TEPL(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s) { asm volatile( - "BSTART.TEPL 0b0100010, %c1\n" + "BSTART.TEPL 0x2B, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "B.IOR [%7],[]\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "i"(tile_type_traits::TilesizeCode), + "Tr"(src0.raw()), + "i"(pto_v057_tile_alloc::CellCountM1), "r"(s) ); } @@ -82,183 +70,191 @@ void TMULS_TEPL(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s) template void TROWMAX_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1000001, %c1\n" + "BSTART.TEPL 0x47, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TSUB_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 1, %c1\n" + "BSTART.TEPL 0x55, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TEXP_TEPL(tile_shape &dst, tile_shape &src) { asm volatile( - "BSTART.TEPL 18, %c1\n" + "BSTART.TEPL 0x1C, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TMUL_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 2, %c1\n" + "BSTART.TEPL 0x2A, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TROWSUM_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1000000, %c1\n" + "BSTART.TEPL 0x4A, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TADD_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 0, %c1\n" + "BSTART.TEPL 0x01, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TRECIP_TEPL(tile_shape &dst, tile_shape &src) { asm volatile( - "BSTART.TEPL 20, %c1\n" + "BSTART.TEPL 0x39, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCAST_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 27, %c1\n" + "BSTART.TEPL 0x19, %c1\n" "B.DATR %c2, RNONE\n" "C.B.DIMI %c3, ->LB0\n" "C.B.DIMI %c4, ->LB1\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TEXPANDSCALAR_TEPL(tile_shape &dst, typename tile_shape::DType s) { asm volatile( - "BSTART.TEPL 0b0111011, %c1\n" + "BSTART.TEPL 0x1D, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [], last, ->%0<%c5>\n" + "B.OTA ->%0<%c5>, last, 0\n" "B.IOR [%6],[]\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "i"(tile_type_traits::TilesizeCode), + "i"(pto_v057_tile_alloc::CellCountM1), "r"(s) ); } @@ -266,101 +262,106 @@ void TEXPANDSCALAR_TEPL(tile_shape &dst, typename tile_shape::DType s) { template void TROWEXPAND_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1000100, %c1\n" + "BSTART.TEPL 0x3F, %c1\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLMAX_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1010001, %c1\n" + "BSTART.TEPL 0x15, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLSUM_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1010000, %c1\n" + "BSTART.TEPL 0x18, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLEXPANDSUB_TEPL(tile_shape_out &dst, tile_shape_out &src0, tile_shape_in &src1) { asm volatile( - "BSTART.TEPL 0b1010110, %c1\n" + "BSTART.TEPL 0x14, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLEXPANDMUL_TEPL(tile_shape_out &dst, tile_shape_out &src0, tile_shape_in &src1) { asm volatile( - "BSTART.TEPL 0b1010111, %c1\n" + "BSTART.TEPL 0x13, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } diff --git a/benchmark-linxisa/test/kernel/control/compile.all b/benchmark-linxisa/test/kernel/control/compile.all index 6ec69f0..b4344c8 100755 --- a/benchmark-linxisa/test/kernel/control/compile.all +++ b/benchmark-linxisa/test/kernel/control/compile.all @@ -5,7 +5,7 @@ for debug in on off; do else debug_define="-DFOR_GFSIM" fi - for num_col in 256 512 1024; do + for num_col in 256 512; do make TESTCASE=hashtable_lookup_simd SUFFIX=_kNum6144_kMaxProbe512_knum_col${num_col}_debug_${debug} EXTRA_DEFINES="-DkNum=6144 -DMAX_PROBE=512 -DNUM_COL=${num_col} ${debug_define}" diss done done diff --git a/benchmark-linxisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh b/benchmark-linxisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh index 2668212..234f7b4 100755 --- a/benchmark-linxisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh +++ b/benchmark-linxisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh @@ -1,5 +1,5 @@ #!/bin/bash -COMPILER_DIR="${COMPILER_DIR:-/remote/lms60/c00622284/janus/linxisa_compiler_v0.55/linx_blockisa_llvm_musl/bin}" +: "${COMPILER_DIR:?COMPILER_DIR must point to the Linx Clang bin directory}" DATA_OBJ_DIR="$1" OUTPUT_DIR="$2" @@ -26,11 +26,11 @@ _binary_${name}_data_end: .equ _binary_${name}_data_size, .-_binary_${name}_data_start EOF - $COMPILER_DIR/clang++ -target linx64v5 -c "$asm_file" -o "$obj_file" + "$COMPILER_DIR/clang++" -mlxbc -c "$asm_file" -o "$obj_file" } build_one "inserted_slot" build_one "lookup_keys" build_one "lookup_values" -echo "Done building data object files" \ No newline at end of file +echo "Done building data object files" diff --git a/benchmark-linxisa/test/kernel/control/hkv/data_obj/build_data_obj.sh b/benchmark-linxisa/test/kernel/control/hkv/data_obj/build_data_obj.sh index ec4819c..f0c7423 100755 --- a/benchmark-linxisa/test/kernel/control/hkv/data_obj/build_data_obj.sh +++ b/benchmark-linxisa/test/kernel/control/hkv/data_obj/build_data_obj.sh @@ -1,5 +1,5 @@ #!/bin/bash -COMPILER_DIR="${COMPILER_DIR:-/remote/lms01/j00827727/jcore/compilers/linx_blockisa_llvm_musl0.56.16/bin}" +: "${COMPILER_DIR:?COMPILER_DIR must point to the Linx Clang bin directory}" DATA_OBJ_DIR="$1" OUTPUT_DIR="$2" @@ -28,7 +28,7 @@ _binary_${sym_name}_end: .equ _binary_${sym_name}_size, .-_binary_${sym_name}_start EOF - $COMPILER_DIR/clang++ -target linx64v5 -c "$asm_file" -o "$obj_file" + "$COMPILER_DIR/clang++" -mlxbc -c "$asm_file" -o "$obj_file" } build_one "buckets.bin" diff --git a/benchmark-linxisa/test/kernel/element_wise/gelu/compile.all b/benchmark-linxisa/test/kernel/element_wise/gelu/compile.all index 84d901c..b1f3da1 100644 --- a/benchmark-linxisa/test/kernel/element_wise/gelu/compile.all +++ b/benchmark-linxisa/test/kernel/element_wise/gelu/compile.all @@ -7,7 +7,7 @@ # BF16, exact mode # ============================================ # 3D shape: (24, 8, 1024) -make TESTCASE=gelu DTYPE=__bf16 tMs=2048 gMs=24*8*1024 \ +make TESTCASE=gelu DTYPE=pto_bf16_t tMs=2048 gMs=24*8*1024 \ SHAPE_NAME=24_8_1024 Approximate=false diss # # 2D shape: (128, 1024) diff --git a/benchmark-linxisa/test/kernel/fa/Makefile b/benchmark-linxisa/test/kernel/fa/Makefile index c9b5c73..4b00c5b 100644 --- a/benchmark-linxisa/test/kernel/fa/Makefile +++ b/benchmark-linxisa/test/kernel/fa/Makefile @@ -14,12 +14,16 @@ ifeq ($(TESTCASE), fa_HIF4_HIF4) Tk = 128 X = 2 Y = 4 + qD = 128 + vD = 128 DEFINES += -DTsq=$(Sq) DEFINES += -DTskv=$(Skv) DEFINES += -DTm=$(Tm) DEFINES += -DTk=$(Tk) DEFINES += -DXdim=$(X) DEFINES += -DYdim=$(Y) + DEFINES += -DTqD=$(qD) + DEFINES += -DTvD=$(vD) ifneq ($(MODE), ) DEFINES += -D$(MODE) endif @@ -44,6 +48,8 @@ ifeq ($(TESTCASE), fa_2d_unroll) Tk = 128 X = 1 Y = 1 + qD = 128 + vD = 128 ifneq ($(X), ) X = $(X) endif @@ -56,7 +62,9 @@ ifeq ($(TESTCASE), fa_2d_unroll) DEFINES += -DTk=$(Tk) DEFINES += -DXdim=$(X) DEFINES += -DYdim=$(Y) - TARGET = $(ELF_HEAD)/$(TESTCASE)_Sq$(Sq)_Skv$(Skv)_Tm$(Tm)_Tk$(Tk)_X$(X)_Y$(Y).elf + DEFINES += -DTqD=$(qD) + DEFINES += -DTvD=$(vD) + TARGET = $(ELF_HEAD)/$(TESTCASE)_Sq$(Sq)_Skv$(Skv)_qD$(qD)_vD$(vD)_Tm$(Tm)_Tk$(Tk)_X$(X)_Y$(Y).elf endif include ../../common/Makefile.common diff --git a/benchmark-linxisa/test/kernel/fa/compile.all b/benchmark-linxisa/test/kernel/fa/compile.all index 52c7464..cad6bdf 100644 --- a/benchmark-linxisa/test/kernel/fa/compile.all +++ b/benchmark-linxisa/test/kernel/fa/compile.all @@ -5,24 +5,25 @@ # fa_2d_unroll: 2D unrolled flash attention # ============================================ # X=1, Y=2 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=2 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=1 Y=2 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=2 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=2 # X=1, Y=4 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=4 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=1 Y=4 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=4 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=4 # X=2, Y=2 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=2 Y=2 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=2 Y=2 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=2 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=2 # X=2, Y=4 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=2 Y=4 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=2 Y=4 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=4 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=4 # ============================================ # fa_HIF4_HIF4: HIF4 quantized flash attention +# Requires block-vector launch migration before it is v0.57-active. # ============================================ -make TESTCASE=fa_HIF4_HIF4 MODE=BF16_NOGATHER Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=1 +# make TESTCASE=fa_HIF4_HIF4 MODE=BF16_NOGATHER Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=1 # make TESTCASE=fa_HIF4_HIF4 MODE=BF16x2 Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=1 # make TESTCASE=fa_HIF4_HIF4 MODE=BF16x2_NOGATHER Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=1 diff --git a/benchmark-linxisa/test/kernel/fa/src/fa_2d_unroll.cpp b/benchmark-linxisa/test/kernel/fa/src/fa_2d_unroll.cpp index d57e0e2..b7024b0 100644 --- a/benchmark-linxisa/test/kernel/fa/src/fa_2d_unroll.cpp +++ b/benchmark-linxisa/test/kernel/fa/src/fa_2d_unroll.cpp @@ -18,8 +18,17 @@ #define Skv Tskv #endif +#ifndef TqD #define qD 128 +#else +#define qD TqD +#endif + +#ifndef TvD #define vD 128 +#else +#define vD TvD +#endif #ifndef Tm #define kTm 128 diff --git a/benchmark-linxisa/test/kernel/gather/compile.all b/benchmark-linxisa/test/kernel/gather/compile.all index f891426..588626f 100755 --- a/benchmark-linxisa/test/kernel/gather/compile.all +++ b/benchmark-linxisa/test/kernel/gather/compile.all @@ -25,4 +25,4 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/outpu # Power-of-2 dimensions make TESTCASE=gather COMPILER_DIR=$COMPILER_DIR DType=__fp32 OType=uint32_t \ gKs=131072 gMs=32 gNs=256 \ - tMs=32 tNs=128 + tMs=32 tNs=32 diff --git a/benchmark-linxisa/test/kernel/matmul/compile.all b/benchmark-linxisa/test/kernel/matmul/compile.all index f48dbfb..3f158c3 100755 --- a/benchmark-linxisa/test/kernel/matmul/compile.all +++ b/benchmark-linxisa/test/kernel/matmul/compile.all @@ -23,39 +23,39 @@ # ============================================ # HIF4_HIF4: MX_NOGATHER variant - 2 configs # ============================================ -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=256 N=2048 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=512 N=1280 K=4096 tM=128 tN=128 tK=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=256 N=2048 K=2048 tM=32 tN=32 tK=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=512 N=1280 K=4096 tM=32 tN=32 tK=128 # ============================================ # HIF4_HIF4: MX_NOGATHER_REUSEA variant - 2 configs # ============================================ -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=256 N=2048 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=512 N=1280 K=4096 tM=128 tN=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=256 N=2048 K=2048 tM=32 tN=32 tK=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=512 N=1280 K=4096 tM=32 tN=32 tK=128 # ============================================ # A16W4: BF16 x FP4 mixed precision - 3 configs # ============================================ -make TESTCASE=matmul TYPE=A16W4 M=256 N=2048 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=A16W4 M=512 N=1280 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=A16W4 M=512 N=512 K=4096 tM=128 tN=256 tK=128 +make TESTCASE=matmul TYPE=A16W4 M=256 N=2048 K=2048 tM=16 tN=16 tK=128 +make TESTCASE=matmul TYPE=A16W4 M=512 N=1280 K=2048 tM=16 tN=16 tK=128 +make TESTCASE=matmul TYPE=A16W4 M=512 N=512 K=4096 tM=16 tN=16 tK=128 # ============================================ # MASK: Generic matmul variants - 8 configs # ============================================ # FP32 variants -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32 M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEA M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEB M=256 N=256 K=256 tM=64 tN=64 tK=64 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32 M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEA M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEB M=256 N=256 K=256 tM=32 tN=32 tK=32 # 黄区能过，蓝区过不了 # make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_DYNAMIC M=256 N=256 K=256 tM=64 tN=64 tK=64 # FP16 variants -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16 M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEA M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEB M=256 N=256 K=256 tM=64 tN=64 tK=64 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16 M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEA M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEB M=256 N=256 K=256 tM=32 tN=32 tK=32 # FP8 variants -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8 M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEA M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEB M=256 N=256 K=256 tM=64 tN=64 tK=64 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8 M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEA M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEB M=256 N=256 K=256 tM=32 tN=32 tK=32 # make TESTCASE=matmul TYPE=MASK MODE=MX_FP8 M=256 N=256 K=256 tM=64 tN=64 tK=64 diff --git a/benchmark-linxisa/test/kernel/matmul/src/A16W4.cpp b/benchmark-linxisa/test/kernel/matmul/src/A16W4.cpp index e6af690..b4a2906 100644 --- a/benchmark-linxisa/test/kernel/matmul/src/A16W4.cpp +++ b/benchmark-linxisa/test/kernel/matmul/src/A16W4.cpp @@ -41,7 +41,7 @@ int main() { // bf16*fp4 using fp4_t = __fp4_e2m1x2; - using bf16_t = __bf16; + using bf16_t = pto_bf16_t; static_assert(tilM % 2 == 0); // 暂时假定tile是偶数的，方便取地址，奇数tile实现需要末尾padding 0对齐地址 static_assert(tilN % 2 == 0); static_assert(tilK == (128)); @@ -68,4 +68,4 @@ int main() { #endif return 0; -} \ No newline at end of file +} diff --git a/benchmark-linxisa/test/kernel/reduction/reducemax_col/compile.all b/benchmark-linxisa/test/kernel/reduction/reducemax_col/compile.all index d54fa35..8f8a50c 100755 --- a/benchmark-linxisa/test/kernel/reduction/reducemax_col/compile.all +++ b/benchmark-linxisa/test/kernel/reduction/reducemax_col/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=256 tN_s=64 \ +make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=2048 gN_s=64 # Large matrix, __half -#make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=256 tN_s=64 \ +#make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ # gM_s=2048 gN_s=64 diff --git a/benchmark-linxisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp b/benchmark-linxisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp index a9cb6ea..dc767c9 100644 --- a/benchmark-linxisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp +++ b/benchmark-linxisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp @@ -11,28 +11,30 @@ #endif -#ifndef tM -#define tM 32 +#ifndef tMs +#define tMs 32 #endif -#ifndef tN -#define tN 128 +#ifndef tNs +#define tNs 128 #endif +#ifndef gIMs +#define gIMs 256 +#endif - - -#define gIM 256 -#define gIN 256 +#ifndef gINs +#define gINs 256 +#endif // ============================================================================ // main // ============================================================================ int main() { using dtype = DType; - dtype input_buf[gIM*gIN]; + dtype input_buf[gIMs*gINs]; // dtype zero_buf[1*gIN]; - dtype output_buf[1*gIN]; + dtype output_buf[1*gINs]; dtype* input=input_buf; // dtype* zero=zero_buf; @@ -42,8 +44,7 @@ int main() { // readBinaryFile("/remote/lms01/q50057645/jcore_project/JanusCoreBench/test/ascpp/reducemax_col/src/data_8192x1024.bin", (uint8_t*)input, gIM * gIN * sizeof(dtype)); // readBinaryFile("/remote/lms01/q50057645/jcore_project/JanusCoreBench/test/ascpp/reducemax_col/src/data1x256_zero.bin", (uint8_t*)zero, gIN * sizeof(dtype)); // reducesum_colsum_rand(input, output); - reducemax_col_rand(input, output); + reducemax_col_rand(input, output); // writeBinaryFile("/remote/lms01/q50057645/jcore_project/JanusCoreBench/test/ascpp/reducemax_col/src/result_max.bin", (uint8_t*)output, gIN * sizeof(dtype)); } - diff --git a/benchmark-linxisa/test/kernel/reduction/reducemax_row/compile.all b/benchmark-linxisa/test/kernel/reduction/reducemax_row/compile.all index 92bfeed..8a0e379 100755 --- a/benchmark-linxisa/test/kernel/reduction/reducemax_row/compile.all +++ b/benchmark-linxisa/test/kernel/reduction/reducemax_row/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=512 \ +make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=16 gN_s=8192 # Large matrix, __half (DISABLED: Tile alignment < 32 bytes for _Float16 with Cols=8) -# make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=512 \ +# make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ # gM_s=16 gN_s=8192 diff --git a/benchmark-linxisa/test/kernel/reduction/reducesum_col/compile.all b/benchmark-linxisa/test/kernel/reduction/reducesum_col/compile.all index c9daba1..d3b6098 100755 --- a/benchmark-linxisa/test/kernel/reduction/reducesum_col/compile.all +++ b/benchmark-linxisa/test/kernel/reduction/reducesum_col/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=256 tN_s=64 \ +make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=2048 gN_s=64 # Large matrix, __half -make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=256 tN_s=64 \ +make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ gM_s=2048 gN_s=64 diff --git a/benchmark-linxisa/test/kernel/reduction/reducesum_row/compile.all b/benchmark-linxisa/test/kernel/reduction/reducesum_row/compile.all index 4bfab51..8f360f3 100755 --- a/benchmark-linxisa/test/kernel/reduction/reducesum_row/compile.all +++ b/benchmark-linxisa/test/kernel/reduction/reducesum_row/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=512 \ +make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=16 gN_s=8192 # Large matrix, __half (DISABLED: Tile alignment < 32 bytes for _Float16 with Cols=8) -# make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=512 \ +# make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ # gM_s=16 gN_s=8192 diff --git a/benchmark-linxisa/test/kernel/sort/topk/data_obj/build_data_obj.sh b/benchmark-linxisa/test/kernel/sort/topk/data_obj/build_data_obj.sh index 8128f1e..4644e7b 100755 --- a/benchmark-linxisa/test/kernel/sort/topk/data_obj/build_data_obj.sh +++ b/benchmark-linxisa/test/kernel/sort/topk/data_obj/build_data_obj.sh @@ -1,5 +1,5 @@ #!/bin/bash -COMPILER_DIR="${COMPILER_DIR:-/remote/lms01/j00827727/jcore/compilers/linx_blockisa_llvm_musl0.56.16/bin}" +: "${COMPILER_DIR:?COMPILER_DIR must point to the Linx Clang bin directory}" DATA_OBJ_DIR="$1" OUTPUT_DIR="$2" @@ -25,10 +25,10 @@ _binary_${name}_data_end: .equ _binary_${name}_data_size, .-_binary_${name}_data_start EOF - $COMPILER_DIR/clang++ -target linx64v5 -c "$asm_file" -o "$obj_file" + "$COMPILER_DIR/clang++" -mlxbc -c "$asm_file" -o "$obj_file" } build_one "input_131072" build_one "top_2048_out" -echo "Done building data object files" \ No newline at end of file +echo "Done building data object files" diff --git a/benchmark-linxisa/test/kernel/sort/topk/topk.cpp b/benchmark-linxisa/test/kernel/sort/topk/topk.cpp index 1f579fd..0856747 100644 --- a/benchmark-linxisa/test/kernel/sort/topk/topk.cpp +++ b/benchmark-linxisa/test/kernel/sort/topk/topk.cpp @@ -42,26 +42,15 @@ int main() { fflush(stdout); #endif - // ------------------------------------------------------------------------- - // Phase 1: SIMT high8 histogram (1 block × 256 lanes, each lane = 1 bucket) - // ------------------------------------------------------------------------- - TileU32 high8HistTile; - TEXPANDSCALAR(high8HistTile, static_cast(0)); - ExtractHigh8Hist_Impl< TileU32 >(high8HistTile, g_input); - - // Copy histogram results out and reduce to global 256-bin histogram - using HistGT = GlobalTensor, Stride<1,1,1,16,1>>; - uint32_t histResult[256]; - HistGT histGlobal(histResult); - TCOPYOUT(histGlobal, high8HistTile); - uint32_t global_high8_hist[256] = {0}; - for (int b = 0; b < 256; b++) { - global_high8_hist[b] = histResult[b]; + for (int i = 0; i < kInputCount; i++) { + uint16_t val = g_input[i]; + uint8_t high8 = static_cast(val >> 8); + global_high8_hist[high8] += 1; } #ifndef FOR_GFSIM - printf("\nPhase 1: high8 histograms built (1 SIMT launch, 256 lanes).\n"); + printf("\nPhase 1: high8 histogram built.\n"); fflush(stdout); #endif @@ -82,21 +71,14 @@ int main() { fflush(stdout); #endif - // ------------------------------------------------------------------------- - // Phase 3: SIMT low8 histogram for kth_bin elements - // ------------------------------------------------------------------------- - TileU32 low8HistTile; - TEXPANDSCALAR(low8HistTile, static_cast(0)); - ExtractLow8HistForKthBin_Impl< TileU32 >(low8HistTile, g_input, - static_cast(kth_bin)); - - uint32_t low8HistResult[256]; - HistGT low8HistGlobal(low8HistResult); - TCOPYOUT(low8HistGlobal, low8HistTile); - uint32_t global_low8_hist_kth[256] = {0}; - for (int b = 0; b < 256; b++) { - global_low8_hist_kth[b] = low8HistResult[b]; + for (int i = 0; i < kInputCount; i++) { + uint16_t val = g_input[i]; + uint8_t high8 = static_cast(val >> 8); + if (high8 == static_cast(kth_bin)) { + uint8_t low8 = static_cast(val & 0xFF); + global_low8_hist_kth[low8] += 1; + } } // ------------------------------------------------------------------------- diff --git a/benchmark-ptoisa/compile_all.sh b/benchmark-ptoisa/compile_all.sh index e677a82..5622fb5 100755 --- a/benchmark-ptoisa/compile_all.sh +++ b/benchmark-ptoisa/compile_all.sh @@ -4,15 +4,19 @@ # Don't use set -e as some operators may fail to compile SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -export COMPILER_DIR=${COMPILER_DIR:-/path/to/pto-toolchain/bin} +LINX_ISA_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +export COMPILER_DIR=${COMPILER_DIR:-$LINX_ISA_ROOT/compiler/llvm/build-linxisa-clang/bin} REPO_ROOT=${REPO_ROOT:-$SCRIPT_DIR} echo "==========================================" echo "[PTO ISA] Starting full compilation" echo "REPO_ROOT: $REPO_ROOT" +echo "COMPILER_DIR: $COMPILER_DIR" echo "==========================================" # Function to compile an operator +FAILURES=0 + compile_operator() { local operator_path=$1 local operator_name=$2 @@ -25,10 +29,15 @@ compile_operator() { if [ ! -d "$operator_path" ]; then echo "Warning: Directory not found: $operator_path" + FAILURES=$((FAILURES + 1)) return 1 fi - cd "$operator_path" + if ! cd "$operator_path"; then + echo "✗ $operator_name compilation failed: cannot enter directory" + FAILURES=$((FAILURES + 1)) + return 1 + fi if [ -f "compile.all" ]; then echo "Running compile.all with baremetal=${baremetal:-off}..." @@ -37,9 +46,12 @@ compile_operator() { echo "✓ $operator_name compilation completed" else echo "✗ $operator_name compilation failed" + FAILURES=$((FAILURES + 1)) + return 1 fi else echo "Warning: No compile.all found in $operator_path" + FAILURES=$((FAILURES + 1)) return 1 fi } @@ -67,3 +79,8 @@ echo "" echo "Generated ELF files:" find "$REPO_ROOT/output" -name "*.elf" -type f | wc -l echo "ELF files are located in: $REPO_ROOT/output/" + +if [ "$FAILURES" -ne 0 ]; then + echo "Compilation failures: $FAILURES" + exit 1 +fi diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast.hpp new file mode 100644 index 0000000..75082ee --- /dev/null +++ b/benchmark-ptoisa/kernels/broadcast/broadcast.hpp @@ -0,0 +1,112 @@ +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_HPP + +#include + +#include +#include +#include + +using namespace pto; + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t *in_shape, + const size_t *out_shape) { + constexpr size_t kBCast = gOM / gIM; + constexpr size_t kElementsPerTile = 512; + constexpr size_t kFullTiles = gOM / kElementsPerTile; + constexpr size_t kTail = gOM % kElementsPerTile; + + static_assert(MAX_DIM >= IN_DIM && MAX_DIM >= OUT_DIM, + "MAX_DIM must cover input and output ranks"); + static_assert(gOM % gIM == 0, + "broadcast output element count must be a multiple of input"); + static_assert(kBCast > 0, "broadcast factor must be positive"); + (void)kTileHint; + + size_t inner = 1; + size_t in_dim = IN_DIM; + size_t out_dim = OUT_DIM; + while (in_dim > 0 && out_dim > 0 && + in_shape[in_dim - 1] == out_shape[out_dim - 1]) { + inner *= out_shape[out_dim - 1]; + --in_dim; + --out_dim; + } + + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + using DataTile = Tile; + using OffsetTile = Tile; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputIterator output_iter(out_ptr); + + auto emit_tile = [&](auto &data_tile, auto &linear, auto &batch, + auto &batch_base, auto &inner_q, auto &inner_base, + auto &inner_idx, auto &inner_bytes, auto &offset, + std::uint32_t base, auto output_addr) { + using LinearTile = std::remove_reference_t; + const auto inner_u32 = static_cast(inner); + const auto group_u32 = static_cast(kBCast * inner); + TCI(linear, base); + + TDIVS(batch, linear, group_u32); + TMULS(batch_base, batch, + static_cast(inner * sizeof(dtype))); + + TDIVS(inner_q, linear, inner_u32); + TMULS(inner_base, inner_q, inner_u32); + TSUB(inner_idx, linear, inner_base); + TMULS(inner_bytes, inner_idx, static_cast(sizeof(dtype))); + + TADD(offset, batch_base, inner_bytes); + MGATHER(data_tile, input_global, offset); + TSTORE(output_addr, data_tile); + }; + + for (size_t tile = 0; tile < kFullTiles; ++tile) { + DataTile data_tile; + OffsetTile linear; + OffsetTile batch; + OffsetTile batch_base; + OffsetTile inner_q; + OffsetTile inner_base; + OffsetTile inner_idx; + OffsetTile inner_bytes; + OffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, + inner_idx, inner_bytes, offset, + static_cast(tile * kElementsPerTile), + output_iter(0, static_cast(tile))); + } + + if constexpr (kTail != 0) { + using TailDataTile = Tile; + using TailOffsetTile = + Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailDataTile data_tile; + TailOffsetTile linear; + TailOffsetTile batch; + TailOffsetTile batch_base; + TailOffsetTile inner_q; + TailOffsetTile inner_base; + TailOffsetTile inner_idx; + TailOffsetTile inner_bytes; + TailOffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, + inner_idx, inner_bytes, offset, + static_cast(kFullTiles * kElementsPerTile), + tail_output_iter(0, static_cast(kFullTiles))); + } +} + +#endif // SUPERNPUBENCH_PTOISA_BROADCAST_HPP diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_pto.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_pto.hpp index 1fb7ab6..cfaf699 100644 --- a/benchmark-ptoisa/kernels/broadcast/broadcast_pto.hpp +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_pto.hpp @@ -190,7 +190,7 @@ void gen_offset_pto( // ---------------------------------------------------------------------------- // broadcast: 接口与原 broadcast.hpp 一致 // ---------------------------------------------------------------------------- -template +template void broadcast( dtype *in_ptr, dtype *out_ptr, diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019.hpp new file mode 100644 index 0000000..e6332b6 --- /dev/null +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019.hpp @@ -0,0 +1,99 @@ +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_VEC_019_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_VEC_019_HPP + +#include + +#include +#include +#include + +using namespace pto; + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t *, const size_t *) { + constexpr size_t kBCast = gOM / gIM; + constexpr size_t kElementsPerTile = 512; + constexpr size_t kFullTiles = gOM / kElementsPerTile; + constexpr size_t kTail = gOM % kElementsPerTile; + + static_assert(MAX_DIM >= IN_DIM && MAX_DIM >= OUT_DIM, + "MAX_DIM must cover input and output ranks"); + static_assert(gOM % gIM == 0, + "broadcast output element count must be a multiple of input"); + static_assert(gIM % kInner == 0, + "input element count must be divisible by kInner"); + static_assert(kBCast > 0, "broadcast factor must be positive"); + static_assert(kTileBatch > 0, "kTileBatch must be positive"); + + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + using DataTile = Tile; + using OffsetTile = Tile; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_ptr); + OutputIterator output_iter(out_ptr); + + auto emit_tile = [&](auto &data_tile, auto &linear, auto &batch, + auto &batch_base, auto &inner_q, auto &inner_base, + auto &inner, auto &inner_bytes, auto &offset, + std::uint32_t base, auto output_addr) { + using LinearTile = std::remove_reference_t; + TCI(linear, base); + + TDIVS(batch, linear, static_cast(kBCast * kInner)); + TMULS(batch_base, batch, + static_cast(kInner * sizeof(dtype))); + + TDIVS(inner_q, linear, static_cast(kInner)); + TMULS(inner_base, inner_q, static_cast(kInner)); + TSUB(inner, linear, inner_base); + TMULS(inner_bytes, inner, static_cast(sizeof(dtype))); + + TADD(offset, batch_base, inner_bytes); + MGATHER(data_tile, input_global, offset); + TSTORE(output_addr, data_tile); + }; + + for (size_t tile = 0; tile < kFullTiles; ++tile) { + DataTile data_tile; + OffsetTile linear; + OffsetTile batch; + OffsetTile batch_base; + OffsetTile inner_q; + OffsetTile inner_base; + OffsetTile inner; + OffsetTile inner_bytes; + OffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, inner, + inner_bytes, offset, + static_cast(tile * kElementsPerTile), + output_iter(0, static_cast(tile))); + } + + if constexpr (kTail != 0) { + using TailDataTile = Tile; + using TailOffsetTile = + Tile; + TailDataTile data_tile; + TailOffsetTile linear; + TailOffsetTile batch; + TailOffsetTile batch_base; + TailOffsetTile inner_q; + TailOffsetTile inner_base; + TailOffsetTile inner; + TailOffsetTile inner_bytes; + TailOffsetTile offset; + emit_tile(data_tile, linear, batch, batch_base, inner_q, inner_base, inner, + inner_bytes, offset, + static_cast(kFullTiles * kElementsPerTile), + output_iter(0, static_cast(kFullTiles))); + } +} + +#endif diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019_pto.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019_pto.hpp index 908e75f..d1aa674 100644 --- a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019_pto.hpp +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_019_pto.hpp @@ -21,8 +21,8 @@ // │ │ 二层实现 │ 当前编译器名 TCOPYIN； │ // │ │ │ jcore/TCopyIn.hpp 用 __vec__ 实现 │ // ├──────────┼──────────────────┼──────────────────────────────────────────┤ -// │ TINSERT │ 完全缺失 │ pto_tileop.hpp 中无此 API； │ -// │ │ │ 当前编译器无 TINSERT 实现 │ +// │ TINSERT │ v0.57 supported │ TileOP API lowers to BSTART.FIXP │ +// │ │ │ with FIXP.Function.TINSERT │ // │ │ │ (仅有反向操作 TEXTRACT) │ // ├──────────┼──────────────────┼──────────────────────────────────────────┤ // │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE； │ @@ -79,7 +79,7 @@ // kInner - inner dimension K (e.g. 49, need not be power-of-2) // ===================================================================== -template void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t * /*in_shape*/, const size_t * /*out_shape*/) { @@ -121,7 +121,7 @@ void broadcast(dtype *in_ptr, dtype *out_ptr, // TINSERT × kBCast: 将输入 tile 插入输出 tile 的 N 个列偏移 // 每次 TINSERT 写入 kInner 列, N 次互不重叠, 合起来填满 N*kInner 列 - // [当前编译器] 完全缺失! pto_tileop.hpp 无 TINSERT API + // v0.57 lowers TINSERT through the FIXP tile block. #pragma clang loop unroll(full) for (size_t c = 0; c < kBCast; c++) { TINSERT(outTile, inTile, /*indexRow=*/0, /*indexCol=*/(uint16_t)(c * kInner)); diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039.hpp new file mode 100644 index 0000000..a918659 --- /dev/null +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039.hpp @@ -0,0 +1,6 @@ +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_VEC_039_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_VEC_039_HPP + +#include "broadcast_vec_019.hpp" + +#endif diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039_pto.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039_pto.hpp index edc99a8..38894d8 100644 --- a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039_pto.hpp +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_039_pto.hpp @@ -22,8 +22,8 @@ // │ │ 二层实现 │ 当前编译器名 TCOPYIN； │ // │ │ │ jcore/TCopyIn.hpp 用 __vec__ 实现 │ // ├──────────┼──────────────────┼──────────────────────────────────────────┤ -// │ TINSERT │ 完全缺失 │ pto_tileop.hpp 中无此 API； │ -// │ │ │ 当前编译器无 TINSERT 实现 │ +// │ TINSERT │ v0.57 supported │ TileOP API lowers to BSTART.FIXP │ +// │ │ │ with FIXP.Function.TINSERT │ // │ │ │ (仅有反向操作 TEXTRACT) │ // ├──────────┼──────────────────┼──────────────────────────────────────────┤ // │ TSTORE │ API 有(名不同)， │ PTO ISA 名 TSTORE； │ @@ -80,7 +80,7 @@ // kInner - inner dimension K, power-of-2 (e.g. 16) // ===================================================================== -template void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t * /*in_shape*/, const size_t * /*out_shape*/) { @@ -122,7 +122,7 @@ void broadcast(dtype *in_ptr, dtype *out_ptr, // TINSERT × kBCast: 将输入 tile 插入输出 tile 的 N 个列偏移 // 每次 TINSERT 写入 kInner 列, N 次互不重叠, 合起来填满 N*kInner 列 - // [当前编译器] 完全缺失! pto_tileop.hpp 无 TINSERT API + // v0.57 lowers TINSERT through the FIXP tile block. #pragma clang loop unroll(full) for (size_t c = 0; c < kBCast; c++) { TINSERT(outTile, inTile, /*indexRow=*/0, /*indexCol=*/(uint16_t)(c * kInner)); diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07.hpp new file mode 100644 index 0000000..e4b8cf6 --- /dev/null +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07.hpp @@ -0,0 +1,69 @@ +#ifndef SUPERNPUBENCH_PTOISA_BROADCAST_VEC_07_HPP +#define SUPERNPUBENCH_PTOISA_BROADCAST_VEC_07_HPP + +#include + +#include + +using namespace pto; + +template +void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t *, const size_t *) { + constexpr size_t kN = gIM; + constexpr size_t kC = gOM / gIM; + constexpr size_t kTileCols = kC; + constexpr size_t kMaxRowsByBytes = + (4096 / (kTileCols * sizeof(dtype))) == 0 + ? 1 + : (4096 / (kTileCols * sizeof(dtype))); + constexpr size_t kRowsPerTile = + kTileRows < kMaxRowsByBytes ? kTileRows : kMaxRowsByBytes; + + static_assert(MAX_DIM >= IN_DIM && MAX_DIM >= OUT_DIM, + "MAX_DIM must cover input and output ranks"); + static_assert(gOM % gIM == 0, + "broadcast output element count must be a multiple of input"); + static_assert(kRowsPerTile > 0, "broadcast tile must contain at least one row"); + + using InputGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + using InputTile = Tile; + using OutputTile = Tile; + using InputIterator = global_iterator; + using OutputIterator = global_iterator; + + InputIterator input_iter(in_ptr); + OutputIterator output_iter(out_ptr); + + constexpr size_t kFullTiles = kN / kRowsPerTile; + constexpr size_t kTail = kN % kRowsPerTile; + + for (size_t tile = 0; tile < kFullTiles; ++tile) { + InputTile input_tile; + OutputTile output_tile; + TLOAD(input_tile, input_iter(static_cast(tile), 0)); + TROWEXPAND(output_tile, input_tile); + TSTORE(output_iter(static_cast(tile), 0), output_tile); + } + + if constexpr (kTail != 0) { + using TailInputTile = Tile; + using TailOutputTile = Tile; + using TailInputIterator = global_iterator; + using TailOutputIterator = global_iterator; + TailInputIterator tail_input_iter(in_ptr); + TailOutputIterator tail_output_iter(out_ptr); + TailInputTile input_tile; + TailOutputTile output_tile; + TLOAD(input_tile, tail_input_iter(static_cast(kFullTiles), 0)); + TROWEXPAND(output_tile, input_tile); + TSTORE(tail_output_iter(static_cast(kFullTiles), 0), output_tile); + } +} + +#endif // SUPERNPUBENCH_PTOISA_BROADCAST_VEC_07_HPP diff --git a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07_pto.hpp b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07_pto.hpp index ee43b44..550a9f4 100644 --- a/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07_pto.hpp +++ b/benchmark-ptoisa/kernels/broadcast/broadcast_vec_07_pto.hpp @@ -70,7 +70,7 @@ // kTileRows - rows per tile, must be power-of-2 (e.g. 1,2,4,..,64) // ===================================================================== -template void broadcast(dtype *in_ptr, dtype *out_ptr, const size_t * /*in_shape*/, const size_t * /*out_shape*/) { diff --git a/benchmark-ptoisa/kernels/concat/concat_gather.hpp b/benchmark-ptoisa/kernels/concat/concat_gather.hpp index e542559..94c738a 100644 --- a/benchmark-ptoisa/kernels/concat/concat_gather.hpp +++ b/benchmark-ptoisa/kernels/concat/concat_gather.hpp @@ -2,7 +2,6 @@ #define CONCAT_GATHER_KERNEL_HPP #include -#include #include #include @@ -22,7 +21,7 @@ using namespace pto; * @tparam DATA_DIM 数据维度数（编译期常量） * @tparam CONCAT_DIM 拼接维度索引（编译期常量） */ -template void concat_gather( DType *in_ptr, diff --git a/benchmark-ptoisa/kernels/concat/concat_scatter.hpp b/benchmark-ptoisa/kernels/concat/concat_scatter.hpp index 142f130..2846f33 100644 --- a/benchmark-ptoisa/kernels/concat/concat_scatter.hpp +++ b/benchmark-ptoisa/kernels/concat/concat_scatter.hpp @@ -2,7 +2,6 @@ #define CONCAT_SCATTER_KERNEL_HPP #include -#include #include #include @@ -25,7 +24,7 @@ using namespace pto; * @tparam DATA_DIM 数据维度数（编译期常量） * @tparam CONCAT_DIM 拼接维度索引（编译期常量） */ -template void concat_scatter( DType *in_ptr, diff --git a/benchmark-ptoisa/kernels/control/hashtable_lookup_simd.hpp b/benchmark-ptoisa/kernels/control/hashtable_lookup_simd.hpp index 19659f3..6f273bc 100644 --- a/benchmark-ptoisa/kernels/control/hashtable_lookup_simd.hpp +++ b/benchmark-ptoisa/kernels/control/hashtable_lookup_simd.hpp @@ -1,6 +1,7 @@ #ifndef HASHTABLE_LOOKUP_SIMD_HPP #define HASHTABLE_LOOKUP_SIMD_HPP +#include #include #include "template_asm.h" // MGATHER #include diff --git a/benchmark-ptoisa/kernels/element_wise/gelu.hpp b/benchmark-ptoisa/kernels/element_wise/gelu.hpp new file mode 100644 index 0000000..76cd3e3 --- /dev/null +++ b/benchmark-ptoisa/kernels/element_wise/gelu.hpp @@ -0,0 +1,77 @@ +#ifndef SUPERNPUBENCH_PTOISA_GELU_HPP +#define SUPERNPUBENCH_PTOISA_GELU_HPP + +#include + +#include + +using namespace pto; + +template +inline void gelu_tile(TileT &dst, TileT &src) { + TileT t; + TileT t2; + TileT p; + TileT tmp; + TileT denom; + TileT recip; + + TMAXS(t, src, -5.75f); + TMINS(t, t, 5.75f); + TMUL(t2, t, t); + + TMULS(p, t2, -3.5123395303315874e-09f); + TADDS(p, p, 2.6452661927578447e-07f); + TMUL(p, p, t2); + TADDS(p, p, -7.9294877650681883e-06f); + TMUL(p, p, t2); + TADDS(p, p, 1.1061238183174282e-04f); + TMUL(p, p, t2); + TADDS(p, p, 6.5189960878342390e-05f); + TMUL(p, p, t2); + TADDS(p, p, -7.2666168212890625e-02f); + TMUL(p, p, t2); + TADDS(p, p, -1.5957698822021484e+00f); + + TMUL(tmp, t, p); + TEXP(tmp, tmp); + TADDS(denom, tmp, 1.0f); + TRECIP(recip, denom); + TMUL(dst, src, recip); +} + +template +void gelu(dtype *in_ptr, dtype *out_ptr, bool = false) { + constexpr int kFullTiles = gM / tM; + constexpr int kTail = gM % tM; + + using Global = global_tensor>; + using TileT = Tile; + using Iterator = global_iterator; + + Iterator input_iter(in_ptr); + Iterator output_iter(out_ptr); + + for (int tile = 0; tile < kFullTiles; ++tile) { + TileT input_tile; + TileT output_tile; + TLOAD(input_tile, input_iter(0, tile)); + gelu_tile(output_tile, input_tile); + TSTORE(output_iter(0, tile), output_tile); + } + + if constexpr (kTail != 0) { + using TailTile = + Tile; + using TailIterator = global_iterator; + TailIterator tail_input_iter(in_ptr); + TailIterator tail_output_iter(out_ptr); + TailTile input_tile; + TailTile output_tile; + TLOAD(input_tile, tail_input_iter(0, kFullTiles)); + gelu_tile(output_tile, input_tile); + TSTORE(tail_output_iter(0, kFullTiles), output_tile); + } +} + +#endif // SUPERNPUBENCH_PTOISA_GELU_HPP diff --git a/benchmark-ptoisa/kernels/fa/fa_2d_unroll.hpp b/benchmark-ptoisa/kernels/fa/fa_2d_unroll.hpp index 296e6e9..709e203 100644 --- a/benchmark-ptoisa/kernels/fa/fa_2d_unroll.hpp +++ b/benchmark-ptoisa/kernels/fa/fa_2d_unroll.hpp @@ -1,6 +1,10 @@ #include "fa_utils.h" #include "fa_fp4_utils.h" +using namespace pto; +using namespace pto::blkv; +using pto::type_traits; + #ifndef Xdim #define Xdim 2 #endif @@ -13,6 +17,11 @@ #define __vbuf__ #endif +#ifndef BLKC_ASSIGN_CAST +#define BLKC_ASSIGN_CAST(tile, idx, value) \ + (pto::blkv::blkv_get_tile_ptr(tile)[(idx)] = (value)) +#endif + template void __vec__ new_max_1src( typename tileMax::TileDType __out__ scale, @@ -797,7 +806,6 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype TEXPANDSCALAR(tSum[x], 0); } - tileO_out tPV_out; tileO tO[Xdim], tPV[Xdim]; tileScale tScale[Xdim]; @@ -840,7 +848,8 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype #if Ydim == 1 #pragma clang loop unroll(full) for(int x=0;x<<>>( + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + flashsoftmax_dn_mout_cast_kernel( tScale[x].data(), tNewMax[x].data(), tNewSum[x].data(), @@ -849,16 +858,19 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype tMax[x].data(), tSum[x].data(), scale); + }); } #elif Ydim == 2 #pragma clang loop unroll(full) for(int x=0;x<<>>( + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_2src( tScale[x].data(), tNewMax[x].data(), tW[x][0].data(), tW[x][1].data(), tMax[x].data(), scale); + }); // src_exp_2src<<>>( // tExpW[x][0].data(), tExpW[x][1].data(), // tW[x][0].data(), tW[x][1].data(), @@ -870,39 +882,49 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype // tSum[x].data(), // tScale[x].data() // ); - src_exp_2src_with_new_sum<<>>( + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_new_sum( tNewSum[x].data(), tExpW[x][0].data(), tExpW[x][1].data(), tW[x][0].data(), tW[x][1].data(), tNewMax[x].data(), tSum[x].data(), tScale[x].data(), scale); + }); } #elif Ydim == 4 tileSum tLocalSum[Xdim][2]; #pragma clang loop unroll(full) for(int x=0;x<<>>( + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_4src( tScale[x].data(), tNewMax[x].data(), tW[x][0].data(), tW[x][1].data(), tW[x][2].data(), tW[x][3].data(), tMax[x].data(), scale); + }); // src_exp_4src<<>>( // tExpW[x][0].data(), tExpW[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), // tW[x][0].data(), tW[x][1].data(), tW[x][2].data(), tW[x][3].data(), // tNewMax[x].data(), // scale); - src_exp_2src_with_local_sum<<>>(tLocalSum[x][0].data(), tExpW[x][0].data(), tExpW[x][1].data(), + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_local_sum(tLocalSum[x][0].data(), tExpW[x][0].data(), tExpW[x][1].data(), tW[x][0].data(), tW[x][1].data(), tNewMax[x].data(), scale); - src_exp_2src_with_local_sum<<>>(tLocalSum[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), + }); + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_local_sum(tLocalSum[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), tW[x][2].data(), tW[x][3].data(), tNewMax[x].data(), scale); + }); // new_sum_4src<<>>( // tNewSum[x].data(), // tExpW[x][0].data(), tExpW[x][1].data(), tExpW[x][2].data(), tExpW[x][3].data(), // tSum[x].data(), // tScale[x].data() // ); - new_sum_of_2_loc_sum<<>>(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tSum[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + new_sum_of_2_loc_sum(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tSum[x].data(), tScale[x].data()); + }); } #elif Ydim == 8 tileMax tLocalMax[Xdim][2]; @@ -912,17 +934,25 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype for(int x=0;x<<>>(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + local_max_4src(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + }); } - new_max_of_2_loc_max<<>>(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tMax[x].data()); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_of_2_loc_max(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tMax[x].data()); + }); #pragma clang loop unroll(full) for(int k=0;k<4;k++){ - src_exp_2src_with_local_sum<<>>(tLocalSum[x][k].data(), tExpW[x][2*k].data(), tExpW[x][2*k+1].data(), + pto::blkv::blkv_for_1d(tileW::ValidRow, [&] { + src_exp_2src_with_local_sum(tLocalSum[x][k].data(), tExpW[x][2*k].data(), tExpW[x][2*k+1].data(), tW[x][2*k].data(), tW[x][2*k+1].data(), tNewMax[x].data(), scale); + }); } - new_sum_of_4_loc_sum<<>>(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + new_sum_of_4_loc_sum(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + }); } #elif Ydim == 16 tileMax tLocalMax[Xdim][4]; @@ -931,25 +961,35 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype #pragma clang loop unroll(full) for(int x=0;x<<>>(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + local_max_4src(tLocalMax[x][k].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), scale); + }); } - new_max_of_4_loc_max<<>>(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tLocalMax[x][2].data(), tLocalMax[x][3].data(), tMax[x].data()); + pto::blkv::blkv_for_1d(tileMax::ValidRow, [&] { + new_max_of_4_loc_max(tScale[x].data(), tNewMax[x].data(), tLocalMax[x][0].data(), tLocalMax[x][1].data(), tLocalMax[x][2].data(), tLocalMax[x][3].data(), tMax[x].data()); + }); #pragma clang loop unroll(full) for(int k=0;k<4;k++){ - src_exp_4src<<>>( + pto::blkv::blkv_for_2d(tileW::ValidRow, tileW::ValidCol, [&] { + src_exp_4src( tExpW[x][4*k].data(), tExpW[x][4*k+1].data(), tExpW[x][4*k+2].data(), tExpW[x][4*k+3].data(), tW[x][4*k].data(), tW[x][4*k+1].data(), tW[x][4*k+2].data(), tW[x][4*k+3].data(), tNewMax[x].data(), scale); + }); } #pragma clang loop unroll(full) for(int k=0;k<4;k++){ - local_sum_4src<<>>(tLocalSum[x][k].data(), tExpW[x][4*k].data(), tExpW[x][4*k+1].data(), tExpW[x][4*k+2].data(), tExpW[x][4*k+3].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + local_sum_4src(tLocalSum[x][k].data(), tExpW[x][4*k].data(), tExpW[x][4*k+1].data(), tExpW[x][4*k+2].data(), tExpW[x][4*k+3].data()); + }); } - new_sum_of_4_loc_sum<<>>(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileSum::ValidRow, [&] { + new_sum_of_4_loc_sum(tNewSum[x].data(), tLocalSum[x][0].data(), tLocalSum[x][1].data(), tLocalSum[x][2].data(), tLocalSum[x][3].data(), tSum[x].data(), tScale[x].data()); + }); } #else #ifdef _2D_UNROLL @@ -978,6 +1018,7 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype tileW_left tW_left[Xdim][Ydim]; #pragma clang loop unroll(full) for(int x=0;x::bits == 4) { @@ -1001,19 +1042,21 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype if(j==0){ #pragma clang loop unroll(full) for(int x=0;x<<>>(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data()); + pto::blkv::blkv_for_1d(tileO::ValidRow, [&] { + global_update(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data()); + }); } } // 更新最大值状态 #pragma clang loop unroll(full) for(int x=0;x::bits == 4) { - normalize_with_last_update_nocast<<>>(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + pto::blkv::blkv_for_2d(tileO::ValidRow, tileO::ValidCol, [&] { + normalize_with_last_update_nocast(tO[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + }); TMOV_NORM(tO_cast[x], tO[x]); } else { - normalize_with_last_update<<>>(tO_cast[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + pto::blkv::blkv_for_2d(tileO::ValidRow, tileO::ValidCol, [&] { + normalize_with_last_update(tO_cast[x].data(), tO[x].data(), tPV[x].data(), tScale[x].data(), tSum[x].data()); + }); } } // 写回全局内存 @@ -1046,4 +1093,6 @@ void flash_attention_2d_unroll(dtype* out_ptr, dtype* q_ptr, dtype* k_ptr, dtype } } +#ifdef _UNALIGN_2D_UNROLL #include "fa_unalign_2d_unroll.hpp" +#endif diff --git a/benchmark-ptoisa/kernels/fa/fa_fp4_utils.h b/benchmark-ptoisa/kernels/fa/fa_fp4_utils.h index d2f776b..ecdbd51 100644 --- a/benchmark-ptoisa/kernels/fa/fa_fp4_utils.h +++ b/benchmark-ptoisa/kernels/fa/fa_fp4_utils.h @@ -1,4 +1,9 @@ +#include + +using namespace pto::blkv; +using pto::type_traits; + template concept is_fp4_tile = requires (T tile) { type_traits::bits == 4; diff --git a/benchmark-ptoisa/kernels/fa/fa_utils.h b/benchmark-ptoisa/kernels/fa/fa_utils.h index 5744a1d..59ee045 100644 --- a/benchmark-ptoisa/kernels/fa/fa_utils.h +++ b/benchmark-ptoisa/kernels/fa/fa_utils.h @@ -1,5 +1,9 @@ // FA utility functions shared across different implementations +#include + +using namespace pto::blkv; + template void __vec__ flashsoftmax_dn_mout_cast_kernel( typename tileScale::TileDType __out__ rescale, diff --git a/benchmark-ptoisa/kernels/fa/linx_blkc.h b/benchmark-ptoisa/kernels/fa/linx_blkc.h new file mode 100644 index 0000000..df68bc9 --- /dev/null +++ b/benchmark-ptoisa/kernels/fa/linx_blkc.h @@ -0,0 +1,121 @@ +#ifndef PTO_FA_LINX_BLKC_H +#define PTO_FA_LINX_BLKC_H + +#include + +#ifndef __bf16 +#define __bf16 pto_bf16_t +#endif + +using pto::blkv::blkv_fexp; +using pto::blkv::blkv_fsqrt; +using pto::blkv::blkv_get_index_x; +using pto::blkv::blkv_get_index_y; +using pto::blkv::blkv_get_index_z; +using pto::blkv::blkv_get_tile_ptr; +using pto::blkv::blkv_max; + +using __fp4_e1m2x2 = pto::fp4_e2m1_t; +using __bf16x2 = uint32_t; + +template +inline To linx_cvt_as(From value) { + return static_cast(value); +} + +template +inline void linx_cvt(To &dst, From value) { + dst = linx_cvt_as(value); +} + +template +inline void linx_cvt_package(To &dst, A a, B b) { + const uint16_t hi = + static_cast(pto::lowp_word_from_bf16(pto::bf16_t{static_cast(a)}) & + 0xffffu); + const uint16_t lo = + static_cast(pto::lowp_word_from_bf16(pto::bf16_t{static_cast(b)}) & + 0xffffu); + if constexpr (sizeof(To) >= 4) { + dst = static_cast((static_cast(hi) << 16) | lo); + } else { + dst = static_cast(pto::float_to_fp4_e2m1(static_cast(a))); + } +} + +inline void blkv_bf16_fmax(__bf16 &dst, __bf16 a, __bf16 b) { + dst = a < b ? b : a; +} + +inline __bf16 blkv_bf16_max(__bf16 a, __bf16 b) { return a < b ? b : a; } +inline __bf16 blkv_bf16_mul(__bf16 a, __bf16 b) { return a * b; } +inline __bf16 blkv_bf16_div(__bf16 a, __bf16 b) { return a / b; } + +inline void blkv_bf16_fadd(__bf16 &dst, __bf16 a, __bf16 b) { dst = a + b; } +inline void blkv_bf16_fsub(__bf16 &dst, __bf16 a, __bf16 b) { dst = a - b; } +inline void blkv_bf16_fmul(__bf16 &dst, __bf16 a, __bf16 b) { dst = a * b; } +inline void blkv_bf16_fdiv(__bf16 &dst, __bf16 a, __bf16 b) { dst = a / b; } +inline void blkv_bf16_fexp(__bf16 &dst, __bf16 a) { + dst = pto::blkv::blkv_fexp(static_cast(a)); +} + +inline float linx_bf16x2_hi(__bf16x2 value) { + return static_cast(pto::bf16_from_lowp_word((value >> 16) & 0xffffu)); +} + +inline float linx_bf16x2_lo(__bf16x2 value) { + return static_cast(pto::bf16_from_lowp_word(value & 0xffffu)); +} + +inline __bf16x2 linx_pack_bf16x2(float hi, float lo) { + return (pto::lowp_word_from_bf16(pto::bf16_t{hi}) << 16) | + pto::lowp_word_from_bf16(pto::bf16_t{lo}); +} + +inline void blkv_bf16x2_fmax(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(pto::blkv::blkv_max(linx_bf16x2_hi(a), linx_bf16x2_hi(b)), + pto::blkv::blkv_max(linx_bf16x2_lo(a), linx_bf16x2_lo(b))); +} + +inline __bf16x2 blkv_bf16x2_mul(__bf16x2 a, __bf16x2 b) { + return linx_pack_bf16x2(linx_bf16x2_hi(a) * linx_bf16x2_hi(b), + linx_bf16x2_lo(a) * linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_mul(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = blkv_bf16x2_mul(a, b); +} + +inline void blkv_bf16x2_fadd(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(linx_bf16x2_hi(a) + linx_bf16x2_hi(b), + linx_bf16x2_lo(a) + linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_fsub(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(linx_bf16x2_hi(a) - linx_bf16x2_hi(b), + linx_bf16x2_lo(a) - linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_fmul(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = blkv_bf16x2_mul(a, b); +} + +inline void blkv_bf16x2_fdiv(__bf16x2 &dst, __bf16x2 a, __bf16x2 b) { + dst = linx_pack_bf16x2(linx_bf16x2_hi(a) / linx_bf16x2_hi(b), + linx_bf16x2_lo(a) / linx_bf16x2_lo(b)); +} + +inline void blkv_bf16x2_fmsub(__bf16x2 &dst, __bf16x2 a, __bf16x2 b, + __bf16x2 c) { + dst = linx_pack_bf16x2((linx_bf16x2_hi(a) * linx_bf16x2_hi(b)) - + linx_bf16x2_hi(c), + (linx_bf16x2_lo(a) * linx_bf16x2_lo(b)) - + linx_bf16x2_lo(c)); +} + +inline void blkv_bf16x2_fexp(__bf16x2 &dst, __bf16x2 a) { + dst = linx_pack_bf16x2(pto::blkv::blkv_fexp(linx_bf16x2_hi(a)), + pto::blkv::blkv_fexp(linx_bf16x2_lo(a))); +} + +#endif // PTO_FA_LINX_BLKC_H diff --git a/benchmark-ptoisa/kernels/gather/gather.hpp b/benchmark-ptoisa/kernels/gather/gather.hpp new file mode 100644 index 0000000..a19ec69 --- /dev/null +++ b/benchmark-ptoisa/kernels/gather/gather.hpp @@ -0,0 +1,92 @@ +#ifndef SUPERNPUBENCH_PTOISA_GATHER_HPP +#define SUPERNPUBENCH_PTOISA_GATHER_HPP + +#include + +#include +#include + +using namespace pto; + +template +void gather(dtype *in_data_ptr, otype *in_offset_ptr, dtype *out_ptr) { + constexpr size_t kFullRows = gM / tM; + constexpr size_t kTailRows = gM % tM; + constexpr size_t kFullCols = gN / tN; + constexpr size_t kTailCols = gN % tN; + + using InputGlobal = global_tensor>; + using OffsetGlobal = global_tensor>; + using OutputGlobal = global_tensor>; + + using OffsetTile = + Tile; + using DataTile = Tile; + using OffsetIterator = global_iterator; + using OutputIterator = global_iterator; + + InputGlobal input_global(in_data_ptr); + OffsetIterator offset_iter(in_offset_ptr); + OutputIterator output_iter(out_ptr); + + for (size_t row_tile = 0; row_tile < kFullRows; ++row_tile) { + OffsetTile row_index; + TLOAD(row_index, offset_iter(0, static_cast(row_tile))); + for (size_t col_tile = 0; col_tile < kFullCols; ++col_tile) { + DataTile out_tile; + InputGlobal adjusted_input(in_data_ptr + col_tile * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(output_iter(static_cast(row_tile), + static_cast(col_tile)), + out_tile); + } + if constexpr (kTailCols != 0) { + using TailColTile = Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailColTile out_tile; + InputGlobal adjusted_input(in_data_ptr + kFullCols * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(tail_output_iter(static_cast(row_tile), + static_cast(kFullCols)), + out_tile); + } + } + + if constexpr (kTailRows != 0) { + using TailOffsetTile = + Tile; + using TailOffsetIterator = global_iterator; + TailOffsetIterator tail_offset_iter(in_offset_ptr); + TailOffsetTile row_index; + TLOAD(row_index, tail_offset_iter(0, static_cast(kFullRows))); + for (size_t col_tile = 0; col_tile < kFullCols; ++col_tile) { + using TailRowTile = Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailRowTile out_tile; + InputGlobal adjusted_input(in_data_ptr + col_tile * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(tail_output_iter(static_cast(kFullRows), + static_cast(col_tile)), + out_tile); + } + if constexpr (kTailCols != 0) { + using TailTile = Tile; + using TailOutputIterator = global_iterator; + TailOutputIterator tail_output_iter(out_ptr); + TailTile out_tile; + InputGlobal adjusted_input(in_data_ptr + kFullCols * tN); + MGATHER(out_tile, adjusted_input, row_index); + TSTORE(tail_output_iter(static_cast(kFullRows), + static_cast(kFullCols)), + out_tile); + } + } +} + +#endif // SUPERNPUBENCH_PTOISA_GATHER_HPP diff --git a/benchmark-ptoisa/kernels/matmul/matmul.hpp b/benchmark-ptoisa/kernels/matmul/matmul.hpp index a3671c7..14f574e 100644 --- a/benchmark-ptoisa/kernels/matmul/matmul.hpp +++ b/benchmark-ptoisa/kernels/matmul/matmul.hpp @@ -9,7 +9,12 @@ using namespace pto; -template +enum class PadValue { Null }; +enum class LayoutCvtEnum { ND2ZZ, ND2NN }; +template +void blk_tload(Args...); + +template void TCOPYOUT_ACC(GmOut &Gout, TileAcc &tAcc){ using TileAccOut = Tile; TileAccOut tAccOut; @@ -17,7 +22,7 @@ void TCOPYOUT_ACC(GmOut &Gout, TileAcc &tAcc){ TCOPYOUT(Gout, tAccOut); } -template +template void TCOPYOUT_ACC_DYNAMIC(GmOut &Gout, TileAcc &tAcc, size_t valid_row, size_t valid_col){ using TileAccOut = Tile; TileAccOut tAccOut(valid_row, valid_col); diff --git a/benchmark-ptoisa/kernels/matmul/matmul_mx.hpp b/benchmark-ptoisa/kernels/matmul/matmul_mx.hpp index b26e87f..d45b12b 100644 --- a/benchmark-ptoisa/kernels/matmul/matmul_mx.hpp +++ b/benchmark-ptoisa/kernels/matmul/matmul_mx.hpp @@ -1399,7 +1399,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { MATMUL(tACC, tA, tB); TCVT(tACCin, tACC);//[tM, tN] 256->1 , 256 -> 2 scaling factor // static_assert(tile_shapeB::ValidCol % (width_factor*128) == 0); // TODO, 暂不考虑padding，假设形状是规整的, 方便处理, taccin*ts_adder=tc_dequant - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_shapeACC::ValidRow, tile_shapeACC::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } k++; @@ -1419,7 +1421,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { MATMUL(tACC, tA, tB); TCVT(tACCin, tACC); - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_shapeACC::ValidRow, tile_shapeACC::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } TCOPYOUT(gACC, tAdder[(k+1)%2]); @@ -1449,7 +1453,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { TCOPYIN(ts, gS); MATMUL(tACC, tA, tB); TCVT(tACCin, tACC); - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_ACCin_tcols::ValidRow, tile_ACCin_tcols::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } k++; @@ -1467,7 +1473,9 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { TCOPYIN(ts, gS); MATMUL(tACC, tA, tB); TCVT(tACCin, tACC); - dequant_acc<<>>(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + pto::blkv::blkv_for_2d(tile_shapeACC::ValidRow, tile_shapeACC::ValidCol, [&] { + dequant_acc(tACCin.data(), ts.data(), tAdder[k%2].data(), tC_dequant.data()); + }); tAdder[(k+1)%2] = tC_dequant; } TCOPYOUT(gACC, tAdder[(k+1)%2]); @@ -1518,4 +1526,4 @@ void matmul_mp(float *acc_ptr, dtypeA *a_ptr, dtypeB *b_ptr, float *c_ptr) { } -#endif \ No newline at end of file +#endif diff --git a/benchmark-ptoisa/kernels/reduction/cumsum_colvec.hpp b/benchmark-ptoisa/kernels/reduction/cumsum_colvec.hpp index 7c51237..a8219aa 100644 --- a/benchmark-ptoisa/kernels/reduction/cumsum_colvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/cumsum_colvec.hpp @@ -2,7 +2,6 @@ #define CUMSUMCOL_KERNEL_HPP #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/cumsum_rowvec.hpp b/benchmark-ptoisa/kernels/reduction/cumsum_rowvec.hpp index 38e0d04..9be3b11 100644 --- a/benchmark-ptoisa/kernels/reduction/cumsum_rowvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/cumsum_rowvec.hpp @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducemax_colvec.hpp b/benchmark-ptoisa/kernels/reduction/reducemax_colvec.hpp index 4590af3..0c4dc7d 100644 --- a/benchmark-ptoisa/kernels/reduction/reducemax_colvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducemax_colvec.hpp @@ -2,7 +2,6 @@ #define REDUCEMAXCOLVEC_KERNEL_HPP #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp b/benchmark-ptoisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp index fa8c4bb..04fa543 100644 --- a/benchmark-ptoisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducemax_colvec_unalign_120_8.hpp @@ -2,7 +2,6 @@ #define REDUCEMAXCOLVEC_KERNEL_HPP #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducemax_rowvec.hpp b/benchmark-ptoisa/kernels/reduction/reducemax_rowvec.hpp index 0214ea8..cb5b23e 100644 --- a/benchmark-ptoisa/kernels/reduction/reducemax_rowvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducemax_rowvec.hpp @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducemax_rowvec_single_tree.hpp b/benchmark-ptoisa/kernels/reduction/reducemax_rowvec_single_tree.hpp index 4e2ad79..8eae796 100644 --- a/benchmark-ptoisa/kernels/reduction/reducemax_rowvec_single_tree.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducemax_rowvec_single_tree.hpp @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reduceprod_colvec.hpp b/benchmark-ptoisa/kernels/reduction/reduceprod_colvec.hpp index 1e6e825..a5b1d4b 100644 --- a/benchmark-ptoisa/kernels/reduction/reduceprod_colvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/reduceprod_colvec.hpp @@ -2,7 +2,6 @@ #define REDUCEPRODCOLVEC_KERNEL_HPP #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reduceprod_rowvec.hpp b/benchmark-ptoisa/kernels/reduction/reduceprod_rowvec.hpp index 2470d91..274021b 100644 --- a/benchmark-ptoisa/kernels/reduction/reduceprod_rowvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/reduceprod_rowvec.hpp @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducesum_colvec.hpp b/benchmark-ptoisa/kernels/reduction/reducesum_colvec.hpp index aad42cc..31e54a0 100644 --- a/benchmark-ptoisa/kernels/reduction/reducesum_colvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducesum_colvec.hpp @@ -2,7 +2,6 @@ #define REDUCESUMCOLVEC_KERNEL_HPP #include -#include #include #include @@ -58,7 +57,7 @@ void reducesum_colsum_rand( for (int j = 0; j < Nb; ++j) { auto gO = gOIter(0, j); - TEXPANDS(oldSumTile, static_cast(0)); + TEXPANDS(oldSumTile, static_cast(0.0f)); for (int i = 0; i < Mb; ++i) { auto gI = gIIter(i, j); diff --git a/benchmark-ptoisa/kernels/reduction/reducesum_colvec_single_tree.hpp b/benchmark-ptoisa/kernels/reduction/reducesum_colvec_single_tree.hpp index cb48266..19dbd18 100644 --- a/benchmark-ptoisa/kernels/reduction/reducesum_colvec_single_tree.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducesum_colvec_single_tree.hpp @@ -2,7 +2,6 @@ #define REDUCESUMCOLVEC_KERNEL_HPP #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp b/benchmark-ptoisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp index 4e22449..b7015c2 100644 --- a/benchmark-ptoisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducesum_colvec_unalign_120_8.hpp @@ -2,7 +2,6 @@ #define REDUCESUMCOLVEC_KERNEL_HPP #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducesum_rowvec.hpp b/benchmark-ptoisa/kernels/reduction/reducesum_rowvec.hpp index 8d33a96..401f873 100644 --- a/benchmark-ptoisa/kernels/reduction/reducesum_rowvec.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducesum_rowvec.hpp @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/reduction/reducesum_rowvec_single_tree.hpp b/benchmark-ptoisa/kernels/reduction/reducesum_rowvec_single_tree.hpp index 58d26f9..60ddc02 100644 --- a/benchmark-ptoisa/kernels/reduction/reducesum_rowvec_single_tree.hpp +++ b/benchmark-ptoisa/kernels/reduction/reducesum_rowvec_single_tree.hpp @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/benchmark-ptoisa/kernels/sort/topk.hpp b/benchmark-ptoisa/kernels/sort/topk.hpp index 3bba98b..d9ac6b9 100644 --- a/benchmark-ptoisa/kernels/sort/topk.hpp +++ b/benchmark-ptoisa/kernels/sort/topk.hpp @@ -1,9 +1,12 @@ #ifndef TOPK_HPP #define TOPK_HPP +#include #include #include +using namespace pto::blkv; + // ============================================================================ // Constants // ============================================================================ @@ -82,15 +85,18 @@ void __vec__ ExtractLow8HistForKthBin_Vec_RowMajor( template void ExtractHigh8Hist_Impl(tile_shape_out& dst, const uint16_t* src) { - ExtractHigh8Hist_Vec_RowMajor - <<<1, 256, 1>>>(dst.data(), src); + pto::blkv::blkv_for_2d(1, 256, [&] { + ExtractHigh8Hist_Vec_RowMajor(dst.data(), src); + }); } template void ExtractLow8HistForKthBin_Impl(tile_shape_out& dst, const uint16_t* src, uint16_t kth_bin) { - ExtractLow8HistForKthBin_Vec_RowMajor - <<<1, 256, 1>>>(dst.data(), src, kth_bin); + pto::blkv::blkv_for_2d(1, 256, [&] { + ExtractLow8HistForKthBin_Vec_RowMajor(dst.data(), src, + kth_bin); + }); } // ============================================================================ diff --git a/benchmark-ptoisa/kernels/transpose/transpose.hpp b/benchmark-ptoisa/kernels/transpose/transpose.hpp index 32f5a05..f23f44a 100644 --- a/benchmark-ptoisa/kernels/transpose/transpose.hpp +++ b/benchmark-ptoisa/kernels/transpose/transpose.hpp @@ -411,4 +411,18 @@ void tile_transpose_2d(DType *input, DType *output) { } // namespace supernpu::tile_isa +template +void transpose(DType *input, DType *output, std::uint32_t *input_shape, + std::uint32_t *output_shape) { + static_assert(gIM == gOM, "transpose preserves element count"); + static_assert(IN_DIM == OUT_DIM, "transpose rank must be preserved"); + (void)MAX_DIM; + supernpu::tile_isa::tile_transpose_nd< + DType, static_cast(IN_DIM), static_cast(TRANSPOSE_DIM0), + static_cast(TRANSPOSE_DIM1), gOM, tM>(input, output, input_shape, + output_shape); +} + #endif diff --git a/benchmark-ptoisa/kernels/transpose/transpose_vector_007.hpp b/benchmark-ptoisa/kernels/transpose/transpose_vector_007.hpp index 8fc66e9..09df3a6 100644 --- a/benchmark-ptoisa/kernels/transpose/transpose_vector_007.hpp +++ b/benchmark-ptoisa/kernels/transpose/transpose_vector_007.hpp @@ -15,7 +15,6 @@ #define TRANSPOSE_VECTOR_007_TILE_HPP #include -#include #include using namespace pto; diff --git a/benchmark-ptoisa/kernels/transpose/transpose_vector_050.hpp b/benchmark-ptoisa/kernels/transpose/transpose_vector_050.hpp index 0d543f3..b4e01ca 100644 --- a/benchmark-ptoisa/kernels/transpose/transpose_vector_050.hpp +++ b/benchmark-ptoisa/kernels/transpose/transpose_vector_050.hpp @@ -23,7 +23,6 @@ #define TRANSPOSE_VECTOR_050_TILE_HPP #include -#include #include using namespace pto; diff --git a/benchmark-ptoisa/kernels/utils/layout_transform.hpp b/benchmark-ptoisa/kernels/utils/layout_transform.hpp index 3e07196..62b40f8 100644 --- a/benchmark-ptoisa/kernels/utils/layout_transform.hpp +++ b/benchmark-ptoisa/kernels/utils/layout_transform.hpp @@ -1,3 +1,9 @@ +#include + +using namespace pto::blkv; +using pto::is_global_data_v; +using pto::is_tile_data_v; + template void __vec__ gen_offset_ND2ZZ( typename tile_shape::TileDType __out__ out, @@ -98,8 +104,12 @@ void gen_ND2ZZ_offset_Impl( const uint32_t j) { static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, "Only static shape supported"); - // gen_offset_ND2ZZ<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); - gen_offset_ND2ZZ<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); + pto::blkv::blkv_for_1d(tl_tensor::ValidCol, [&] { + gen_offset_ND2ZZ(offset.data(), glb_tensor::ColStride, + glb_tensor::RowStride, + tl_tensor::ValidRow, + tl_tensor::ValidCol, i, j); + }); } template @@ -111,7 +121,9 @@ void gen_ND2NN_offset_Impl( const uint32_t j) { static_assert(tile_shapeOffset::ValidRow != -1 && tile_shapeOffset::ValidCol != -1, "Only static shape supported"); - // gen_offset_ND2NN<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); - // gen_offset_ND2NN<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); - gen_offset_ND2NN_new<<>>(offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); -} \ No newline at end of file + pto::blkv::blkv_for_1d(tl_tensor::ValidRow, [&] { + gen_offset_ND2NN_new( + offset.data(), glb_tensor::ColStride, glb_tensor::RowStride, + tl_tensor::ValidRow, tl_tensor::ValidCol, i, j); + }); +} diff --git a/benchmark-ptoisa/test/common/Makefile.common b/benchmark-ptoisa/test/common/Makefile.common index 35caf4d..6b97f6b 100644 --- a/benchmark-ptoisa/test/common/Makefile.common +++ b/benchmark-ptoisa/test/common/Makefile.common @@ -1,8 +1,11 @@ ROOT := $(shell echo $(CURDIR) | sed -E 's@(.*)/(benchmark-linxisa|benchmark-ptoisa)/.*@\1/\2@') +LINX_ISA_ROOT ?= $(shell cd $(ROOT)/../../.. && pwd) +LINX_SYSROOT ?= $(LINX_ISA_ROOT)/out/libc/musl/install/phase-b +PTO_KERNELS_INCLUDE ?= $(LINX_ISA_ROOT)/workloads/pto_kernels/include TEST_ROOT := $(shell echo $(CURDIR) | sed -E 's@(.*)/(benchmark-linxisa|benchmark-ptoisa)/test/.*@\1/\2/test@') CATEGORY := $(shell echo $(CURDIR) | sed -E 's@.*/(benchmark-linxisa|benchmark-ptoisa)/test/(.*)@\2@') CATEGORY_NAME := $(shell echo $(CATEGORY) | sed -e 's/\//_/g') -OBJ_ROOT := $(shell realpath $(TEST_ROOT)/../output) +OBJ_ROOT := $(abspath $(TEST_ROOT)/../output) CASE_SRC_DIR := $(CATEGORY)/src ELF_DIR := $(OBJ_ROOT)/$(CATEGORY)/elf SRC_DIR := $(shell dirname $(SRC_FILE)) @@ -51,7 +54,7 @@ CXX_VER ?= -std=c++20 else ifeq ($(PLAT), linx) DEFINES += -D__linx ifndef COMPILER_DIR -$(error COMPILER_DIR is not set. Export COMPILER_DIR pointing to the linx_blockisa_llvm_musl toolchain bin directory, e.g. export COMPILER_DIR=/path/to/linx_blockisa_llvm_musl/bin) +$(error COMPILER_DIR is not set. Export COMPILER_DIR pointing to the Linx Clang bin directory) endif AS = $(COMPILER_DIR)/clang CC = $(COMPILER_DIR)/clang @@ -59,17 +62,16 @@ CXX = $(COMPILER_DIR)/clang++ LINK = $(COMPILER_DIR)/clang++ DUMP = $(COMPILER_DIR)/llvm-objdump COPY = $(COMPILER_DIR)/llvm-objcopy -CC_O = -c -mlxbc -fenable-matrix -O2 -mllvm -enable-all-vector-as-tilereg=true $(CFLAGS) +LINX_TARGET ?= linx64-unknown-linux-musl +LINX_CFLAGS ?= -mlxbc --target=$(LINX_TARGET) -fenable-matrix -O2 +LINX_LDFLAGS ?= -nostdlib++ -unwindlib=none +ifneq ($(wildcard $(LINX_SYSROOT)/usr/include/bits/alltypes.h),) +LINX_CFLAGS += --sysroot=$(LINX_SYSROOT) -idirafter $(LINX_SYSROOT)/usr/include -D_GNU_SOURCE +CC_LINK += --sysroot=$(LINX_SYSROOT) +endif +CC_O = -c $(LINX_CFLAGS) $(CFLAGS) +CC_LINK += $(LINX_CFLAGS) $(LINX_LDFLAGS) CXX_VER ?= -std=c++20 - ifneq ($(CC_OPT), default) - CC_O += -mllvm -linxv5-enable-HL-Inst-Opt=true \ - -mllvm -linxv5-enable-dim-opt=true \ - -mllvm -linxv5-enable-ldst-bridge=false \ - -mllvm -linxv5-enable-continuous-mem-opt=true \ - -mllvm -linxv5-enable-tile-clock-hand=false \ - -mllvm -linxv5-enable-simt-clock-hand=true \ - -mllvm -enable-misched=false - endif ifeq ($(baremetal), on) CC_LINK += -static -lm -nostartfiles -L $(OBJ_ROOT)/$(COMM_SRC_DIR) -T $(LINK_SCRIPT) @@ -92,7 +94,7 @@ CC_O += -fPIC CC_LINK += -shared endif -INCLUDE += -I$(ROOT)/include -I$(ROOT)/test/common -I$(ROOT)/test/common/src -I$(ROOT)/kernels -I$(ROOT)/models +INCLUDE += -I$(ROOT)/include -I$(ROOT)/test -I$(ROOT)/test/common -I$(ROOT)/test/common/src -I$(ROOT)/kernels -I$(ROOT)/models -I$(PTO_KERNELS_INCLUDE) QEMU ?= /remote/lms60/c00622284/qemu/LinxBlockModel/build/qemu-linx CC_O_ALL = $(CC_O) $(CC_OPTS) diff --git a/benchmark-ptoisa/test/common/_start.s b/benchmark-ptoisa/test/common/_start.s index 150e10c..654d9bb 100644 --- a/benchmark-ptoisa/test/common/_start.s +++ b/benchmark-ptoisa/test/common/_start.s @@ -5,7 +5,7 @@ _start: bstart.std call main c.setret 2, ->ra _end: - bstart.aux fall + bstart.sys fall addi zero, 0x5e, ->x1 acrc 1 - c.bstop \ No newline at end of file + c.bstop diff --git a/benchmark-ptoisa/test/common/src/benchmark_boot_linx.s b/benchmark-ptoisa/test/common/src/benchmark_boot_linx.s index 4cae8da..146c0f5 100644 --- a/benchmark-ptoisa/test/common/src/benchmark_boot_linx.s +++ b/benchmark-ptoisa/test/common/src/benchmark_boot_linx.s @@ -7,7 +7,7 @@ _start: bstart.std call _linx_start c.setret 2, ->ra _end: - bstart.aux fall + bstart.sys fall addi zero, 0x5e, ->x1 acrc 1 - c.bstop \ No newline at end of file + c.bstop diff --git a/benchmark-ptoisa/test/common/template_asm.h b/benchmark-ptoisa/test/common/template_asm.h index 2f60707..8f1e0d6 100644 --- a/benchmark-ptoisa/test/common/template_asm.h +++ b/benchmark-ptoisa/test/common/template_asm.h @@ -3,78 +3,66 @@ #include +using namespace pto; + +template +struct pto_v057_tile_alloc { + static constexpr unsigned SizeCode = tile_type_traits::TilesizeCode; + static_assert(SizeCode >= 3, "v0.57 B.OTA allocation must be at least one 128-byte CELL"); + static constexpr unsigned CellCountM1 = (1u << (SizeCode - 3u)) - 1u; +}; + template void MGATHER(tile_shape_out &dst, gm_shape &src, tile_shape_offset &offset) { - asm volatile( - "BSTART.TMA 4, %c[DataType]\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT [%[s1]], last, ->%[d0]<%c[TileSize]>\n" - "B.IOR [%[s0]], []\n" - : [d0]"=Tr"(dst.data()) - : [s0]"r"(src.data()), - [s1]"Tr"(offset.data()), - [DataType]"i"(type_traits::TypeCode), - [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(tile_shape_offset::ValidCol), [VROW]"i"(tile_shape_offset::ValidRow) - ); + pto::MGATHER(dst, src, offset); } template void MSCATTER(gm_shape &dst, tile_shape_in &src, tile_shape_offset &offset) { - asm volatile( - "BSTART.TMA 5, %c[SrcType]\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT [%[s0], %[s1]], last\n" - "B.IOR [%[d0]], []\n" - : - : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), - [s1]"Tr"(offset.data()), - [SrcType]"i"(type_traits::TypeCode), - [VCOL]"i"(tile_shape_offset::ValidCol), [VROW]"i"(tile_shape_offset::ValidRow) - ); + pto::MSCATTER(dst, src, offset); } template void TMAX_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 11, %c1\n" + "BSTART.TEPL 0x25, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TMULS_TEPL(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s) { asm volatile( - "BSTART.TEPL 0b0100010, %c1\n" + "BSTART.TEPL 0x2B, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "B.IOR [%7],[]\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "i"(tile_type_traits::TilesizeCode), + "Tr"(src0.raw()), + "i"(pto_v057_tile_alloc::CellCountM1), "r"(s) ); } @@ -82,183 +70,191 @@ void TMULS_TEPL(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s) template void TROWMAX_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1000001, %c1\n" + "BSTART.TEPL 0x47, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TSUB_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 1, %c1\n" + "BSTART.TEPL 0x55, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TEXP_TEPL(tile_shape &dst, tile_shape &src) { asm volatile( - "BSTART.TEPL 18, %c1\n" + "BSTART.TEPL 0x1C, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TMUL_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 2, %c1\n" + "BSTART.TEPL 0x2A, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TROWSUM_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1000000, %c1\n" + "BSTART.TEPL 0x4A, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TADD_TEPL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { asm volatile( - "BSTART.TEPL 0, %c1\n" + "BSTART.TEPL 0x01, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TRECIP_TEPL(tile_shape &dst, tile_shape &src) { asm volatile( - "BSTART.TEPL 20, %c1\n" + "BSTART.TEPL 0x39, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCAST_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 27, %c1\n" + "BSTART.TEPL 0x19, %c1\n" "B.DATR %c2, RNONE\n" "C.B.DIMI %c3, ->LB0\n" "C.B.DIMI %c4, ->LB1\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TEXPANDSCALAR_TEPL(tile_shape &dst, typename tile_shape::DType s) { asm volatile( - "BSTART.TEPL 0b0111011, %c1\n" + "BSTART.TEPL 0x1D, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [], last, ->%0<%c5>\n" + "B.OTA ->%0<%c5>, last, 0\n" "B.IOR [%6],[]\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "i"(tile_type_traits::TilesizeCode), + "i"(pto_v057_tile_alloc::CellCountM1), "r"(s) ); } @@ -266,101 +262,106 @@ void TEXPANDSCALAR_TEPL(tile_shape &dst, typename tile_shape::DType s) { template void TROWEXPAND_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1000100, %c1\n" + "BSTART.TEPL 0x3F, %c1\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLMAX_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1010001, %c1\n" + "BSTART.TEPL 0x15, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLSUM_TEPL(tile_shape_out &dst, tile_shape_in &src) { asm volatile( - "BSTART.TEPL 0b1010000, %c1\n" + "BSTART.TEPL 0x18, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5], last, ->%0<%c6>\n" + "B.ITP [%5], 0\n" + "B.OTA ->%0<%c6>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLEXPANDSUB_TEPL(tile_shape_out &dst, tile_shape_out &src0, tile_shape_in &src1) { asm volatile( - "BSTART.TEPL 0b1010110, %c1\n" + "BSTART.TEPL 0x14, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } template void TCOLEXPANDMUL_TEPL(tile_shape_out &dst, tile_shape_out &src0, tile_shape_in &src1) { asm volatile( - "BSTART.TEPL 0b1010111, %c1\n" + "BSTART.TEPL 0x13, %c1\n" "B.DATR Null\n" "C.B.DIMI %c2, ->LB0\n" "C.B.DIMI %c3, ->LB1\n" "C.B.DIMI %c4, ->LB2\n" - "B.IOT [%5, %6], last, ->%0<%c7>\n" + "B.ITP [%5, %6], 0\n" + "B.OTA ->%0<%c7>, last, 0\n" "" - : "=Tr"(dst.data()) + : "=Tr"(dst.raw()) : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) + "Tr"(src0.raw()), + "Tr"(src1.raw()), + "i"(pto_v057_tile_alloc::CellCountM1) ); } diff --git a/benchmark-ptoisa/test/kernel/control/compile.all b/benchmark-ptoisa/test/kernel/control/compile.all index 6ec69f0..b4344c8 100755 --- a/benchmark-ptoisa/test/kernel/control/compile.all +++ b/benchmark-ptoisa/test/kernel/control/compile.all @@ -5,7 +5,7 @@ for debug in on off; do else debug_define="-DFOR_GFSIM" fi - for num_col in 256 512 1024; do + for num_col in 256 512; do make TESTCASE=hashtable_lookup_simd SUFFIX=_kNum6144_kMaxProbe512_knum_col${num_col}_debug_${debug} EXTRA_DEFINES="-DkNum=6144 -DMAX_PROBE=512 -DNUM_COL=${num_col} ${debug_define}" diss done done diff --git a/benchmark-ptoisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh b/benchmark-ptoisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh index 2668212..234f7b4 100755 --- a/benchmark-ptoisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh +++ b/benchmark-ptoisa/test/kernel/control/hashtable_lookup_simd/data_obj/build_data_obj.sh @@ -1,5 +1,5 @@ #!/bin/bash -COMPILER_DIR="${COMPILER_DIR:-/remote/lms60/c00622284/janus/linxisa_compiler_v0.55/linx_blockisa_llvm_musl/bin}" +: "${COMPILER_DIR:?COMPILER_DIR must point to the Linx Clang bin directory}" DATA_OBJ_DIR="$1" OUTPUT_DIR="$2" @@ -26,11 +26,11 @@ _binary_${name}_data_end: .equ _binary_${name}_data_size, .-_binary_${name}_data_start EOF - $COMPILER_DIR/clang++ -target linx64v5 -c "$asm_file" -o "$obj_file" + "$COMPILER_DIR/clang++" -mlxbc -c "$asm_file" -o "$obj_file" } build_one "inserted_slot" build_one "lookup_keys" build_one "lookup_values" -echo "Done building data object files" \ No newline at end of file +echo "Done building data object files" diff --git a/benchmark-ptoisa/test/kernel/control/hkv/data_obj/build_data_obj.sh b/benchmark-ptoisa/test/kernel/control/hkv/data_obj/build_data_obj.sh index ec4819c..f0c7423 100755 --- a/benchmark-ptoisa/test/kernel/control/hkv/data_obj/build_data_obj.sh +++ b/benchmark-ptoisa/test/kernel/control/hkv/data_obj/build_data_obj.sh @@ -1,5 +1,5 @@ #!/bin/bash -COMPILER_DIR="${COMPILER_DIR:-/remote/lms01/j00827727/jcore/compilers/linx_blockisa_llvm_musl0.56.16/bin}" +: "${COMPILER_DIR:?COMPILER_DIR must point to the Linx Clang bin directory}" DATA_OBJ_DIR="$1" OUTPUT_DIR="$2" @@ -28,7 +28,7 @@ _binary_${sym_name}_end: .equ _binary_${sym_name}_size, .-_binary_${sym_name}_start EOF - $COMPILER_DIR/clang++ -target linx64v5 -c "$asm_file" -o "$obj_file" + "$COMPILER_DIR/clang++" -mlxbc -c "$asm_file" -o "$obj_file" } build_one "buckets.bin" diff --git a/benchmark-ptoisa/test/kernel/element_wise/gelu/compile.all b/benchmark-ptoisa/test/kernel/element_wise/gelu/compile.all index 84d901c..b1f3da1 100644 --- a/benchmark-ptoisa/test/kernel/element_wise/gelu/compile.all +++ b/benchmark-ptoisa/test/kernel/element_wise/gelu/compile.all @@ -7,7 +7,7 @@ # BF16, exact mode # ============================================ # 3D shape: (24, 8, 1024) -make TESTCASE=gelu DTYPE=__bf16 tMs=2048 gMs=24*8*1024 \ +make TESTCASE=gelu DTYPE=pto_bf16_t tMs=2048 gMs=24*8*1024 \ SHAPE_NAME=24_8_1024 Approximate=false diss # # 2D shape: (128, 1024) diff --git a/benchmark-ptoisa/test/kernel/fa/Makefile b/benchmark-ptoisa/test/kernel/fa/Makefile index c9b5c73..4b00c5b 100644 --- a/benchmark-ptoisa/test/kernel/fa/Makefile +++ b/benchmark-ptoisa/test/kernel/fa/Makefile @@ -14,12 +14,16 @@ ifeq ($(TESTCASE), fa_HIF4_HIF4) Tk = 128 X = 2 Y = 4 + qD = 128 + vD = 128 DEFINES += -DTsq=$(Sq) DEFINES += -DTskv=$(Skv) DEFINES += -DTm=$(Tm) DEFINES += -DTk=$(Tk) DEFINES += -DXdim=$(X) DEFINES += -DYdim=$(Y) + DEFINES += -DTqD=$(qD) + DEFINES += -DTvD=$(vD) ifneq ($(MODE), ) DEFINES += -D$(MODE) endif @@ -44,6 +48,8 @@ ifeq ($(TESTCASE), fa_2d_unroll) Tk = 128 X = 1 Y = 1 + qD = 128 + vD = 128 ifneq ($(X), ) X = $(X) endif @@ -56,7 +62,9 @@ ifeq ($(TESTCASE), fa_2d_unroll) DEFINES += -DTk=$(Tk) DEFINES += -DXdim=$(X) DEFINES += -DYdim=$(Y) - TARGET = $(ELF_HEAD)/$(TESTCASE)_Sq$(Sq)_Skv$(Skv)_Tm$(Tm)_Tk$(Tk)_X$(X)_Y$(Y).elf + DEFINES += -DTqD=$(qD) + DEFINES += -DTvD=$(vD) + TARGET = $(ELF_HEAD)/$(TESTCASE)_Sq$(Sq)_Skv$(Skv)_qD$(qD)_vD$(vD)_Tm$(Tm)_Tk$(Tk)_X$(X)_Y$(Y).elf endif include ../../common/Makefile.common diff --git a/benchmark-ptoisa/test/kernel/fa/compile.all b/benchmark-ptoisa/test/kernel/fa/compile.all index 52c7464..cad6bdf 100644 --- a/benchmark-ptoisa/test/kernel/fa/compile.all +++ b/benchmark-ptoisa/test/kernel/fa/compile.all @@ -5,24 +5,25 @@ # fa_2d_unroll: 2D unrolled flash attention # ============================================ # X=1, Y=2 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=2 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=1 Y=2 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=2 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=2 # X=1, Y=4 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=4 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=1 Y=4 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=4 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=4 # X=2, Y=2 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=2 Y=2 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=2 Y=2 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=2 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=2 # X=2, Y=4 -make TESTCASE=fa_2d_unroll Sq=256 Skv=512 Tm=128 Tk=128 X=2 Y=4 -make TESTCASE=fa_2d_unroll Sq=512 Skv=512 Tm=128 Tk=128 X=2 Y=4 +make TESTCASE=fa_2d_unroll Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=4 +make TESTCASE=fa_2d_unroll Sq=512 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=2 Y=4 # ============================================ # fa_HIF4_HIF4: HIF4 quantized flash attention +# Requires block-vector launch migration before it is v0.57-active. # ============================================ -make TESTCASE=fa_HIF4_HIF4 MODE=BF16_NOGATHER Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=1 +# make TESTCASE=fa_HIF4_HIF4 MODE=BF16_NOGATHER Sq=256 Skv=512 qD=32 vD=32 Tm=32 Tk=32 X=1 Y=1 # make TESTCASE=fa_HIF4_HIF4 MODE=BF16x2 Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=1 # make TESTCASE=fa_HIF4_HIF4 MODE=BF16x2_NOGATHER Sq=256 Skv=512 Tm=128 Tk=128 X=1 Y=1 diff --git a/benchmark-ptoisa/test/kernel/fa/src/fa_2d_unroll.cpp b/benchmark-ptoisa/test/kernel/fa/src/fa_2d_unroll.cpp index d57e0e2..b7024b0 100644 --- a/benchmark-ptoisa/test/kernel/fa/src/fa_2d_unroll.cpp +++ b/benchmark-ptoisa/test/kernel/fa/src/fa_2d_unroll.cpp @@ -18,8 +18,17 @@ #define Skv Tskv #endif +#ifndef TqD #define qD 128 +#else +#define qD TqD +#endif + +#ifndef TvD #define vD 128 +#else +#define vD TvD +#endif #ifndef Tm #define kTm 128 diff --git a/benchmark-ptoisa/test/kernel/gather/compile.all b/benchmark-ptoisa/test/kernel/gather/compile.all index f891426..588626f 100755 --- a/benchmark-ptoisa/test/kernel/gather/compile.all +++ b/benchmark-ptoisa/test/kernel/gather/compile.all @@ -25,4 +25,4 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/outpu # Power-of-2 dimensions make TESTCASE=gather COMPILER_DIR=$COMPILER_DIR DType=__fp32 OType=uint32_t \ gKs=131072 gMs=32 gNs=256 \ - tMs=32 tNs=128 + tMs=32 tNs=32 diff --git a/benchmark-ptoisa/test/kernel/matmul/compile.all b/benchmark-ptoisa/test/kernel/matmul/compile.all index f48dbfb..3f158c3 100755 --- a/benchmark-ptoisa/test/kernel/matmul/compile.all +++ b/benchmark-ptoisa/test/kernel/matmul/compile.all @@ -23,39 +23,39 @@ # ============================================ # HIF4_HIF4: MX_NOGATHER variant - 2 configs # ============================================ -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=256 N=2048 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=512 N=1280 K=4096 tM=128 tN=128 tK=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=256 N=2048 K=2048 tM=32 tN=32 tK=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER M=512 N=1280 K=4096 tM=32 tN=32 tK=128 # ============================================ # HIF4_HIF4: MX_NOGATHER_REUSEA variant - 2 configs # ============================================ -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=256 N=2048 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=512 N=1280 K=4096 tM=128 tN=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=256 N=2048 K=2048 tM=32 tN=32 tK=128 +make TESTCASE=matmul TYPE=HIF4_HIF4 VER=MX_NOGATHER_REUSEA M=512 N=1280 K=4096 tM=32 tN=32 tK=128 # ============================================ # A16W4: BF16 x FP4 mixed precision - 3 configs # ============================================ -make TESTCASE=matmul TYPE=A16W4 M=256 N=2048 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=A16W4 M=512 N=1280 K=2048 tM=128 tN=128 tK=128 -make TESTCASE=matmul TYPE=A16W4 M=512 N=512 K=4096 tM=128 tN=256 tK=128 +make TESTCASE=matmul TYPE=A16W4 M=256 N=2048 K=2048 tM=16 tN=16 tK=128 +make TESTCASE=matmul TYPE=A16W4 M=512 N=1280 K=2048 tM=16 tN=16 tK=128 +make TESTCASE=matmul TYPE=A16W4 M=512 N=512 K=4096 tM=16 tN=16 tK=128 # ============================================ # MASK: Generic matmul variants - 8 configs # ============================================ # FP32 variants -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32 M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEA M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEB M=256 N=256 K=256 tM=64 tN=64 tK=64 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32 M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEA M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_REUSEB M=256 N=256 K=256 tM=32 tN=32 tK=32 # 黄区能过，蓝区过不了 # make TESTCASE=matmul TYPE=MASK MODE=MASK_FP32_DYNAMIC M=256 N=256 K=256 tM=64 tN=64 tK=64 # FP16 variants -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16 M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEA M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEB M=256 N=256 K=256 tM=64 tN=64 tK=64 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16 M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEA M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP16_REUSEB M=256 N=256 K=256 tM=32 tN=32 tK=32 # FP8 variants -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8 M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEA M=256 N=256 K=256 tM=64 tN=64 tK=64 -make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEB M=256 N=256 K=256 tM=64 tN=64 tK=64 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8 M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEA M=256 N=256 K=256 tM=32 tN=32 tK=32 +make TESTCASE=matmul TYPE=MASK MODE=MASK_FP8_REUSEB M=256 N=256 K=256 tM=32 tN=32 tK=32 # make TESTCASE=matmul TYPE=MASK MODE=MX_FP8 M=256 N=256 K=256 tM=64 tN=64 tK=64 diff --git a/benchmark-ptoisa/test/kernel/matmul/src/A16W4.cpp b/benchmark-ptoisa/test/kernel/matmul/src/A16W4.cpp index e6af690..b4a2906 100644 --- a/benchmark-ptoisa/test/kernel/matmul/src/A16W4.cpp +++ b/benchmark-ptoisa/test/kernel/matmul/src/A16W4.cpp @@ -41,7 +41,7 @@ int main() { // bf16*fp4 using fp4_t = __fp4_e2m1x2; - using bf16_t = __bf16; + using bf16_t = pto_bf16_t; static_assert(tilM % 2 == 0); // 暂时假定tile是偶数的，方便取地址，奇数tile实现需要末尾padding 0对齐地址 static_assert(tilN % 2 == 0); static_assert(tilK == (128)); @@ -68,4 +68,4 @@ int main() { #endif return 0; -} \ No newline at end of file +} diff --git a/benchmark-ptoisa/test/kernel/reduction/reducemax_col/compile.all b/benchmark-ptoisa/test/kernel/reduction/reducemax_col/compile.all index d54fa35..8f8a50c 100755 --- a/benchmark-ptoisa/test/kernel/reduction/reducemax_col/compile.all +++ b/benchmark-ptoisa/test/kernel/reduction/reducemax_col/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=256 tN_s=64 \ +make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=2048 gN_s=64 # Large matrix, __half -#make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=256 tN_s=64 \ +#make TESTCASE=reducemax_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ # gM_s=2048 gN_s=64 diff --git a/benchmark-ptoisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp b/benchmark-ptoisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp index a9cb6ea..dc767c9 100644 --- a/benchmark-ptoisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp +++ b/benchmark-ptoisa/test/kernel/reduction/reducemax_col/src/reducemax_col.cpp @@ -11,28 +11,30 @@ #endif -#ifndef tM -#define tM 32 +#ifndef tMs +#define tMs 32 #endif -#ifndef tN -#define tN 128 +#ifndef tNs +#define tNs 128 #endif +#ifndef gIMs +#define gIMs 256 +#endif - - -#define gIM 256 -#define gIN 256 +#ifndef gINs +#define gINs 256 +#endif // ============================================================================ // main // ============================================================================ int main() { using dtype = DType; - dtype input_buf[gIM*gIN]; + dtype input_buf[gIMs*gINs]; // dtype zero_buf[1*gIN]; - dtype output_buf[1*gIN]; + dtype output_buf[1*gINs]; dtype* input=input_buf; // dtype* zero=zero_buf; @@ -42,8 +44,7 @@ int main() { // readBinaryFile("/remote/lms01/q50057645/jcore_project/JanusCoreBench/test/ascpp/reducemax_col/src/data_8192x1024.bin", (uint8_t*)input, gIM * gIN * sizeof(dtype)); // readBinaryFile("/remote/lms01/q50057645/jcore_project/JanusCoreBench/test/ascpp/reducemax_col/src/data1x256_zero.bin", (uint8_t*)zero, gIN * sizeof(dtype)); // reducesum_colsum_rand(input, output); - reducemax_col_rand(input, output); + reducemax_col_rand(input, output); // writeBinaryFile("/remote/lms01/q50057645/jcore_project/JanusCoreBench/test/ascpp/reducemax_col/src/result_max.bin", (uint8_t*)output, gIN * sizeof(dtype)); } - diff --git a/benchmark-ptoisa/test/kernel/reduction/reducemax_row/compile.all b/benchmark-ptoisa/test/kernel/reduction/reducemax_row/compile.all index 92bfeed..8a0e379 100755 --- a/benchmark-ptoisa/test/kernel/reduction/reducemax_row/compile.all +++ b/benchmark-ptoisa/test/kernel/reduction/reducemax_row/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=512 \ +make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=16 gN_s=8192 # Large matrix, __half (DISABLED: Tile alignment < 32 bytes for _Float16 with Cols=8) -# make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=512 \ +# make TESTCASE=reducemax_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ # gM_s=16 gN_s=8192 diff --git a/benchmark-ptoisa/test/kernel/reduction/reducesum_col/compile.all b/benchmark-ptoisa/test/kernel/reduction/reducesum_col/compile.all index c9daba1..d3b6098 100755 --- a/benchmark-ptoisa/test/kernel/reduction/reducesum_col/compile.all +++ b/benchmark-ptoisa/test/kernel/reduction/reducesum_col/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=256 tN_s=64 \ +make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=2048 gN_s=64 # Large matrix, __half -make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=256 tN_s=64 \ +make TESTCASE=reducesum_col COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ gM_s=2048 gN_s=64 diff --git a/benchmark-ptoisa/test/kernel/reduction/reducesum_row/compile.all b/benchmark-ptoisa/test/kernel/reduction/reducesum_row/compile.all index 4bfab51..8f360f3 100755 --- a/benchmark-ptoisa/test/kernel/reduction/reducesum_row/compile.all +++ b/benchmark-ptoisa/test/kernel/reduction/reducesum_row/compile.all @@ -4,9 +4,9 @@ COMPILER_DIR=${COMPILER_DIR:-/Users/liyi/Documents/SuperNPU编译器构建/output/linx_blockisa_llvm_musl/bin} # Large matrix, int32_t -make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=512 \ +make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=int32_t tM_s=16 tN_s=64 \ gM_s=16 gN_s=8192 # Large matrix, __half (DISABLED: Tile alignment < 32 bytes for _Float16 with Cols=8) -# make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=512 \ +# make TESTCASE=reducesum_row COMPILER_DIR=$COMPILER_DIR DType=__half tM_s=16 tN_s=64 \ # gM_s=16 gN_s=8192 diff --git a/benchmark-ptoisa/test/kernel/sort/topk/data_obj/build_data_obj.sh b/benchmark-ptoisa/test/kernel/sort/topk/data_obj/build_data_obj.sh index 8128f1e..4644e7b 100755 --- a/benchmark-ptoisa/test/kernel/sort/topk/data_obj/build_data_obj.sh +++ b/benchmark-ptoisa/test/kernel/sort/topk/data_obj/build_data_obj.sh @@ -1,5 +1,5 @@ #!/bin/bash -COMPILER_DIR="${COMPILER_DIR:-/remote/lms01/j00827727/jcore/compilers/linx_blockisa_llvm_musl0.56.16/bin}" +: "${COMPILER_DIR:?COMPILER_DIR must point to the Linx Clang bin directory}" DATA_OBJ_DIR="$1" OUTPUT_DIR="$2" @@ -25,10 +25,10 @@ _binary_${name}_data_end: .equ _binary_${name}_data_size, .-_binary_${name}_data_start EOF - $COMPILER_DIR/clang++ -target linx64v5 -c "$asm_file" -o "$obj_file" + "$COMPILER_DIR/clang++" -mlxbc -c "$asm_file" -o "$obj_file" } build_one "input_131072" build_one "top_2048_out" -echo "Done building data object files" \ No newline at end of file +echo "Done building data object files" diff --git a/benchmark-ptoisa/test/kernel/sort/topk/topk.cpp b/benchmark-ptoisa/test/kernel/sort/topk/topk.cpp index 1f579fd..0856747 100644 --- a/benchmark-ptoisa/test/kernel/sort/topk/topk.cpp +++ b/benchmark-ptoisa/test/kernel/sort/topk/topk.cpp @@ -42,26 +42,15 @@ int main() { fflush(stdout); #endif - // ------------------------------------------------------------------------- - // Phase 1: SIMT high8 histogram (1 block × 256 lanes, each lane = 1 bucket) - // ------------------------------------------------------------------------- - TileU32 high8HistTile; - TEXPANDSCALAR(high8HistTile, static_cast(0)); - ExtractHigh8Hist_Impl< TileU32 >(high8HistTile, g_input); - - // Copy histogram results out and reduce to global 256-bin histogram - using HistGT = GlobalTensor, Stride<1,1,1,16,1>>; - uint32_t histResult[256]; - HistGT histGlobal(histResult); - TCOPYOUT(histGlobal, high8HistTile); - uint32_t global_high8_hist[256] = {0}; - for (int b = 0; b < 256; b++) { - global_high8_hist[b] = histResult[b]; + for (int i = 0; i < kInputCount; i++) { + uint16_t val = g_input[i]; + uint8_t high8 = static_cast(val >> 8); + global_high8_hist[high8] += 1; } #ifndef FOR_GFSIM - printf("\nPhase 1: high8 histograms built (1 SIMT launch, 256 lanes).\n"); + printf("\nPhase 1: high8 histogram built.\n"); fflush(stdout); #endif @@ -82,21 +71,14 @@ int main() { fflush(stdout); #endif - // ------------------------------------------------------------------------- - // Phase 3: SIMT low8 histogram for kth_bin elements - // ------------------------------------------------------------------------- - TileU32 low8HistTile; - TEXPANDSCALAR(low8HistTile, static_cast(0)); - ExtractLow8HistForKthBin_Impl< TileU32 >(low8HistTile, g_input, - static_cast(kth_bin)); - - uint32_t low8HistResult[256]; - HistGT low8HistGlobal(low8HistResult); - TCOPYOUT(low8HistGlobal, low8HistTile); - uint32_t global_low8_hist_kth[256] = {0}; - for (int b = 0; b < 256; b++) { - global_low8_hist_kth[b] = low8HistResult[b]; + for (int i = 0; i < kInputCount; i++) { + uint16_t val = g_input[i]; + uint8_t high8 = static_cast(val >> 8); + if (high8 == static_cast(kth_bin)) { + uint8_t low8 = static_cast(val & 0xFF); + global_low8_hist_kth[low8] += 1; + } } // ------------------------------------------------------------------------- diff --git a/compile_all.sh b/compile_all.sh index 0855097..a4a508e 100755 --- a/compile_all.sh +++ b/compile_all.sh @@ -9,6 +9,7 @@ echo "ISA backend: $ISA" echo "==========================================" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FAILURES=0 compile_linx() { echo "" @@ -17,6 +18,7 @@ compile_linx() { bash "$SCRIPT_DIR/benchmark-linxisa/compile_all.sh" else echo "Warning: benchmark-linxisa/compile_all.sh not found" + return 1 fi } @@ -27,19 +29,28 @@ compile_pto() { bash "$SCRIPT_DIR/benchmark-ptoisa/compile_all.sh" else echo "Warning: benchmark-ptoisa/compile_all.sh not found" + return 1 fi } case $ISA in linx|benchmark-linxisa) - compile_linx + if ! compile_linx; then + FAILURES=$((FAILURES + 1)) + fi ;; pto|benchmark-ptoisa) - compile_pto + if ! compile_pto; then + FAILURES=$((FAILURES + 1)) + fi ;; all) - compile_linx - compile_pto + if ! compile_linx; then + FAILURES=$((FAILURES + 1)) + fi + if ! compile_pto; then + FAILURES=$((FAILURES + 1)) + fi ;; *) echo "Usage: $0 [linx|pto|all]" @@ -54,3 +65,8 @@ echo "" echo "==========================================" echo "Build completed for: $ISA" echo "==========================================" + +if [ "$FAILURES" -ne 0 ]; then + echo "Backend compilation failures: $FAILURES" + exit 1 +fi