From c0840c304052d9e22b1aa45be5b11ab053675a18 Mon Sep 17 00:00:00 2001 From: Basem Mohammed <95645899+Basemism@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:43:25 +0200 Subject: [PATCH] arch-amdgpu: support ACC-selected DS data operands Decode the CDNA DS ACC bit and map DS data operands, but not address operands, through the AGPR window of the unified vector register file. Apply the same accumulator offset during operand mapping and instruction execution for 32-, 64-, 96-, and 128-bit DS reads and writes. Also accept all DS primary-encoding variants used by the ACC form and check that mapped AGPR operands remain inside the wavefront's reserved vector register allocation. --- src/arch/amdgpu/vega/gpu_decoder.cc | 8 +-- src/arch/amdgpu/vega/gpu_decoder.hh | 4 +- src/arch/amdgpu/vega/insts/ds.cc | 82 ++++++++++++++-------- src/arch/amdgpu/vega/insts/op_encodings.cc | 4 ++ src/arch/amdgpu/vega/insts/op_encodings.hh | 8 +++ src/gpu-compute/gpu_static_inst.cc | 21 ++++++ src/gpu-compute/hsa_queue_entry.hh | 5 +- src/gpu-compute/operand_info.hh | 8 ++- 8 files changed, 102 insertions(+), 38 deletions(-) diff --git a/src/arch/amdgpu/vega/gpu_decoder.cc b/src/arch/amdgpu/vega/gpu_decoder.cc index 4574e88a98e..6f26338bb7c 100644 --- a/src/arch/amdgpu/vega/gpu_decoder.cc +++ b/src/arch/amdgpu/vega/gpu_decoder.cc @@ -512,10 +512,10 @@ IsaDecodeMethod Decoder::tableDecodePrimary[] = { &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_DS, - &Decoder::decode_invalid, - &Decoder::decode_invalid, - &Decoder::decode_invalid, - &Decoder::decode_invalid, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_FLAT, &Decoder::subDecode_OP_FLAT, &Decoder::subDecode_OP_FLAT, diff --git a/src/arch/amdgpu/vega/gpu_decoder.hh b/src/arch/amdgpu/vega/gpu_decoder.hh index 36b653e33a8..049baa721bf 100644 --- a/src/arch/amdgpu/vega/gpu_decoder.hh +++ b/src/arch/amdgpu/vega/gpu_decoder.hh @@ -1782,7 +1782,9 @@ struct InFmt_DS unsigned int OFFSET1 : 8; unsigned int GDS : 1; unsigned int OP : 8; - unsigned int pad_25 : 1; + // CDNA ACC selects the AGPR bank for DS data registers. ADDR remains + // a VGPR. + unsigned int ACC : 1; unsigned int ENCODING : 6; }; diff --git a/src/arch/amdgpu/vega/insts/ds.cc b/src/arch/amdgpu/vega/insts/ds.cc index a5118f66187..035014261ee 100644 --- a/src/arch/amdgpu/vega/insts/ds.cc +++ b/src/arch/amdgpu/vega/insts/ds.cc @@ -406,7 +406,8 @@ Inst_DS__DS_WRITE_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data(gpuDynInst, extData.DATA0); + ConstVecOperandU32 data(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); addr.read(); data.read(); @@ -471,8 +472,10 @@ Inst_DS__DS_WRITE2_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU32 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU32 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -539,8 +542,10 @@ Inst_DS__DS_WRITE2ST64_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU32 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU32 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -1621,7 +1626,8 @@ Inst_DS__DS_READ_B32::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B32::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst(gpuDynInst, extData.VDST); + Wavefront *wf = gpuDynInst->wavefront(); + VecOperandU32 vdst(gpuDynInst, extData.VDST + accDataOffset(wf)); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -1686,8 +1692,10 @@ Inst_DS__DS_READ2_B32::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ2_B32::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -2762,7 +2770,8 @@ Inst_DS__DS_WRITE_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data(gpuDynInst, extData.DATA0); + ConstVecOperandU64 data(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); addr.read(); data.read(); @@ -2827,8 +2836,10 @@ Inst_DS__DS_WRITE2_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU64 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU64 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU64 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -2895,8 +2906,10 @@ Inst_DS__DS_WRITE2ST64_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU64 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU64 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU64 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -3486,7 +3499,8 @@ Inst_DS__DS_READ_B64::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B64::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU64 vdst(gpuDynInst, extData.VDST); + Wavefront *wf = gpuDynInst->wavefront(); + VecOperandU64 vdst(gpuDynInst, extData.VDST + accDataOffset(wf)); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -3551,8 +3565,10 @@ Inst_DS__DS_READ2_B64::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ2_B64::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU64 vdst0(gpuDynInst, extData.VDST); - VecOperandU64 vdst1(gpuDynInst, extData.VDST + 2); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU64 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU64 vdst1(gpuDynInst, extData.VDST + acc + 2); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -4543,9 +4559,10 @@ Inst_DS__DS_WRITE_B96::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1); - ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2); + unsigned acc = accDataOffset(wf); + ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc); + ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1); + ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2); addr.read(); data0.read(); @@ -4607,10 +4624,11 @@ Inst_DS__DS_WRITE_B128::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1); - ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2); - ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + 3); + unsigned acc = accDataOffset(wf); + ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc); + ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1); + ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2); + ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + acc + 3); addr.read(); data0.read(); @@ -4697,9 +4715,11 @@ Inst_DS__DS_READ_B96::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B96::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); - VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); + VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -4762,10 +4782,12 @@ Inst_DS__DS_READ_B128::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B128::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); - VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2); - VecOperandU32 vdst3(gpuDynInst, extData.VDST + 3); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); + VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2); + VecOperandU32 vdst3(gpuDynInst, extData.VDST + acc + 3); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { diff --git a/src/arch/amdgpu/vega/insts/op_encodings.cc b/src/arch/amdgpu/vega/insts/op_encodings.cc index fb49cb5b5ce..4b743f8ff20 100644 --- a/src/arch/amdgpu/vega/insts/op_encodings.cc +++ b/src/arch/amdgpu/vega/insts/op_encodings.cc @@ -1345,6 +1345,8 @@ Inst_DS::initOperandInfo() for (opIdx = 0; opIdx < numSrcRegOperands(); opIdx++) { srcOps.emplace_back(srcs[opIdx], getOperandSize(opIdx), true, false, true, false); + if (instData.ACC && opIdx > 0) + srcOps.back().setAccum(); } if (numDstRegOperands()) { @@ -1352,6 +1354,8 @@ Inst_DS::initOperandInfo() int reg = extData.VDST; dstOps.emplace_back(reg, getOperandSize(opIdx), false, false, true, false); + if (instData.ACC) + dstOps.back().setAccum(); } assert(srcOps.size() == numSrcRegOperands()); diff --git a/src/arch/amdgpu/vega/insts/op_encodings.hh b/src/arch/amdgpu/vega/insts/op_encodings.hh index 96fd9fd4a98..40c68f2b6b4 100644 --- a/src/arch/amdgpu/vega/insts/op_encodings.hh +++ b/src/arch/amdgpu/vega/insts/op_encodings.hh @@ -983,6 +983,14 @@ class Inst_DS : public VEGAGPUStaticInst void initOperandInfo() override; protected: + // CDNA ACC data registers are mapped into the AGPR window of the + // unified vector register file. ADDR remains an ordinary VGPR. + unsigned + accDataOffset(Wavefront *wf) const + { + return instData.ACC ? wf->accumOffset : 0; + } + template void initMemRead(GPUDynInstPtr gpuDynInst, Addr offset) diff --git a/src/gpu-compute/gpu_static_inst.cc b/src/gpu-compute/gpu_static_inst.cc index 1919a70a8ed..0b28adf7551 100644 --- a/src/gpu-compute/gpu_static_inst.cc +++ b/src/gpu-compute/gpu_static_inst.cc @@ -71,9 +71,30 @@ GPUStaticInst::generateVirtToPhysMap(Wavefront *wf, ComputeUnit *cu, int num_dwords = op.sizeInDWords(); int virt_idx = op.registerIndex(wf->reservedScalarRegs); + // CDNA ACC bit: the operand's data registers live in the accumulator + // (AGPR) window of the unified VRF, which begins at wf->accumOffset. + // KEEP IN SYNC with Inst_DS::accDataOffset() (op_encodings.hh), which + // applies the identical offset on the data path during execute(). + if (op.isAccum()) { + virt_idx += wf->accumOffset; + } + int phys_idx = -1; for (int i = 0; i < num_dwords; i++) { if (opType == OpType::SRC_VEC || opType == OpType::DST_VEC) { + const int this_virt_idx = virt_idx + i; + panic_if(this_virt_idx >= wf->reservedVectorRegs, + "%s maps %s vector %s operand out of reserved VGPR " + "range: virt_idx=%d raw_reg=%d base_reg=%d dword=%d/%d " + "reserved=%d accum_offset=%u is_accum=%d " + "reserved_sgprs=%d\n", + disassemble().c_str(), + opType == OpType::SRC_VEC ? "src" : "dst", + op.isAccum() ? "AGPR" : "VGPR", + this_virt_idx, op.rawRegisterIndex(), + op.registerIndex(wf->reservedScalarRegs), i, num_dwords, + wf->reservedVectorRegs, wf->accumOffset, op.isAccum(), + wf->reservedScalarRegs); phys_idx = cu->registerManager->mapVgpr(wf, virt_idx + i); } else { assert(opType == OpType::SRC_SCALAR || diff --git a/src/gpu-compute/hsa_queue_entry.hh b/src/gpu-compute/hsa_queue_entry.hh index f5e1d833b00..5385e8f664c 100644 --- a/src/gpu-compute/hsa_queue_entry.hh +++ b/src/gpu-compute/hsa_queue_entry.hh @@ -104,8 +104,9 @@ class HSAQueueEntry // LLVM docs: https://www.llvm.org/docs/AMDGPUUsage.html // #code-object-v3-kernel-descriptor // - // Currently, gem5 supported gfx version use a multiplier of 8. The - // only exception is gfx900 (Vega10). + // CDNA's VGPR field is encoded from LLVM's total unified allocation: + // alignTo(num_arch_vgprs, 4) + num_agprs. ACCUM_OFFSET only identifies + // where AGPR operands begin; it must not be added to this allocation. if (gfx_version == GfxVersion::gfx90a || gfx_version == GfxVersion::gfx942 || gfx_version == GfxVersion::gfx950) { diff --git a/src/gpu-compute/operand_info.hh b/src/gpu-compute/operand_info.hh index e28f2c2ddd7..05d13a1355f 100644 --- a/src/gpu-compute/operand_info.hh +++ b/src/gpu-compute/operand_info.hh @@ -157,6 +157,9 @@ class OperandInfo return flags.isSet(FLAT); } + void setAccum() { flags.set(ACCUM); } + bool isAccum() const { return flags.isSet(ACCUM); } + void setVirtToPhysMapping(std::vector v, std::vector p) { @@ -234,7 +237,10 @@ class OperandInfo CONSTANT = 0x00000100, // If the constant is positive or negative - POS_CONST = 0x00000200 + POS_CONST = 0x00000200, + + // Operand resides in the CDNA accumulator (AGPR) window. + ACCUM = 0x00000400 }; Flags flags;