diff --git a/src/arch/amdgpu/vega/gpu_decoder.cc b/src/arch/amdgpu/vega/gpu_decoder.cc index 4574e88a98e..6f26338bb7c 100644 --- a/src/arch/amdgpu/vega/gpu_decoder.cc +++ b/src/arch/amdgpu/vega/gpu_decoder.cc @@ -512,10 +512,10 @@ IsaDecodeMethod Decoder::tableDecodePrimary[] = { &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_DS, - &Decoder::decode_invalid, - &Decoder::decode_invalid, - &Decoder::decode_invalid, - &Decoder::decode_invalid, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, + &Decoder::subDecode_OP_DS, &Decoder::subDecode_OP_FLAT, &Decoder::subDecode_OP_FLAT, &Decoder::subDecode_OP_FLAT, diff --git a/src/arch/amdgpu/vega/gpu_decoder.hh b/src/arch/amdgpu/vega/gpu_decoder.hh index 36b653e33a8..049baa721bf 100644 --- a/src/arch/amdgpu/vega/gpu_decoder.hh +++ b/src/arch/amdgpu/vega/gpu_decoder.hh @@ -1782,7 +1782,9 @@ struct InFmt_DS unsigned int OFFSET1 : 8; unsigned int GDS : 1; unsigned int OP : 8; - unsigned int pad_25 : 1; + // CDNA ACC selects the AGPR bank for DS data registers. ADDR remains + // a VGPR. + unsigned int ACC : 1; unsigned int ENCODING : 6; }; diff --git a/src/arch/amdgpu/vega/insts/ds.cc b/src/arch/amdgpu/vega/insts/ds.cc index a5118f66187..035014261ee 100644 --- a/src/arch/amdgpu/vega/insts/ds.cc +++ b/src/arch/amdgpu/vega/insts/ds.cc @@ -406,7 +406,8 @@ Inst_DS__DS_WRITE_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data(gpuDynInst, extData.DATA0); + ConstVecOperandU32 data(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); addr.read(); data.read(); @@ -471,8 +472,10 @@ Inst_DS__DS_WRITE2_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU32 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU32 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -539,8 +542,10 @@ Inst_DS__DS_WRITE2ST64_B32::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU32 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU32 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -1621,7 +1626,8 @@ Inst_DS__DS_READ_B32::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B32::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst(gpuDynInst, extData.VDST); + Wavefront *wf = gpuDynInst->wavefront(); + VecOperandU32 vdst(gpuDynInst, extData.VDST + accDataOffset(wf)); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -1686,8 +1692,10 @@ Inst_DS__DS_READ2_B32::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ2_B32::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -2762,7 +2770,8 @@ Inst_DS__DS_WRITE_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data(gpuDynInst, extData.DATA0); + ConstVecOperandU64 data(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); addr.read(); data.read(); @@ -2827,8 +2836,10 @@ Inst_DS__DS_WRITE2_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU64 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU64 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU64 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -2895,8 +2906,10 @@ Inst_DS__DS_WRITE2ST64_B64::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU64 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU64 data1(gpuDynInst, extData.DATA1); + ConstVecOperandU64 data0(gpuDynInst, + extData.DATA0 + accDataOffset(wf)); + ConstVecOperandU64 data1(gpuDynInst, + extData.DATA1 + accDataOffset(wf)); addr.read(); data0.read(); @@ -3486,7 +3499,8 @@ Inst_DS__DS_READ_B64::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B64::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU64 vdst(gpuDynInst, extData.VDST); + Wavefront *wf = gpuDynInst->wavefront(); + VecOperandU64 vdst(gpuDynInst, extData.VDST + accDataOffset(wf)); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -3551,8 +3565,10 @@ Inst_DS__DS_READ2_B64::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ2_B64::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU64 vdst0(gpuDynInst, extData.VDST); - VecOperandU64 vdst1(gpuDynInst, extData.VDST + 2); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU64 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU64 vdst1(gpuDynInst, extData.VDST + acc + 2); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -4543,9 +4559,10 @@ Inst_DS__DS_WRITE_B96::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1); - ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2); + unsigned acc = accDataOffset(wf); + ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc); + ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1); + ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2); addr.read(); data0.read(); @@ -4607,10 +4624,11 @@ Inst_DS__DS_WRITE_B128::execute(GPUDynInstPtr gpuDynInst) gpuDynInst->latency.set( gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24))); ConstVecOperandU32 addr(gpuDynInst, extData.ADDR); - ConstVecOperandU32 data0(gpuDynInst, extData.DATA0); - ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1); - ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2); - ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + 3); + unsigned acc = accDataOffset(wf); + ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc); + ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1); + ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2); + ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + acc + 3); addr.read(); data0.read(); @@ -4697,9 +4715,11 @@ Inst_DS__DS_READ_B96::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B96::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); - VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); + VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { @@ -4762,10 +4782,12 @@ Inst_DS__DS_READ_B128::initiateAcc(GPUDynInstPtr gpuDynInst) void Inst_DS__DS_READ_B128::completeAcc(GPUDynInstPtr gpuDynInst) { - VecOperandU32 vdst0(gpuDynInst, extData.VDST); - VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1); - VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2); - VecOperandU32 vdst3(gpuDynInst, extData.VDST + 3); + Wavefront *wf = gpuDynInst->wavefront(); + unsigned acc = accDataOffset(wf); + VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc); + VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1); + VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2); + VecOperandU32 vdst3(gpuDynInst, extData.VDST + acc + 3); for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) { if (gpuDynInst->exec_mask[lane]) { diff --git a/src/arch/amdgpu/vega/insts/op_encodings.cc b/src/arch/amdgpu/vega/insts/op_encodings.cc index fb49cb5b5ce..4b743f8ff20 100644 --- a/src/arch/amdgpu/vega/insts/op_encodings.cc +++ b/src/arch/amdgpu/vega/insts/op_encodings.cc @@ -1345,6 +1345,8 @@ Inst_DS::initOperandInfo() for (opIdx = 0; opIdx < numSrcRegOperands(); opIdx++) { srcOps.emplace_back(srcs[opIdx], getOperandSize(opIdx), true, false, true, false); + if (instData.ACC && opIdx > 0) + srcOps.back().setAccum(); } if (numDstRegOperands()) { @@ -1352,6 +1354,8 @@ Inst_DS::initOperandInfo() int reg = extData.VDST; dstOps.emplace_back(reg, getOperandSize(opIdx), false, false, true, false); + if (instData.ACC) + dstOps.back().setAccum(); } assert(srcOps.size() == numSrcRegOperands()); diff --git a/src/arch/amdgpu/vega/insts/op_encodings.hh b/src/arch/amdgpu/vega/insts/op_encodings.hh index 96fd9fd4a98..40c68f2b6b4 100644 --- a/src/arch/amdgpu/vega/insts/op_encodings.hh +++ b/src/arch/amdgpu/vega/insts/op_encodings.hh @@ -983,6 +983,14 @@ class Inst_DS : public VEGAGPUStaticInst void initOperandInfo() override; protected: + // CDNA ACC data registers are mapped into the AGPR window of the + // unified vector register file. ADDR remains an ordinary VGPR. + unsigned + accDataOffset(Wavefront *wf) const + { + return instData.ACC ? wf->accumOffset : 0; + } + template void initMemRead(GPUDynInstPtr gpuDynInst, Addr offset) diff --git a/src/gpu-compute/gpu_static_inst.cc b/src/gpu-compute/gpu_static_inst.cc index 1919a70a8ed..0b28adf7551 100644 --- a/src/gpu-compute/gpu_static_inst.cc +++ b/src/gpu-compute/gpu_static_inst.cc @@ -71,9 +71,30 @@ GPUStaticInst::generateVirtToPhysMap(Wavefront *wf, ComputeUnit *cu, int num_dwords = op.sizeInDWords(); int virt_idx = op.registerIndex(wf->reservedScalarRegs); + // CDNA ACC bit: the operand's data registers live in the accumulator + // (AGPR) window of the unified VRF, which begins at wf->accumOffset. + // KEEP IN SYNC with Inst_DS::accDataOffset() (op_encodings.hh), which + // applies the identical offset on the data path during execute(). + if (op.isAccum()) { + virt_idx += wf->accumOffset; + } + int phys_idx = -1; for (int i = 0; i < num_dwords; i++) { if (opType == OpType::SRC_VEC || opType == OpType::DST_VEC) { + const int this_virt_idx = virt_idx + i; + panic_if(this_virt_idx >= wf->reservedVectorRegs, + "%s maps %s vector %s operand out of reserved VGPR " + "range: virt_idx=%d raw_reg=%d base_reg=%d dword=%d/%d " + "reserved=%d accum_offset=%u is_accum=%d " + "reserved_sgprs=%d\n", + disassemble().c_str(), + opType == OpType::SRC_VEC ? "src" : "dst", + op.isAccum() ? "AGPR" : "VGPR", + this_virt_idx, op.rawRegisterIndex(), + op.registerIndex(wf->reservedScalarRegs), i, num_dwords, + wf->reservedVectorRegs, wf->accumOffset, op.isAccum(), + wf->reservedScalarRegs); phys_idx = cu->registerManager->mapVgpr(wf, virt_idx + i); } else { assert(opType == OpType::SRC_SCALAR || diff --git a/src/gpu-compute/hsa_queue_entry.hh b/src/gpu-compute/hsa_queue_entry.hh index f5e1d833b00..5385e8f664c 100644 --- a/src/gpu-compute/hsa_queue_entry.hh +++ b/src/gpu-compute/hsa_queue_entry.hh @@ -104,8 +104,9 @@ class HSAQueueEntry // LLVM docs: https://www.llvm.org/docs/AMDGPUUsage.html // #code-object-v3-kernel-descriptor // - // Currently, gem5 supported gfx version use a multiplier of 8. The - // only exception is gfx900 (Vega10). + // CDNA's VGPR field is encoded from LLVM's total unified allocation: + // alignTo(num_arch_vgprs, 4) + num_agprs. ACCUM_OFFSET only identifies + // where AGPR operands begin; it must not be added to this allocation. if (gfx_version == GfxVersion::gfx90a || gfx_version == GfxVersion::gfx942 || gfx_version == GfxVersion::gfx950) { diff --git a/src/gpu-compute/operand_info.hh b/src/gpu-compute/operand_info.hh index e28f2c2ddd7..05d13a1355f 100644 --- a/src/gpu-compute/operand_info.hh +++ b/src/gpu-compute/operand_info.hh @@ -157,6 +157,9 @@ class OperandInfo return flags.isSet(FLAT); } + void setAccum() { flags.set(ACCUM); } + bool isAccum() const { return flags.isSet(ACCUM); } + void setVirtToPhysMapping(std::vector v, std::vector p) { @@ -234,7 +237,10 @@ class OperandInfo CONSTANT = 0x00000100, // If the constant is positive or negative - POS_CONST = 0x00000200 + POS_CONST = 0x00000200, + + // Operand resides in the CDNA accumulator (AGPR) window. + ACCUM = 0x00000400 }; Flags flags;