Skip to content

arch-amdgpu: support ACC-selected DS data operands #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Basemism wants to merge 1 commit into
base: staging
Choose a base branch
from
Open

arch-amdgpu: support ACC-selected DS data operands #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/arch/amdgpu/vega/gpu_decoder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,10 +512,10 @@ IsaDecodeMethod Decoder::tableDecodePrimary[] = {
&Decoder::subDecode_OP_DS,
&Decoder::subDecode_OP_DS,
&Decoder::subDecode_OP_DS,
&Decoder::decode_invalid,
&Decoder::decode_invalid,
&Decoder::decode_invalid,
&Decoder::decode_invalid,
&Decoder::subDecode_OP_DS,
&Decoder::subDecode_OP_DS,
&Decoder::subDecode_OP_DS,
&Decoder::subDecode_OP_DS,
&Decoder::subDecode_OP_FLAT,
&Decoder::subDecode_OP_FLAT,
&Decoder::subDecode_OP_FLAT,
Expand Down
4 changes: 3 additions & 1 deletion src/arch/amdgpu/vega/gpu_decoder.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1782,7 +1782,9 @@ struct InFmt_DS
unsigned int OFFSET1 : 8;
unsigned int GDS : 1;
unsigned int OP : 8;
unsigned int pad_25 : 1;
// CDNA ACC selects the AGPR bank for DS data registers. ADDR remains
// a VGPR.
unsigned int ACC : 1;
unsigned int ENCODING : 6;
};

Expand Down
82 changes: 52 additions & 30 deletions src/arch/amdgpu/vega/insts/ds.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,8 @@ Inst_DS__DS_WRITE_B32::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU32 data(gpuDynInst, extData.DATA0);
ConstVecOperandU32 data(gpuDynInst,
extData.DATA0 + accDataOffset(wf));

addr.read();
data.read();
Expand Down Expand Up @@ -471,8 +472,10 @@ Inst_DS__DS_WRITE2_B32::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU32 data0(gpuDynInst, extData.DATA0);
ConstVecOperandU32 data1(gpuDynInst, extData.DATA1);
ConstVecOperandU32 data0(gpuDynInst,
extData.DATA0 + accDataOffset(wf));
ConstVecOperandU32 data1(gpuDynInst,
extData.DATA1 + accDataOffset(wf));

addr.read();
data0.read();
Expand Down Expand Up @@ -539,8 +542,10 @@ Inst_DS__DS_WRITE2ST64_B32::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU32 data0(gpuDynInst, extData.DATA0);
ConstVecOperandU32 data1(gpuDynInst, extData.DATA1);
ConstVecOperandU32 data0(gpuDynInst,
extData.DATA0 + accDataOffset(wf));
ConstVecOperandU32 data1(gpuDynInst,
extData.DATA1 + accDataOffset(wf));

addr.read();
data0.read();
Expand Down Expand Up @@ -1621,7 +1626,8 @@ Inst_DS__DS_READ_B32::initiateAcc(GPUDynInstPtr gpuDynInst)
void
Inst_DS__DS_READ_B32::completeAcc(GPUDynInstPtr gpuDynInst)
{
VecOperandU32 vdst(gpuDynInst, extData.VDST);
Wavefront *wf = gpuDynInst->wavefront();
VecOperandU32 vdst(gpuDynInst, extData.VDST + accDataOffset(wf));

for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
if (gpuDynInst->exec_mask[lane]) {
Expand Down Expand Up @@ -1686,8 +1692,10 @@ Inst_DS__DS_READ2_B32::initiateAcc(GPUDynInstPtr gpuDynInst)
void
Inst_DS__DS_READ2_B32::completeAcc(GPUDynInstPtr gpuDynInst)
{
VecOperandU32 vdst0(gpuDynInst, extData.VDST);
VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1);
Wavefront *wf = gpuDynInst->wavefront();
unsigned acc = accDataOffset(wf);
VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc);
VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1);

for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
if (gpuDynInst->exec_mask[lane]) {
Expand Down Expand Up @@ -2762,7 +2770,8 @@ Inst_DS__DS_WRITE_B64::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU64 data(gpuDynInst, extData.DATA0);
ConstVecOperandU64 data(gpuDynInst,
extData.DATA0 + accDataOffset(wf));

addr.read();
data.read();
Expand Down Expand Up @@ -2827,8 +2836,10 @@ Inst_DS__DS_WRITE2_B64::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU64 data0(gpuDynInst, extData.DATA0);
ConstVecOperandU64 data1(gpuDynInst, extData.DATA1);
ConstVecOperandU64 data0(gpuDynInst,
extData.DATA0 + accDataOffset(wf));
ConstVecOperandU64 data1(gpuDynInst,
extData.DATA1 + accDataOffset(wf));

addr.read();
data0.read();
Expand Down Expand Up @@ -2895,8 +2906,10 @@ Inst_DS__DS_WRITE2ST64_B64::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU64 data0(gpuDynInst, extData.DATA0);
ConstVecOperandU64 data1(gpuDynInst, extData.DATA1);
ConstVecOperandU64 data0(gpuDynInst,
extData.DATA0 + accDataOffset(wf));
ConstVecOperandU64 data1(gpuDynInst,
extData.DATA1 + accDataOffset(wf));

addr.read();
data0.read();
Expand Down Expand Up @@ -3486,7 +3499,8 @@ Inst_DS__DS_READ_B64::initiateAcc(GPUDynInstPtr gpuDynInst)
void
Inst_DS__DS_READ_B64::completeAcc(GPUDynInstPtr gpuDynInst)
{
VecOperandU64 vdst(gpuDynInst, extData.VDST);
Wavefront *wf = gpuDynInst->wavefront();
VecOperandU64 vdst(gpuDynInst, extData.VDST + accDataOffset(wf));

for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
if (gpuDynInst->exec_mask[lane]) {
Expand Down Expand Up @@ -3551,8 +3565,10 @@ Inst_DS__DS_READ2_B64::initiateAcc(GPUDynInstPtr gpuDynInst)
void
Inst_DS__DS_READ2_B64::completeAcc(GPUDynInstPtr gpuDynInst)
{
VecOperandU64 vdst0(gpuDynInst, extData.VDST);
VecOperandU64 vdst1(gpuDynInst, extData.VDST + 2);
Wavefront *wf = gpuDynInst->wavefront();
unsigned acc = accDataOffset(wf);
VecOperandU64 vdst0(gpuDynInst, extData.VDST + acc);
VecOperandU64 vdst1(gpuDynInst, extData.VDST + acc + 2);

for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
if (gpuDynInst->exec_mask[lane]) {
Expand Down Expand Up @@ -4543,9 +4559,10 @@ Inst_DS__DS_WRITE_B96::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU32 data0(gpuDynInst, extData.DATA0);
ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1);
ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2);
unsigned acc = accDataOffset(wf);
ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc);
ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1);
ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2);

addr.read();
data0.read();
Expand Down Expand Up @@ -4607,10 +4624,11 @@ Inst_DS__DS_WRITE_B128::execute(GPUDynInstPtr gpuDynInst)
gpuDynInst->latency.set(
gpuDynInst->computeUnit()->cyclesToTicks(Cycles(24)));
ConstVecOperandU32 addr(gpuDynInst, extData.ADDR);
ConstVecOperandU32 data0(gpuDynInst, extData.DATA0);
ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + 1);
ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + 2);
ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + 3);
unsigned acc = accDataOffset(wf);
ConstVecOperandU32 data0(gpuDynInst, extData.DATA0 + acc);
ConstVecOperandU32 data1(gpuDynInst, extData.DATA0 + acc + 1);
ConstVecOperandU32 data2(gpuDynInst, extData.DATA0 + acc + 2);
ConstVecOperandU32 data3(gpuDynInst, extData.DATA0 + acc + 3);

addr.read();
data0.read();
Expand Down Expand Up @@ -4697,9 +4715,11 @@ Inst_DS__DS_READ_B96::initiateAcc(GPUDynInstPtr gpuDynInst)
void
Inst_DS__DS_READ_B96::completeAcc(GPUDynInstPtr gpuDynInst)
{
VecOperandU32 vdst0(gpuDynInst, extData.VDST);
VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1);
VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2);
Wavefront *wf = gpuDynInst->wavefront();
unsigned acc = accDataOffset(wf);
VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc);
VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1);
VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2);

for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
if (gpuDynInst->exec_mask[lane]) {
Expand Down Expand Up @@ -4762,10 +4782,12 @@ Inst_DS__DS_READ_B128::initiateAcc(GPUDynInstPtr gpuDynInst)
void
Inst_DS__DS_READ_B128::completeAcc(GPUDynInstPtr gpuDynInst)
{
VecOperandU32 vdst0(gpuDynInst, extData.VDST);
VecOperandU32 vdst1(gpuDynInst, extData.VDST + 1);
VecOperandU32 vdst2(gpuDynInst, extData.VDST + 2);
VecOperandU32 vdst3(gpuDynInst, extData.VDST + 3);
Wavefront *wf = gpuDynInst->wavefront();
unsigned acc = accDataOffset(wf);
VecOperandU32 vdst0(gpuDynInst, extData.VDST + acc);
VecOperandU32 vdst1(gpuDynInst, extData.VDST + acc + 1);
VecOperandU32 vdst2(gpuDynInst, extData.VDST + acc + 2);
VecOperandU32 vdst3(gpuDynInst, extData.VDST + acc + 3);

for (int lane = 0; lane < NumVecElemPerVecReg; ++lane) {
if (gpuDynInst->exec_mask[lane]) {
Expand Down
4 changes: 4 additions & 0 deletions src/arch/amdgpu/vega/insts/op_encodings.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1345,13 +1345,17 @@ Inst_DS::initOperandInfo()
for (opIdx = 0; opIdx < numSrcRegOperands(); opIdx++) {
srcOps.emplace_back(srcs[opIdx], getOperandSize(opIdx), true, false,
true, false);
if (instData.ACC && opIdx > 0)
srcOps.back().setAccum();
}

if (numDstRegOperands()) {
// Needed because can't take addr of bitfield
int reg = extData.VDST;
dstOps.emplace_back(reg, getOperandSize(opIdx), false, false, true,
false);
if (instData.ACC)
dstOps.back().setAccum();
}

assert(srcOps.size() == numSrcRegOperands());
Expand Down
8 changes: 8 additions & 0 deletions src/arch/amdgpu/vega/insts/op_encodings.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -983,6 +983,14 @@ class Inst_DS : public VEGAGPUStaticInst
void initOperandInfo() override;

protected:
// CDNA ACC data registers are mapped into the AGPR window of the
// unified vector register file. ADDR remains an ordinary VGPR.
unsigned
accDataOffset(Wavefront *wf) const
{
return instData.ACC ? wf->accumOffset : 0;
}

template <typename T>
void
initMemRead(GPUDynInstPtr gpuDynInst, Addr offset)
Expand Down
21 changes: 21 additions & 0 deletions src/gpu-compute/gpu_static_inst.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,30 @@ GPUStaticInst::generateVirtToPhysMap(Wavefront *wf, ComputeUnit *cu,
int num_dwords = op.sizeInDWords();
int virt_idx = op.registerIndex(wf->reservedScalarRegs);

// CDNA ACC bit: the operand's data registers live in the accumulator
// (AGPR) window of the unified VRF, which begins at wf->accumOffset.
// KEEP IN SYNC with Inst_DS::accDataOffset() (op_encodings.hh), which
// applies the identical offset on the data path during execute().
if (op.isAccum()) {
virt_idx += wf->accumOffset;
}

int phys_idx = -1;
for (int i = 0; i < num_dwords; i++) {
if (opType == OpType::SRC_VEC || opType == OpType::DST_VEC) {
const int this_virt_idx = virt_idx + i;
panic_if(this_virt_idx >= wf->reservedVectorRegs,
"%s maps %s vector %s operand out of reserved VGPR "
"range: virt_idx=%d raw_reg=%d base_reg=%d dword=%d/%d "
"reserved=%d accum_offset=%u is_accum=%d "
"reserved_sgprs=%d\n",
disassemble().c_str(),
opType == OpType::SRC_VEC ? "src" : "dst",
op.isAccum() ? "AGPR" : "VGPR",
this_virt_idx, op.rawRegisterIndex(),
op.registerIndex(wf->reservedScalarRegs), i, num_dwords,
wf->reservedVectorRegs, wf->accumOffset, op.isAccum(),
wf->reservedScalarRegs);
phys_idx = cu->registerManager->mapVgpr(wf, virt_idx + i);
} else {
assert(opType == OpType::SRC_SCALAR ||
Expand Down
5 changes: 3 additions & 2 deletions src/gpu-compute/hsa_queue_entry.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,9 @@ class HSAQueueEntry
// LLVM docs: https://www.llvm.org/docs/AMDGPUUsage.html
// #code-object-v3-kernel-descriptor
//
// Currently, gem5 supported gfx version use a multiplier of 8. The
// only exception is gfx900 (Vega10).
// CDNA's VGPR field is encoded from LLVM's total unified allocation:
// alignTo(num_arch_vgprs, 4) + num_agprs. ACCUM_OFFSET only identifies
// where AGPR operands begin; it must not be added to this allocation.
if (gfx_version == GfxVersion::gfx90a ||
gfx_version == GfxVersion::gfx942 ||
gfx_version == GfxVersion::gfx950) {
Expand Down
8 changes: 7 additions & 1 deletion src/gpu-compute/operand_info.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ class OperandInfo
return flags.isSet(FLAT);
}

void setAccum() { flags.set(ACCUM); }
bool isAccum() const { return flags.isSet(ACCUM); }

void
setVirtToPhysMapping(std::vector<int> v, std::vector<int> p)
{
Expand Down Expand Up @@ -234,7 +237,10 @@ class OperandInfo
CONSTANT = 0x00000100,

// If the constant is positive or negative
POS_CONST = 0x00000200
POS_CONST = 0x00000200,

// Operand resides in the CDNA accumulator (AGPR) window.
ACCUM = 0x00000400
};

Flags flags;
Expand Down