gpu-compute, arch-gcn3: Change how waitcnts are implemented
Use single counters per memory operation type and increment
them upon issue, not execute.
Change-Id: I6afc0b66b21882538ef90a14a57a3ab3cc7bd6f3
diff --git a/src/arch/gcn3/insts/instructions.cc b/src/arch/gcn3/insts/instructions.cc
index 19c6e63..bf8f60f 100644
--- a/src/arch/gcn3/insts/instructions.cc
+++ b/src/arch/gcn3/insts/instructions.cc
@@ -32565,6 +32565,7 @@
vdst.write();
+ wf->decLGKMInstsIssued();
wf->rdLmReqsInPipe--;
wf->validateRequestCounters();
} // execute
@@ -32635,6 +32636,7 @@
vdst.write();
+ wf->decLGKMInstsIssued();
wf->rdLmReqsInPipe--;
wf->validateRequestCounters();
} // execute
@@ -39400,6 +39402,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->rdGmReqsInPipe--;
wf->rdLmReqsInPipe--;
return;
@@ -39496,6 +39500,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->rdGmReqsInPipe--;
wf->rdLmReqsInPipe--;
return;
@@ -39592,6 +39598,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->rdGmReqsInPipe--;
wf->rdLmReqsInPipe--;
return;
@@ -39660,6 +39668,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->rdGmReqsInPipe--;
wf->rdLmReqsInPipe--;
return;
@@ -39728,6 +39738,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->rdGmReqsInPipe--;
wf->rdLmReqsInPipe--;
return;
@@ -39805,6 +39817,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->rdGmReqsInPipe--;
wf->rdLmReqsInPipe--;
}
@@ -39884,6 +39898,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->wrLmReqsInPipe--;
return;
@@ -39952,6 +39968,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->wrLmReqsInPipe--;
return;
@@ -40021,6 +40039,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->wrLmReqsInPipe--;
return;
@@ -40090,6 +40110,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->wrLmReqsInPipe--;
return;
@@ -40159,6 +40181,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->wrLmReqsInPipe--;
return;
@@ -40237,6 +40261,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->wrLmReqsInPipe--;
return;
@@ -40325,6 +40351,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->rdGmReqsInPipe--;
return;
@@ -40425,6 +40453,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->rdGmReqsInPipe--;
return;
@@ -40526,6 +40556,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->rdGmReqsInPipe--;
return;
@@ -40893,6 +40925,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->rdGmReqsInPipe--;
return;
@@ -40995,6 +41029,8 @@
Wavefront *wf = gpuDynInst->wavefront();
if (wf->execMask().none()) {
+ wf->decVMemInstsIssued();
+ wf->decLGKMInstsIssued();
wf->wrGmReqsInPipe--;
wf->rdGmReqsInPipe--;
return;
diff --git a/src/gpu-compute/global_memory_pipeline.cc b/src/gpu-compute/global_memory_pipeline.cc
index 327b5d1..eedd938 100644
--- a/src/gpu-compute/global_memory_pipeline.cc
+++ b/src/gpu-compute/global_memory_pipeline.cc
@@ -133,6 +133,7 @@
DPRINTF(GPUMem, "CU%d: WF[%d][%d]: Completing global mem instr %s\n",
m->cu_id, m->simdId, m->wfSlotId, m->disassemble());
m->completeAcc(m);
+ w->decVMemInstsIssued();
if (m->isLoad() || m->isAtomicRet()) {
w->computeUnit->vrf[w->simdId]->
diff --git a/src/gpu-compute/gpu_dyn_inst.cc b/src/gpu-compute/gpu_dyn_inst.cc
index ef5f5dd..debbc37 100644
--- a/src/gpu-compute/gpu_dyn_inst.cc
+++ b/src/gpu-compute/gpu_dyn_inst.cc
@@ -821,6 +821,7 @@
if (executedAs() == Enums::SC_GLOBAL) {
// no transormation for global segment
wavefront()->execUnitId = wavefront()->flatGmUnitId;
+ wavefront()->decLGKMInstsIssued();
if (isLoad()) {
wavefront()->rdLmReqsInPipe--;
} else if (isStore()) {
@@ -840,6 +841,7 @@
}
}
wavefront()->execUnitId = wavefront()->flatLmUnitId;
+ wavefront()->decVMemInstsIssued();
if (isLoad()) {
wavefront()->rdGmReqsInPipe--;
} else if (isStore()) {
@@ -899,6 +901,7 @@
}
}
wavefront()->execUnitId = wavefront()->flatLmUnitId;
+ wavefront()->decLGKMInstsIssued();
if (isLoad()) {
wavefront()->rdGmReqsInPipe--;
} else if (isStore()) {
diff --git a/src/gpu-compute/local_memory_pipeline.cc b/src/gpu-compute/local_memory_pipeline.cc
index e34627c..a4f62a6 100644
--- a/src/gpu-compute/local_memory_pipeline.cc
+++ b/src/gpu-compute/local_memory_pipeline.cc
@@ -78,6 +78,7 @@
DPRINTF(GPUMem, "CU%d: WF[%d][%d]: Completing local mem instr %s\n",
m->cu_id, m->simdId, m->wfSlotId, m->disassemble());
m->completeAcc(m);
+ w->decLGKMInstsIssued();
if (m->isLoad() || m->isAtomicRet()) {
w->computeUnit->vrf[w->simdId]->
diff --git a/src/gpu-compute/scalar_memory_pipeline.cc b/src/gpu-compute/scalar_memory_pipeline.cc
index 35b4ca5..5e4496d 100644
--- a/src/gpu-compute/scalar_memory_pipeline.cc
+++ b/src/gpu-compute/scalar_memory_pipeline.cc
@@ -85,6 +85,7 @@
}
m->completeAcc(m);
+ w->decLGKMInstsIssued();
if (m->isLoad() || m->isAtomic()) {
returnedLoads.pop();
diff --git a/src/gpu-compute/schedule_stage.cc b/src/gpu-compute/schedule_stage.cc
index 1f4a96d..0ee6a0f 100644
--- a/src/gpu-compute/schedule_stage.cc
+++ b/src/gpu-compute/schedule_stage.cc
@@ -139,6 +139,15 @@
// this wave spends in SCH stage.
wf->schCycles++;
addToSchListStalls[j]++;
+ } else {
+ if (gpu_dyn_inst->isScalar() || gpu_dyn_inst->isGroupSeg()) {
+ wf->incLGKMInstsIssued();
+ } else {
+ wf->incVMemInstsIssued();
+ if (gpu_dyn_inst->isFlat()) {
+ wf->incLGKMInstsIssued();
+ }
+ }
}
}
diff --git a/src/gpu-compute/wavefront.cc b/src/gpu-compute/wavefront.cc
index 6f3b755..10a1505 100644
--- a/src/gpu-compute/wavefront.cc
+++ b/src/gpu-compute/wavefront.cc
@@ -55,6 +55,7 @@
: SimObject(p), wfSlotId(p->wf_slot_id), simdId(p->simdId),
maxIbSize(p->max_ib_size), _gpuISA(*this),
vmWaitCnt(-1), expWaitCnt(-1), lgkmWaitCnt(-1),
+ vmemInstsIssued(0), expInstsIssued(0), lgkmInstsIssued(0),
barId(WFBarrier::InvalidID)
{
lastTrace = 0;
@@ -1255,37 +1256,27 @@
return false;
}
- // If we reach here, that means waitCnt instruction is executed and
- // the waitcnts are set by the execute method. Check if waitcnts are
- // satisfied.
-
- // current number of vector memory ops in flight
- int vm_cnt = outstandingReqsWrGm + outstandingReqsRdGm;
-
- // current number of export insts or vector memory writes in flight
- int exp_cnt = outstandingReqsWrGm;
-
- // current number of scalar/LDS memory ops in flight
- // we do not consider GDS/message ops
- int lgkm_cnt = outstandingReqsWrLm + outstandingReqsRdLm +
- scalarOutstandingReqsRdGm + scalarOutstandingReqsWrGm;
-
+ /**
+ * If we reach here, that means an s_waitcnt instruction was executed
+ * and the waitcnts are set by the execute method. Check if waitcnts
+ * are satisfied.
+ */
if (vmWaitCnt != -1) {
- if (vm_cnt > vmWaitCnt) {
+ if (vmemInstsIssued > vmWaitCnt) {
// vmWaitCnt not satisfied
return false;
}
}
if (expWaitCnt != -1) {
- if (exp_cnt > expWaitCnt) {
+ if (expInstsIssued > expWaitCnt) {
// expWaitCnt not satisfied
return false;
}
}
if (lgkmWaitCnt != -1) {
- if (lgkm_cnt > lgkmWaitCnt) {
+ if (lgkmInstsIssued > lgkmWaitCnt) {
// lgkmWaitCnt not satisfied
return false;
}
@@ -1357,6 +1348,42 @@
status = S_RUNNING;
}
+void
+Wavefront::incVMemInstsIssued()
+{
+ ++vmemInstsIssued;
+}
+
+void
+Wavefront::incExpInstsIssued()
+{
+ ++expInstsIssued;
+}
+
+void
+Wavefront::incLGKMInstsIssued()
+{
+ ++lgkmInstsIssued;
+}
+
+void
+Wavefront::decVMemInstsIssued()
+{
+ --vmemInstsIssued;
+}
+
+void
+Wavefront::decExpInstsIssued()
+{
+ --expInstsIssued;
+}
+
+void
+Wavefront::decLGKMInstsIssued()
+{
+ --lgkmInstsIssued;
+}
+
Addr
Wavefront::pc() const
{
diff --git a/src/gpu-compute/wavefront.hh b/src/gpu-compute/wavefront.hh
index 59a6108..f05c17a 100644
--- a/src/gpu-compute/wavefront.hh
+++ b/src/gpu-compute/wavefront.hh
@@ -306,6 +306,13 @@
void setWaitCnts(int vm_wait_cnt, int exp_wait_cnt, int lgkm_wait_cnt);
void clearWaitCnts();
+ void incVMemInstsIssued();
+ void incExpInstsIssued();
+ void incLGKMInstsIssued();
+ void decVMemInstsIssued();
+ void decExpInstsIssued();
+ void decLGKMInstsIssued();
+
/** Freeing VRF space */
void freeRegisterFile();
@@ -345,6 +352,9 @@
int vmWaitCnt;
int expWaitCnt;
int lgkmWaitCnt;
+ int vmemInstsIssued;
+ int expInstsIssued;
+ int lgkmInstsIssued;
status_e status;
Addr _pc;
VectorMask _execMask;
