| /* | |
| * Copyright (c) 2011-2017 Advanced Micro Devices, Inc. | |
| * All rights reserved. | |
| * | |
| * Redistribution and use in source and binary forms, with or without | |
| * modification, are permitted provided that the following conditions are met: | |
| * | |
| * 1. Redistributions of source code must retain the above copyright notice, | |
| * this list of conditions and the following disclaimer. | |
| * | |
| * 2. Redistributions in binary form must reproduce the above copyright notice, | |
| * this list of conditions and the following disclaimer in the documentation | |
| * and/or other materials provided with the distribution. | |
| * | |
| * 3. Neither the name of the copyright holder nor the names of its | |
| * contributors may be used to endorse or promote products derived from this | |
| * software without specific prior written permission. | |
| * | |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE | |
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |
| * POSSIBILITY OF SUCH DAMAGE. | |
| */ | |
| #include "gpu-compute/wavefront.hh" | |
| #include "base/bitfield.hh" | |
| #include "debug/GPUExec.hh" | |
| #include "debug/GPUInitAbi.hh" | |
| #include "debug/WavefrontStack.hh" | |
| #include "gpu-compute/compute_unit.hh" | |
| #include "gpu-compute/gpu_dyn_inst.hh" | |
| #include "gpu-compute/scalar_register_file.hh" | |
| #include "gpu-compute/shader.hh" | |
| #include "gpu-compute/simple_pool_manager.hh" | |
| #include "gpu-compute/vector_register_file.hh" | |
| namespace gem5 | |
| { | |
| Wavefront::Wavefront(const Params &p) | |
| : SimObject(p), wfSlotId(p.wf_slot_id), simdId(p.simdId), | |
| maxIbSize(p.max_ib_size), _gpuISA(*this), | |
| vmWaitCnt(-1), expWaitCnt(-1), lgkmWaitCnt(-1), | |
| vmemInstsIssued(0), expInstsIssued(0), lgkmInstsIssued(0), | |
| sleepCnt(0), barId(WFBarrier::InvalidID), stats(this) | |
| { | |
| lastTrace = 0; | |
| execUnitId = -1; | |
| status = S_STOPPED; | |
| reservedVectorRegs = 0; | |
| reservedScalarRegs = 0; | |
| startVgprIndex = 0; | |
| startSgprIndex = 0; | |
| outstandingReqs = 0; | |
| outstandingReqsWrGm = 0; | |
| outstandingReqsWrLm = 0; | |
| outstandingReqsRdGm = 0; | |
| outstandingReqsRdLm = 0; | |
| rdLmReqsInPipe = 0; | |
| rdGmReqsInPipe = 0; | |
| wrLmReqsInPipe = 0; | |
| wrGmReqsInPipe = 0; | |
| scalarRdGmReqsInPipe = 0; | |
| scalarWrGmReqsInPipe = 0; | |
| scalarOutstandingReqsRdGm = 0; | |
| scalarOutstandingReqsWrGm = 0; | |
| lastNonIdleTick = 0; | |
| ldsChunk = nullptr; | |
| memTraceBusy = 0; | |
| oldVgprTcnt = 0xffffffffffffffffll; | |
| oldDgprTcnt = 0xffffffffffffffffll; | |
| oldVgpr.resize(p.wf_size); | |
| pendingFetch = false; | |
| dropFetch = false; | |
| maxVgprs = 0; | |
| maxSgprs = 0; | |
| lastAddr.resize(p.wf_size); | |
| workItemFlatId.resize(p.wf_size); | |
| oldDgpr.resize(p.wf_size); | |
| for (int i = 0; i < 3; ++i) { | |
| workItemId[i].resize(p.wf_size); | |
| } | |
| _execMask.set(); | |
| rawDist.clear(); | |
| lastInstExec = 0; | |
| vecReads.clear(); | |
| } | |
| void | |
| Wavefront::init() | |
| { | |
| reservedVectorRegs = 0; | |
| reservedScalarRegs = 0; | |
| startVgprIndex = 0; | |
| startSgprIndex = 0; | |
| scalarAlu = computeUnit->mapWaveToScalarAlu(this); | |
| scalarAluGlobalIdx = computeUnit->mapWaveToScalarAluGlobalIdx(this); | |
| globalMem = computeUnit->mapWaveToGlobalMem(this); | |
| localMem = computeUnit->mapWaveToLocalMem(this); | |
| scalarMem = computeUnit->mapWaveToScalarMem(this); | |
| } | |
| void | |
| Wavefront::initRegState(HSAQueueEntry *task, int wgSizeInWorkItems) | |
| { | |
| int regInitIdx = 0; | |
| // Iterate over all the init fields and check which | |
| // bits are enabled. Useful information can be found here: | |
| // https://github.com/ROCm-Developer-Tools/ROCm-ComputeABI-Doc/ | |
| // blob/master/AMDGPU-ABI.md | |
| for (int en_bit = 0; en_bit < NumScalarInitFields; ++en_bit) { | |
| if (task->sgprBitEnabled(en_bit)) { | |
| int physSgprIdx = 0; | |
| uint32_t wiCount = 0; | |
| uint32_t firstWave = 0; | |
| int orderedAppendTerm = 0; | |
| int numWfsInWg = 0; | |
| uint32_t finalValue = 0; | |
| Addr host_disp_pkt_addr = task->hostDispPktAddr(); | |
| Addr kernarg_addr = task->kernargAddr(); | |
| Addr hidden_priv_base(0); | |
| switch (en_bit) { | |
| case PrivateSegBuf: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[0]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting PrivateSegBuffer: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[0]); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[1]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting PrivateSegBuffer: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[1]); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[2]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting PrivateSegBuffer: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[2]); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[3]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting PrivateSegBuffer: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->amdQueue.scratch_resource_descriptor[3]); | |
| break; | |
| case DispatchPtr: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| bits(host_disp_pkt_addr, 31, 0)); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting DispatchPtr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| bits(host_disp_pkt_addr, 31, 0)); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| bits(host_disp_pkt_addr, 63, 32)); | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting DispatchPtr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| bits(host_disp_pkt_addr, 63, 32)); | |
| ++regInitIdx; | |
| break; | |
| case QueuePtr: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| bits(task->hostAMDQueueAddr, 31, 0)); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting QueuePtr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| bits(task->hostAMDQueueAddr, 31, 0)); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| bits(task->hostAMDQueueAddr, 63, 32)); | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting QueuePtr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| bits(task->hostAMDQueueAddr, 63, 32)); | |
| ++regInitIdx; | |
| break; | |
| case KernargSegPtr: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| bits(kernarg_addr, 31, 0)); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting KernargSegPtr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| bits(kernarg_addr, 31, 0)); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| bits(kernarg_addr, 63, 32)); | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting KernargSegPtr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| bits(kernarg_addr, 63, 32)); | |
| ++regInitIdx; | |
| break; | |
| case DispatchId: | |
| physSgprIdx | |
| = computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->dispatchId()); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting DispatchId: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->dispatchId()); | |
| // Dispatch ID in gem5 is an int. Set upper 32-bits to zero. | |
| physSgprIdx | |
| = computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, 0); | |
| ++regInitIdx; | |
| break; | |
| case FlatScratchInit: | |
| physSgprIdx | |
| = computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| (TheGpuISA::ScalarRegU32)(task->amdQueue | |
| .scratch_backing_memory_location & 0xffffffff)); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting FlatScratch Addr: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| (TheGpuISA::ScalarRegU32)(task->amdQueue | |
| .scratch_backing_memory_location & 0xffffffff)); | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| // This vallue should be sizeof(DWORD) aligned, that is | |
| // 4 byte aligned | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->amdQueue.scratch_workitem_byte_size); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting FlatScratch size: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->amdQueue.scratch_workitem_byte_size); | |
| /** | |
| * Since flat scratch init is needed for this kernel, this | |
| * kernel is going to have flat memory instructions and we | |
| * need to initialize the hidden private base for this queue. | |
| * scratch_resource_descriptor[0] has this queue's scratch | |
| * base address. scratch_backing_memory_location has the | |
| * offset to this queue's scratch base address from the | |
| * SH_HIDDEN_PRIVATE_BASE_VMID. Ideally, we only require this | |
| * queue's scratch base address for address calculation | |
| * (stored in scratch_resource_descriptor[0]). But that | |
| * address calculation shoule be done by first finding the | |
| * queue's scratch base address using the calculation | |
| * "SH_HIDDEN_PRIVATE_BASE_VMID + offset". So, we initialize | |
| * SH_HIDDEN_PRIVATE_BASE_VMID. | |
| * | |
| * For more details see: | |
| * http://rocm-documentation.readthedocs.io/en/latest/ | |
| * ROCm_Compiler_SDK/ROCm-Native-ISA.html#flat-scratch | |
| * | |
| * https://github.com/ROCm-Developer-Tools/ | |
| * ROCm-ComputeABI-Doc/blob/master/AMDGPU-ABI.md | |
| * #flat-addressing | |
| */ | |
| hidden_priv_base = | |
| (uint64_t)task->amdQueue.scratch_resource_descriptor[0] | | |
| (((uint64_t)task->amdQueue.scratch_resource_descriptor[1] | |
| & 0x000000000000ffff) << 32); | |
| computeUnit->shader->initShHiddenPrivateBase( | |
| hidden_priv_base, | |
| task->amdQueue.scratch_backing_memory_location); | |
| break; | |
| case PrivateSegSize: | |
| physSgprIdx | |
| = computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| task->privMemPerItem()); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting private segment size: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| task->privMemPerItem()); | |
| break; | |
| case GridWorkgroupCountX: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| wiCount = ((task->gridSize(0) + | |
| task->wgSize(0) - 1) / | |
| task->wgSize(0)); | |
| computeUnit->srf[simdId]->write(physSgprIdx, wiCount); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting num WG X: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, wiCount); | |
| break; | |
| case GridWorkgroupCountY: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| wiCount = ((task->gridSize(1) + | |
| task->wgSize(1) - 1) / | |
| task->wgSize(1)); | |
| computeUnit->srf[simdId]->write(physSgprIdx, wiCount); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting num WG Y: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, wiCount); | |
| break; | |
| case GridWorkgroupCountZ: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| wiCount = ((task->gridSize(2) + | |
| task->wgSize(2) - 1) / | |
| task->wgSize(2)); | |
| computeUnit->srf[simdId]->write(physSgprIdx, wiCount); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting num WG Z: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, wiCount); | |
| break; | |
| case WorkgroupIdX: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| workGroupId[0]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting WG ID X: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, workGroupId[0]); | |
| break; | |
| case WorkgroupIdY: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| workGroupId[1]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting WG ID Y: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, workGroupId[1]); | |
| break; | |
| case WorkgroupIdZ: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]->write(physSgprIdx, | |
| workGroupId[2]); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting WG ID Z: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, workGroupId[2]); | |
| break; | |
| case PrivSegWaveByteOffset: | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| /** | |
| * the compute_tmpring_size_wavesize specifies the number of | |
| * kB allocated per wavefront, hence the multiplication by | |
| * 1024. | |
| * | |
| * to get the per wavefront offset into the scratch | |
| * memory, we also multiply this by the wfId. the wfId stored | |
| * in the Wavefront class, however, is the wave ID within the | |
| * WG, whereas here we need the global WFID because the | |
| * scratch space will be divided amongst all waves in the | |
| * kernel. to get the global ID we multiply the WGID by | |
| * the WG size, then add the WFID of the wave within its WG. | |
| */ | |
| computeUnit->srf[simdId]->write(physSgprIdx, 1024 * | |
| (wgId * (wgSz / 64) + wfId) * | |
| task->amdQueue.compute_tmpring_size_wavesize); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting Private Seg Offset: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, | |
| 1024 * (wgId * (wgSz / 64) + wfId) * | |
| task->amdQueue.compute_tmpring_size_wavesize); | |
| break; | |
| case WorkgroupInfo: | |
| firstWave = (wfId == 0) ? 1 : 0; | |
| numWfsInWg = divCeil(wgSizeInWorkItems, | |
| computeUnit->wfSize()); | |
| finalValue = firstWave << ((sizeof(uint32_t) * 8) - 1); | |
| finalValue |= (orderedAppendTerm << 6); | |
| finalValue |= numWfsInWg; | |
| physSgprIdx = | |
| computeUnit->registerManager->mapSgpr(this, regInitIdx); | |
| computeUnit->srf[simdId]-> | |
| write(physSgprIdx, finalValue); | |
| ++regInitIdx; | |
| DPRINTF(GPUInitAbi, "CU%d: WF[%d][%d]: wave[%d] " | |
| "Setting WG Info: s[%d] = %x\n", | |
| computeUnit->cu_id, simdId, | |
| wfSlotId, wfDynId, physSgprIdx, finalValue); | |
| break; | |
| default: | |
| fatal("SGPR enable bit %i not supported\n", en_bit); | |
| break; | |
| } | |
| } | |
| } | |
| regInitIdx = 0; | |
| // iterate over all the init fields and check which | |
| // bits are enabled | |
| for (int en_bit = 0; en_bit < NumVectorInitFields; ++en_bit) { | |
| if (task->vgprBitEnabled(en_bit)) { | |
| uint32_t physVgprIdx = 0; | |
| TheGpuISA::VecRegContainerU32 raw_vgpr; | |
| switch (en_bit) { | |
| case WorkitemIdX: | |
| { | |
| physVgprIdx = computeUnit->registerManager | |
| ->mapVgpr(this, regInitIdx); | |
| TheGpuISA::VecElemU32 *vgpr_x | |
| = raw_vgpr.as<TheGpuISA::VecElemU32>(); | |
| for (int lane = 0; lane < workItemId[0].size(); ++lane) { | |
| vgpr_x[lane] = workItemId[0][lane]; | |
| } | |
| computeUnit->vrf[simdId]->write(physVgprIdx, raw_vgpr); | |
| rawDist[regInitIdx] = 0; | |
| ++regInitIdx; | |
| } | |
| break; | |
| case WorkitemIdY: | |
| { | |
| physVgprIdx = computeUnit->registerManager | |
| ->mapVgpr(this, regInitIdx); | |
| TheGpuISA::VecElemU32 *vgpr_y | |
| = raw_vgpr.as<TheGpuISA::VecElemU32>(); | |
| for (int lane = 0; lane < workItemId[1].size(); ++lane) { | |
| vgpr_y[lane] = workItemId[1][lane]; | |
| } | |
| computeUnit->vrf[simdId]->write(physVgprIdx, raw_vgpr); | |
| rawDist[regInitIdx] = 0; | |
| ++regInitIdx; | |
| } | |
| break; | |
| case WorkitemIdZ: | |
| { | |
| physVgprIdx = computeUnit->registerManager-> | |
| mapVgpr(this, regInitIdx); | |
| TheGpuISA::VecElemU32 *vgpr_z | |
| = raw_vgpr.as<TheGpuISA::VecElemU32>(); | |
| for (int lane = 0; lane < workItemId[2].size(); ++lane) { | |
| vgpr_z[lane] = workItemId[2][lane]; | |
| } | |
| computeUnit->vrf[simdId]->write(physVgprIdx, raw_vgpr); | |
| rawDist[regInitIdx] = 0; | |
| ++regInitIdx; | |
| } | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| void | |
| Wavefront::resizeRegFiles(int num_vregs, int num_sregs) | |
| { | |
| maxVgprs = num_vregs; | |
| maxSgprs = num_sregs; | |
| } | |
| Wavefront::~Wavefront() | |
| { | |
| } | |
| void | |
| Wavefront::setStatus(status_e newStatus) | |
| { | |
| if (computeUnit->idleCUTimeout > 0) { | |
| // Wavefront's status transitions to stalled or stopped | |
| if ((newStatus == S_STOPPED || newStatus == S_STALLED || | |
| newStatus == S_WAITCNT || newStatus == S_BARRIER) && | |
| (status != newStatus)) { | |
| computeUnit->idleWfs++; | |
| assert(computeUnit->idleWfs <= | |
| (computeUnit->shader->n_wf * computeUnit->numVectorALUs)); | |
| if (computeUnit->idleWfs == | |
| (computeUnit->shader->n_wf * computeUnit->numVectorALUs)) { | |
| lastNonIdleTick = curTick(); | |
| } | |
| // Wavefront's status transitions to an active state (from | |
| // a stopped or stalled state) | |
| } else if ((status == S_STOPPED || status == S_STALLED || | |
| status == S_WAITCNT || status == S_BARRIER) && | |
| (status != newStatus)) { | |
| // if all WFs in the CU were idle then check if the idleness | |
| // period exceeded the timeout threshold | |
| if (computeUnit->idleWfs == | |
| (computeUnit->shader->n_wf * computeUnit->numVectorALUs)) { | |
| panic_if((curTick() - lastNonIdleTick) >= | |
| computeUnit->idleCUTimeout, | |
| "CU%d has been idle for %d ticks at tick %d", | |
| computeUnit->cu_id, computeUnit->idleCUTimeout, | |
| curTick()); | |
| } | |
| computeUnit->idleWfs--; | |
| assert(computeUnit->idleWfs >= 0); | |
| } | |
| } | |
| status = newStatus; | |
| } | |
| void | |
| Wavefront::start(uint64_t _wf_dyn_id, Addr init_pc) | |
| { | |
| wfDynId = _wf_dyn_id; | |
| _pc = init_pc; | |
| status = S_RUNNING; | |
| vecReads.resize(maxVgprs, 0); | |
| } | |
| bool | |
| Wavefront::isGmInstruction(GPUDynInstPtr ii) | |
| { | |
| if (ii->isGlobalMem() || | |
| (ii->isFlat() && ii->executedAs() == enums::SC_GLOBAL)) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isLmInstruction(GPUDynInstPtr ii) | |
| { | |
| if (ii->isLocalMem() || | |
| (ii->isFlat() && ii->executedAs() == enums::SC_GROUP)) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstSleep() | |
| { | |
| if (instructionBuffer.empty()) | |
| return false; | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (ii->isSleep()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstWaitcnt() | |
| { | |
| if (instructionBuffer.empty()) | |
| return false; | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (ii->isWaitcnt()) { | |
| // waitcnt is a scalar | |
| assert(ii->isScalar()); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstScalarALU() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && ii->isScalar() && (ii->isNop() || ii->isReturn() | |
| || ii->isEndOfKernel() || ii->isBranch() || ii->isALU() || | |
| (ii->isKernArgSeg() && ii->isLoad()))) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstVectorALU() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && !ii->isScalar() && (ii->isNop() || | |
| ii->isReturn() || ii->isBranch() || ii->isALU() || ii->isEndOfKernel() | |
| || (ii->isKernArgSeg() && ii->isLoad()))) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstBarrier() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && ii->isBarrier()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstGMem() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && !ii->isScalar() && ii->isGlobalMem()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstScalarMem() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && ii->isScalar() && ii->isGlobalMem()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstLMem() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && ii->isLocalMem()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstPrivMem() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && ii->isPrivateSeg()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::isOldestInstFlatMem() | |
| { | |
| assert(!instructionBuffer.empty()); | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| if (status != S_STOPPED && ii->isFlat()) { | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool | |
| Wavefront::stopFetch() | |
| { | |
| for (auto it : instructionBuffer) { | |
| GPUDynInstPtr ii = it; | |
| if (ii->isReturn() || ii->isBranch() || | |
| ii->isEndOfKernel()) { | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| void | |
| Wavefront::freeResources() | |
| { | |
| execUnitId = -1; | |
| } | |
| void Wavefront::validateRequestCounters() | |
| { | |
| panic_if(wrGmReqsInPipe < 0 || rdGmReqsInPipe < 0 || | |
| wrLmReqsInPipe < 0 || rdLmReqsInPipe < 0 || | |
| outstandingReqs < 0, | |
| "Negative requests in pipe for WF%d for slot%d" | |
| " and SIMD%d: Rd GlobalMem Reqs=%d, Wr GlobalMem Reqs=%d," | |
| " Rd LocalMem Reqs=%d, Wr LocalMem Reqs=%d," | |
| " Outstanding Reqs=%d\n", | |
| wfDynId, wfSlotId, simdId, rdGmReqsInPipe, wrGmReqsInPipe, | |
| rdLmReqsInPipe, wrLmReqsInPipe, outstandingReqs); | |
| } | |
| void | |
| Wavefront::reserveGmResource(GPUDynInstPtr ii) | |
| { | |
| if (!ii->isScalar()) { | |
| if (ii->isLoad()) { | |
| rdGmReqsInPipe++; | |
| } else if (ii->isStore()) { | |
| wrGmReqsInPipe++; | |
| } else if (ii->isAtomic() || ii->isMemSync()) { | |
| rdGmReqsInPipe++; | |
| wrGmReqsInPipe++; | |
| } else { | |
| panic("Invalid memory operation!\n"); | |
| } | |
| execUnitId = globalMem; | |
| } else { | |
| if (ii->isLoad()) { | |
| scalarRdGmReqsInPipe++; | |
| } else if (ii->isStore()) { | |
| scalarWrGmReqsInPipe++; | |
| } else if (ii->isAtomic() || ii->isMemSync()) { | |
| scalarWrGmReqsInPipe++; | |
| scalarRdGmReqsInPipe++; | |
| } else { | |
| panic("Invalid memory operation!\n"); | |
| } | |
| execUnitId = scalarMem; | |
| } | |
| } | |
| void | |
| Wavefront::reserveLmResource(GPUDynInstPtr ii) | |
| { | |
| fatal_if(ii->isScalar(), | |
| "Scalar instructions can not access Shared memory!!!"); | |
| if (ii->isLoad()) { | |
| rdLmReqsInPipe++; | |
| } else if (ii->isStore()) { | |
| wrLmReqsInPipe++; | |
| } else if (ii->isAtomic() || ii->isMemSync()) { | |
| wrLmReqsInPipe++; | |
| rdLmReqsInPipe++; | |
| } else { | |
| panic("Invalid memory operation!\n"); | |
| } | |
| execUnitId = localMem; | |
| } | |
| std::vector<int> | |
| Wavefront::reserveResources() | |
| { | |
| // vector of execution unit IDs to return to schedule stage | |
| // this return is only used for debugging and an assertion... | |
| std::vector<int> execUnitIds; | |
| // Get current instruction | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| assert(ii); | |
| // Single precision ALU or Branch or Return or Special instruction | |
| if (ii->isALU() || ii->isSpecialOp() || | |
| ii->isBranch() || ii->isNop() || | |
| (ii->isKernArgSeg() && ii->isLoad()) || ii->isArgSeg() || | |
| ii->isReturn() || ii->isEndOfKernel()) { | |
| if (!ii->isScalar()) { | |
| execUnitId = simdId; | |
| } else { | |
| execUnitId = scalarAluGlobalIdx; | |
| } | |
| // this is to enforce a fixed number of cycles per issue slot per SIMD | |
| } else if (ii->isBarrier()) { | |
| execUnitId = ii->isScalar() ? scalarAluGlobalIdx : simdId; | |
| } else if (ii->isFlat()) { | |
| assert(!ii->isScalar()); | |
| reserveLmResource(ii); | |
| // add execUnitId, reserved by reserveLmResource, list before it is | |
| // overwriten by reserveGmResource | |
| execUnitIds.push_back(execUnitId); | |
| flatLmUnitId = execUnitId; | |
| reserveGmResource(ii); | |
| flatGmUnitId = execUnitId; | |
| execUnitIds.push_back(flatGmUnitId); | |
| execUnitId = -1; | |
| } else if (ii->isGlobalMem()) { | |
| reserveGmResource(ii); | |
| } else if (ii->isLocalMem()) { | |
| reserveLmResource(ii); | |
| } else if (ii->isPrivateSeg()) { | |
| fatal_if(ii->isScalar(), | |
| "Scalar instructions can not access Private memory!!!"); | |
| reserveGmResource(ii); | |
| } else { | |
| panic("reserveResources -> Couldn't process op!\n"); | |
| } | |
| if (execUnitId != -1) { | |
| execUnitIds.push_back(execUnitId); | |
| } | |
| assert(execUnitIds.size()); | |
| return execUnitIds; | |
| } | |
| void | |
| Wavefront::exec() | |
| { | |
| // ---- Exit if wavefront is inactive ----------------------------- // | |
| if (status == S_STOPPED || status == S_RETURNING || | |
| status==S_STALLED || instructionBuffer.empty()) { | |
| return; | |
| } | |
| if (status == S_WAITCNT) { | |
| /** | |
| * if this wave is in S_WAITCNT state, then | |
| * it should enter exec() precisely one time | |
| * before the waitcnts are satisfied, in order | |
| * to execute the waitcnt instruction itself | |
| * thus we assert that the waitcnt is the | |
| * oldest instruction. if we enter exec() with | |
| * active waitcnts, and we're not executing | |
| * the waitcnt instruction, something must be | |
| * wrong | |
| */ | |
| assert(isOldestInstWaitcnt()); | |
| } | |
| // Get current instruction | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| const Addr old_pc = pc(); | |
| DPRINTF(GPUExec, "CU%d: WF[%d][%d]: wave[%d] Executing inst: %s " | |
| "(pc: %#x; seqNum: %d)\n", computeUnit->cu_id, simdId, wfSlotId, | |
| wfDynId, ii->disassemble(), old_pc, ii->seqNum()); | |
| ii->execute(ii); | |
| // delete the dynamic instruction from the pipeline map | |
| computeUnit->deleteFromPipeMap(this); | |
| // update the instruction stats in the CU | |
| computeUnit->updateInstStats(ii); | |
| // inform VRF of instruction execution to schedule write-back | |
| // and scoreboard ready for registers | |
| if (!ii->isScalar()) { | |
| computeUnit->vrf[simdId]->waveExecuteInst(this, ii); | |
| } | |
| computeUnit->srf[simdId]->waveExecuteInst(this, ii); | |
| computeUnit->shader->incVectorInstSrcOperand(ii->numSrcVecRegOperands()); | |
| computeUnit->shader->incVectorInstDstOperand(ii->numDstVecRegOperands()); | |
| computeUnit->stats.numInstrExecuted++; | |
| stats.numInstrExecuted++; | |
| computeUnit->instExecPerSimd[simdId]++; | |
| computeUnit->stats.execRateDist.sample( | |
| computeUnit->stats.totalCycles.value() - | |
| computeUnit->lastExecCycle[simdId]); | |
| computeUnit->lastExecCycle[simdId] = | |
| computeUnit->stats.totalCycles.value(); | |
| if (lastInstExec) { | |
| computeUnit->stats.instInterleave[simdId]. | |
| sample(computeUnit->instExecPerSimd[simdId] - lastInstExec); | |
| } | |
| lastInstExec = computeUnit->instExecPerSimd[simdId]; | |
| // want to track: | |
| // number of reads that occur per value written | |
| // vector RAW dependency tracking | |
| for (const auto& srcVecOp : ii->srcVecRegOperands()) { | |
| for (const auto& virtIdx : srcVecOp.virtIndices()) { | |
| // This check should never fail, but to be safe we check | |
| if (rawDist.find(virtIdx) != rawDist.end()) { | |
| stats.vecRawDistance.sample(stats.numInstrExecuted.value() - | |
| rawDist[virtIdx]); | |
| } | |
| // increment number of reads to this register | |
| vecReads[virtIdx]++; | |
| } | |
| } | |
| for (const auto& dstVecOp : ii->dstVecRegOperands()) { | |
| for (const auto& virtIdx : dstVecOp.virtIndices()) { | |
| // rawDist is set on writes, but will not be set for the first | |
| // write to each physical register | |
| if (rawDist.find(virtIdx) != rawDist.end()) { | |
| // Sample the number of reads that were performed | |
| stats.readsPerWrite.sample(vecReads[virtIdx]); | |
| } | |
| // on a write, reset count of reads to 0 | |
| vecReads[virtIdx] = 0; | |
| rawDist[virtIdx] = stats.numInstrExecuted.value(); | |
| } | |
| } | |
| if (pc() == old_pc) { | |
| // PC not modified by instruction, proceed to next | |
| _gpuISA.advancePC(ii); | |
| instructionBuffer.pop_front(); | |
| } else { | |
| DPRINTF(GPUExec, "CU%d: WF[%d][%d]: wave%d %s taken branch\n", | |
| computeUnit->cu_id, simdId, wfSlotId, wfDynId, | |
| ii->disassemble()); | |
| discardFetch(); | |
| } | |
| DPRINTF(GPUExec, "CU%d: WF[%d][%d]: wave[%d] (pc: %#x)\n", | |
| computeUnit->cu_id, simdId, wfSlotId, wfDynId, pc()); | |
| if (computeUnit->shader->hsail_mode==Shader::SIMT) { | |
| const int num_active_lanes = execMask().count(); | |
| computeUnit->stats.controlFlowDivergenceDist.sample(num_active_lanes); | |
| computeUnit->stats.numVecOpsExecuted += num_active_lanes; | |
| if (ii->isF16() && ii->isALU()) { | |
| if (ii->isF32() || ii->isF64()) { | |
| fatal("Instruction is tagged as both (1) F16, and (2)" | |
| "either F32 or F64."); | |
| } | |
| computeUnit->stats.numVecOpsExecutedF16 += num_active_lanes; | |
| if (ii->isFMA()) { | |
| computeUnit->stats.numVecOpsExecutedFMA16 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| else if (ii->isMAC()) { | |
| computeUnit->stats.numVecOpsExecutedMAC16 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| else if (ii->isMAD()) { | |
| computeUnit->stats.numVecOpsExecutedMAD16 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| } | |
| if (ii->isF32() && ii->isALU()) { | |
| if (ii->isF16() || ii->isF64()) { | |
| fatal("Instruction is tagged as both (1) F32, and (2)" | |
| "either F16 or F64."); | |
| } | |
| computeUnit->stats.numVecOpsExecutedF32 += num_active_lanes; | |
| if (ii->isFMA()) { | |
| computeUnit->stats.numVecOpsExecutedFMA32 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| else if (ii->isMAC()) { | |
| computeUnit->stats.numVecOpsExecutedMAC32 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| else if (ii->isMAD()) { | |
| computeUnit->stats.numVecOpsExecutedMAD32 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| } | |
| if (ii->isF64() && ii->isALU()) { | |
| if (ii->isF16() || ii->isF32()) { | |
| fatal("Instruction is tagged as both (1) F64, and (2)" | |
| "either F16 or F32."); | |
| } | |
| computeUnit->stats.numVecOpsExecutedF64 += num_active_lanes; | |
| if (ii->isFMA()) { | |
| computeUnit->stats.numVecOpsExecutedFMA64 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| else if (ii->isMAC()) { | |
| computeUnit->stats.numVecOpsExecutedMAC64 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| else if (ii->isMAD()) { | |
| computeUnit->stats.numVecOpsExecutedMAD64 += num_active_lanes; | |
| computeUnit->stats.numVecOpsExecutedTwoOpFP | |
| += num_active_lanes; | |
| } | |
| } | |
| if (isGmInstruction(ii)) { | |
| computeUnit->stats.activeLanesPerGMemInstrDist.sample( | |
| num_active_lanes); | |
| } else if (isLmInstruction(ii)) { | |
| computeUnit->stats.activeLanesPerLMemInstrDist.sample( | |
| num_active_lanes); | |
| } | |
| } | |
| /** | |
| * we return here to avoid spurious errors related to flat insts | |
| * and their address segment resolution. | |
| */ | |
| if (execMask().none() && ii->isFlat()) { | |
| computeUnit->getTokenManager()->recvTokens(1); | |
| return; | |
| } | |
| // Update Vector ALU pipeline and other resources | |
| bool flat_as_gm = false; | |
| bool flat_as_lm = false; | |
| if (ii->isFlat()) { | |
| flat_as_gm = (ii->executedAs() == enums::SC_GLOBAL) || | |
| (ii->executedAs() == enums::SC_PRIVATE); | |
| flat_as_lm = (ii->executedAs() == enums::SC_GROUP); | |
| } | |
| // Single precision ALU or Branch or Return or Special instruction | |
| // Note, we use the same timing regardless of SP or DP ALU operation. | |
| if (ii->isALU() || ii->isSpecialOp() || | |
| ii->isBranch() || ii->isNop() || | |
| (ii->isKernArgSeg() && ii->isLoad()) || | |
| ii->isArgSeg() || ii->isEndOfKernel() || ii->isReturn()) { | |
| // this is to enforce a fixed number of cycles per issue slot per SIMD | |
| if (!ii->isScalar()) { | |
| computeUnit->vectorALUs[simdId].set(computeUnit-> | |
| cyclesToTicks(computeUnit->issuePeriod)); | |
| } else { | |
| computeUnit->scalarALUs[scalarAlu].set(computeUnit-> | |
| cyclesToTicks(computeUnit->issuePeriod)); | |
| } | |
| // Barrier on Scalar ALU | |
| } else if (ii->isBarrier()) { | |
| computeUnit->scalarALUs[scalarAlu].set(computeUnit-> | |
| cyclesToTicks(computeUnit->issuePeriod)); | |
| // GM or Flat as GM Load | |
| } else if (ii->isLoad() && (ii->isGlobalMem() || flat_as_gm)) { | |
| if (!ii->isScalar()) { | |
| computeUnit->vrfToGlobalMemPipeBus.set( | |
| computeUnit->cyclesToTicks(computeUnit->vrf_gm_bus_latency)); | |
| computeUnit->vectorGlobalMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesVMemPerSimd[simdId] += | |
| computeUnit->vrf_gm_bus_latency; | |
| } else { | |
| computeUnit->srfToScalarMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(computeUnit->srf_scm_bus_latency)); | |
| computeUnit->scalarMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesScMemPerSimd[simdId] += | |
| computeUnit->srf_scm_bus_latency; | |
| } | |
| // GM or Flat as GM Store | |
| } else if (ii->isStore() && (ii->isGlobalMem() || flat_as_gm)) { | |
| if (!ii->isScalar()) { | |
| computeUnit->vrfToGlobalMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(Cycles(2 * computeUnit->vrf_gm_bus_latency))); | |
| computeUnit->vectorGlobalMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesVMemPerSimd[simdId] += | |
| (2 * computeUnit->vrf_gm_bus_latency); | |
| } else { | |
| computeUnit->srfToScalarMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(Cycles(2 * computeUnit->srf_scm_bus_latency))); | |
| computeUnit->scalarMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesScMemPerSimd[simdId] += | |
| (2 * computeUnit->srf_scm_bus_latency); | |
| } | |
| } else if ((ii->isAtomic() || ii->isMemSync()) && | |
| (ii->isGlobalMem() || flat_as_gm)) { | |
| if (!ii->isScalar()) { | |
| computeUnit->vrfToGlobalMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(Cycles(2 * computeUnit->vrf_gm_bus_latency))); | |
| computeUnit->vectorGlobalMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesVMemPerSimd[simdId] += | |
| (2 * computeUnit->vrf_gm_bus_latency); | |
| } else { | |
| computeUnit->srfToScalarMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(Cycles(2 * computeUnit->srf_scm_bus_latency))); | |
| computeUnit->scalarMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesScMemPerSimd[simdId] += | |
| (2 * computeUnit->srf_scm_bus_latency); | |
| } | |
| // LM or Flat as LM Load | |
| } else if (ii->isLoad() && (ii->isLocalMem() || flat_as_lm)) { | |
| computeUnit->vrfToLocalMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(computeUnit->vrf_lm_bus_latency)); | |
| computeUnit->vectorSharedMemUnit. | |
| set(computeUnit->shader->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesLdsPerSimd[simdId] += | |
| computeUnit->vrf_lm_bus_latency; | |
| // LM or Flat as LM Store | |
| } else if (ii->isStore() && (ii->isLocalMem() || flat_as_lm)) { | |
| computeUnit->vrfToLocalMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(Cycles(2 * computeUnit->vrf_lm_bus_latency))); | |
| computeUnit->vectorSharedMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesLdsPerSimd[simdId] += | |
| (2 * computeUnit->vrf_lm_bus_latency); | |
| // LM or Flat as LM, Atomic or MemFence | |
| } else if ((ii->isAtomic() || ii->isMemSync()) && | |
| (ii->isLocalMem() || flat_as_lm)) { | |
| computeUnit->vrfToLocalMemPipeBus.set(computeUnit-> | |
| cyclesToTicks(Cycles(2 * computeUnit->vrf_lm_bus_latency))); | |
| computeUnit->vectorSharedMemUnit. | |
| set(computeUnit->cyclesToTicks(computeUnit->issuePeriod)); | |
| computeUnit->stats.instCyclesLdsPerSimd[simdId] += | |
| (2 * computeUnit->vrf_lm_bus_latency); | |
| } else { | |
| panic("Bad instruction type!\n"); | |
| } | |
| } | |
| GPUDynInstPtr | |
| Wavefront::nextInstr() | |
| { | |
| // Read next instruction from instruction buffer | |
| GPUDynInstPtr ii = instructionBuffer.front(); | |
| // if the WF has been dispatched in the schedule stage then | |
| // check the next oldest instruction for readiness | |
| if (computeUnit->pipeMap.find(ii->seqNum()) != | |
| computeUnit->pipeMap.end()) { | |
| if (instructionBuffer.size() > 1) { | |
| auto it = instructionBuffer.begin() + 1; | |
| return *it; | |
| } else { // No new instructions to check | |
| return nullptr; | |
| } | |
| } | |
| return ii; | |
| } | |
| void | |
| Wavefront::discardFetch() | |
| { | |
| instructionBuffer.clear(); | |
| dropFetch |= pendingFetch; | |
| /** | |
| * clear the fetch buffer for this wave in order to | |
| * remove any stale inst data | |
| */ | |
| computeUnit->fetchStage.fetchUnit(simdId).flushBuf(wfSlotId); | |
| } | |
| bool | |
| Wavefront::waitCntsSatisfied() | |
| { | |
| // Both vmWaitCnt && lgkmWaitCnt uninitialized means | |
| // waitCnt instruction has been dispatched but not executed yet: next | |
| // instruction should be blocked until waitCnt is executed. | |
| if (vmWaitCnt == -1 && expWaitCnt == -1 && lgkmWaitCnt == -1) { | |
| return false; | |
| } | |
| /** | |
| * If we reach here, that means an s_waitcnt instruction was executed | |
| * and the waitcnts are set by the execute method. Check if waitcnts | |
| * are satisfied. | |
| */ | |
| if (vmWaitCnt != -1) { | |
| if (vmemInstsIssued > vmWaitCnt) { | |
| // vmWaitCnt not satisfied | |
| return false; | |
| } | |
| } | |
| if (expWaitCnt != -1) { | |
| if (expInstsIssued > expWaitCnt) { | |
| // expWaitCnt not satisfied | |
| return false; | |
| } | |
| } | |
| if (lgkmWaitCnt != -1) { | |
| if (lgkmInstsIssued > lgkmWaitCnt) { | |
| // lgkmWaitCnt not satisfied | |
| return false; | |
| } | |
| } | |
| // if we get here all outstanding waitcnts must | |
| // be satisfied, so we resume normal operation | |
| clearWaitCnts(); | |
| return true; | |
| } | |
| bool | |
| Wavefront::sleepDone() | |
| { | |
| assert(status == S_STALLED_SLEEP); | |
| // if the sleep count has not been set, then the sleep instruction has not | |
| // been executed yet, so we will return true without setting the wavefront | |
| // status | |
| if (sleepCnt == 0) | |
| return false; | |
| sleepCnt--; | |
| if (sleepCnt != 0) | |
| return false; | |
| status = S_RUNNING; | |
| return true; | |
| } | |
| void | |
| Wavefront::setSleepTime(int sleep_time) | |
| { | |
| assert(sleepCnt == 0); | |
| sleepCnt = sleep_time; | |
| } | |
| void | |
| Wavefront::setWaitCnts(int vm_wait_cnt, int exp_wait_cnt, int lgkm_wait_cnt) | |
| { | |
| // the scoreboard should have set the status | |
| // to S_WAITCNT once a waitcnt instruction | |
| // was marked as ready | |
| assert(status == S_WAITCNT); | |
| // waitcnt instruction shouldn't be sending | |
| // negative counts | |
| assert(vm_wait_cnt >= 0); | |
| assert(exp_wait_cnt >= 0); | |
| assert(lgkm_wait_cnt >= 0); | |
| // waitcnts are a max of 15 because we have | |
| // only 1 nibble (4 bits) to set the counts | |
| assert(vm_wait_cnt <= 0xf); | |
| assert(exp_wait_cnt <= 0x7); | |
| assert(lgkm_wait_cnt <= 0x1f); | |
| /** | |
| * prior waitcnts should be satisfied, | |
| * at which time the WF resets them | |
| * back to -1, indicating they are no | |
| * longer active | |
| */ | |
| assert(vmWaitCnt == -1); | |
| assert(expWaitCnt == -1); | |
| assert(lgkmWaitCnt == -1); | |
| /** | |
| * if the instruction encoding | |
| * indicates a waitcnt of 0xf, | |
| * that means the waitcnt is | |
| * not being used | |
| */ | |
| if (vm_wait_cnt != 0xf) | |
| vmWaitCnt = vm_wait_cnt; | |
| if (exp_wait_cnt != 0x7) | |
| expWaitCnt = exp_wait_cnt; | |
| if (lgkm_wait_cnt != 0x1f) | |
| lgkmWaitCnt = lgkm_wait_cnt; | |
| } | |
| void | |
| Wavefront::clearWaitCnts() | |
| { | |
| // reset the waitcnts back to | |
| // -1, indicating they are no | |
| // longer valid | |
| vmWaitCnt = -1; | |
| expWaitCnt = -1; | |
| lgkmWaitCnt = -1; | |
| // resume running normally | |
| status = S_RUNNING; | |
| } | |
| void | |
| Wavefront::incVMemInstsIssued() | |
| { | |
| ++vmemInstsIssued; | |
| } | |
| void | |
| Wavefront::incExpInstsIssued() | |
| { | |
| ++expInstsIssued; | |
| } | |
| void | |
| Wavefront::incLGKMInstsIssued() | |
| { | |
| ++lgkmInstsIssued; | |
| } | |
| void | |
| Wavefront::decVMemInstsIssued() | |
| { | |
| --vmemInstsIssued; | |
| } | |
| void | |
| Wavefront::decExpInstsIssued() | |
| { | |
| --expInstsIssued; | |
| } | |
| void | |
| Wavefront::decLGKMInstsIssued() | |
| { | |
| --lgkmInstsIssued; | |
| } | |
| Addr | |
| Wavefront::pc() const | |
| { | |
| return _pc; | |
| } | |
| void | |
| Wavefront::pc(Addr new_pc) | |
| { | |
| _pc = new_pc; | |
| } | |
| VectorMask& | |
| Wavefront::execMask() | |
| { | |
| return _execMask; | |
| } | |
| bool | |
| Wavefront::execMask(int lane) const | |
| { | |
| return _execMask[lane]; | |
| } | |
| void | |
| Wavefront::freeRegisterFile() | |
| { | |
| /* clear busy registers */ | |
| for (int i=0; i < maxVgprs; i++) { | |
| int vgprIdx = computeUnit->registerManager->mapVgpr(this, i); | |
| computeUnit->vrf[simdId]->markReg(vgprIdx, false); | |
| } | |
| /* Free registers used by this wavefront */ | |
| uint32_t endIndex = (startVgprIndex + reservedVectorRegs - 1) % | |
| computeUnit->vrf[simdId]->numRegs(); | |
| computeUnit->registerManager->vrfPoolMgrs[simdId]-> | |
| freeRegion(startVgprIndex, endIndex); | |
| } | |
| void | |
| Wavefront::computeActualWgSz(HSAQueueEntry *task) | |
| { | |
| actualWgSzTotal = 1; | |
| for (int d = 0; d < HSAQueueEntry::MAX_DIM; ++d) { | |
| actualWgSz[d] = std::min(workGroupSz[d], gridSz[d] | |
| - task->wgId(d) * workGroupSz[d]); | |
| actualWgSzTotal *= actualWgSz[d]; | |
| } | |
| } | |
| void | |
| Wavefront::barrierId(int bar_id) | |
| { | |
| assert(bar_id >= WFBarrier::InvalidID); | |
| assert(bar_id < computeUnit->numBarrierSlots()); | |
| barId = bar_id; | |
| } | |
| int | |
| Wavefront::barrierId() const | |
| { | |
| return barId; | |
| } | |
| bool | |
| Wavefront::hasBarrier() const | |
| { | |
| return barId > WFBarrier::InvalidID; | |
| } | |
| void | |
| Wavefront::releaseBarrier() | |
| { | |
| barId = WFBarrier::InvalidID; | |
| } | |
| Wavefront::WavefrontStats::WavefrontStats(statistics::Group *parent) | |
| : statistics::Group(parent), | |
| ADD_STAT(numInstrExecuted, | |
| "number of instructions executed by this WF slot"), | |
| ADD_STAT(schCycles, "number of cycles spent in schedule stage"), | |
| ADD_STAT(schStalls, "number of cycles WF is stalled in SCH stage"), | |
| ADD_STAT(schRfAccessStalls, "number of cycles wave selected in SCH but " | |
| "RF denied adding instruction"), | |
| ADD_STAT(schResourceStalls, "number of cycles stalled in sch by resource" | |
| " not available"), | |
| ADD_STAT(schOpdNrdyStalls, "number of cycles stalled in sch waiting for " | |
| "RF reads to complete"), | |
| ADD_STAT(schLdsArbStalls, | |
| "number of cycles wave stalled due to LDS-VRF arbitration"), | |
| // FIXME: the name of the WF needs to be unique | |
| ADD_STAT(numTimesBlockedDueWAXDependencies, "number of times the wf's " | |
| "instructions are blocked due to WAW or WAR dependencies"), | |
| // FIXME: the name of the WF needs to be unique | |
| ADD_STAT(numTimesBlockedDueRAWDependencies, "number of times the wf's " | |
| "instructions are blocked due to RAW dependencies"), | |
| ADD_STAT(vecRawDistance, | |
| "Count of RAW distance in dynamic instructions for this WF"), | |
| ADD_STAT(readsPerWrite, "Count of Vector reads per write for this WF") | |
| { | |
| vecRawDistance.init(0, 20, 1); | |
| readsPerWrite.init(0, 4, 1); | |
| } | |
| } // namespace gem5 |