blob: 27c169193ba1c42bb47b177d2dd17928ecc9e408 [file] [log] [blame]
/*
* Copyright (c) 2021 Advanced Micro Devices, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __DEV_AMDGPU_SDMA_ENGINE_HH__
#define __DEV_AMDGPU_SDMA_ENGINE_HH__
#include "base/bitunion.hh"
#include "dev/amdgpu/amdgpu_device.hh"
#include "dev/amdgpu/pm4_queues.hh"
#include "dev/amdgpu/sdma_packets.hh"
#include "dev/dma_virt_device.hh"
#include "params/SDMAEngine.hh"
namespace gem5
{
/**
* System DMA Engine class for AMD dGPU.
*/
class SDMAEngine : public DmaVirtDevice
{
enum SDMAType
{
SDMAGfx,
SDMAPage
};
class SDMAQueue
{
Addr _base;
Addr _rptr;
Addr _wptr;
Addr _size;
Addr _rptr_wb_addr = 0;
Addr _global_rptr = 0;
bool _valid;
bool _processing;
SDMAQueue *_parent;
SDMAQueue *_ib;
SDMAType _type;
SDMAQueueDesc *_mqd;
Addr _mqd_addr = 0;
public:
SDMAQueue() : _rptr(0), _wptr(0), _valid(false), _processing(false),
_parent(nullptr), _ib(nullptr), _type(SDMAGfx), _mqd(nullptr) {}
Addr base() { return _base; }
Addr rptr() { return _base + _rptr; }
Addr getRptr() { return _rptr; }
Addr wptr() { return _base + _wptr; }
Addr getWptr() { return _wptr; }
Addr size() { return _size; }
Addr rptrWbAddr() { return _rptr_wb_addr; }
Addr globalRptr() { return _global_rptr; }
bool valid() { return _valid; }
bool processing() { return _processing; }
SDMAQueue* parent() { return _parent; }
SDMAQueue* ib() { return _ib; }
SDMAType queueType() { return _type; }
SDMAQueueDesc* getMQD() { return _mqd; }
Addr getMQDAddr() { return _mqd_addr; }
void base(Addr value) { _base = value; }
void
incRptr(uint32_t value)
{
_rptr = (_rptr + value) % _size;
_global_rptr += value;
}
void
rptr(Addr value)
{
_rptr = value;
_global_rptr = value;
}
void
setWptr(Addr value)
{
_wptr = value % _size;
}
void wptr(Addr value) { _wptr = value; }
void size(Addr value) { _size = value; }
void rptrWbAddr(Addr value) { _rptr_wb_addr = value; }
void valid(bool v) { _valid = v; }
void processing(bool value) { _processing = value; }
void parent(SDMAQueue* q) { _parent = q; }
void ib(SDMAQueue* ib) { _ib = ib; }
void queueType(SDMAType type) { _type = type; }
void setMQD(SDMAQueueDesc *mqd) { _mqd = mqd; }
void setMQDAddr(Addr mqdAddr) { _mqd_addr = mqdAddr; }
};
/* SDMA Engine ID */
int id;
/**
* Each SDMAEngine processes four queues: paging, gfx, rlc0, and rlc1,
* where RLC stands for Run List Controller. Each one of these
* can have one indirect buffer associated at any particular time.
* The switching order between queues is supposed to be page -> gfx ->
* rlc0 -> page -> gfx -> rlc1, skipping empty queues.
*/
SDMAQueue gfx, page, gfxIb, pageIb;
SDMAQueue rlc0, rlc0Ib, rlc1, rlc1Ib;
/* Gfx ring buffer registers */
uint64_t gfxBase;
uint64_t gfxRptr;
uint64_t gfxDoorbell;
uint64_t gfxDoorbellOffset;
uint64_t gfxWptr;
/* Page ring buffer registers */
uint64_t pageBase;
uint64_t pageRptr;
uint64_t pageDoorbell;
uint64_t pageDoorbellOffset;
uint64_t pageWptr;
AMDGPUDevice *gpuDevice;
VegaISA::Walker *walker;
/* processRLC will select the correct queue for the doorbell */
std::array<Addr, 2> rlcInfo{};
void processRLC0(Addr wptrOffset);
void processRLC1(Addr wptrOffset);
public:
SDMAEngine(const SDMAEngineParams &p);
void setGPUDevice(AMDGPUDevice *gpu_device);
void setId(int _id) { id = _id; }
/**
* Returns the client id for the Interrupt Handler.
*/
int getIHClientId();
/**
* Methods for translation.
*/
Addr getGARTAddr(Addr addr) const;
TranslationGenPtr translate(Addr vaddr, Addr size) override;
/**
* Translate an address in an SDMA packet. Return the device address if
* address in the packet is on the device and 0 if the the address in the
* packet is on the host/system memory.
*/
Addr getDeviceAddress(Addr raw_addr);
/**
* Inherited methods.
*/
Tick write(PacketPtr pkt) override { return 0; }
Tick read(PacketPtr pkt) override { return 0; }
AddrRangeList getAddrRanges() const override;
void serialize(CheckpointOut &cp) const override;
void unserialize(CheckpointIn &cp) override;
/**
* Given a new write ptr offset, communicated to the GPU through a doorbell
* write, the SDMA engine processes the page, gfx, rlc0, or rlc1 queue.
*/
void processGfx(Addr wptrOffset);
void processPage(Addr wptrOffset);
void processRLC(Addr doorbellOffset, Addr wptrOffset);
/**
* This method checks read and write pointers and starts decoding
* packets if the read pointer is less than the write pointer.
* It also marks a queue a being currently processing, in case the
* doorbell is rung again, the newly enqueued packets will be decoded once
* the currently processing once are finished. This is achieved by calling
* decodeNext once an entire SDMA packet has been processed.
*/
void decodeNext(SDMAQueue *q);
/**
* Reads the first DW (32 bits) (i.e., header) of an SDMA packet, which
* encodes the opcode and sub-opcode of the packet. It also creates an
* SDMA packet object and calls the associated processing function.
*/
void decodeHeader(SDMAQueue *q, uint32_t data);
/**
* Methods that implement processing of SDMA packets
*/
void write(SDMAQueue *q, sdmaWrite *pkt);
void writeReadData(SDMAQueue *q, sdmaWrite *pkt, uint32_t *dmaBuffer);
void writeDone(SDMAQueue *q, sdmaWrite *pkt, uint32_t *dmaBuffer);
void copy(SDMAQueue *q, sdmaCopy *pkt);
void copyReadData(SDMAQueue *q, sdmaCopy *pkt, uint8_t *dmaBuffer);
void copyDone(SDMAQueue *q, sdmaCopy *pkt, uint8_t *dmaBuffer);
void indirectBuffer(SDMAQueue *q, sdmaIndirectBuffer *pkt);
void fence(SDMAQueue *q, sdmaFence *pkt);
void fenceDone(SDMAQueue *q, sdmaFence *pkt);
void trap(SDMAQueue *q, sdmaTrap *pkt);
void srbmWrite(SDMAQueue *q, sdmaSRBMWriteHeader *header,
sdmaSRBMWrite *pkt);
void pollRegMem(SDMAQueue *q, sdmaPollRegMemHeader *header,
sdmaPollRegMem *pkt);
void pollRegMemRead(SDMAQueue *q, sdmaPollRegMemHeader *header,
sdmaPollRegMem *pkt, uint32_t dma_buffer, int count);
bool pollRegMemFunc(uint32_t value, uint32_t reference, uint32_t func);
void ptePde(SDMAQueue *q, sdmaPtePde *pkt);
void ptePdeDone(SDMAQueue *q, sdmaPtePde *pkt, uint64_t *dmaBuffer);
void atomic(SDMAQueue *q, sdmaAtomicHeader *header, sdmaAtomic *pkt);
void atomicData(SDMAQueue *q, sdmaAtomicHeader *header, sdmaAtomic *pkt,
uint64_t *dmaBuffer);
void atomicDone(SDMAQueue *q, sdmaAtomicHeader *header, sdmaAtomic *pkt,
uint64_t *dmaBuffer);
/**
* Methods for getting the values of SDMA MMIO registers.
*/
uint64_t getGfxBase() { return gfxBase; }
uint64_t getGfxRptr() { return gfxRptr; }
uint64_t getGfxDoorbell() { return gfxDoorbell; }
uint64_t getGfxDoorbellOffset() { return gfxDoorbellOffset; }
uint64_t getGfxWptr() { return gfxWptr; }
uint64_t getPageBase() { return pageBase; }
uint64_t getPageRptr() { return pageRptr; }
uint64_t getPageDoorbell() { return pageDoorbell; }
uint64_t getPageDoorbellOffset() { return pageDoorbellOffset; }
uint64_t getPageWptr() { return pageWptr; }
/**
* Methods for setting the values of SDMA MMIO registers.
*/
void writeMMIO(PacketPtr pkt, Addr mmio_offset);
void setGfxBaseLo(uint32_t data);
void setGfxBaseHi(uint32_t data);
void setGfxRptrLo(uint32_t data);
void setGfxRptrHi(uint32_t data);
void setGfxDoorbellLo(uint32_t data);
void setGfxDoorbellHi(uint32_t data);
void setGfxDoorbellOffsetLo(uint32_t data);
void setGfxDoorbellOffsetHi(uint32_t data);
void setGfxSize(uint64_t data);
void setGfxWptrLo(uint32_t data);
void setGfxWptrHi(uint32_t data);
void setPageBaseLo(uint32_t data);
void setPageBaseHi(uint32_t data);
void setPageRptrLo(uint32_t data);
void setPageRptrHi(uint32_t data);
void setPageDoorbellLo(uint32_t data);
void setPageDoorbellHi(uint32_t data);
void setPageDoorbellOffsetLo(uint32_t data);
void setPageDoorbellOffsetHi(uint32_t data);
void setPageSize(uint64_t data);
void setPageWptrLo(uint32_t data);
void setPageWptrHi(uint32_t data);
/**
* Methods for RLC queues
*/
void registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd);
void unregisterRLCQueue(Addr doorbell);
void deallocateRLCQueues();
int cur_vmid = 0;
};
} // namespace gem5
#endif // __DEV_AMDGPU_SDMA_ENGINE_HH__