src/gpu-compute/gpu_command_processor.hh - public/gem5 - Git at Google

 /*
  * Copyright (c) 2018 Advanced Micro Devices, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  * 1. Redistributions of source code must retain the above copyright notice,
  * this list of conditions and the following disclaimer.
  *
  * 2. Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
  *
  * 3. Neither the name of the copyright holder nor the names of its
  * contributors may be used to endorse or promote products derived from this
  * software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */

 /**
  * @file
  * The GPUCommandProcessor (CP) is responsible for accepting commands, in
  * the form of HSA AQL packets, from the HSA packet processor (HSAPP). The CP
  * works with several components, including the HSAPP and the dispatcher.
  * When the HSAPP sends a ready task to the CP, it will perform the necessary
  * operations to extract relevant data structures from memory, such as the
  * AQL queue descriptor and AQL packet, and initializes register state for the
  * task's wavefronts.
  */

 #ifndef __DEV_HSA_GPU_COMMAND_PROCESSOR_HH__
 #define __DEV_HSA_GPU_COMMAND_PROCESSOR_HH__

 #include <cstdint>
 #include <functional>

 #include "base/logging.hh"
 #include "base/trace.hh"
 #include "base/types.hh"
 #include "debug/GPUCommandProc.hh"
 #include "dev/dma_virt_device.hh"
 #include "dev/hsa/hsa_packet_processor.hh"
 #include "dev/hsa/hsa_signal.hh"
 #include "gpu-compute/dispatcher.hh"
 #include "gpu-compute/gpu_compute_driver.hh"
 #include "gpu-compute/hsa_queue_entry.hh"
 #include "params/GPUCommandProcessor.hh"

 namespace gem5
 {

 struct GPUCommandProcessorParams;
 class GPUComputeDriver;
 class GPUDispatcher;
 class Shader;

 class GPUCommandProcessor : public DmaVirtDevice
 {
   public:
     typedef GPUCommandProcessorParams Params;
     typedef std::function<void(const uint64_t &)> HsaSignalCallbackFunction;

     GPUCommandProcessor() = delete;
     GPUCommandProcessor(const Params &p);

     HSAPacketProcessor& hsaPacketProc();

     void setShader(Shader *shader);
     Shader* shader();
     GPUComputeDriver* driver();

     enum AgentCmd
     {
       Nop = 0,
       Steal = 1
     };

     void submitAgentDispatchPkt(void *raw_pkt, uint32_t queue_id,
                            Addr host_pkt_addr);
     void submitDispatchPkt(void *raw_pkt, uint32_t queue_id,
                            Addr host_pkt_addr);
     void submitVendorPkt(void *raw_pkt, uint32_t queue_id,
                          Addr host_pkt_addr);
     void attachDriver(GPUComputeDriver *driver);

     void dispatchPkt(HSAQueueEntry *task);
     void signalWakeupEvent(uint32_t event_id);

     Tick write(PacketPtr pkt) override { return 0; }
     Tick read(PacketPtr pkt) override { return 0; }
     AddrRangeList getAddrRanges() const override;
     System *system();

     void updateHsaSignal(Addr signal_handle, uint64_t signal_value,
                          HsaSignalCallbackFunction function =
                             [] (const uint64_t &) { });

     uint64_t functionalReadHsaSignal(Addr signal_handle);

     Addr getHsaSignalValueAddr(Addr signal_handle)
     {
         return signal_handle + offsetof(amd_signal_t, value);
     }

     Addr getHsaSignalMailboxAddr(Addr signal_handle)
     {
         return signal_handle + offsetof(amd_signal_t, event_mailbox_ptr);
     }

     Addr getHsaSignalEventAddr(Addr signal_handle)
     {
         return signal_handle + offsetof(amd_signal_t, event_id);
     }

   private:
     Shader *_shader;
     GPUDispatcher &dispatcher;
     GPUComputeDriver *_driver;

     // Typedefing dmaRead and dmaWrite function pointer
     typedef void (DmaDevice::*DmaFnPtr)(Addr, int, Event*, uint8_t*, Tick);
     void initABI(HSAQueueEntry *task);
     HSAPacketProcessor *hsaPP;
     TranslationGenPtr translate(Addr vaddr, Addr size) override;

     /**
      * Perform a DMA read of the read_dispatch_id_field_base_byte_offset
      * field, which follows directly after the read_dispatch_id (the read
      * pointer) in the amd_hsa_queue_t struct (aka memory queue descriptor
      * (MQD)), to find the base address of the MQD. The MQD is the runtime's
      * soft representation of a HW queue descriptor (HQD).
      *
      * Any fields below the read dispatch ID in the amd_hsa_queue_t should
      * not change according to the HSA standard, therefore we should be able
      * to get them based on their known relative position to the read dispatch
      * ID.
      */
     void
     ReadDispIdOffsetDmaEvent(HSAQueueEntry *task,
                              const uint32_t &readDispIdOffset)
     {
         /**
          * Now that the read pointer's offset from the base of
          * the MQD is known, we can use that to calculate the
          * the address of the MQD itself, the dispatcher will
          * DMA that into the HSAQueueEntry when a kernel is
          * launched.
          */
         task->hostAMDQueueAddr = hsaPP->getQueueDesc(
             task->queueId())->hostReadIndexPtr - readDispIdOffset;

         /**
          * DMA a copy of the MQD into the task. some fields of
          * the MQD will be used to initialize register state in VI
          */
         auto *mqdDmaEvent = new DmaVirtCallback<int>(
             [ = ] (const int &) { MQDDmaEvent(task); });

         dmaReadVirt(task->hostAMDQueueAddr,
                     sizeof(_amd_queue_t), mqdDmaEvent, &task->amdQueue);
     }

     /**
      * Perform a DMA read of the MQD that corresponds to a hardware
      * queue descriptor (HQD). We store a copy of the MQD in the
      * HSAQueueEntry object so we can send a copy of it along with
      * a dispatch packet, which is needed to initialize register
      * state.
      */
      void
      MQDDmaEvent(HSAQueueEntry *task)
      {
         /**
          *  dGPUs on any version of ROCm and APUs starting with ROCm 2.2
          *  can perform lazy allocation of private segment (scratch) memory,
          *  where the runtime will intentianally underallocate scratch
          *  resources to save framebuffer (or system on APU) memory.
          *  If we don't have enough scratch memory to launch this kernel,
          *  we need to raise a recoverable error code to the runtime by
          *  asserting queue_inactive_signal for the queue.  The runtime will
          *  then try to allocate more scratch and reset this signal.  When
          *  the signal is reset we should check that the runtime was
          *  successful and then proceed to launch the kernel.
          */
         if (task->privMemPerItem() >
             task->amdQueue.compute_tmpring_size_wavesize * 1024) {
             // TODO: Raising this signal will potentially nuke scratch
             // space for in-flight kernels that were launched from this
             // queue.  We need to drain all kernels and deschedule the
             // queue before raising this signal. For now, just assert if
             // there are any in-flight kernels and tell the user that this
             // feature still needs to be implemented.
             fatal_if(hsaPP->inFlightPkts(task->queueId()) > 1,
                         "Needed more scratch, but kernels are in flight for "
                         "this queue and it is unsafe to reallocate scratch. "
                         "We need to implement additional intelligence in the "
                         "hardware scheduling logic to support CP-driven "
                         "queue draining and scheduling.");
             DPRINTF(GPUCommandProc, "Not enough scratch space to launch "
                     "kernel (%x available, %x requested bytes per "
                     "workitem). Asking host runtime to allocate more "
                     "space.\n",
                     task->amdQueue.compute_tmpring_size_wavesize * 1024,
                     task->privMemPerItem());

             updateHsaSignal(task->amdQueue.queue_inactive_signal.handle, 1,
                             [ = ] (const uint64_t &dma_buffer)
                                 { WaitScratchDmaEvent(task, dma_buffer); });

         } else {
             DPRINTF(GPUCommandProc, "Sufficient scratch space, launching "
                     "kernel (%x available, %x requested bytes per "
                     "workitem).\n",
                     task->amdQueue.compute_tmpring_size_wavesize * 1024,
                     task->privMemPerItem());
             dispatchPkt(task);
         }
     }

     /**
      * Poll on queue_inactive signal until the runtime can get around to
      * taking care of our lack of scratch space.
      */
     void
     WaitScratchDmaEvent(HSAQueueEntry *task, const uint64_t &dmaBuffer)
     {
         if (dmaBuffer == 0) {
             DPRINTF(GPUCommandProc, "Host scratch allocation complete. "
                     "Attempting to re-read MQD\n");
             /**
             * Runtime will have updated the MQD to give us more scratch
             * space.  Read it out and continue to pester the runtime until
             * we get all that we need to launch.
             *
             * TODO: Technically only need to update private segment fields
             * since other MQD entries won't change since we last read them.
             */
             auto cb = new DmaVirtCallback<int>(
                 [ = ] (const int &) { MQDDmaEvent(task); });

             dmaReadVirt(task->hostAMDQueueAddr, sizeof(_amd_queue_t), cb,
                         &task->amdQueue);
         } else {
             /**
             * Poll until runtime signals us that scratch space has been
             * allocated.
             */
             Addr value_addr = getHsaSignalValueAddr(
                 task->amdQueue.queue_inactive_signal.handle);
             DPRINTF(GPUCommandProc, "Polling queue inactive signal at "
                     "%p.\n", value_addr);
             auto cb = new DmaVirtCallback<uint64_t>(
                 [ = ] (const uint64_t &dma_buffer)
                 { WaitScratchDmaEvent(task, dma_buffer); } );
             dmaReadVirt(value_addr, sizeof(Addr), cb, &cb->dmaBuffer);
         }
     }
 };

 } // namespace gem5

 #endif // __DEV_HSA_GPU_COMMAND_PROCESSOR_HH__
	/*
	* Copyright (c) 2018 Advanced Micro Devices, Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are met:
	*
	* 1. Redistributions of source code must retain the above copyright notice,
	* this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright notice,
	* this list of conditions and the following disclaimer in the documentation
	* and/or other materials provided with the distribution.
	*
	* 3. Neither the name of the copyright holder nor the names of its
	* contributors may be used to endorse or promote products derived from this
	* software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/

	/**
	* @file
	* The GPUCommandProcessor (CP) is responsible for accepting commands, in
	* the form of HSA AQL packets, from the HSA packet processor (HSAPP). The CP
	* works with several components, including the HSAPP and the dispatcher.
	* When the HSAPP sends a ready task to the CP, it will perform the necessary
	* operations to extract relevant data structures from memory, such as the
	* AQL queue descriptor and AQL packet, and initializes register state for the
	* task's wavefronts.
	*/

	#ifndef __DEV_HSA_GPU_COMMAND_PROCESSOR_HH__
	#define __DEV_HSA_GPU_COMMAND_PROCESSOR_HH__

	#include <cstdint>
	#include <functional>

	#include "base/logging.hh"
	#include "base/trace.hh"
	#include "base/types.hh"
	#include "debug/GPUCommandProc.hh"
	#include "dev/dma_virt_device.hh"
	#include "dev/hsa/hsa_packet_processor.hh"
	#include "dev/hsa/hsa_signal.hh"
	#include "gpu-compute/dispatcher.hh"
	#include "gpu-compute/gpu_compute_driver.hh"
	#include "gpu-compute/hsa_queue_entry.hh"
	#include "params/GPUCommandProcessor.hh"

	namespace gem5
	{

	struct GPUCommandProcessorParams;
	class GPUComputeDriver;
	class GPUDispatcher;
	class Shader;

	class GPUCommandProcessor : public DmaVirtDevice
	{
	public:
	typedef GPUCommandProcessorParams Params;
	typedef std::function<void(const uint64_t &)> HsaSignalCallbackFunction;

	GPUCommandProcessor() = delete;
	GPUCommandProcessor(const Params &p);

	HSAPacketProcessor& hsaPacketProc();

	void setShader(Shader *shader);
	Shader* shader();
	GPUComputeDriver* driver();

	enum AgentCmd
	{
	Nop = 0,
	Steal = 1
	};

	void submitAgentDispatchPkt(void *raw_pkt, uint32_t queue_id,
	Addr host_pkt_addr);
	void submitDispatchPkt(void *raw_pkt, uint32_t queue_id,
	Addr host_pkt_addr);
	void submitVendorPkt(void *raw_pkt, uint32_t queue_id,
	Addr host_pkt_addr);
	void attachDriver(GPUComputeDriver *driver);

	void dispatchPkt(HSAQueueEntry *task);
	void signalWakeupEvent(uint32_t event_id);

	Tick write(PacketPtr pkt) override { return 0; }
	Tick read(PacketPtr pkt) override { return 0; }
	AddrRangeList getAddrRanges() const override;
	System *system();

	void updateHsaSignal(Addr signal_handle, uint64_t signal_value,
	HsaSignalCallbackFunction function =
	[] (const uint64_t &) { });

	uint64_t functionalReadHsaSignal(Addr signal_handle);

	Addr getHsaSignalValueAddr(Addr signal_handle)
	{
	return signal_handle + offsetof(amd_signal_t, value);
	}

	Addr getHsaSignalMailboxAddr(Addr signal_handle)
	{
	return signal_handle + offsetof(amd_signal_t, event_mailbox_ptr);
	}

	Addr getHsaSignalEventAddr(Addr signal_handle)
	{
	return signal_handle + offsetof(amd_signal_t, event_id);
	}

	private:
	Shader *_shader;
	GPUDispatcher &dispatcher;
	GPUComputeDriver *_driver;

	// Typedefing dmaRead and dmaWrite function pointer
	typedef void (DmaDevice::DmaFnPtr)(Addr, int, Event, uint8_t*, Tick);
	void initABI(HSAQueueEntry *task);
	HSAPacketProcessor *hsaPP;
	TranslationGenPtr translate(Addr vaddr, Addr size) override;

	/**
	* Perform a DMA read of the read_dispatch_id_field_base_byte_offset
	* field, which follows directly after the read_dispatch_id (the read
	* pointer) in the amd_hsa_queue_t struct (aka memory queue descriptor
	* (MQD)), to find the base address of the MQD. The MQD is the runtime's
	* soft representation of a HW queue descriptor (HQD).
	*
	* Any fields below the read dispatch ID in the amd_hsa_queue_t should
	* not change according to the HSA standard, therefore we should be able
	* to get them based on their known relative position to the read dispatch
	* ID.
	*/
	void
	ReadDispIdOffsetDmaEvent(HSAQueueEntry *task,
	const uint32_t &readDispIdOffset)
	{
	/**
	* Now that the read pointer's offset from the base of
	* the MQD is known, we can use that to calculate the
	* the address of the MQD itself, the dispatcher will
	* DMA that into the HSAQueueEntry when a kernel is
	* launched.
	*/
	task->hostAMDQueueAddr = hsaPP->getQueueDesc(
	task->queueId())->hostReadIndexPtr - readDispIdOffset;

	/**
	* DMA a copy of the MQD into the task. some fields of
	* the MQD will be used to initialize register state in VI
	*/
	auto *mqdDmaEvent = new DmaVirtCallback<int>(
	[ = ] (const int &) { MQDDmaEvent(task); });

	dmaReadVirt(task->hostAMDQueueAddr,
	sizeof(_amd_queue_t), mqdDmaEvent, &task->amdQueue);
	}

	/**
	* Perform a DMA read of the MQD that corresponds to a hardware
	* queue descriptor (HQD). We store a copy of the MQD in the
	* HSAQueueEntry object so we can send a copy of it along with
	* a dispatch packet, which is needed to initialize register
	* state.
	*/
	void
	MQDDmaEvent(HSAQueueEntry *task)
	{
	/**
	* dGPUs on any version of ROCm and APUs starting with ROCm 2.2
	* can perform lazy allocation of private segment (scratch) memory,
	* where the runtime will intentianally underallocate scratch
	* resources to save framebuffer (or system on APU) memory.
	* If we don't have enough scratch memory to launch this kernel,
	* we need to raise a recoverable error code to the runtime by
	* asserting queue_inactive_signal for the queue. The runtime will
	* then try to allocate more scratch and reset this signal. When
	* the signal is reset we should check that the runtime was
	* successful and then proceed to launch the kernel.
	*/
	if (task->privMemPerItem() >
	task->amdQueue.compute_tmpring_size_wavesize * 1024) {
	// TODO: Raising this signal will potentially nuke scratch
	// space for in-flight kernels that were launched from this
	// queue. We need to drain all kernels and deschedule the
	// queue before raising this signal. For now, just assert if
	// there are any in-flight kernels and tell the user that this
	// feature still needs to be implemented.
	fatal_if(hsaPP->inFlightPkts(task->queueId()) > 1,
	"Needed more scratch, but kernels are in flight for "
	"this queue and it is unsafe to reallocate scratch. "
	"We need to implement additional intelligence in the "
	"hardware scheduling logic to support CP-driven "
	"queue draining and scheduling.");
	DPRINTF(GPUCommandProc, "Not enough scratch space to launch "
	"kernel (%x available, %x requested bytes per "
	"workitem). Asking host runtime to allocate more "
	"space.\n",
	task->amdQueue.compute_tmpring_size_wavesize * 1024,
	task->privMemPerItem());

	updateHsaSignal(task->amdQueue.queue_inactive_signal.handle, 1,
	[ = ] (const uint64_t &dma_buffer)
	{ WaitScratchDmaEvent(task, dma_buffer); });

	} else {
	DPRINTF(GPUCommandProc, "Sufficient scratch space, launching "
	"kernel (%x available, %x requested bytes per "
	"workitem).\n",
	task->amdQueue.compute_tmpring_size_wavesize * 1024,
	task->privMemPerItem());
	dispatchPkt(task);
	}
	}

	/**
	* Poll on queue_inactive signal until the runtime can get around to
	* taking care of our lack of scratch space.
	*/
	void
	WaitScratchDmaEvent(HSAQueueEntry *task, const uint64_t &dmaBuffer)
	{
	if (dmaBuffer == 0) {
	DPRINTF(GPUCommandProc, "Host scratch allocation complete. "
	"Attempting to re-read MQD\n");
	/**
	* Runtime will have updated the MQD to give us more scratch
	* space. Read it out and continue to pester the runtime until
	* we get all that we need to launch.
	*
	* TODO: Technically only need to update private segment fields
	* since other MQD entries won't change since we last read them.
	*/
	auto cb = new DmaVirtCallback<int>(
	[ = ] (const int &) { MQDDmaEvent(task); });

	dmaReadVirt(task->hostAMDQueueAddr, sizeof(_amd_queue_t), cb,
	&task->amdQueue);
	} else {
	/**
	* Poll until runtime signals us that scratch space has been
	* allocated.
	*/
	Addr value_addr = getHsaSignalValueAddr(
	task->amdQueue.queue_inactive_signal.handle);
	DPRINTF(GPUCommandProc, "Polling queue inactive signal at "
	"%p.\n", value_addr);
	auto cb = new DmaVirtCallback<uint64_t>(
	[ = ] (const uint64_t &dma_buffer)
	{ WaitScratchDmaEvent(task, dma_buffer); } );
	dmaReadVirt(value_addr, sizeof(Addr), cb, &cb->dmaBuffer);
	}
	}
	};

	} // namespace gem5

	#endif // __DEV_HSA_GPU_COMMAND_PROCESSOR_HH__