mem-ruby: Add support for multiple DMA engines in MESI_Two_Level

Previously the MESI_Two_Level protocol supported systems with a single
DMA engine and responses from the directory to DMA requests were
routed back to the only DMA engine. This changeset adds support for
multiple DMA engines in the system by routing the response to the DMA
engine that originally sent the request.

Change-Id: I10ceda682ea29746636862ec8ef2a9c4220ca045
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/7161
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>
diff --git a/src/mem/protocol/MESI_Two_Level-dir.sm b/src/mem/protocol/MESI_Two_Level-dir.sm
index 440114a..991de5a 100644
--- a/src/mem/protocol/MESI_Two_Level-dir.sm
+++ b/src/mem/protocol/MESI_Two_Level-dir.sm
@@ -83,6 +83,7 @@
     State TBEState,        desc="Transient State";
     DataBlock DataBlk,     desc="Data to be written (DMA write only)";
     int Len,               desc="...";
+    MachineID Requestor,   desc="The DMA engine that sent the request";
   }
 
   structure(TBETable, external="yes") {
@@ -330,10 +331,11 @@
   action(dr_sendDMAData, "dr", desc="Send Data to DMA controller from directory") {
     peek(memQueue_in, MemoryMsg) {
       enqueue(responseNetwork_out, ResponseMsg, to_mem_ctrl_latency) {
+        assert(is_valid(tbe));
         out_msg.addr := address;
         out_msg.Type := CoherenceResponseType:DATA;
         out_msg.DataBlk := in_msg.DataBlk;   // we send the entire data block and rely on the dma controller to split it up if need be
-        out_msg.Destination.add(map_Address_to_DMA(address));
+        out_msg.Destination.add(tbe.Requestor);
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
     }
@@ -349,9 +351,10 @@
 
   action(da_sendDMAAck, "da", desc="Send Ack to DMA controller") {
       enqueue(responseNetwork_out, ResponseMsg, to_mem_ctrl_latency) {
+        assert(is_valid(tbe));
         out_msg.addr := address;
         out_msg.Type := CoherenceResponseType:ACK;
-        out_msg.Destination.add(map_Address_to_DMA(address));
+        out_msg.Destination.add(tbe.Requestor);
         out_msg.MessageSize := MessageSizeType:Writeback_Control;
       }
   }
@@ -380,10 +383,11 @@
   action(drp_sendDMAData, "drp", desc="Send Data to DMA controller from incoming PUTX") {
     peek(responseNetwork_in, ResponseMsg) {
       enqueue(responseNetwork_out, ResponseMsg, to_mem_ctrl_latency) {
+        assert(is_valid(tbe));
         out_msg.addr := address;
         out_msg.Type := CoherenceResponseType:DATA;
         out_msg.DataBlk := in_msg.DataBlk;   // we send the entire data block and rely on the dma controller to split it up if need be
-        out_msg.Destination.add(map_Address_to_DMA(address));
+        out_msg.Destination.add(tbe.Requestor);
         out_msg.MessageSize := MessageSizeType:Response_Data;
       }
     }
@@ -396,6 +400,7 @@
       tbe.DataBlk := in_msg.DataBlk;
       tbe.PhysicalAddress := in_msg.addr;
       tbe.Len := in_msg.Len;
+      tbe.Requestor := in_msg.Requestor;
     }
   }
 
@@ -451,23 +456,27 @@
 
 //added by SS for dma support
   transition(I, DMA_READ, ID) {
+    v_allocateTBE;
     qf_queueMemoryFetchRequestDMA;
     j_popIncomingRequestQueue;
   }
 
   transition(ID, Memory_Data, I) {
     dr_sendDMAData;
+    w_deallocateTBE;
     l_popMemQueue;
     kd_wakeUpDependents;
   }
 
   transition(I, DMA_WRITE, ID_W) {
+    v_allocateTBE;
     qw_queueMemoryWBRequest_partial;
     j_popIncomingRequestQueue;
   }
 
   transition(ID_W, Memory_Ack, I) {
     da_sendDMAAck;
+    w_deallocateTBE;
     l_popMemQueue;
     kd_wakeUpDependents;
   }
@@ -482,12 +491,14 @@
 
 
   transition(M, DMA_READ, M_DRD) {
+    v_allocateTBE;
     inv_sendCacheInvalidate;
     j_popIncomingRequestQueue;
   }
 
   transition(M_DRD, Data, M_DRDI) {
     drp_sendDMAData;
+    w_deallocateTBE;
     qw_queueMemoryWBRequest;
     k_popIncomingResponseQueue;
   }
diff --git a/src/mem/protocol/MESI_Two_Level-dma.sm b/src/mem/protocol/MESI_Two_Level-dma.sm
index 73a1fa2..5d0b885 100644
--- a/src/mem/protocol/MESI_Two_Level-dma.sm
+++ b/src/mem/protocol/MESI_Two_Level-dma.sm
@@ -137,6 +137,7 @@
       enqueue(requestToDir_out, RequestMsg, request_latency) {
         out_msg.addr := in_msg.PhysicalAddress;
         out_msg.Type := CoherenceRequestType:DMA_READ;
+        out_msg.Requestor := machineID;
         out_msg.DataBlk := in_msg.DataBlk;
         out_msg.Len := in_msg.Len;
         out_msg.Destination.add(mapAddressToMachine(address, MachineType:Directory));
@@ -150,6 +151,7 @@
       enqueue(requestToDir_out, RequestMsg, request_latency) {
           out_msg.addr := in_msg.PhysicalAddress;
           out_msg.Type := CoherenceRequestType:DMA_WRITE;
+          out_msg.Requestor := machineID;
           out_msg.DataBlk := in_msg.DataBlk;
           out_msg.Len := in_msg.Len;
           out_msg.Destination.add(mapAddressToMachine(address, MachineType:Directory));