configs/common/HMC.py - public/gem5 - Git at Google

 # Copyright (c) 2012-2013 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
 # not be construed as granting a license to any other intellectual
 # property including but not limited to intellectual property relating
 # to a hardware implementation of the functionality of the software
 # licensed hereunder.  You may use the software subject to the license
 # terms below provided that you ensure that this notice is replicated
 # unmodified and in its entirety in all distributions of the software,
 # modified or unmodified, in source code or in binary form.
 #
 # Copyright (c) 2015 The University of Bologna
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met: redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer;
 # redistributions in binary form must reproduce the above copyright
 # notice, this list of conditions and the following disclaimer in the
 # documentation and/or other materials provided with the distribution;
 # neither the name of the copyright holders nor the names of its
 # contributors may be used to endorse or promote products derived from
 # this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 # A Simplified model of a complete HMC device. Based on:
 #  [1] http://www.hybridmemorycube.org/specification-download/
 #  [2] High performance AXI-4.0 based interconnect for extensible smart memory
 #      cubes(E. Azarkhish et. al)
 #  [3] Low-Power Hybrid Memory Cubes With Link Power Management and Two-Level
 #      Prefetching (J. Ahn et. al)
 #  [4] Memory-centric system interconnect design with Hybrid Memory Cubes
 #      (G. Kim et. al)
 #  [5] Near Data Processing, Are we there yet? (M. Gokhale)
 #      http://www.cs.utah.edu/wondp/gokhale.pdf
 #  [6] openHMC - A Configurable Open-Source Hybrid Memory Cube Controller
 #      (J. Schmidt)
 #  [7] Hybrid Memory Cube performance characterization on data-centric
 #      workloads (M. Gokhale)
 #
 # This script builds a complete HMC device composed of vault controllers,
 # serial links, the main internal crossbar, and an external hmc controller.
 #
 # - VAULT CONTROLLERS:
 #   Instances of the HMC_2500_1x32 class with their functionality specified in
 #   dram_ctrl.cc
 #
 # - THE MAIN XBAR:
 #   This component is simply an instance of the NoncoherentXBar class, and its
 #   parameters are tuned to [2].
 #
 # - SERIAL LINKS CONTROLLER:
 #   SerialLink is a simple variation of the Bridge class, with the ability to
 #   account for the latency of packet serialization and controller latency. We
 #   assume that the serializer component at the transmitter side does not need
 #   to receive the whole packet to start the serialization. But the
 #   deserializer waits for the complete packet to check its integrity first.
 #
 #   * Bandwidth of the serial links is not modeled in the SerialLink component
 #     itself.
 #
 #   * Latency of serial link controller is composed of SerDes latency + link
 #     controller
 #
 #   * It is inferred from the standard [1] and the literature [3] that serial
 #     links share the same address range and packets can travel over any of
 #     them so a load distribution mechanism is required among them.
 #
 #   -----------------------------------------
 #   | Host/HMC Controller                   |
 #   |        ----------------------         |
 #   |        |  Link Aggregator   |  opt    |
 #   |        ----------------------         |
 #   |        ----------------------         |
 #   |        |  Serial Link + Ser | * 4     |
 #   |        ----------------------         |
 #   |---------------------------------------
 #   -----------------------------------------
 #   | Device
 #   |        ----------------------         |
 #   |        |       Xbar         | * 4     |
 #   |        ----------------------         |
 #   |        ----------------------         |
 #   |        |  Vault Controller  | * 16    |
 #   |        ----------------------         |
 #   |        ----------------------         |
 #   |        |     Memory         |         |
 #   |        ----------------------         |
 #   |---------------------------------------|
 #
 #   In this version we have present 3 different HMC archiecture along with
 #   alongwith their corresponding test script.
 #
 #   same: It has 4 crossbars in HMC memory. All the crossbars are connected
 #   to each other, providing complete memory range. This archicture also covers
 #   the added latency for sending a request to non-local vault(bridge in b/t
 #   crossbars). All the 4 serial links can access complete memory. So each
 #   link can be connected to separate processor.
 #
 #   distributed: It has 4 crossbars inside the HMC. Crossbars are not
 #   connected.Through each crossbar only local vaults can be accessed. But to
 #   support this architecture we need a crossbar between serial links and
 #   processor.
 #
 #   mixed: This is a hybrid architecture. It has 4 crossbars inside the HMC.
 #   2 Crossbars are connected to only local vaults. From other 2 crossbar, a
 #   request can be forwarded to any other vault.

 import argparse

 import m5
 from m5.objects import *
 from m5.util import *


 def add_options(parser):
     # *****************************CROSSBAR PARAMETERS*************************
     # Flit size of the main interconnect [1]
     parser.add_argument(
         "--xbar-width",
         default=32,
         action="store",
         type=int,
         help="Data width of the main XBar (Bytes)",
     )

     # Clock frequency of the main interconnect [1]
     # This crossbar, is placed on the logic-based of the HMC and it has its
     # own voltage and clock domains, different from the DRAM dies or from the
     # host.
     parser.add_argument(
         "--xbar-frequency",
         default="1GHz",
         type=str,
         help="Clock Frequency of the main XBar",
     )

     # Arbitration latency of the HMC XBar [1]
     parser.add_argument(
         "--xbar-frontend-latency",
         default=1,
         action="store",
         type=int,
         help="Arbitration latency of the XBar",
     )

     # Latency to forward a packet via the interconnect [1](two levels of FIFOs
     # at the input and output of the inteconnect)
     parser.add_argument(
         "--xbar-forward-latency",
         default=2,
         action="store",
         type=int,
         help="Forward latency of the XBar",
     )

     # Latency to forward a response via the interconnect [1](two levels of
     # FIFOs at the input and output of the inteconnect)
     parser.add_argument(
         "--xbar-response-latency",
         default=2,
         action="store",
         type=int,
         help="Response latency of the XBar",
     )

     # number of cross which connects 16 Vaults to serial link[7]
     parser.add_argument(
         "--number-mem-crossbar",
         default=4,
         action="store",
         type=int,
         help="Number of crossbar in HMC",
     )

     # *****************************SERIAL LINK PARAMETERS**********************
     # Number of serial links controllers [1]
     parser.add_argument(
         "--num-links-controllers",
         default=4,
         action="store",
         type=int,
         help="Number of serial links",
     )

     # Number of packets (not flits) to store at the request side of the serial
     #  link. This number should be adjusted to achive required bandwidth
     parser.add_argument(
         "--link-buffer-size-req",
         default=10,
         action="store",
         type=int,
         help="Number of packets to buffer at the\
                         request side of the serial link",
     )

     # Number of packets (not flits) to store at the response side of the serial
     #  link. This number should be adjusted to achive required bandwidth
     parser.add_argument(
         "--link-buffer-size-rsp",
         default=10,
         action="store",
         type=int,
         help="Number of packets to buffer at the\
                         response side of the serial link",
     )

     # Latency of the serial link composed by SER/DES latency (1.6ns [4]) plus
     # the PCB trace latency (3ns Estimated based on [5])
     parser.add_argument(
         "--link-latency",
         default="4.6ns",
         type=str,
         help="Latency of the serial links",
     )

     # Clock frequency of the each serial link(SerDes) [1]
     parser.add_argument(
         "--link-frequency",
         default="10GHz",
         type=str,
         help="Clock Frequency of the serial links",
     )

     # Clock frequency of serial link Controller[6]
     # clk_hmc[Mhz]= num_lanes_per_link * lane_speed [Gbits/s] /
     # data_path_width * 10^6
     # clk_hmc[Mhz]= 16 * 10 Gbps / 256 * 10^6 = 625 Mhz
     parser.add_argument(
         "--link-controller-frequency",
         default="625MHz",
         type=str,
         help="Clock Frequency of the link\
                         controller",
     )

     # Latency of the serial link controller to process the packets[1][6]
     # (ClockDomain = 625 Mhz )
     # used here for calculations only
     parser.add_argument(
         "--link-ctrl-latency",
         default=4,
         action="store",
         type=int,
         help="The number of cycles required for the\
                         controller to process the packet",
     )

     # total_ctrl_latency = link_ctrl_latency + link_latency
     # total_ctrl_latency = 4(Cycles) * 1.6 ns +  4.6 ns
     parser.add_argument(
         "--total-ctrl-latency",
         default="11ns",
         type=str,
         help="The latency experienced by every packet\
                         regardless of size of packet",
     )

     # Number of parallel lanes in each serial link [1]
     parser.add_argument(
         "--num-lanes-per-link",
         default=16,
         action="store",
         type=int,
         help="Number of lanes per each link",
     )

     # Number of serial links [1]
     parser.add_argument(
         "--num-serial-links",
         default=4,
         action="store",
         type=int,
         help="Number of serial links",
     )

     # speed of each lane of serial link - SerDes serial interface 10 Gb/s
     parser.add_argument(
         "--serial-link-speed",
         default=10,
         action="store",
         type=int,
         help="Gbs/s speed of each lane of serial\
                         link",
     )

     # address range for each of the serial links
     parser.add_argument(
         "--serial-link-addr-range",
         default="1GB",
         type=str,
         help="memory range for each of the serial links.\
                         Default: 1GB",
     )

     # *****************************PERFORMANCE MONITORING*********************
     # The main monitor behind the HMC Controller
     parser.add_argument(
         "--enable-global-monitor",
         action="store_true",
         help="The main monitor behind the HMC Controller",
     )

     # The link performance monitors
     parser.add_argument(
         "--enable-link-monitor", action="store_true", help="The link monitors"
     )

     # link aggregator enable - put a cross between buffers & links
     parser.add_argument(
         "--enable-link-aggr",
         action="store_true",
         help="The\
                         crossbar between port and Link Controller",
     )

     parser.add_argument(
         "--enable-buff-div",
         action="store_true",
         help="Memory Range of Buffer is ivided between total\
                         range",
     )

     # *****************************HMC ARCHITECTURE **************************
     # Memory chunk for 16 vault - numbers of vault / number of crossbars
     parser.add_argument(
         "--mem-chunk",
         default=4,
         action="store",
         type=int,
         help="Chunk of memory range for each cross bar in\
                         arch 0",
     )

     # size of req buffer within crossbar, used for modelling extra latency
     # when the reuqest go to non-local vault
     parser.add_argument(
         "--xbar-buffer-size-req",
         default=10,
         action="store",
         type=int,
         help="Number of packets to buffer at the\
                         request side of the crossbar",
     )

     # size of response buffer within crossbar, used for modelling extra latency
     # when the response received from non-local vault
     parser.add_argument(
         "--xbar-buffer-size-resp",
         default=10,
         action="store",
         type=int,
         help="Number of packets to buffer at the\
                         response side of the crossbar",
     )
     # HMC device architecture. It affects the HMC host controller as well
     parser.add_argument(
         "--arch",
         type=str,
         choices=["same", "distributed", "mixed"],
         default="distributed",
         help="same: HMC with\
                         4 links, all with same range.\ndistributed: HMC with\
                         4 links with distributed range.\nmixed: mixed with\
                         same and distributed range.\nDefault: distributed",
     )
     # HMC device - number of vaults
     parser.add_argument(
         "--hmc-dev-num-vaults",
         default=16,
         action="store",
         type=int,
         help="number of independent vaults within\
                         the HMC device. Note: each vault has a memory\
                         controller (valut controller)\nDefault: 16",
     )
     # HMC device - vault capacity or size
     parser.add_argument(
         "--hmc-dev-vault-size",
         default="256MB",
         type=str,
         help="vault storage capacity in bytes. Default:\
                         256MB",
     )
     parser.add_argument(
         "--mem-type",
         type=str,
         choices=["HMC_2500_1x32"],
         default="HMC_2500_1x32",
         help="type of HMC memory to\
                         use. Default: HMC_2500_1x32",
     )
     parser.add_argument(
         "--mem-channels",
         default=1,
         action="store",
         type=int,
         help="Number of memory channels",
     )
     parser.add_argument(
         "--mem-ranks",
         default=1,
         action="store",
         type=int,
         help="Number of ranks to iterate across",
     )
     parser.add_argument(
         "--burst-length",
         default=256,
         action="store",
         type=int,
         help="burst length in bytes. Note: the\
                         cache line size will be set to this value.\nDefault:\
                         256",
     )


 # configure HMC host controller
 def config_hmc_host_ctrl(opt, system):

     # create HMC host controller
     system.hmc_host = SubSystem()

     # Create additional crossbar for arch1
     if opt.arch == "distributed" or opt.arch == "mixed":
         clk = "100GHz"
         vd = VoltageDomain(voltage="1V")
         # Create additional crossbar for arch1
         system.membus = NoncoherentXBar(width=8)
         system.membus.badaddr_responder = BadAddr()
         system.membus.default = Self.badaddr_responder.pio
         system.membus.width = 8
         system.membus.frontend_latency = 3
         system.membus.forward_latency = 4
         system.membus.response_latency = 2
         cd = SrcClockDomain(clock=clk, voltage_domain=vd)
         system.membus.clk_domain = cd

     # create memory ranges for the serial links
     slar = convert.toMemorySize(opt.serial_link_addr_range)
     # Memmory ranges of serial link for arch-0. Same as the ranges of vault
     # controllers (4 vaults to 1 serial link)
     if opt.arch == "same":
         ser_ranges = [
             AddrRange(0, (4 * slar) - 1) for i in range(opt.num_serial_links)
         ]
     # Memmory ranges of serial link for arch-1. Distributed range accross
     # links
     if opt.arch == "distributed":
         ser_ranges = [
             AddrRange(i * slar, ((i + 1) * slar) - 1)
             for i in range(opt.num_serial_links)
         ]
     # Memmory ranges of serial link for arch-2 'Mixed' address distribution
     # over links
     if opt.arch == "mixed":
         ser_range0 = AddrRange(0, (1 * slar) - 1)
         ser_range1 = AddrRange(1 * slar, 2 * slar - 1)
         ser_range2 = AddrRange(0, (4 * slar) - 1)
         ser_range3 = AddrRange(0, (4 * slar) - 1)
         ser_ranges = [ser_range0, ser_range1, ser_range2, ser_range3]

     # Serial link Controller with 16 SerDes links at 10 Gbps with serial link
     # ranges w.r.t to architecture
     sl = [
         SerialLink(
             ranges=ser_ranges[i],
             req_size=opt.link_buffer_size_req,
             resp_size=opt.link_buffer_size_rsp,
             num_lanes=opt.num_lanes_per_link,
             link_speed=opt.serial_link_speed,
             delay=opt.total_ctrl_latency,
         )
         for i in range(opt.num_serial_links)
     ]
     system.hmc_host.seriallink = sl

     # enable global monitor
     if opt.enable_global_monitor:
         system.hmc_host.lmonitor = [
             CommMonitor() for i in range(opt.num_serial_links)
         ]

     # set the clock frequency for serial link
     for i in range(opt.num_serial_links):
         clk = opt.link_controller_frequency
         vd = VoltageDomain(voltage="1V")
         scd = SrcClockDomain(clock=clk, voltage_domain=vd)
         system.hmc_host.seriallink[i].clk_domain = scd

     # Connect membus/traffic gen to Serial Link Controller for differrent HMC
     # architectures
     hh = system.hmc_host
     if opt.arch == "distributed":
         mb = system.membus
         for i in range(opt.num_links_controllers):
             if opt.enable_global_monitor:
                 mb.mem_side_ports = hh.lmonitor[i].cpu_side_port
                 hh.lmonitor[i].mem_side_port = hh.seriallink[i].cpu_side_port
             else:
                 mb.mem_side_ports = hh.seriallink[i].cpu_side_port
     if opt.arch == "mixed":
         mb = system.membus
         if opt.enable_global_monitor:
             mb.mem_side_ports = hh.lmonitor[0].cpu_side_port
             hh.lmonitor[0].mem_side_port = hh.seriallink[0].cpu_side_port
             mb.mem_side_ports = hh.lmonitor[1].cpu_side_port
             hh.lmonitor[1].mem_side_port = hh.seriallink[1].cpu_side_port
         else:
             mb.mem_side_ports = hh.seriallink[0].cpu_side_port
             mb.mem_side_ports = hh.seriallink[1].cpu_side_port

     if opt.arch == "same":
         for i in range(opt.num_links_controllers):
             if opt.enable_global_monitor:
                 hh.lmonitor[i].mem_side_port = hh.seriallink[i].cpu_side_port

     return system


 # Create an HMC device
 def config_hmc_dev(opt, system, hmc_host):

     # create HMC device
     system.hmc_dev = SubSystem()

     # create memory ranges for the vault controllers
     arv = convert.toMemorySize(opt.hmc_dev_vault_size)
     addr_ranges_vaults = [
         AddrRange(i * arv, ((i + 1) * arv - 1))
         for i in range(opt.hmc_dev_num_vaults)
     ]
     system.mem_ranges = addr_ranges_vaults

     if opt.enable_link_monitor:
         lm = [CommMonitor() for i in range(opt.num_links_controllers)]
         system.hmc_dev.lmonitor = lm

     # 4 HMC Crossbars located in its logic-base (LoB)
     xb = [
         NoncoherentXBar(
             width=opt.xbar_width,
             frontend_latency=opt.xbar_frontend_latency,
             forward_latency=opt.xbar_forward_latency,
             response_latency=opt.xbar_response_latency,
         )
         for i in range(opt.number_mem_crossbar)
     ]
     system.hmc_dev.xbar = xb

     for i in range(opt.number_mem_crossbar):
         clk = opt.xbar_frequency
         vd = VoltageDomain(voltage="1V")
         scd = SrcClockDomain(clock=clk, voltage_domain=vd)
         system.hmc_dev.xbar[i].clk_domain = scd

     # Attach 4 serial link to 4 crossbar/s
     for i in range(opt.num_serial_links):
         if opt.enable_link_monitor:
             system.hmc_host.seriallink[
                 i
             ].mem_side_port = system.hmc_dev.lmonitor[i].cpu_side_port
             system.hmc_dev.lmonitor[i].mem_side_port = system.hmc_dev.xbar[
                 i
             ].cpu_side_ports
         else:
             system.hmc_host.seriallink[i].mem_side_port = system.hmc_dev.xbar[
                 i
             ].cpu_side_ports

     # Connecting xbar with each other for request arriving at the wrong xbar,
     # then it will be forward to correct xbar. Bridge is used to connect xbars
     if opt.arch == "same":
         numx = len(system.hmc_dev.xbar)

         # create a list of buffers
         system.hmc_dev.buffers = [
             Bridge(
                 req_size=opt.xbar_buffer_size_req,
                 resp_size=opt.xbar_buffer_size_resp,
             )
             for i in range(numx * (opt.mem_chunk - 1))
         ]

         # Buffer iterator
         it = iter(list(range(len(system.hmc_dev.buffers))))

         # necesarry to add system_port to one of the xbar
         system.system_port = system.hmc_dev.xbar[3].cpu_side_ports

         # iterate over all the crossbars and connect them as required
         for i in range(numx):
             for j in range(numx):
                 # connect xbar to all other xbars except itself
                 if i != j:
                     # get the next index of buffer
                     index = next(it)

                     # Change the default values for ranges of bridge
                     system.hmc_dev.buffers[index].ranges = system.mem_ranges[
                         j * int(opt.mem_chunk) : (j + 1) * int(opt.mem_chunk)
                     ]

                     # Connect the bridge between corssbars
                     system.hmc_dev.xbar[
                         i
                     ].mem_side_ports = system.hmc_dev.buffers[
                         index
                     ].cpu_side_port
                     system.hmc_dev.buffers[
                         index
                     ].mem_side_port = system.hmc_dev.xbar[j].cpu_side_ports
                 else:
                     # Don't connect the xbar to itself
                     pass

     # Two crossbars are connected to all other crossbars-Other 2 vault
     # can only direct traffic to it local vaults
     if opt.arch == "mixed":
         system.hmc_dev.buffer30 = Bridge(ranges=system.mem_ranges[0:4])
         system.hmc_dev.xbar[
             3
         ].mem_side_ports = system.hmc_dev.buffer30.cpu_side_port
         system.hmc_dev.buffer30.mem_side_port = system.hmc_dev.xbar[
             0
         ].cpu_side_ports

         system.hmc_dev.buffer31 = Bridge(ranges=system.mem_ranges[4:8])
         system.hmc_dev.xbar[
             3
         ].mem_side_ports = system.hmc_dev.buffer31.cpu_side_port
         system.hmc_dev.buffer31.mem_side_port = system.hmc_dev.xbar[
             1
         ].cpu_side_ports

         system.hmc_dev.buffer32 = Bridge(ranges=system.mem_ranges[8:12])
         system.hmc_dev.xbar[
             3
         ].mem_side_ports = system.hmc_dev.buffer32.cpu_side_port
         system.hmc_dev.buffer32.mem_side_port = system.hmc_dev.xbar[
             2
         ].cpu_side_ports

         system.hmc_dev.buffer20 = Bridge(ranges=system.mem_ranges[0:4])
         system.hmc_dev.xbar[
             2
         ].mem_side_ports = system.hmc_dev.buffer20.cpu_side_port
         system.hmc_dev.buffer20.mem_side_port = system.hmc_dev.xbar[
             0
         ].cpu_side_ports

         system.hmc_dev.buffer21 = Bridge(ranges=system.mem_ranges[4:8])
         system.hmc_dev.xbar[
             2
         ].mem_side_ports = system.hmc_dev.buffer21.cpu_side_port
         system.hmc_dev.buffer21.mem_side_port = system.hmc_dev.xbar[
             1
         ].cpu_side_ports

         system.hmc_dev.buffer23 = Bridge(ranges=system.mem_ranges[12:16])
         system.hmc_dev.xbar[
             2
         ].mem_side_ports = system.hmc_dev.buffer23.cpu_side_port
         system.hmc_dev.buffer23.mem_side_port = system.hmc_dev.xbar[
             3
         ].cpu_side_ports
	# Copyright (c) 2012-2013 ARM Limited
	# All rights reserved.
	#
	# The license below extends only to copyright in the software and shall
	# not be construed as granting a license to any other intellectual
	# property including but not limited to intellectual property relating
	# to a hardware implementation of the functionality of the software
	# licensed hereunder. You may use the software subject to the license
	# terms below provided that you ensure that this notice is replicated
	# unmodified and in its entirety in all distributions of the software,
	# modified or unmodified, in source code or in binary form.
	#
	# Copyright (c) 2015 The University of Bologna
	# All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions are
	# met: redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer;
	# redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution;
	# neither the name of the copyright holders nor the names of its
	# contributors may be used to endorse or promote products derived from
	# this software without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	# A Simplified model of a complete HMC device. Based on:
	# [1] http://www.hybridmemorycube.org/specification-download/
	# [2] High performance AXI-4.0 based interconnect for extensible smart memory
	# cubes(E. Azarkhish et. al)
	# [3] Low-Power Hybrid Memory Cubes With Link Power Management and Two-Level
	# Prefetching (J. Ahn et. al)
	# [4] Memory-centric system interconnect design with Hybrid Memory Cubes
	# (G. Kim et. al)
	# [5] Near Data Processing, Are we there yet? (M. Gokhale)
	# http://www.cs.utah.edu/wondp/gokhale.pdf
	# [6] openHMC - A Configurable Open-Source Hybrid Memory Cube Controller
	# (J. Schmidt)
	# [7] Hybrid Memory Cube performance characterization on data-centric
	# workloads (M. Gokhale)
	#
	# This script builds a complete HMC device composed of vault controllers,
	# serial links, the main internal crossbar, and an external hmc controller.
	#
	# - VAULT CONTROLLERS:
	# Instances of the HMC_2500_1x32 class with their functionality specified in
	# dram_ctrl.cc
	#
	# - THE MAIN XBAR:
	# This component is simply an instance of the NoncoherentXBar class, and its
	# parameters are tuned to [2].
	#
	# - SERIAL LINKS CONTROLLER:
	# SerialLink is a simple variation of the Bridge class, with the ability to
	# account for the latency of packet serialization and controller latency. We
	# assume that the serializer component at the transmitter side does not need
	# to receive the whole packet to start the serialization. But the
	# deserializer waits for the complete packet to check its integrity first.
	#
	# * Bandwidth of the serial links is not modeled in the SerialLink component
	# itself.
	#
	# * Latency of serial link controller is composed of SerDes latency + link
	# controller
	#
	# * It is inferred from the standard [1] and the literature [3] that serial
	# links share the same address range and packets can travel over any of
	# them so a load distribution mechanism is required among them.
	#
	# -----------------------------------------
	# \| Host/HMC Controller \|
	# \| ---------------------- \|
	# \| \| Link Aggregator \| opt \|
	# \| ---------------------- \|
	# \| ---------------------- \|
	# \| \| Serial Link + Ser \| * 4 \|
	# \| ---------------------- \|
	# \|---------------------------------------
	# -----------------------------------------
	# \| Device
	# \| ---------------------- \|
	# \| \| Xbar \| * 4 \|
	# \| ---------------------- \|
	# \| ---------------------- \|
	# \| \| Vault Controller \| * 16 \|
	# \| ---------------------- \|
	# \| ---------------------- \|
	# \| \| Memory \| \|
	# \| ---------------------- \|
	# \|---------------------------------------\|
	#
	# In this version we have present 3 different HMC archiecture along with
	# alongwith their corresponding test script.
	#
	# same: It has 4 crossbars in HMC memory. All the crossbars are connected
	# to each other, providing complete memory range. This archicture also covers
	# the added latency for sending a request to non-local vault(bridge in b/t
	# crossbars). All the 4 serial links can access complete memory. So each
	# link can be connected to separate processor.
	#
	# distributed: It has 4 crossbars inside the HMC. Crossbars are not
	# connected.Through each crossbar only local vaults can be accessed. But to
	# support this architecture we need a crossbar between serial links and
	# processor.
	#
	# mixed: This is a hybrid architecture. It has 4 crossbars inside the HMC.
	# 2 Crossbars are connected to only local vaults. From other 2 crossbar, a
	# request can be forwarded to any other vault.

	import argparse

	import m5
	from m5.objects import *
	from m5.util import *


	def add_options(parser):
	# ***************************CROSSBAR PARAMETERS***********************
	# Flit size of the main interconnect [1]
	parser.add_argument(
	"--xbar-width",
	default=32,
	action="store",
	type=int,
	help="Data width of the main XBar (Bytes)",
	)

	# Clock frequency of the main interconnect [1]
	# This crossbar, is placed on the logic-based of the HMC and it has its
	# own voltage and clock domains, different from the DRAM dies or from the
	# host.
	parser.add_argument(
	"--xbar-frequency",
	default="1GHz",
	type=str,
	help="Clock Frequency of the main XBar",
	)

	# Arbitration latency of the HMC XBar [1]
	parser.add_argument(
	"--xbar-frontend-latency",
	default=1,
	action="store",
	type=int,
	help="Arbitration latency of the XBar",
	)

	# Latency to forward a packet via the interconnect [1](two levels of FIFOs
	# at the input and output of the inteconnect)
	parser.add_argument(
	"--xbar-forward-latency",
	default=2,
	action="store",
	type=int,
	help="Forward latency of the XBar",
	)

	# Latency to forward a response via the interconnect [1](two levels of
	# FIFOs at the input and output of the inteconnect)
	parser.add_argument(
	"--xbar-response-latency",
	default=2,
	action="store",
	type=int,
	help="Response latency of the XBar",
	)

	# number of cross which connects 16 Vaults to serial link[7]
	parser.add_argument(
	"--number-mem-crossbar",
	default=4,
	action="store",
	type=int,
	help="Number of crossbar in HMC",
	)

	# ***************************SERIAL LINK PARAMETERS********************
	# Number of serial links controllers [1]
	parser.add_argument(
	"--num-links-controllers",
	default=4,
	action="store",
	type=int,
	help="Number of serial links",
	)

	# Number of packets (not flits) to store at the request side of the serial
	# link. This number should be adjusted to achive required bandwidth
	parser.add_argument(
	"--link-buffer-size-req",
	default=10,
	action="store",
	type=int,
	help="Number of packets to buffer at the\
	request side of the serial link",
	)

	# Number of packets (not flits) to store at the response side of the serial
	# link. This number should be adjusted to achive required bandwidth
	parser.add_argument(
	"--link-buffer-size-rsp",
	default=10,
	action="store",
	type=int,
	help="Number of packets to buffer at the\
	response side of the serial link",
	)

	# Latency of the serial link composed by SER/DES latency (1.6ns [4]) plus
	# the PCB trace latency (3ns Estimated based on [5])
	parser.add_argument(
	"--link-latency",
	default="4.6ns",
	type=str,
	help="Latency of the serial links",
	)

	# Clock frequency of the each serial link(SerDes) [1]
	parser.add_argument(
	"--link-frequency",
	default="10GHz",
	type=str,
	help="Clock Frequency of the serial links",
	)

	# Clock frequency of serial link Controller[6]
	# clk_hmc[Mhz]= num_lanes_per_link * lane_speed [Gbits/s] /
	# data_path_width * 10^6
	# clk_hmc[Mhz]= 16 * 10 Gbps / 256 * 10^6 = 625 Mhz
	parser.add_argument(
	"--link-controller-frequency",
	default="625MHz",
	type=str,
	help="Clock Frequency of the link\
	controller",
	)

	# Latency of the serial link controller to process the packets[1][6]
	# (ClockDomain = 625 Mhz )
	# used here for calculations only
	parser.add_argument(
	"--link-ctrl-latency",
	default=4,
	action="store",
	type=int,
	help="The number of cycles required for the\
	controller to process the packet",
	)

	# total_ctrl_latency = link_ctrl_latency + link_latency
	# total_ctrl_latency = 4(Cycles) * 1.6 ns + 4.6 ns
	parser.add_argument(
	"--total-ctrl-latency",
	default="11ns",
	type=str,
	help="The latency experienced by every packet\
	regardless of size of packet",
	)

	# Number of parallel lanes in each serial link [1]
	parser.add_argument(
	"--num-lanes-per-link",
	default=16,
	action="store",
	type=int,
	help="Number of lanes per each link",
	)

	# Number of serial links [1]
	parser.add_argument(
	"--num-serial-links",
	default=4,
	action="store",
	type=int,
	help="Number of serial links",
	)

	# speed of each lane of serial link - SerDes serial interface 10 Gb/s
	parser.add_argument(
	"--serial-link-speed",
	default=10,
	action="store",
	type=int,
	help="Gbs/s speed of each lane of serial\
	link",
	)

	# address range for each of the serial links
	parser.add_argument(
	"--serial-link-addr-range",
	default="1GB",
	type=str,
	help="memory range for each of the serial links.\
	Default: 1GB",
	)

	# ***************************PERFORMANCE MONITORING*******************
	# The main monitor behind the HMC Controller
	parser.add_argument(
	"--enable-global-monitor",
	action="store_true",
	help="The main monitor behind the HMC Controller",
	)

	# The link performance monitors
	parser.add_argument(
	"--enable-link-monitor", action="store_true", help="The link monitors"
	)

	# link aggregator enable - put a cross between buffers & links
	parser.add_argument(
	"--enable-link-aggr",
	action="store_true",
	help="The\
	crossbar between port and Link Controller",
	)

	parser.add_argument(
	"--enable-buff-div",
	action="store_true",
	help="Memory Range of Buffer is ivided between total\
	range",
	)

	# ***************************HMC ARCHITECTURE ************************
	# Memory chunk for 16 vault - numbers of vault / number of crossbars
	parser.add_argument(
	"--mem-chunk",
	default=4,
	action="store",
	type=int,
	help="Chunk of memory range for each cross bar in\
	arch 0",
	)

	# size of req buffer within crossbar, used for modelling extra latency
	# when the reuqest go to non-local vault
	parser.add_argument(
	"--xbar-buffer-size-req",
	default=10,
	action="store",
	type=int,
	help="Number of packets to buffer at the\
	request side of the crossbar",
	)

	# size of response buffer within crossbar, used for modelling extra latency
	# when the response received from non-local vault
	parser.add_argument(
	"--xbar-buffer-size-resp",
	default=10,
	action="store",
	type=int,
	help="Number of packets to buffer at the\
	response side of the crossbar",
	)
	# HMC device architecture. It affects the HMC host controller as well
	parser.add_argument(
	"--arch",
	type=str,
	choices=["same", "distributed", "mixed"],
	default="distributed",
	help="same: HMC with\
	4 links, all with same range.\ndistributed: HMC with\
	4 links with distributed range.\nmixed: mixed with\
	same and distributed range.\nDefault: distributed",
	)
	# HMC device - number of vaults
	parser.add_argument(
	"--hmc-dev-num-vaults",
	default=16,
	action="store",
	type=int,
	help="number of independent vaults within\
	the HMC device. Note: each vault has a memory\
	controller (valut controller)\nDefault: 16",
	)
	# HMC device - vault capacity or size
	parser.add_argument(
	"--hmc-dev-vault-size",
	default="256MB",
	type=str,
	help="vault storage capacity in bytes. Default:\
	256MB",
	)
	parser.add_argument(
	"--mem-type",
	type=str,
	choices=["HMC_2500_1x32"],
	default="HMC_2500_1x32",
	help="type of HMC memory to\
	use. Default: HMC_2500_1x32",
	)
	parser.add_argument(
	"--mem-channels",
	default=1,
	action="store",
	type=int,
	help="Number of memory channels",
	)
	parser.add_argument(
	"--mem-ranks",
	default=1,
	action="store",
	type=int,
	help="Number of ranks to iterate across",
	)
	parser.add_argument(
	"--burst-length",
	default=256,
	action="store",
	type=int,
	help="burst length in bytes. Note: the\
	cache line size will be set to this value.\nDefault:\
	256",
	)


	# configure HMC host controller
	def config_hmc_host_ctrl(opt, system):

	# create HMC host controller
	system.hmc_host = SubSystem()

	# Create additional crossbar for arch1
	if opt.arch == "distributed" or opt.arch == "mixed":
	clk = "100GHz"
	vd = VoltageDomain(voltage="1V")
	# Create additional crossbar for arch1
	system.membus = NoncoherentXBar(width=8)
	system.membus.badaddr_responder = BadAddr()
	system.membus.default = Self.badaddr_responder.pio
	system.membus.width = 8
	system.membus.frontend_latency = 3
	system.membus.forward_latency = 4
	system.membus.response_latency = 2
	cd = SrcClockDomain(clock=clk, voltage_domain=vd)
	system.membus.clk_domain = cd

	# create memory ranges for the serial links
	slar = convert.toMemorySize(opt.serial_link_addr_range)
	# Memmory ranges of serial link for arch-0. Same as the ranges of vault
	# controllers (4 vaults to 1 serial link)
	if opt.arch == "same":
	ser_ranges = [
	AddrRange(0, (4 * slar) - 1) for i in range(opt.num_serial_links)
	]
	# Memmory ranges of serial link for arch-1. Distributed range accross
	# links
	if opt.arch == "distributed":
	ser_ranges = [
	AddrRange(i * slar, ((i + 1) * slar) - 1)
	for i in range(opt.num_serial_links)
	]
	# Memmory ranges of serial link for arch-2 'Mixed' address distribution
	# over links
	if opt.arch == "mixed":
	ser_range0 = AddrRange(0, (1 * slar) - 1)
	ser_range1 = AddrRange(1 * slar, 2 * slar - 1)
	ser_range2 = AddrRange(0, (4 * slar) - 1)
	ser_range3 = AddrRange(0, (4 * slar) - 1)
	ser_ranges = [ser_range0, ser_range1, ser_range2, ser_range3]

	# Serial link Controller with 16 SerDes links at 10 Gbps with serial link
	# ranges w.r.t to architecture
	sl = [
	SerialLink(
	ranges=ser_ranges[i],
	req_size=opt.link_buffer_size_req,
	resp_size=opt.link_buffer_size_rsp,
	num_lanes=opt.num_lanes_per_link,
	link_speed=opt.serial_link_speed,
	delay=opt.total_ctrl_latency,
	)
	for i in range(opt.num_serial_links)
	]
	system.hmc_host.seriallink = sl

	# enable global monitor
	if opt.enable_global_monitor:
	system.hmc_host.lmonitor = [
	CommMonitor() for i in range(opt.num_serial_links)
	]

	# set the clock frequency for serial link
	for i in range(opt.num_serial_links):
	clk = opt.link_controller_frequency
	vd = VoltageDomain(voltage="1V")
	scd = SrcClockDomain(clock=clk, voltage_domain=vd)
	system.hmc_host.seriallink[i].clk_domain = scd

	# Connect membus/traffic gen to Serial Link Controller for differrent HMC
	# architectures
	hh = system.hmc_host
	if opt.arch == "distributed":
	mb = system.membus
	for i in range(opt.num_links_controllers):
	if opt.enable_global_monitor:
	mb.mem_side_ports = hh.lmonitor[i].cpu_side_port
	hh.lmonitor[i].mem_side_port = hh.seriallink[i].cpu_side_port
	else:
	mb.mem_side_ports = hh.seriallink[i].cpu_side_port
	if opt.arch == "mixed":
	mb = system.membus
	if opt.enable_global_monitor:
	mb.mem_side_ports = hh.lmonitor[0].cpu_side_port
	hh.lmonitor[0].mem_side_port = hh.seriallink[0].cpu_side_port
	mb.mem_side_ports = hh.lmonitor[1].cpu_side_port
	hh.lmonitor[1].mem_side_port = hh.seriallink[1].cpu_side_port
	else:
	mb.mem_side_ports = hh.seriallink[0].cpu_side_port
	mb.mem_side_ports = hh.seriallink[1].cpu_side_port

	if opt.arch == "same":
	for i in range(opt.num_links_controllers):
	if opt.enable_global_monitor:
	hh.lmonitor[i].mem_side_port = hh.seriallink[i].cpu_side_port

	return system


	# Create an HMC device
	def config_hmc_dev(opt, system, hmc_host):

	# create HMC device
	system.hmc_dev = SubSystem()

	# create memory ranges for the vault controllers
	arv = convert.toMemorySize(opt.hmc_dev_vault_size)
	addr_ranges_vaults = [
	AddrRange(i * arv, ((i + 1) * arv - 1))
	for i in range(opt.hmc_dev_num_vaults)
	]
	system.mem_ranges = addr_ranges_vaults

	if opt.enable_link_monitor:
	lm = [CommMonitor() for i in range(opt.num_links_controllers)]
	system.hmc_dev.lmonitor = lm

	# 4 HMC Crossbars located in its logic-base (LoB)
	xb = [
	NoncoherentXBar(
	width=opt.xbar_width,
	frontend_latency=opt.xbar_frontend_latency,
	forward_latency=opt.xbar_forward_latency,
	response_latency=opt.xbar_response_latency,
	)
	for i in range(opt.number_mem_crossbar)
	]
	system.hmc_dev.xbar = xb

	for i in range(opt.number_mem_crossbar):
	clk = opt.xbar_frequency
	vd = VoltageDomain(voltage="1V")
	scd = SrcClockDomain(clock=clk, voltage_domain=vd)
	system.hmc_dev.xbar[i].clk_domain = scd

	# Attach 4 serial link to 4 crossbar/s
	for i in range(opt.num_serial_links):
	if opt.enable_link_monitor:
	system.hmc_host.seriallink[
	i
	].mem_side_port = system.hmc_dev.lmonitor[i].cpu_side_port
	system.hmc_dev.lmonitor[i].mem_side_port = system.hmc_dev.xbar[
	i
	].cpu_side_ports
	else:
	system.hmc_host.seriallink[i].mem_side_port = system.hmc_dev.xbar[
	i
	].cpu_side_ports

	# Connecting xbar with each other for request arriving at the wrong xbar,
	# then it will be forward to correct xbar. Bridge is used to connect xbars
	if opt.arch == "same":
	numx = len(system.hmc_dev.xbar)

	# create a list of buffers
	system.hmc_dev.buffers = [
	Bridge(
	req_size=opt.xbar_buffer_size_req,
	resp_size=opt.xbar_buffer_size_resp,
	)
	for i in range(numx * (opt.mem_chunk - 1))
	]

	# Buffer iterator
	it = iter(list(range(len(system.hmc_dev.buffers))))

	# necesarry to add system_port to one of the xbar
	system.system_port = system.hmc_dev.xbar[3].cpu_side_ports

	# iterate over all the crossbars and connect them as required
	for i in range(numx):
	for j in range(numx):
	# connect xbar to all other xbars except itself
	if i != j:
	# get the next index of buffer
	index = next(it)

	# Change the default values for ranges of bridge
	system.hmc_dev.buffers[index].ranges = system.mem_ranges[
	j * int(opt.mem_chunk) : (j + 1) * int(opt.mem_chunk)
	]

	# Connect the bridge between corssbars
	system.hmc_dev.xbar[
	i
	].mem_side_ports = system.hmc_dev.buffers[
	index
	].cpu_side_port
	system.hmc_dev.buffers[
	index
	].mem_side_port = system.hmc_dev.xbar[j].cpu_side_ports
	else:
	# Don't connect the xbar to itself
	pass

	# Two crossbars are connected to all other crossbars-Other 2 vault
	# can only direct traffic to it local vaults
	if opt.arch == "mixed":
	system.hmc_dev.buffer30 = Bridge(ranges=system.mem_ranges[0:4])
	system.hmc_dev.xbar[
	3
	].mem_side_ports = system.hmc_dev.buffer30.cpu_side_port
	system.hmc_dev.buffer30.mem_side_port = system.hmc_dev.xbar[
	0
	].cpu_side_ports

	system.hmc_dev.buffer31 = Bridge(ranges=system.mem_ranges[4:8])
	system.hmc_dev.xbar[
	3
	].mem_side_ports = system.hmc_dev.buffer31.cpu_side_port
	system.hmc_dev.buffer31.mem_side_port = system.hmc_dev.xbar[
	1
	].cpu_side_ports

	system.hmc_dev.buffer32 = Bridge(ranges=system.mem_ranges[8:12])
	system.hmc_dev.xbar[
	3
	].mem_side_ports = system.hmc_dev.buffer32.cpu_side_port
	system.hmc_dev.buffer32.mem_side_port = system.hmc_dev.xbar[
	2
	].cpu_side_ports

	system.hmc_dev.buffer20 = Bridge(ranges=system.mem_ranges[0:4])
	system.hmc_dev.xbar[
	2
	].mem_side_ports = system.hmc_dev.buffer20.cpu_side_port
	system.hmc_dev.buffer20.mem_side_port = system.hmc_dev.xbar[
	0
	].cpu_side_ports

	system.hmc_dev.buffer21 = Bridge(ranges=system.mem_ranges[4:8])
	system.hmc_dev.xbar[
	2
	].mem_side_ports = system.hmc_dev.buffer21.cpu_side_port
	system.hmc_dev.buffer21.mem_side_port = system.hmc_dev.xbar[
	1
	].cpu_side_ports

	system.hmc_dev.buffer23 = Bridge(ranges=system.mem_ranges[12:16])
	system.hmc_dev.xbar[
	2
	].mem_side_ports = system.hmc_dev.buffer23.cpu_side_port
	system.hmc_dev.buffer23.mem_side_port = system.hmc_dev.xbar[
	3
	].cpu_side_ports