# Copyright (c) 2012-2013 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2015 The University of Bologna
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# A Simplified model of a complete HMC device. Based on:
#  [1] http://www.hybridmemorycube.org/specification-download/
#  [2] High performance AXI-4.0 based interconnect for extensible smart memory
#      cubes(E. Azarkhish et. al)
#  [3] Low-Power Hybrid Memory Cubes With Link Power Management and Two-Level
#      Prefetching (J. Ahn et. al)
#  [4] Memory-centric system interconnect design with Hybrid Memory Cubes
#      (G. Kim et. al)
#  [5] Near Data Processing, Are we there yet? (M. Gokhale)
#      http://www.cs.utah.edu/wondp/gokhale.pdf
#  [6] openHMC - A Configurable Open-Source Hybrid Memory Cube Controller
#      (J. Schmidt)
#  [7] Hybrid Memory Cube performance characterization on data-centric
#      workloads (M. Gokhale)
#
# This script builds a complete HMC device composed of vault controllers,
# serial links, the main internal crossbar, and an external hmc controller.
#
# - VAULT CONTROLLERS:
#   Instances of the HMC_2500_1x32 class with their functionality specified in
#   dram_ctrl.cc
#
# - THE MAIN XBAR:
#   This component is simply an instance of the NoncoherentXBar class, and its
#   parameters are tuned to [2].
#
# - SERIAL LINKS CONTROLLER:
#   SerialLink is a simple variation of the Bridge class, with the ability to
#   account for the latency of packet serialization and controller latency. We
#   assume that the serializer component at the transmitter side does not need
#   to receive the whole packet to start the serialization. But the
#   deserializer waits for the complete packet to check its integrity first.
#
#   * Bandwidth of the serial links is not modeled in the SerialLink component
#     itself.
#
#   * Latency of serial link controller is composed of SerDes latency + link
#     controller
#
#   * It is inferred from the standard [1] and the literature [3] that serial
#     links share the same address range and packets can travel over any of
#     them so a load distribution mechanism is required among them.
#
#   -----------------------------------------
#   | Host/HMC Controller                   |
#   |        ----------------------         |
#   |        |  Link Aggregator   |  opt    |
#   |        ----------------------         |
#   |        ----------------------         |
#   |        |  Serial Link + Ser | * 4     |
#   |        ----------------------         |
#   |---------------------------------------
#   -----------------------------------------
#   | Device
#   |        ----------------------         |
#   |        |       Xbar         | * 4     |
#   |        ----------------------         |
#   |        ----------------------         |
#   |        |  Vault Controller  | * 16    |
#   |        ----------------------         |
#   |        ----------------------         |
#   |        |     Memory         |         |
#   |        ----------------------         |
#   |---------------------------------------|
#
#   In this version we have present 3 different HMC archiecture along with
#   alongwith their corresponding test script.
#
#   same: It has 4 crossbars in HMC memory. All the crossbars are connected
#   to each other, providing complete memory range. This archicture also covers
#   the added latency for sending a request to non-local vault(bridge in b/t
#   crossbars). All the 4 serial links can access complete memory. So each
#   link can be connected to separate processor.
#
#   distributed: It has 4 crossbars inside the HMC. Crossbars are not
#   connected.Through each crossbar only local vaults can be accessed. But to
#   support this architecture we need a crossbar between serial links and
#   processor.
#
#   mixed: This is a hybrid architecture. It has 4 crossbars inside the HMC.
#   2 Crossbars are connected to only local vaults. From other 2 crossbar, a
#   request can be forwarded to any other vault.

import argparse

import m5
from m5.objects import *
from m5.util import *


def add_options(parser):
    # *****************************CROSSBAR PARAMETERS*************************
    # Flit size of the main interconnect [1]
    parser.add_argument(
        "--xbar-width",
        default=32,
        action="store",
        type=int,
        help="Data width of the main XBar (Bytes)",
    )

    # Clock frequency of the main interconnect [1]
    # This crossbar, is placed on the logic-based of the HMC and it has its
    # own voltage and clock domains, different from the DRAM dies or from the
    # host.
    parser.add_argument(
        "--xbar-frequency",
        default="1GHz",
        type=str,
        help="Clock Frequency of the main XBar",
    )

    # Arbitration latency of the HMC XBar [1]
    parser.add_argument(
        "--xbar-frontend-latency",
        default=1,
        action="store",
        type=int,
        help="Arbitration latency of the XBar",
    )

    # Latency to forward a packet via the interconnect [1](two levels of FIFOs
    # at the input and output of the inteconnect)
    parser.add_argument(
        "--xbar-forward-latency",
        default=2,
        action="store",
        type=int,
        help="Forward latency of the XBar",
    )

    # Latency to forward a response via the interconnect [1](two levels of
    # FIFOs at the input and output of the inteconnect)
    parser.add_argument(
        "--xbar-response-latency",
        default=2,
        action="store",
        type=int,
        help="Response latency of the XBar",
    )

    # number of cross which connects 16 Vaults to serial link[7]
    parser.add_argument(
        "--number-mem-crossbar",
        default=4,
        action="store",
        type=int,
        help="Number of crossbar in HMC",
    )

    # *****************************SERIAL LINK PARAMETERS**********************
    # Number of serial links controllers [1]
    parser.add_argument(
        "--num-links-controllers",
        default=4,
        action="store",
        type=int,
        help="Number of serial links",
    )

    # Number of packets (not flits) to store at the request side of the serial
    #  link. This number should be adjusted to achive required bandwidth
    parser.add_argument(
        "--link-buffer-size-req",
        default=10,
        action="store",
        type=int,
        help="Number of packets to buffer at the\
                        request side of the serial link",
    )

    # Number of packets (not flits) to store at the response side of the serial
    #  link. This number should be adjusted to achive required bandwidth
    parser.add_argument(
        "--link-buffer-size-rsp",
        default=10,
        action="store",
        type=int,
        help="Number of packets to buffer at the\
                        response side of the serial link",
    )

    # Latency of the serial link composed by SER/DES latency (1.6ns [4]) plus
    # the PCB trace latency (3ns Estimated based on [5])
    parser.add_argument(
        "--link-latency",
        default="4.6ns",
        type=str,
        help="Latency of the serial links",
    )

    # Clock frequency of the each serial link(SerDes) [1]
    parser.add_argument(
        "--link-frequency",
        default="10GHz",
        type=str,
        help="Clock Frequency of the serial links",
    )

    # Clock frequency of serial link Controller[6]
    # clk_hmc[Mhz]= num_lanes_per_link * lane_speed [Gbits/s] /
    # data_path_width * 10^6
    # clk_hmc[Mhz]= 16 * 10 Gbps / 256 * 10^6 = 625 Mhz
    parser.add_argument(
        "--link-controller-frequency",
        default="625MHz",
        type=str,
        help="Clock Frequency of the link\
                        controller",
    )

    # Latency of the serial link controller to process the packets[1][6]
    # (ClockDomain = 625 Mhz )
    # used here for calculations only
    parser.add_argument(
        "--link-ctrl-latency",
        default=4,
        action="store",
        type=int,
        help="The number of cycles required for the\
                        controller to process the packet",
    )

    # total_ctrl_latency = link_ctrl_latency + link_latency
    # total_ctrl_latency = 4(Cycles) * 1.6 ns +  4.6 ns
    parser.add_argument(
        "--total-ctrl-latency",
        default="11ns",
        type=str,
        help="The latency experienced by every packet\
                        regardless of size of packet",
    )

    # Number of parallel lanes in each serial link [1]
    parser.add_argument(
        "--num-lanes-per-link",
        default=16,
        action="store",
        type=int,
        help="Number of lanes per each link",
    )

    # Number of serial links [1]
    parser.add_argument(
        "--num-serial-links",
        default=4,
        action="store",
        type=int,
        help="Number of serial links",
    )

    # speed of each lane of serial link - SerDes serial interface 10 Gb/s
    parser.add_argument(
        "--serial-link-speed",
        default=10,
        action="store",
        type=int,
        help="Gbs/s speed of each lane of serial\
                        link",
    )

    # address range for each of the serial links
    parser.add_argument(
        "--serial-link-addr-range",
        default="1GB",
        type=str,
        help="memory range for each of the serial links.\
                        Default: 1GB",
    )

    # *****************************PERFORMANCE MONITORING*********************
    # The main monitor behind the HMC Controller
    parser.add_argument(
        "--enable-global-monitor",
        action="store_true",
        help="The main monitor behind the HMC Controller",
    )

    # The link performance monitors
    parser.add_argument(
        "--enable-link-monitor", action="store_true", help="The link monitors"
    )

    # link aggregator enable - put a cross between buffers & links
    parser.add_argument(
        "--enable-link-aggr",
        action="store_true",
        help="The\
                        crossbar between port and Link Controller",
    )

    parser.add_argument(
        "--enable-buff-div",
        action="store_true",
        help="Memory Range of Buffer is ivided between total\
                        range",
    )

    # *****************************HMC ARCHITECTURE **************************
    # Memory chunk for 16 vault - numbers of vault / number of crossbars
    parser.add_argument(
        "--mem-chunk",
        default=4,
        action="store",
        type=int,
        help="Chunk of memory range for each cross bar in\
                        arch 0",
    )

    # size of req buffer within crossbar, used for modelling extra latency
    # when the reuqest go to non-local vault
    parser.add_argument(
        "--xbar-buffer-size-req",
        default=10,
        action="store",
        type=int,
        help="Number of packets to buffer at the\
                        request side of the crossbar",
    )

    # size of response buffer within crossbar, used for modelling extra latency
    # when the response received from non-local vault
    parser.add_argument(
        "--xbar-buffer-size-resp",
        default=10,
        action="store",
        type=int,
        help="Number of packets to buffer at the\
                        response side of the crossbar",
    )
    # HMC device architecture. It affects the HMC host controller as well
    parser.add_argument(
        "--arch",
        type=str,
        choices=["same", "distributed", "mixed"],
        default="distributed",
        help="same: HMC with\
                        4 links, all with same range.\ndistributed: HMC with\
                        4 links with distributed range.\nmixed: mixed with\
                        same and distributed range.\nDefault: distributed",
    )
    # HMC device - number of vaults
    parser.add_argument(
        "--hmc-dev-num-vaults",
        default=16,
        action="store",
        type=int,
        help="number of independent vaults within\
                        the HMC device. Note: each vault has a memory\
                        controller (valut controller)\nDefault: 16",
    )
    # HMC device - vault capacity or size
    parser.add_argument(
        "--hmc-dev-vault-size",
        default="256MB",
        type=str,
        help="vault storage capacity in bytes. Default:\
                        256MB",
    )
    parser.add_argument(
        "--mem-type",
        type=str,
        choices=["HMC_2500_1x32"],
        default="HMC_2500_1x32",
        help="type of HMC memory to\
                        use. Default: HMC_2500_1x32",
    )
    parser.add_argument(
        "--mem-channels",
        default=1,
        action="store",
        type=int,
        help="Number of memory channels",
    )
    parser.add_argument(
        "--mem-ranks",
        default=1,
        action="store",
        type=int,
        help="Number of ranks to iterate across",
    )
    parser.add_argument(
        "--burst-length",
        default=256,
        action="store",
        type=int,
        help="burst length in bytes. Note: the\
                        cache line size will be set to this value.\nDefault:\
                        256",
    )


# configure HMC host controller
def config_hmc_host_ctrl(opt, system):

    # create HMC host controller
    system.hmc_host = SubSystem()

    # Create additional crossbar for arch1
    if opt.arch == "distributed" or opt.arch == "mixed":
        clk = "100GHz"
        vd = VoltageDomain(voltage="1V")
        # Create additional crossbar for arch1
        system.membus = NoncoherentXBar(width=8)
        system.membus.badaddr_responder = BadAddr()
        system.membus.default = Self.badaddr_responder.pio
        system.membus.width = 8
        system.membus.frontend_latency = 3
        system.membus.forward_latency = 4
        system.membus.response_latency = 2
        cd = SrcClockDomain(clock=clk, voltage_domain=vd)
        system.membus.clk_domain = cd

    # create memory ranges for the serial links
    slar = convert.toMemorySize(opt.serial_link_addr_range)
    # Memmory ranges of serial link for arch-0. Same as the ranges of vault
    # controllers (4 vaults to 1 serial link)
    if opt.arch == "same":
        ser_ranges = [
            AddrRange(0, (4 * slar) - 1) for i in range(opt.num_serial_links)
        ]
    # Memmory ranges of serial link for arch-1. Distributed range accross
    # links
    if opt.arch == "distributed":
        ser_ranges = [
            AddrRange(i * slar, ((i + 1) * slar) - 1)
            for i in range(opt.num_serial_links)
        ]
    # Memmory ranges of serial link for arch-2 'Mixed' address distribution
    # over links
    if opt.arch == "mixed":
        ser_range0 = AddrRange(0, (1 * slar) - 1)
        ser_range1 = AddrRange(1 * slar, 2 * slar - 1)
        ser_range2 = AddrRange(0, (4 * slar) - 1)
        ser_range3 = AddrRange(0, (4 * slar) - 1)
        ser_ranges = [ser_range0, ser_range1, ser_range2, ser_range3]

    # Serial link Controller with 16 SerDes links at 10 Gbps with serial link
    # ranges w.r.t to architecture
    sl = [
        SerialLink(
            ranges=ser_ranges[i],
            req_size=opt.link_buffer_size_req,
            resp_size=opt.link_buffer_size_rsp,
            num_lanes=opt.num_lanes_per_link,
            link_speed=opt.serial_link_speed,
            delay=opt.total_ctrl_latency,
        )
        for i in range(opt.num_serial_links)
    ]
    system.hmc_host.seriallink = sl

    # enable global monitor
    if opt.enable_global_monitor:
        system.hmc_host.lmonitor = [
            CommMonitor() for i in range(opt.num_serial_links)
        ]

    # set the clock frequency for serial link
    for i in range(opt.num_serial_links):
        clk = opt.link_controller_frequency
        vd = VoltageDomain(voltage="1V")
        scd = SrcClockDomain(clock=clk, voltage_domain=vd)
        system.hmc_host.seriallink[i].clk_domain = scd

    # Connect membus/traffic gen to Serial Link Controller for differrent HMC
    # architectures
    hh = system.hmc_host
    if opt.arch == "distributed":
        mb = system.membus
        for i in range(opt.num_links_controllers):
            if opt.enable_global_monitor:
                mb.mem_side_ports = hh.lmonitor[i].cpu_side_port
                hh.lmonitor[i].mem_side_port = hh.seriallink[i].cpu_side_port
            else:
                mb.mem_side_ports = hh.seriallink[i].cpu_side_port
    if opt.arch == "mixed":
        mb = system.membus
        if opt.enable_global_monitor:
            mb.mem_side_ports = hh.lmonitor[0].cpu_side_port
            hh.lmonitor[0].mem_side_port = hh.seriallink[0].cpu_side_port
            mb.mem_side_ports = hh.lmonitor[1].cpu_side_port
            hh.lmonitor[1].mem_side_port = hh.seriallink[1].cpu_side_port
        else:
            mb.mem_side_ports = hh.seriallink[0].cpu_side_port
            mb.mem_side_ports = hh.seriallink[1].cpu_side_port

    if opt.arch == "same":
        for i in range(opt.num_links_controllers):
            if opt.enable_global_monitor:
                hh.lmonitor[i].mem_side_port = hh.seriallink[i].cpu_side_port

    return system


# Create an HMC device
def config_hmc_dev(opt, system, hmc_host):

    # create HMC device
    system.hmc_dev = SubSystem()

    # create memory ranges for the vault controllers
    arv = convert.toMemorySize(opt.hmc_dev_vault_size)
    addr_ranges_vaults = [
        AddrRange(i * arv, ((i + 1) * arv - 1))
        for i in range(opt.hmc_dev_num_vaults)
    ]
    system.mem_ranges = addr_ranges_vaults

    if opt.enable_link_monitor:
        lm = [CommMonitor() for i in range(opt.num_links_controllers)]
        system.hmc_dev.lmonitor = lm

    # 4 HMC Crossbars located in its logic-base (LoB)
    xb = [
        NoncoherentXBar(
            width=opt.xbar_width,
            frontend_latency=opt.xbar_frontend_latency,
            forward_latency=opt.xbar_forward_latency,
            response_latency=opt.xbar_response_latency,
        )
        for i in range(opt.number_mem_crossbar)
    ]
    system.hmc_dev.xbar = xb

    for i in range(opt.number_mem_crossbar):
        clk = opt.xbar_frequency
        vd = VoltageDomain(voltage="1V")
        scd = SrcClockDomain(clock=clk, voltage_domain=vd)
        system.hmc_dev.xbar[i].clk_domain = scd

    # Attach 4 serial link to 4 crossbar/s
    for i in range(opt.num_serial_links):
        if opt.enable_link_monitor:
            system.hmc_host.seriallink[
                i
            ].mem_side_port = system.hmc_dev.lmonitor[i].cpu_side_port
            system.hmc_dev.lmonitor[i].mem_side_port = system.hmc_dev.xbar[
                i
            ].cpu_side_ports
        else:
            system.hmc_host.seriallink[i].mem_side_port = system.hmc_dev.xbar[
                i
            ].cpu_side_ports

    # Connecting xbar with each other for request arriving at the wrong xbar,
    # then it will be forward to correct xbar. Bridge is used to connect xbars
    if opt.arch == "same":
        numx = len(system.hmc_dev.xbar)

        # create a list of buffers
        system.hmc_dev.buffers = [
            Bridge(
                req_size=opt.xbar_buffer_size_req,
                resp_size=opt.xbar_buffer_size_resp,
            )
            for i in range(numx * (opt.mem_chunk - 1))
        ]

        # Buffer iterator
        it = iter(list(range(len(system.hmc_dev.buffers))))

        # necesarry to add system_port to one of the xbar
        system.system_port = system.hmc_dev.xbar[3].cpu_side_ports

        # iterate over all the crossbars and connect them as required
        for i in range(numx):
            for j in range(numx):
                # connect xbar to all other xbars except itself
                if i != j:
                    # get the next index of buffer
                    index = next(it)

                    # Change the default values for ranges of bridge
                    system.hmc_dev.buffers[index].ranges = system.mem_ranges[
                        j * int(opt.mem_chunk) : (j + 1) * int(opt.mem_chunk)
                    ]

                    # Connect the bridge between corssbars
                    system.hmc_dev.xbar[
                        i
                    ].mem_side_ports = system.hmc_dev.buffers[
                        index
                    ].cpu_side_port
                    system.hmc_dev.buffers[
                        index
                    ].mem_side_port = system.hmc_dev.xbar[j].cpu_side_ports
                else:
                    # Don't connect the xbar to itself
                    pass

    # Two crossbars are connected to all other crossbars-Other 2 vault
    # can only direct traffic to it local vaults
    if opt.arch == "mixed":
        system.hmc_dev.buffer30 = Bridge(ranges=system.mem_ranges[0:4])
        system.hmc_dev.xbar[
            3
        ].mem_side_ports = system.hmc_dev.buffer30.cpu_side_port
        system.hmc_dev.buffer30.mem_side_port = system.hmc_dev.xbar[
            0
        ].cpu_side_ports

        system.hmc_dev.buffer31 = Bridge(ranges=system.mem_ranges[4:8])
        system.hmc_dev.xbar[
            3
        ].mem_side_ports = system.hmc_dev.buffer31.cpu_side_port
        system.hmc_dev.buffer31.mem_side_port = system.hmc_dev.xbar[
            1
        ].cpu_side_ports

        system.hmc_dev.buffer32 = Bridge(ranges=system.mem_ranges[8:12])
        system.hmc_dev.xbar[
            3
        ].mem_side_ports = system.hmc_dev.buffer32.cpu_side_port
        system.hmc_dev.buffer32.mem_side_port = system.hmc_dev.xbar[
            2
        ].cpu_side_ports

        system.hmc_dev.buffer20 = Bridge(ranges=system.mem_ranges[0:4])
        system.hmc_dev.xbar[
            2
        ].mem_side_ports = system.hmc_dev.buffer20.cpu_side_port
        system.hmc_dev.buffer20.mem_side_port = system.hmc_dev.xbar[
            0
        ].cpu_side_ports

        system.hmc_dev.buffer21 = Bridge(ranges=system.mem_ranges[4:8])
        system.hmc_dev.xbar[
            2
        ].mem_side_ports = system.hmc_dev.buffer21.cpu_side_port
        system.hmc_dev.buffer21.mem_side_port = system.hmc_dev.xbar[
            1
        ].cpu_side_ports

        system.hmc_dev.buffer23 = Bridge(ranges=system.mem_ranges[12:16])
        system.hmc_dev.xbar[
            2
        ].mem_side_ports = system.hmc_dev.buffer23.cpu_side_port
        system.hmc_dev.buffer23.mem_side_port = system.hmc_dev.xbar[
            3
        ].cpu_side_ports