src/mem/qos/mem_ctrl.hh

/*
 * Copyright (c) 2019 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "debug/QOS.hh"
#include "mem/abstract_mem.hh"
#include "mem/qos/q_policy.hh"
#include "mem/qos/policy.hh"
#include "params/QoSMemCtrl.hh"
#include "sim/system.hh"

#include <unordered_map>
#include <vector>
#include <deque>

#ifndef __MEM_QOS_MEM_CTRL_HH__
#define __MEM_QOS_MEM_CTRL_HH__

namespace QoS {

/**
 * The QoS::MemCtrl is a base class for Memory objects
 * which support QoS - it provides access to a set of QoS
 * scheduling policies
 */
class MemCtrl: public AbstractMemory
{
  public:
    /** Bus Direction */
    enum BusState { READ, WRITE };

  protected:
    /** QoS Policy, assigns QoS priority to the incoming packets */
    const std::unique_ptr<Policy> policy;

    /** QoS Bus Turnaround Policy: selects the bus direction (READ/WRITE) */
    const std::unique_ptr<TurnaroundPolicy> turnPolicy;

    /** QoS Queue Policy: selects packet among same-priority queue */
    const std::unique_ptr<QueuePolicy> queuePolicy;

    /** Number of configured QoS priorities */
    const uint8_t _numPriorities;

    /** Enables QoS priority escalation */
    const bool qosPriorityEscalation;

    /**
     * Enables QoS synchronized scheduling invokes the QoS scheduler
     * on all masters, at every packet arrival.
     */
    const bool qosSyncroScheduler;

    /** Hash of master ID - master name */
    std::unordered_map<MasterID, const std::string> masters;

    /** Hash of masters - number of packets queued per priority */
    std::unordered_map<MasterID, std::vector<uint64_t> > packetPriorities;

    /** Hash of masters - address of request - queue of times of request */
    std::unordered_map<MasterID,
            std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes;

    /**
     * Vector of QoS priorities/last service time. Refreshed at every
     * qosSchedule call.
     */
    std::vector<Tick> serviceTick;

    /** Read request packets queue length in #packets, per QoS priority */
    std::vector<uint64_t> readQueueSizes;

    /** Write request packets queue length in #packets, per QoS priority */
    std::vector<uint64_t> writeQueueSizes;

    /** Total read request packets queue length in #packets */
    uint64_t totalReadQueueSize;

    /** Total write request packets queue length in #packets */
    uint64_t totalWriteQueueSize;

    /**
     * Bus state used to control the read/write switching and drive
     * the scheduling of the next request.
     */
    BusState busState;

    /** bus state for next request event triggered */
    BusState busStateNext;

    struct MemCtrlStats : public Stats::Group
    {
        MemCtrlStats(MemCtrl &mc);

        void regStats() override;

        const MemCtrl &memCtrl;

        /** per-master average QoS priority */
        Stats::VectorStandardDeviation avgPriority;
        /**
         * per-master average QoS distance between assigned and
         * queued values
         */
        Stats::VectorStandardDeviation avgPriorityDistance;

        /** per-priority minimum latency */
        Stats::Vector priorityMinLatency;
        /** per-priority maximum latency */
        Stats::Vector priorityMaxLatency;
        /** Count the number of turnarounds READ to WRITE */
        Stats::Scalar numReadWriteTurnArounds;
        /** Count the number of turnarounds WRITE to READ */
        Stats::Scalar numWriteReadTurnArounds;
        /** Count the number of times bus staying in READ state */
        Stats::Scalar numStayReadState;
        /** Count the number of times bus staying in WRITE state */
        Stats::Scalar numStayWriteState;
    } stats;

    /**
     * Initializes dynamically counters and
     * statistics for a given Master
     *
     * @param m_id the master ID
     */
    void addMaster(const MasterID m_id);

    /**
     * Called upon receiving a request or
     * updates statistics and updates queues status
     *
     * @param dir request direction
     * @param m_id master id
     * @param qos packet qos value
     * @param addr packet address
     * @param entries number of entries to record
     */
    void logRequest(BusState dir, MasterID m_id, uint8_t qos,
                    Addr addr, uint64_t entries);

    /**
     * Called upon receiving a response,
     * updates statistics and updates queues status
     *
     * @param dir response direction
     * @param m_id master id
     * @param qos packet qos value
     * @param addr packet address
     * @param entries number of entries to record
     * @param delay response delay
     */
    void logResponse(BusState dir, MasterID m_id, uint8_t qos,
                     Addr addr, uint64_t entries, double delay);

    /**
     * Assign priority to a packet by executing
     * the configured QoS policy.
     *
     * @param queues_ptr list of pointers to packet queues
     * @param queue_entry_size size in bytes per each packet in the queue
     * @param pkt pointer to the Packet
     * @return a QoS priority value
     */
    template<typename Queues>
    uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr,
                        uint64_t queue_entry_size, const PacketPtr pkt);

    using SimObject::schedule;
    uint8_t schedule(MasterID m_id, uint64_t data);
    uint8_t schedule(const PacketPtr pkt);

    /**
     * Returns next bus direction (READ or WRITE)
     * based on configured policy.
     */
    BusState selectNextBusState();

    /**
     * Set current bus direction (READ or WRITE)
     * from next selected one
     */
    void setCurrentBusState() { busState = busStateNext; }

    /**
     * Record statistics on turnarounds based on
     * busStateNext and busState values
     */
    void recordTurnaroundStats();

    /**
     * Escalates/demotes priority of all packets
     * belonging to the passed master to given
     * priority value
     *
     * @param queues list of pointers to packet queues
     * @param queue_entry_size size of an entry in the queue
     * @param m_id master whose packets priority will change
     * @param tgt_prio target priority value
     */
    template<typename Queues>
    void escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  MasterID m_id, uint8_t tgt_prio);

    /**
     * Escalates/demotes priority of all packets
     * belonging to the passed master to given
     * priority value in a specified cluster of queues
     * (e.g. read queues or write queues) which is passed
     * as an argument to the function.
     * The curr_prio/tgt_prio parameters are queue selectors in the
     * queue cluster.
     *
     * @param queues reference to packet queues
     * @param queue_entry_size size of an entry in the queue
     * @param m_id master whose packets priority will change
     * @param curr_prio source queue priority value
     * @param tgt_prio target queue priority value
     */
    template<typename Queues>
    void escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio);

  public:
    /**
     * QoS Memory base class
     *
     * @param p pointer to QoSMemCtrl parameters
     */
    MemCtrl(const QoSMemCtrlParams*);

    virtual ~MemCtrl();

    /**
     * Initializes this object
     */
    void init() override;

    /**
     * Gets the current bus state
     *
     * @return current bus state
     */
    BusState getBusState() const { return busState; }

    /**
     * Gets the next bus state
     *
     * @return next bus state
     */
    BusState getBusStateNext() const { return busStateNext; }

    /**
     * hasMaster returns true if the selected master(ID) has
     * been registered in the memory controller, which happens if
     * the memory controller has received at least a packet from
     * that master.
     *
     * @param m_id master id to lookup
     * @return true if the memory controller has received a packet
     *         from the master, false otherwise.
     */
    bool hasMaster(MasterID m_id) const
    {
        return masters.find(m_id) != masters.end();
    }

    /**
     * Gets a READ queue size
     *
     * @param prio QoS Priority of the queue
     * @return queue size in packets
     */
    uint64_t getReadQueueSize(const uint8_t prio) const
    { return readQueueSizes[prio]; }

    /**
     * Gets a WRITE queue size
     *
     * @param prio QoS Priority of the queue
     * @return queue size in packets
     */
    uint64_t getWriteQueueSize(const uint8_t prio) const
    { return writeQueueSizes[prio]; }

    /**
     * Gets the total combined READ queues size
     *
     * @return total queues size in packets
     */
    uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; }

    /**
     * Gets the total combined WRITE queues size
     *
     * @return total queues size in packets
     */
    uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; }

    /**
     * Gets the last service tick related to a QoS Priority
     *
     * @param prio QoS Priority
     * @return tick
     */
    Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; }

    /**
     * Gets the total number of priority levels in the
     * QoS memory controller.
     *
     * @return total number of priority levels
     */
    uint8_t numPriorities() const { return _numPriorities; }
};

template<typename Queues>
void
MemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio)
{
    auto it = queues[curr_prio].begin();
    while (it != queues[curr_prio].end()) {
        // No packets left to move
        if (packetPriorities[m_id][curr_prio] == 0)
            break;

        auto pkt = *it;

        DPRINTF(QOS,
                "QoSMemCtrl::escalate checking priority %d packet "
                "m_id %d address %d\n", curr_prio,
                pkt->masterId(), pkt->getAddr());

        // Found a packet to move
        if (pkt->masterId() == m_id) {

            uint64_t moved_entries = divCeil(pkt->getSize(),
                                             queue_entry_size);

            DPRINTF(QOS,
                    "QoSMemCtrl::escalate Master %s [id %d] moving "
                    "packet addr %d size %d (p size %d) from priority %d "
                    "to priority %d - "
                    "this master packets %d (entries to move %d)\n",
                    masters[m_id], m_id, pkt->getAddr(),
                    pkt->getSize(),
                    queue_entry_size, curr_prio, tgt_prio,
                    packetPriorities[m_id][curr_prio], moved_entries);


            if (pkt->isRead()) {
                panic_if(readQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate master %s negative READ "
                         "packets for priority %d",
                        masters[m_id], tgt_prio);
                readQueueSizes[curr_prio] -= moved_entries;
                readQueueSizes[tgt_prio] += moved_entries;
            } else if (pkt->isWrite()) {
                panic_if(writeQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate master %s negative WRITE "
                         "packets for priority %d",
                        masters[m_id], tgt_prio);
                writeQueueSizes[curr_prio] -= moved_entries;
                writeQueueSizes[tgt_prio] += moved_entries;
            }

            // Change QoS priority and move packet
            pkt->qosValue(tgt_prio);
            queues[tgt_prio].push_back(pkt);

            // Erase element from source packet queue, this will
            // increment the iterator
            it = queues[curr_prio].erase(it);
            panic_if(packetPriorities[m_id][curr_prio] < moved_entries,
                     "QoSMemCtrl::escalate master %s negative packets "
                     "for priority %d",
                     masters[m_id], tgt_prio);

            packetPriorities[m_id][curr_prio] -= moved_entries;
            packetPriorities[m_id][tgt_prio] += moved_entries;
        } else {
            // Increment iterator to next location in the queue
            it++;
        }
    }
}

template<typename Queues>
void
MemCtrl::escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  MasterID m_id, uint8_t tgt_prio)
{
    // If needed, initialize all counters and statistics
    // for this master
    addMaster(m_id);

    DPRINTF(QOS,
            "QoSMemCtrl::escalate Master %s [id %d] to priority "
            "%d (currently %d packets)\n",masters[m_id], m_id, tgt_prio,
            packetPriorities[m_id][tgt_prio]);

    for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) {
        // Skip target priority
        if (curr_prio == tgt_prio)
            continue;

        // Process other priority packet
        while (packetPriorities[m_id][curr_prio] > 0) {
            DPRINTF(QOS,
                    "QoSMemCtrl::escalate MID %d checking priority %d "
                    "(packets %d)- current packets in prio %d:  %d\n"
                    "\t(source read %d source write %d target read %d, "
                    "target write %d)\n",
                    m_id, curr_prio, packetPriorities[m_id][curr_prio],
                    tgt_prio, packetPriorities[m_id][tgt_prio],
                    readQueueSizes[curr_prio],
                    writeQueueSizes[curr_prio], readQueueSizes[tgt_prio],
                    writeQueueSizes[tgt_prio]);

            // Check both read and write queue
            for (auto q : queues) {
                escalateQueues(*q, queue_entry_size, m_id,
                               curr_prio, tgt_prio);
            }
        }
    }

    DPRINTF(QOS,
            "QoSMemCtrl::escalate Completed master %s [id %d] to priority %d "
            "(now %d packets)\n\t(total read %d, total write %d)\n",
            masters[m_id], m_id, tgt_prio, packetPriorities[m_id][tgt_prio],
            readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]);
}

template<typename Queues>
uint8_t
MemCtrl::qosSchedule(std::initializer_list<Queues*> queues,
                     const uint64_t queue_entry_size,
                     const PacketPtr pkt)
{
    // Schedule packet.
    uint8_t pkt_priority = schedule(pkt);

    assert(pkt_priority < numPriorities());

    pkt->qosValue(pkt_priority);

    if (qosSyncroScheduler) {
        // Call the scheduling function on all other masters.
        for (const auto& m : masters) {

            if (m.first == pkt->masterId())
                continue;

            uint8_t prio = schedule(m.first, 0);

            if (qosPriorityEscalation) {
                DPRINTF(QOS,
                        "QoSMemCtrl::qosSchedule: (syncro) escalating "
                        "MASTER %s to assigned priority %d\n",
                        _system->getMasterName(m.first),
                        prio);
                escalate(queues, queue_entry_size, m.first, prio);
            }
        }
    }

    if (qosPriorityEscalation) {
        DPRINTF(QOS,
                "QoSMemCtrl::qosSchedule: escalating "
                "MASTER %s to assigned priority %d\n",
                _system->getMasterName(pkt->masterId()),
                pkt_priority);
        escalate(queues, queue_entry_size, pkt->masterId(), pkt_priority);
    }

    // Update last service tick for selected priority
    serviceTick[pkt_priority] = curTick();

    return pkt_priority;
}

} // namespace QoS

#endif /* __MEM_QOS_MEM_CTRL_HH__ */