src/arch/amdgpu/vega/tlb.cc

/*
 * Copyright (c) 2021 Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "arch/amdgpu/vega/tlb.hh"

#include <cmath>
#include <cstring>

#include "arch/amdgpu/common/gpu_translation_state.hh"
#include "arch/amdgpu/vega/faults.hh"
#include "arch/amdgpu/vega/pagetable_walker.hh"
#include "debug/GPUPrefetch.hh"
#include "debug/GPUTLB.hh"
#include "dev/amdgpu/amdgpu_device.hh"

namespace gem5
{
namespace VegaISA
{

// we have no limit for the number of translations we send
// downstream as we depend on the limit of the coalescer
// above us
GpuTLB::GpuTLB(const VegaGPUTLBParams &p)
    :  ClockedObject(p), walker(p.walker),
      gpuDevice(p.gpu_device), size(p.size), stats(this),
      cleanupEvent([this]{ cleanup(); }, name(), false,
                   Event::Maximum_Pri)
{
    assoc = p.assoc;
    assert(assoc <= size);
    numSets = size/assoc;
    allocationPolicy = p.allocationPolicy;
    hasMemSidePort = false;

    tlb.assign(size, VegaTlbEntry());

    freeList.resize(numSets);
    entryList.resize(numSets);

    for (int set = 0; set < numSets; ++set) {
        for (int way = 0; way < assoc; ++way) {
            int x = set * assoc + way;
            freeList[set].push_back(&tlb.at(x));
        }
    }

    FA = (size == assoc);
    setMask = numSets - 1;

    maxCoalescedReqs = p.maxOutstandingReqs;


    outstandingReqs = 0;
    hitLatency = p.hitLatency;
    missLatency1 = p.missLatency1;
    missLatency2 = p.missLatency2;

    // create the response ports based on the number of connected ports
    for (size_t i = 0; i < p.port_cpu_side_ports_connection_count; ++i) {
        cpuSidePort.push_back(new CpuSidePort(csprintf("%s-port%d",
                              name(), i), this, i));
    }

    // create the requestor ports based on the number of connected ports
    for (size_t i = 0; i < p.port_mem_side_ports_connection_count; ++i) {
        memSidePort.push_back(new MemSidePort(csprintf("%s-port%d",
                              name(), i), this, i));
    }

    // assuming one walker per TLB, set our walker's TLB to this TLB.
    walker->setTLB(this);

    // gpuDevice should be non-null in full system only and is set by GpuTLB
    // params from the config file.
    if (gpuDevice) {
        gpuDevice->getVM().registerTLB(this);
    }
}

GpuTLB::~GpuTLB()
{
}

Port &
GpuTLB::getPort(const std::string &if_name, PortID idx)
{
    if (if_name == "cpu_side_ports") {
        if (idx >= static_cast<PortID>(cpuSidePort.size())) {
            panic("TLBCoalescer::getPort: unknown index %d\n", idx);
        }

        return *cpuSidePort[idx];
    } else if (if_name == "mem_side_ports") {
        if (idx >= static_cast<PortID>(memSidePort.size())) {
            panic("TLBCoalescer::getPort: unknown index %d\n", idx);
        }

        hasMemSidePort = true;

        return *memSidePort[idx];
    } else {
        panic("TLBCoalescer::getPort: unknown port %s\n", if_name);
    }
}

Fault
GpuTLB::createPagefault(Addr vaddr, Mode mode)
{
    DPRINTF(GPUTLB, "GPUTLB: Raising page fault.\n");
    ExceptionCode code;
    if (mode == BaseMMU::Read)
        code = ExceptionCode::LOAD_PAGE;
    else if (mode == BaseMMU::Write)
        code = ExceptionCode::STORE_PAGE;
    else
        code = ExceptionCode::INST_PAGE;
    return std::make_shared<PageFault>(vaddr, code, true, mode, true);
}

Addr
GpuTLB::pageAlign(Addr vaddr)
{
    Addr pageMask = mask(VegaISA::PageShift);
    return (vaddr & ~pageMask);
}

VegaTlbEntry*
GpuTLB::insert(Addr vpn, VegaTlbEntry &entry)
{
    VegaTlbEntry *newEntry = nullptr;

    int set = (entry.vaddr >> VegaISA::PageShift) & setMask;

    if (!freeList[set].empty()) {
        newEntry = freeList[set].front();
        freeList[set].pop_front();
    } else {
        newEntry = entryList[set].back();
        entryList[set].pop_back();
    }

    *newEntry = entry;
    entryList[set].push_front(newEntry);

    DPRINTF(GPUTLB, "Inserted %#lx -> %#lx of size %#lx into set %d\n",
            newEntry->vaddr, newEntry->paddr, entry.size(), set);

    return newEntry;
}

GpuTLB::EntryList::iterator
GpuTLB::lookupIt(Addr va, bool update_lru)
{
    int set = (va >> VegaISA::PageShift) & setMask;

    if (FA) {
        assert(!set);
    }

    auto entry = entryList[set].begin();
    for (; entry != entryList[set].end(); ++entry) {
        int page_size = (*entry)->size();

        if ((*entry)->vaddr <= va && (*entry)->vaddr + page_size > va) {
            DPRINTF(GPUTLB, "Matched vaddr %#x to entry starting at %#x "
                    "with size %#x.\n", va, (*entry)->vaddr, page_size);

            if (update_lru) {
                entryList[set].push_front(*entry);
                entryList[set].erase(entry);
                entry = entryList[set].begin();
            }

            break;
        }
    }

    return entry;
}

VegaTlbEntry*
GpuTLB::lookup(Addr va, bool update_lru)
{
    int set = (va >> VegaISA::PageShift) & setMask;

    auto entry = lookupIt(va, update_lru);

    if (entry == entryList[set].end())
        return nullptr;
    else
        return *entry;
}

void
GpuTLB::invalidateAll()
{
    DPRINTF(GPUTLB, "Invalidating all entries.\n");

    for (int i = 0; i < numSets; ++i) {
        while (!entryList[i].empty()) {
            VegaTlbEntry *entry = entryList[i].front();
            entryList[i].pop_front();
            freeList[i].push_back(entry);
        }
    }
}

void
GpuTLB::demapPage(Addr va, uint64_t asn)
{

    int set = (va >> VegaISA::PageShift) & setMask;
    auto entry = lookupIt(va, false);

    if (entry != entryList[set].end()) {
        freeList[set].push_back(*entry);
        entryList[set].erase(entry);
    }
}



/**
 * TLB_lookup will only perform a TLB lookup returning the TLB entry on a TLB
 * hit and nullptr on a TLB miss.
 * Many of the checks about different modes have been converted to
 * assertions, since these parts of the code are not really used.
 * On a hit it will update the LRU stack.
 */
VegaTlbEntry *
GpuTLB::tlbLookup(const RequestPtr &req, bool update_stats)
{
    Addr vaddr = req->getVaddr();
    Addr alignedVaddr = pageAlign(vaddr);
    DPRINTF(GPUTLB, "TLB Lookup for vaddr %#x.\n", vaddr);

    //update LRU stack on a hit
    VegaTlbEntry *entry = lookup(alignedVaddr, true);

    if (!update_stats) {
        // functional tlb access for memory initialization
        // i.e., memory seeding or instr. seeding -> don't update
        // TLB and stats
        return entry;
    }

    stats.localNumTLBAccesses++;

    if (!entry) {
        stats.localNumTLBMisses++;
    } else {
        stats.localNumTLBHits++;
    }

    return entry;
}

Walker*
GpuTLB::getWalker()
{
    return walker;
}


void
GpuTLB::serialize(CheckpointOut &cp) const
{
}

void
GpuTLB::unserialize(CheckpointIn &cp)
{
}

/**
 * Do the TLB lookup for this coalesced request and schedule
 * another event <TLB access latency> cycles later.
 */

void
GpuTLB::issueTLBLookup(PacketPtr pkt)
{
    assert(pkt);
    assert(pkt->senderState);

    /**
     * The page size is not fixed in Vega and tracking events by VPN could
     * potentially lead to redundant page walks by using the smallest page
     * size. The actual VPN can be determined after the first walk is done
     * and fixed up later.
     */
    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);

    GpuTranslationState *sender_state =
            safe_cast<GpuTranslationState*>(pkt->senderState);

    bool update_stats = !sender_state->isPrefetch;

    DPRINTF(GPUTLB, "Translation req. for virt. page addr %#x\n",
            virt_page_addr);

    int req_cnt = sender_state->reqCnt.back();

    if (update_stats) {
        stats.accessCycles -= (curCycle() * req_cnt);
        stats.localCycles -= curCycle();
        stats.globalNumTLBAccesses += req_cnt;
    }

    tlbOutcome lookup_outcome = TLB_MISS;
    const RequestPtr &tmp_req = pkt->req;

    // Access the TLB and figure out if it's a hit or a miss.
    auto entry = tlbLookup(tmp_req, update_stats);

    if (entry) {
        lookup_outcome = TLB_HIT;
        // Put the entry in SenderState
        VegaTlbEntry *entry = lookup(virt_page_addr, false);
        assert(entry);

        // Set if this is a system request
        pkt->req->setSystemReq(entry->pte.s);

        Addr alignedPaddr = pageAlign(entry->paddr);
        sender_state->tlbEntry =
            new VegaTlbEntry(1 /* VMID */, virt_page_addr, alignedPaddr,
                            entry->logBytes, entry->pte);

        if (update_stats) {
            // the reqCnt has an entry per level, so its size tells us
            // which level we are in
            sender_state->hitLevel = sender_state->reqCnt.size();
            stats.globalNumTLBHits += req_cnt;
        }
    } else {
        if (update_stats)
            stats.globalNumTLBMisses += req_cnt;
    }

    /*
     * We now know the TLB lookup outcome (if it's a hit or a miss), as
     * well as the TLB access latency.
     *
     * We create and schedule a new TLBEvent which will help us take the
     * appropriate actions (e.g., update TLB on a hit, send request to
     * lower level TLB on a miss, or start a page walk if this was the
     * last-level TLB)
     */
    TLBEvent *tlb_event =
        new TLBEvent(this, virt_page_addr, lookup_outcome, pkt);

    if (translationReturnEvent.count(virt_page_addr)) {
        panic("Virtual Page Address %#x already has a return event\n",
              virt_page_addr);
    }

    translationReturnEvent[virt_page_addr] = tlb_event;
    assert(tlb_event);

    DPRINTF(GPUTLB, "schedule translationReturnEvent @ curTick %d\n",
            curTick() + cyclesToTicks(Cycles(hitLatency)));

    schedule(tlb_event, curTick() + cyclesToTicks(Cycles(hitLatency)));
}

GpuTLB::TLBEvent::TLBEvent(GpuTLB* _tlb, Addr _addr,
    tlbOutcome tlb_outcome, PacketPtr _pkt)
        : Event(CPU_Tick_Pri), tlb(_tlb), virtPageAddr(_addr),
          outcome(tlb_outcome), pkt(_pkt)
{
}

/**
 * Do Paging protection checks. If we encounter a page fault, then
 * an assertion is fired.
 */
void
GpuTLB::pagingProtectionChecks(PacketPtr pkt, VegaTlbEntry * tlb_entry,
        Mode mode)
{
    // Do paging protection checks.
    bool badWrite = (!tlb_entry->writable());

    if (mode == BaseMMU::Write && badWrite) {
        // The page must have been present to get into the TLB in
        // the first place. We'll assume the reserved bits are
        // fine even though we're not checking them.
        fatal("Page fault on addr %lx PTE=%#lx", pkt->req->getVaddr(),
                (uint64_t)tlb_entry->pte);
    }
}

void
GpuTLB::walkerResponse(VegaTlbEntry& entry, PacketPtr pkt)
{
    DPRINTF(GPUTLB, "WalkerResponse for %#lx. Entry: (%#lx, %#lx, %#lx)\n",
            pkt->req->getVaddr(), entry.vaddr, entry.paddr, entry.size());

    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);

    Addr page_addr = entry.pte.ppn << VegaISA::PageShift;
    Addr paddr = page_addr + (entry.vaddr & mask(entry.logBytes));
    pkt->req->setPaddr(paddr);
    pkt->req->setSystemReq(entry.pte.s);

    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);
    sender_state->tlbEntry = new VegaTlbEntry(entry);

    handleTranslationReturn(virt_page_addr, TLB_MISS, pkt);
}

/**
 * handleTranslationReturn is called on a TLB hit,
 * when a TLB miss returns or when a page fault returns.
 * The latter calls handelHit with TLB miss as tlbOutcome.
 */
void
GpuTLB::handleTranslationReturn(Addr virt_page_addr,
    tlbOutcome tlb_outcome, PacketPtr pkt)
{
    assert(pkt);
    Addr vaddr = pkt->req->getVaddr();

    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);

    Mode mode = sender_state->tlbMode;

    VegaTlbEntry *local_entry, *new_entry;

    int req_cnt = sender_state->reqCnt.back();
    bool update_stats = !sender_state->isPrefetch;

    if (update_stats) {
        stats.accessCycles += (req_cnt * curCycle());
        stats.localCycles += curCycle();
    }

    if (tlb_outcome == TLB_HIT) {
        DPRINTF(GPUTLB, "Translation Done - TLB Hit for addr %#x\n",
            vaddr);
        local_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
    } else {
        DPRINTF(GPUTLB, "Translation Done - TLB Miss for addr %#x\n",
                vaddr);

        /**
         * We are returning either from a page walk or from a hit at a
         * lower TLB level. The senderState should be "carrying" a pointer
         * to the correct TLBEntry.
         */
        new_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
        assert(new_entry);
        local_entry = new_entry;

        if (allocationPolicy) {
            assert(new_entry->pte);
            DPRINTF(GPUTLB, "allocating entry w/ addr %#lx of size %#lx\n",
                    virt_page_addr, new_entry->size());

            local_entry = insert(virt_page_addr, *new_entry);
        }

        assert(local_entry);
    }

    /**
     * At this point the packet carries an up-to-date tlbEntry pointer
     * in its senderState.
     * Next step is to do the paging protection checks.
     */
    DPRINTF(GPUTLB, "Entry found with vaddr %#x,  doing protection checks "
            "while paddr was %#x.\n", local_entry->vaddr,
            local_entry->paddr);

    pagingProtectionChecks(pkt, local_entry, mode);
    int page_size = local_entry->size();
    Addr paddr = local_entry->paddr + (vaddr & (page_size - 1));
    DPRINTF(GPUTLB, "Translated %#x -> %#x.\n", vaddr, paddr);

    // Since this packet will be sent through the cpu side port, it must be
    // converted to a response pkt if it is not one already
    if (pkt->isRequest()) {
        pkt->makeTimingResponse();
    }

    pkt->req->setPaddr(paddr);

    if (local_entry->uncacheable()) {
         pkt->req->setFlags(Request::UNCACHEABLE);
    }

    //send packet back to coalescer
    cpuSidePort[0]->sendTimingResp(pkt);
    //schedule cleanup event
    cleanupQueue.push(virt_page_addr);

    DPRINTF(GPUTLB, "Scheduled %#lx for cleanup\n", virt_page_addr);

    // schedule this only once per cycle.
    // The check is required because we might have multiple translations
    // returning the same cycle
    // this is a maximum priority event and must be on the same cycle
    // as the cleanup event in TLBCoalescer to avoid a race with
    // IssueProbeEvent caused by TLBCoalescer::MemSidePort::recvReqRetry
    if (!cleanupEvent.scheduled())
        schedule(cleanupEvent, curTick());
}

/**
 * Here we take the appropriate actions based on the result of the
 * TLB lookup.
 */
void
GpuTLB::translationReturn(Addr virtPageAddr, tlbOutcome outcome,
                          PacketPtr pkt)
{
    DPRINTF(GPUTLB, "Triggered TLBEvent for addr %#x\n", virtPageAddr);

    assert(translationReturnEvent[virtPageAddr]);
    assert(pkt);

    GpuTranslationState *tmp_sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);

    int req_cnt = tmp_sender_state->reqCnt.back();
    bool update_stats = !tmp_sender_state->isPrefetch;


    if (outcome == TLB_HIT) {
        handleTranslationReturn(virtPageAddr, TLB_HIT, pkt);

    } else if (outcome == TLB_MISS) {

        DPRINTF(GPUTLB, "This is a TLB miss\n");
        if (hasMemSidePort) {
            // the one cyle added here represent the delay from when we get
            // the reply back till when we propagate it to the coalescer
            // above.

            /**
             * There is a TLB below. Send the coalesced request.
             * We actually send the very first packet of all the
             * pending packets for this virtual page address.
             */
            tmp_sender_state->deviceId = 1;
            tmp_sender_state->pasId = 0;

            if (!memSidePort[0]->sendTimingReq(pkt)) {
                DPRINTF(GPUTLB, "Failed sending translation request to "
                        "lower level TLB for addr %#x\n", virtPageAddr);

                memSidePort[0]->retries.push_back(pkt);
            } else {
                DPRINTF(GPUTLB, "Sent translation request to lower level "
                        "TLB for addr %#x\n", virtPageAddr);
            }
        } else {
            //this is the last level TLB. Start a page walk
            DPRINTF(GPUTLB, "Last level TLB - start a page walk for "
                    "addr %#x\n", virtPageAddr);

            if (update_stats)
                stats.pageTableCycles -= (req_cnt*curCycle());

            TLBEvent *tlb_event = translationReturnEvent[virtPageAddr];
            assert(tlb_event);
            tlb_event->updateOutcome(PAGE_WALK);
            schedule(tlb_event,
                     curTick() + cyclesToTicks(Cycles(missLatency2)));
        }
    } else if (outcome == PAGE_WALK) {
        if (update_stats)
            stats.pageTableCycles += (req_cnt*curCycle());

        // Need to access the page table and update the TLB
        DPRINTF(GPUTLB, "Doing a page walk for address %#x\n",
                virtPageAddr);

        Addr base = gpuDevice->getVM().getPageTableBase(1);
        Addr vaddr = pkt->req->getVaddr();
        walker->setDevRequestor(gpuDevice->vramRequestorId());

        // Do page table walk
        walker->startTiming(pkt, base, vaddr, BaseMMU::Mode::Read);
    } else if (outcome == MISS_RETURN) {
        /** we add an extra cycle in the return path of the translation
         * requests in between the various TLB levels.
         */
        handleTranslationReturn(virtPageAddr, TLB_MISS, pkt);
    } else {
        panic("Unexpected TLB outcome %d", outcome);
    }
}

void
GpuTLB::TLBEvent::process()
{
    tlb->translationReturn(virtPageAddr, outcome, pkt);
}

const char*
GpuTLB::TLBEvent::description() const
{
    return "trigger translationDoneEvent";
}

void
GpuTLB::TLBEvent::updateOutcome(tlbOutcome _outcome)
{
    outcome = _outcome;
}

Addr
GpuTLB::TLBEvent::getTLBEventVaddr()
{
    return virtPageAddr;
}

/**
 * recvTiming receives a coalesced timing request from a TLBCoalescer
 * and it calls issueTLBLookup()
 * It only rejects the packet if we have exceeded the max
 * outstanding number of requests for the TLB
 */
bool
GpuTLB::CpuSidePort::recvTimingReq(PacketPtr pkt)
{
    bool ret = false;
    [[maybe_unused]] Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                                     VegaISA::PageBytes);

    if (tlb->outstandingReqs < tlb->maxCoalescedReqs) {
        assert(!tlb->translationReturnEvent.count(virt_page_addr));
        tlb->issueTLBLookup(pkt);
        // update number of outstanding translation requests
        tlb->outstandingReqs++;
        ret = true;
    } else {
        DPRINTF(GPUTLB, "Reached maxCoalescedReqs number %d\n",
                tlb->outstandingReqs);
        tlb->stats.maxDownstreamReached++;
        ret = false;

    }

    if (tlb->outstandingReqs > tlb->stats.outstandingReqsMax.value())
        tlb->stats.outstandingReqsMax = tlb->outstandingReqs;

    return ret;
}

/**
 * handleFuncTranslationReturn is called on a TLB hit,
 * when a TLB miss returns or when a page fault returns.
 * It updates LRU, inserts the TLB entry on a miss
 * depending on the allocation policy and does the required
 * protection checks. It does NOT create a new packet to
 * update the packet's addr; this is done in hsail-gpu code.
 */
void
GpuTLB::handleFuncTranslationReturn(PacketPtr pkt, tlbOutcome tlb_outcome)
{
    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);

    Mode mode = sender_state->tlbMode;
    Addr vaddr = pkt->req->getVaddr();

    VegaTlbEntry *local_entry, *new_entry;

    if (tlb_outcome == TLB_HIT) {
        DPRINTF(GPUTLB, "Functional Translation Done - TLB hit for addr "
                "%#x\n", vaddr);

        local_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
    } else {
        DPRINTF(GPUTLB, "Functional Translation Done - TLB miss for addr "
                "%#x\n", vaddr);

        /**
         * We are returning either from a page walk or from a hit at a
         * lower TLB level. The senderState should be "carrying" a pointer
         * to the correct TLBEntry.
         */
        new_entry = safe_cast<VegaTlbEntry *>(sender_state->tlbEntry);
        assert(new_entry);
        local_entry = new_entry;

        if (allocationPolicy) {
            Addr virt_page_addr = roundDown(vaddr, VegaISA::PageBytes);

            DPRINTF(GPUTLB, "allocating entry w/ addr %#lx\n",
                    virt_page_addr);

            local_entry = insert(virt_page_addr, *new_entry);
        }

        assert(local_entry);
    }

    DPRINTF(GPUTLB, "Entry found with vaddr %#x, doing protection checks "
            "while paddr was %#x.\n", local_entry->vaddr,
            local_entry->paddr);

    /**
     * Do paging checks if it's a normal functional access.  If it's for a
     * prefetch, then sometimes you can try to prefetch something that
     * won't pass protection. We don't actually want to fault becuase there
     * is no demand access to deem this a violation.  Just put it in the
     * TLB and it will fault if indeed a future demand access touches it in
     * violation.
     *
     * This feature could be used to explore security issues around
     * speculative memory accesses.
     */
    if (!sender_state->isPrefetch && sender_state->tlbEntry)
        pagingProtectionChecks(pkt, local_entry, mode);

    int page_size = local_entry->size();
    Addr paddr = local_entry->paddr + (vaddr & (page_size - 1));
    DPRINTF(GPUTLB, "Translated %#x -> %#x.\n", vaddr, paddr);

    pkt->req->setPaddr(paddr);

    if (local_entry->uncacheable())
         pkt->req->setFlags(Request::UNCACHEABLE);
}

// This is used for atomic translations. Need to
// make it all happen during the same cycle.
void
GpuTLB::CpuSidePort::recvFunctional(PacketPtr pkt)
{
    GpuTranslationState *sender_state =
        safe_cast<GpuTranslationState*>(pkt->senderState);

    bool update_stats = !sender_state->isPrefetch;

    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);

    // do the TLB lookup without updating the stats
    bool success = tlb->tlbLookup(pkt->req, update_stats);
    tlbOutcome tlb_outcome = success ? TLB_HIT : TLB_MISS;

    // functional mode means no coalescing
    // global metrics are the same as the local metrics
    if (update_stats) {
        tlb->stats.globalNumTLBAccesses++;

        if (success) {
            sender_state->hitLevel = sender_state->reqCnt.size();
            tlb->stats.globalNumTLBHits++;
        } else {
            tlb->stats.globalNumTLBMisses++;
        }
    }

    if (!success) {
        if (tlb->hasMemSidePort) {
            // there is a TLB below -> propagate down the TLB hierarchy
            tlb->memSidePort[0]->sendFunctional(pkt);
            // If no valid translation from a prefetch, then just return
            if (sender_state->isPrefetch && !pkt->req->hasPaddr())
                return;
        } else {
            // Need to access the page table and update the TLB
            DPRINTF(GPUTLB, "Doing a page walk for address %#x\n",
                    virt_page_addr);

            Addr vaddr = pkt->req->getVaddr();
            [[maybe_unused]] Addr alignedVaddr =
                tlb->pageAlign(virt_page_addr);
            assert(alignedVaddr == virt_page_addr);

            unsigned logBytes;
            PageTableEntry pte;

            // Initialize walker state for VMID
            Addr base = tlb->gpuDevice->getVM().getPageTableBase(1);
            tlb->walker->setDevRequestor(tlb->gpuDevice->vramRequestorId());

            // Do page table walk
            Fault fault = tlb->walker->startFunctional(base, vaddr, pte,
                                                       logBytes,
                                                       BaseMMU::Mode::Read);
            if (fault != NoFault) {
                fatal("Translation fault in TLB at %d!", __LINE__);
            }

            // PPN is already shifted by fragment so we only shift by native
            // page size. Fragment is still used via logBytes to select lower
            // bits from vaddr.
            Addr page_addr = pte.ppn << PageShift;
            Addr paddr = page_addr + (vaddr & mask(logBytes));
            Addr alignedPaddr = tlb->pageAlign(paddr);
            pkt->req->setPaddr(paddr);
            pkt->req->setSystemReq(pte.s);

            if (!sender_state->isPrefetch) {
                assert(paddr);

                DPRINTF(GPUTLB, "Mapping %#x to %#x\n", vaddr, paddr);

                sender_state->tlbEntry =
                    new VegaTlbEntry(1 /* VMID */, virt_page_addr,
                                 alignedPaddr, logBytes, pte);
            } else {
                // If this was a prefetch, then do the normal thing if it
                // was a successful translation.  Otherwise, send an empty
                // TLB entry back so that it can be figured out as empty
                // and handled accordingly.
                if (paddr) {
                    DPRINTF(GPUTLB, "Mapping %#x to %#x\n", vaddr, paddr);

                    sender_state->tlbEntry =
                        new VegaTlbEntry(1 /* VMID */, virt_page_addr,
                                     alignedPaddr, logBytes, pte);
                } else {
                    DPRINTF(GPUPrefetch, "Prefetch failed %#x\n", vaddr);

                    sender_state->tlbEntry = nullptr;

                    return;
                }
            }
        }
    } else {
        VegaTlbEntry *entry = tlb->lookup(virt_page_addr, update_stats);
        assert(entry);

        if (sender_state->isPrefetch) {
            DPRINTF(GPUPrefetch, "Functional Hit for vaddr %#x\n",
                    entry->vaddr);
        }

        sender_state->tlbEntry = new VegaTlbEntry(1 /* VMID */, entry->vaddr,
                                                 entry->paddr, entry->logBytes,
                                                 entry->pte);
    }

    // This is the function that would populate pkt->req with the paddr of
    // the translation. But if no translation happens (i.e Prefetch fails)
    // then the early returns in the above code wiill keep this function
    // from executing.
    tlb->handleFuncTranslationReturn(pkt, tlb_outcome);
}

void
GpuTLB::CpuSidePort::recvReqRetry()
{
    // The CPUSidePort never sends anything but replies. No retries
    // expected.
    panic("recvReqRetry called");
}

AddrRangeList
GpuTLB::CpuSidePort::getAddrRanges() const
{
    // currently not checked by the requestor
    AddrRangeList ranges;

    return ranges;
}

/**
 * MemSidePort receives the packet back.
 * We need to call the handleTranslationReturn
 * and propagate up the hierarchy.
 */
bool
GpuTLB::MemSidePort::recvTimingResp(PacketPtr pkt)
{
    Addr virt_page_addr = roundDown(pkt->req->getVaddr(),
                                    VegaISA::PageBytes);

    DPRINTF(GPUTLB, "MemSidePort recvTiming for virt_page_addr %#x\n",
            virt_page_addr);

    TLBEvent *tlb_event = tlb->translationReturnEvent[virt_page_addr];
    assert(tlb_event);
    assert(virt_page_addr == tlb_event->getTLBEventVaddr());

    tlb_event->updateOutcome(MISS_RETURN);
    tlb->schedule(tlb_event, curTick()+tlb->clockPeriod());

    return true;
}

void
GpuTLB::MemSidePort::recvReqRetry()
{
    // No retries should reach the TLB. The retries
    // should only reach the TLBCoalescer.
    panic("recvReqRetry called");
}

void
GpuTLB::cleanup()
{
    while (!cleanupQueue.empty()) {
        Addr cleanup_addr = cleanupQueue.front();
        cleanupQueue.pop();

        DPRINTF(GPUTLB, "Deleting return event for %#lx\n", cleanup_addr);

        // delete TLBEvent
        TLBEvent * old_tlb_event = translationReturnEvent[cleanup_addr];
        delete old_tlb_event;
        translationReturnEvent.erase(cleanup_addr);

        // update number of outstanding requests
        outstandingReqs--;
    }

    /** the higher level coalescer should retry if it has
     * any pending requests.
     */
    for (int i = 0; i < cpuSidePort.size(); ++i) {
        cpuSidePort[i]->sendRetryReq();
    }
}

GpuTLB::VegaTLBStats::VegaTLBStats(statistics::Group *parent)
    : statistics::Group(parent),
      ADD_STAT(maxDownstreamReached, "Number of refused translation requests"),
      ADD_STAT(outstandingReqsMax, "Maximum count in coalesced request queue"),
      ADD_STAT(localNumTLBAccesses, "Number of TLB accesses"),
      ADD_STAT(localNumTLBHits, "Number of TLB hits"),
      ADD_STAT(localNumTLBMisses, "Number of TLB misses"),
      ADD_STAT(localTLBMissRate, "TLB miss rate"),
      ADD_STAT(globalNumTLBAccesses, "Number of TLB accesses"),
      ADD_STAT(globalNumTLBHits, "Number of TLB hits"),
      ADD_STAT(globalNumTLBMisses, "Number of TLB misses"),
      ADD_STAT(globalTLBMissRate, "TLB miss rate"),
      ADD_STAT(accessCycles, "Cycles spent accessing this TLB level"),
      ADD_STAT(pageTableCycles, "Cycles spent accessing the page table"),
      ADD_STAT(localCycles, "Number of cycles spent in queue for all "
               "incoming reqs"),
      ADD_STAT(localLatency, "Avg. latency over incoming coalesced reqs")
{
    localTLBMissRate = 100 * localNumTLBMisses / localNumTLBAccesses;
    globalTLBMissRate = 100 * globalNumTLBMisses / globalNumTLBAccesses;

    localLatency = localCycles / localNumTLBAccesses;
}

} // namespace VegaISA
} // namespace gem5