src/arch/riscv/tlb.cc - public/gem5 - Git at Google

 /*
  * Copyright (c) 2001-2005 The Regents of The University of Michigan
  * Copyright (c) 2007 MIPS Technologies, Inc.
  * Copyright (c) 2020 Barkhausen Institut
  * Copyright (c) 2021 Huawei International
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met: redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer;
  * redistributions in binary form must reproduce the above copyright
  * notice, this list of conditions and the following disclaimer in the
  * documentation and/or other materials provided with the distribution;
  * neither the name of the copyright holders nor the names of its
  * contributors may be used to endorse or promote products derived from
  * this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 #include "arch/riscv/tlb.hh"

 #include <string>
 #include <vector>

 #include "arch/riscv/faults.hh"
 #include "arch/riscv/mmu.hh"
 #include "arch/riscv/pagetable.hh"
 #include "arch/riscv/pagetable_walker.hh"
 #include "arch/riscv/pma_checker.hh"
 #include "arch/riscv/pra_constants.hh"
 #include "arch/riscv/utility.hh"
 #include "base/inifile.hh"
 #include "base/str.hh"
 #include "base/trace.hh"
 #include "cpu/thread_context.hh"
 #include "debug/TLB.hh"
 #include "debug/TLBVerbose.hh"
 #include "mem/page_table.hh"
 #include "params/RiscvTLB.hh"
 #include "sim/full_system.hh"
 #include "sim/process.hh"
 #include "sim/system.hh"

 using namespace RiscvISA;

 ///////////////////////////////////////////////////////////////////////
 //
 //  RISC-V TLB
 //

 static Addr
 buildKey(Addr vpn, uint16_t asid)
 {
     return (static_cast<Addr>(asid) << 48) | vpn;
 }

 TLB::TLB(const Params &p) :
     BaseTLB(p), size(p.size), tlb(size),
     lruSeq(0), stats(this), pma(p.pma_checker)
 {
     for (size_t x = 0; x < size; x++) {
         tlb[x].trieHandle = NULL;
         freeList.push_back(&tlb[x]);
     }

     walker = p.walker;
     walker->setTLB(this);
 }

 Walker *
 TLB::getWalker()
 {
     return walker;
 }

 void
 TLB::evictLRU()
 {
     // Find the entry with the lowest (and hence least recently updated)
     // sequence number.

     size_t lru = 0;
     for (size_t i = 1; i < size; i++) {
         if (tlb[i].lruSeq < tlb[lru].lruSeq)
             lru = i;
     }

     remove(lru);
 }

 TlbEntry *
 TLB::lookup(Addr vpn, uint16_t asid, Mode mode, bool hidden)
 {
     TlbEntry *entry = trie.lookup(buildKey(vpn, asid));

     if (!hidden) {
         if (entry)
             entry->lruSeq = nextSeq();

         if (mode == Write)
             stats.writeAccesses++;
         else
             stats.readAccesses++;

         if (!entry) {
             if (mode == Write)
                 stats.writeMisses++;
             else
                 stats.readMisses++;
         }
         else {
             if (mode == Write)
                 stats.writeHits++;
             else
                 stats.readHits++;
         }

         DPRINTF(TLBVerbose, "lookup(vpn=%#x, asid=%#x): %s ppn %#x\n",
                 vpn, asid, entry ? "hit" : "miss", entry ? entry->paddr : 0);
     }

     return entry;
 }

 TlbEntry *
 TLB::insert(Addr vpn, const TlbEntry &entry)
 {
     DPRINTF(TLB, "insert(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n",
         vpn, entry.asid, entry.paddr, entry.pte, entry.size());

     // If somebody beat us to it, just use that existing entry.
     TlbEntry *newEntry = lookup(vpn, entry.asid, Mode::Read, true);
     if (newEntry) {
         // update PTE flags (maybe we set the dirty/writable flag)
         newEntry->pte = entry.pte;
         assert(newEntry->vaddr == vpn);
         return newEntry;
     }

     if (freeList.empty())
         evictLRU();

     newEntry = freeList.front();
     freeList.pop_front();

     Addr key = buildKey(vpn, entry.asid);
     *newEntry = entry;
     newEntry->lruSeq = nextSeq();
     newEntry->vaddr = vpn;
     newEntry->trieHandle =
     trie.insert(key, TlbEntryTrie::MaxBits - entry.logBytes, newEntry);
     return newEntry;
 }

 void
 TLB::demapPage(Addr vpn, uint64_t asid)
 {
     asid &= 0xFFFF;

     if (vpn == 0 && asid == 0)
         flushAll();
     else {
         DPRINTF(TLB, "flush(vpn=%#x, asid=%#x)\n", vpn, asid);
         if (vpn != 0 && asid != 0) {
             TlbEntry *newEntry = lookup(vpn, asid, Mode::Read, true);
             if (newEntry)
                 remove(newEntry - tlb.data());
         }
         else {
             for (size_t i = 0; i < size; i++) {
                 if (tlb[i].trieHandle) {
                     Addr mask = ~(tlb[i].size() - 1);
                     if ((vpn == 0 || (vpn & mask) == tlb[i].vaddr) &&
                         (asid == 0 || tlb[i].asid == asid))
                         remove(i);
                 }
             }
         }
     }
 }

 void
 TLB::flushAll()
 {
     DPRINTF(TLB, "flushAll()\n");
     for (size_t i = 0; i < size; i++) {
         if (tlb[i].trieHandle)
             remove(i);
     }
 }

 void
 TLB::remove(size_t idx)
 {
     DPRINTF(TLB, "remove(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n",
         tlb[idx].vaddr, tlb[idx].asid, tlb[idx].paddr, tlb[idx].pte,
         tlb[idx].size());

     assert(tlb[idx].trieHandle);
     trie.remove(tlb[idx].trieHandle);
     tlb[idx].trieHandle = NULL;
     freeList.push_back(&tlb[idx]);
 }

 Fault
 TLB::checkPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr,
                       Mode mode, PTESv39 pte)
 {
     Fault fault = NoFault;

     if (mode == TLB::Read && !pte.r) {
         DPRINTF(TLB, "PTE has no read perm, raising PF\n");
         fault = createPagefault(vaddr, mode);
     }
     else if (mode == TLB::Write && !pte.w) {
         DPRINTF(TLB, "PTE has no write perm, raising PF\n");
         fault = createPagefault(vaddr, mode);
     }
     else if (mode == TLB::Execute && !pte.x) {
         DPRINTF(TLB, "PTE has no exec perm, raising PF\n");
         fault = createPagefault(vaddr, mode);
     }

     if (fault == NoFault) {
         // check pte.u
         if (pmode == PrivilegeMode::PRV_U && !pte.u) {
             DPRINTF(TLB, "PTE is not user accessible, raising PF\n");
             fault = createPagefault(vaddr, mode);
         }
         else if (pmode == PrivilegeMode::PRV_S && pte.u && status.sum == 0) {
             DPRINTF(TLB, "PTE is only user accessible, raising PF\n");
             fault = createPagefault(vaddr, mode);
         }
     }

     return fault;
 }

 Fault
 TLB::createPagefault(Addr vaddr, Mode mode)
 {
     ExceptionCode code;
     if (mode == TLB::Read)
         code = ExceptionCode::LOAD_PAGE;
     else if (mode == TLB::Write)
         code = ExceptionCode::STORE_PAGE;
     else
         code = ExceptionCode::INST_PAGE;
     return std::make_shared<AddressFault>(vaddr, code);
 }

 Addr
 TLB::translateWithTLB(Addr vaddr, uint16_t asid, Mode mode)
 {
     TlbEntry *e = lookup(vaddr, asid, mode, false);
     assert(e != nullptr);
     return e->paddr << PageShift | (vaddr & mask(e->logBytes));
 }

 Fault
 TLB::doTranslate(const RequestPtr &req, ThreadContext *tc,
                  Translation *translation, Mode mode, bool &delayed)
 {
     delayed = false;

     Addr vaddr = req->getVaddr() & ((static_cast<Addr>(1) << VADDR_BITS) - 1);
     SATP satp = tc->readMiscReg(MISCREG_SATP);

     TlbEntry *e = lookup(vaddr, satp.asid, mode, false);
     if (!e) {
         Fault fault = walker->start(tc, translation, req, mode);
         if (translation != nullptr || fault != NoFault) {
             // This gets ignored in atomic mode.
             delayed = true;
             return fault;
         }
         e = lookup(vaddr, satp.asid, mode, false);
         assert(e != nullptr);
     }

     STATUS status = tc->readMiscReg(MISCREG_STATUS);
     PrivilegeMode pmode = getMemPriv(tc, mode);
     Fault fault = checkPermissions(status, pmode, vaddr, mode, e->pte);
     if (fault != NoFault) {
         // if we want to write and it isn't writable, do a page table walk
         // again to update the dirty flag.
         if (mode == TLB::Write && !e->pte.w) {
             DPRINTF(TLB, "Dirty bit not set, repeating PT walk\n");
             fault = walker->start(tc, translation, req, mode);
             if (translation != nullptr || fault != NoFault) {
                 delayed = true;
                 return fault;
             }
         }
         if (fault != NoFault)
             return fault;
     }

     Addr paddr = e->paddr << PageShift | (vaddr & mask(e->logBytes));
     DPRINTF(TLBVerbose, "translate(vpn=%#x, asid=%#x): %#x\n",
             vaddr, satp.asid, paddr);
     req->setPaddr(paddr);

     return NoFault;
 }

 PrivilegeMode
 TLB::getMemPriv(ThreadContext *tc, Mode mode)
 {
     STATUS status = (STATUS)tc->readMiscReg(MISCREG_STATUS);
     PrivilegeMode pmode = (PrivilegeMode)tc->readMiscReg(MISCREG_PRV);
     if (mode != Mode::Execute && status.mprv == 1)
         pmode = (PrivilegeMode)(RegVal)status.mpp;
     return pmode;
 }

 Fault
 TLB::translate(const RequestPtr &req, ThreadContext *tc,
                Translation *translation, Mode mode, bool &delayed)
 {
     delayed = false;

     if (FullSystem) {
         PrivilegeMode pmode = getMemPriv(tc, mode);
         SATP satp = tc->readMiscReg(MISCREG_SATP);
         if (pmode == PrivilegeMode::PRV_M || satp.mode == AddrXlateMode::BARE)
             req->setFlags(Request::PHYSICAL);

         Fault fault;
         if (req->getFlags() & Request::PHYSICAL) {
             /**
              * we simply set the virtual address to physical address
              */
             req->setPaddr(req->getVaddr());
             fault = NoFault;
         } else {
             fault = doTranslate(req, tc, translation, mode, delayed);
         }

         // according to the RISC-V tests, negative physical addresses trigger
         // an illegal address exception.
         // TODO where is that written in the manual?
         if (!delayed && fault == NoFault && bits(req->getPaddr(), 63)) {
             ExceptionCode code;
             if (mode == TLB::Read)
                 code = ExceptionCode::LOAD_ACCESS;
             else if (mode == TLB::Write)
                 code = ExceptionCode::STORE_ACCESS;
             else
                 code = ExceptionCode::INST_ACCESS;
             fault = std::make_shared<AddressFault>(req->getVaddr(), code);
         }

         if (!delayed && fault == NoFault) {
             pma->check(req);
         }

         return fault;
     } else {
         // In the O3 CPU model, sometimes a memory access will be speculatively
         // executed along a branch that will end up not being taken where the
         // address is invalid.  In that case, return a fault rather than trying
         // to translate it (which will cause a panic).  Since RISC-V allows
         // unaligned memory accesses, this should only happen if the request's
         // length is long enough to wrap around from the end of the memory to
         // the start.
         assert(req->getSize() > 0);
         if (req->getVaddr() + req->getSize() - 1 < req->getVaddr())
             return std::make_shared<GenericPageTableFault>(req->getVaddr());

         Process * p = tc->getProcessPtr();

         Fault fault = p->pTable->translate(req);
         if (fault != NoFault)
             return fault;

         return NoFault;
     }
 }

 Fault
 TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode)
 {
     bool delayed;
     return translate(req, tc, nullptr, mode, delayed);
 }

 void
 TLB::translateTiming(const RequestPtr &req, ThreadContext *tc,
                      Translation *translation, Mode mode)
 {
     bool delayed;
     assert(translation);
     Fault fault = translate(req, tc, translation, mode, delayed);
     if (!delayed)
         translation->finish(fault, req, tc, mode);
     else
         translation->markDelayed();
 }

 Fault
 TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode)
 {
     const Addr vaddr = req->getVaddr();
     Addr paddr = vaddr;

     if (FullSystem) {
         MMU *mmu = static_cast<MMU *>(tc->getMMUPtr());

         PrivilegeMode pmode = mmu->getMemPriv(tc, mode);
         SATP satp = tc->readMiscReg(MISCREG_SATP);
         if (pmode != PrivilegeMode::PRV_M &&
             satp.mode != AddrXlateMode::BARE) {
             Walker *walker = mmu->getDataWalker();
             unsigned logBytes;
             Fault fault = walker->startFunctional(
                     tc, paddr, logBytes, mode);
             if (fault != NoFault)
                 return fault;

             Addr masked_addr = vaddr & mask(logBytes);
             paddr |= masked_addr;
         }
     }
     else {
         Process *process = tc->getProcessPtr();
         const auto *pte = process->pTable->lookup(vaddr);

         if (!pte && mode != Execute) {
             // Check if we just need to grow the stack.
             if (process->fixupFault(vaddr)) {
                 // If we did, lookup the entry for the new page.
                 pte = process->pTable->lookup(vaddr);
             }
         }

         if (!pte)
             return std::make_shared<GenericPageTableFault>(req->getVaddr());

         paddr = pte->paddr | process->pTable->pageOffset(vaddr);
     }

     DPRINTF(TLB, "Translated (functional) %#x -> %#x.\n", vaddr, paddr);
     req->setPaddr(paddr);
     return NoFault;
 }

 Fault
 TLB::finalizePhysical(const RequestPtr &req,
                       ThreadContext *tc, Mode mode) const
 {
     return NoFault;
 }

 void
 TLB::serialize(CheckpointOut &cp) const
 {
     // Only store the entries in use.
     uint32_t _size = size - freeList.size();
     SERIALIZE_SCALAR(_size);
     SERIALIZE_SCALAR(lruSeq);

     uint32_t _count = 0;
     for (uint32_t x = 0; x < size; x++) {
         if (tlb[x].trieHandle != NULL)
             tlb[x].serializeSection(cp, csprintf("Entry%d", _count++));
     }
 }

 void
 TLB::unserialize(CheckpointIn &cp)
 {
     // Do not allow to restore with a smaller tlb.
     uint32_t _size;
     UNSERIALIZE_SCALAR(_size);
     if (_size > size) {
         fatal("TLB size less than the one in checkpoint!");
     }

     UNSERIALIZE_SCALAR(lruSeq);

     for (uint32_t x = 0; x < _size; x++) {
         TlbEntry *newEntry = freeList.front();
         freeList.pop_front();

         newEntry->unserializeSection(cp, csprintf("Entry%d", x));
         Addr key = buildKey(newEntry->vaddr, newEntry->asid);
         newEntry->trieHandle = trie.insert(key,
             TlbEntryTrie::MaxBits - newEntry->logBytes, newEntry);
     }
 }

 TLB::TlbStats::TlbStats(Stats::Group *parent)
   : Stats::Group(parent),
     ADD_STAT(readHits, UNIT_COUNT, "read hits"),
     ADD_STAT(readMisses, UNIT_COUNT, "read misses"),
     ADD_STAT(readAccesses, UNIT_COUNT, "read accesses"),
     ADD_STAT(writeHits, UNIT_COUNT, "write hits"),
     ADD_STAT(writeMisses, UNIT_COUNT, "write misses"),
     ADD_STAT(writeAccesses, UNIT_COUNT, "write accesses"),
     ADD_STAT(hits, UNIT_COUNT, "Total TLB (read and write) hits",
              readHits + writeHits),
     ADD_STAT(misses, UNIT_COUNT, "Total TLB (read and write) misses",
              readMisses + writeMisses),
     ADD_STAT(accesses, UNIT_COUNT, "Total TLB (read and write) accesses",
              readAccesses + writeAccesses)
 {
 }

 Port *
 TLB::getTableWalkerPort()
 {
     return &walker->getPort("port");
 }
	/*
	* Copyright (c) 2001-2005 The Regents of The University of Michigan
	* Copyright (c) 2007 MIPS Technologies, Inc.
	* Copyright (c) 2020 Barkhausen Institut
	* Copyright (c) 2021 Huawei International
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are
	* met: redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer;
	* redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution;
	* neither the name of the copyright holders nor the names of its
	* contributors may be used to endorse or promote products derived from
	* this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include "arch/riscv/tlb.hh"

	#include <string>
	#include <vector>

	#include "arch/riscv/faults.hh"
	#include "arch/riscv/mmu.hh"
	#include "arch/riscv/pagetable.hh"
	#include "arch/riscv/pagetable_walker.hh"
	#include "arch/riscv/pma_checker.hh"
	#include "arch/riscv/pra_constants.hh"
	#include "arch/riscv/utility.hh"
	#include "base/inifile.hh"
	#include "base/str.hh"
	#include "base/trace.hh"
	#include "cpu/thread_context.hh"
	#include "debug/TLB.hh"
	#include "debug/TLBVerbose.hh"
	#include "mem/page_table.hh"
	#include "params/RiscvTLB.hh"
	#include "sim/full_system.hh"
	#include "sim/process.hh"
	#include "sim/system.hh"

	using namespace RiscvISA;

	///////////////////////////////////////////////////////////////////////
	//
	// RISC-V TLB
	//

	static Addr
	buildKey(Addr vpn, uint16_t asid)
	{
	return (static_cast<Addr>(asid) << 48) \| vpn;
	}

	TLB::TLB(const Params &p) :
	BaseTLB(p), size(p.size), tlb(size),
	lruSeq(0), stats(this), pma(p.pma_checker)
	{
	for (size_t x = 0; x < size; x++) {
	tlb[x].trieHandle = NULL;
	freeList.push_back(&tlb[x]);
	}

	walker = p.walker;
	walker->setTLB(this);
	}

	Walker *
	TLB::getWalker()
	{
	return walker;
	}

	void
	TLB::evictLRU()
	{
	// Find the entry with the lowest (and hence least recently updated)
	// sequence number.

	size_t lru = 0;
	for (size_t i = 1; i < size; i++) {
	if (tlb[i].lruSeq < tlb[lru].lruSeq)
	lru = i;
	}

	remove(lru);
	}

	TlbEntry *
	TLB::lookup(Addr vpn, uint16_t asid, Mode mode, bool hidden)
	{
	TlbEntry *entry = trie.lookup(buildKey(vpn, asid));

	if (!hidden) {
	if (entry)
	entry->lruSeq = nextSeq();

	if (mode == Write)
	stats.writeAccesses++;
	else
	stats.readAccesses++;

	if (!entry) {
	if (mode == Write)
	stats.writeMisses++;
	else
	stats.readMisses++;
	}
	else {
	if (mode == Write)
	stats.writeHits++;
	else
	stats.readHits++;
	}

	DPRINTF(TLBVerbose, "lookup(vpn=%#x, asid=%#x): %s ppn %#x\n",
	vpn, asid, entry ? "hit" : "miss", entry ? entry->paddr : 0);
	}

	return entry;
	}

	TlbEntry *
	TLB::insert(Addr vpn, const TlbEntry &entry)
	{
	DPRINTF(TLB, "insert(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n",
	vpn, entry.asid, entry.paddr, entry.pte, entry.size());

	// If somebody beat us to it, just use that existing entry.
	TlbEntry *newEntry = lookup(vpn, entry.asid, Mode::Read, true);
	if (newEntry) {
	// update PTE flags (maybe we set the dirty/writable flag)
	newEntry->pte = entry.pte;
	assert(newEntry->vaddr == vpn);
	return newEntry;
	}

	if (freeList.empty())
	evictLRU();

	newEntry = freeList.front();
	freeList.pop_front();

	Addr key = buildKey(vpn, entry.asid);
	*newEntry = entry;
	newEntry->lruSeq = nextSeq();
	newEntry->vaddr = vpn;
	newEntry->trieHandle =
	trie.insert(key, TlbEntryTrie::MaxBits - entry.logBytes, newEntry);
	return newEntry;
	}

	void
	TLB::demapPage(Addr vpn, uint64_t asid)
	{
	asid &= 0xFFFF;

	if (vpn == 0 && asid == 0)
	flushAll();
	else {
	DPRINTF(TLB, "flush(vpn=%#x, asid=%#x)\n", vpn, asid);
	if (vpn != 0 && asid != 0) {
	TlbEntry *newEntry = lookup(vpn, asid, Mode::Read, true);
	if (newEntry)
	remove(newEntry - tlb.data());
	}
	else {
	for (size_t i = 0; i < size; i++) {
	if (tlb[i].trieHandle) {
	Addr mask = ~(tlb[i].size() - 1);
	if ((vpn == 0 \|\| (vpn & mask) == tlb[i].vaddr) &&
	(asid == 0 \|\| tlb[i].asid == asid))
	remove(i);
	}
	}
	}
	}
	}

	void
	TLB::flushAll()
	{
	DPRINTF(TLB, "flushAll()\n");
	for (size_t i = 0; i < size; i++) {
	if (tlb[i].trieHandle)
	remove(i);
	}
	}

	void
	TLB::remove(size_t idx)
	{
	DPRINTF(TLB, "remove(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n",
	tlb[idx].vaddr, tlb[idx].asid, tlb[idx].paddr, tlb[idx].pte,
	tlb[idx].size());

	assert(tlb[idx].trieHandle);
	trie.remove(tlb[idx].trieHandle);
	tlb[idx].trieHandle = NULL;
	freeList.push_back(&tlb[idx]);
	}

	Fault
	TLB::checkPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr,
	Mode mode, PTESv39 pte)
	{
	Fault fault = NoFault;

	if (mode == TLB::Read && !pte.r) {
	DPRINTF(TLB, "PTE has no read perm, raising PF\n");
	fault = createPagefault(vaddr, mode);
	}
	else if (mode == TLB::Write && !pte.w) {
	DPRINTF(TLB, "PTE has no write perm, raising PF\n");
	fault = createPagefault(vaddr, mode);
	}
	else if (mode == TLB::Execute && !pte.x) {
	DPRINTF(TLB, "PTE has no exec perm, raising PF\n");
	fault = createPagefault(vaddr, mode);
	}

	if (fault == NoFault) {
	// check pte.u
	if (pmode == PrivilegeMode::PRV_U && !pte.u) {
	DPRINTF(TLB, "PTE is not user accessible, raising PF\n");
	fault = createPagefault(vaddr, mode);
	}
	else if (pmode == PrivilegeMode::PRV_S && pte.u && status.sum == 0) {
	DPRINTF(TLB, "PTE is only user accessible, raising PF\n");
	fault = createPagefault(vaddr, mode);
	}
	}

	return fault;
	}

	Fault
	TLB::createPagefault(Addr vaddr, Mode mode)
	{
	ExceptionCode code;
	if (mode == TLB::Read)
	code = ExceptionCode::LOAD_PAGE;
	else if (mode == TLB::Write)
	code = ExceptionCode::STORE_PAGE;
	else
	code = ExceptionCode::INST_PAGE;
	return std::make_shared<AddressFault>(vaddr, code);
	}

	Addr
	TLB::translateWithTLB(Addr vaddr, uint16_t asid, Mode mode)
	{
	TlbEntry *e = lookup(vaddr, asid, mode, false);
	assert(e != nullptr);
	return e->paddr << PageShift \| (vaddr & mask(e->logBytes));
	}

	Fault
	TLB::doTranslate(const RequestPtr &req, ThreadContext *tc,
	Translation *translation, Mode mode, bool &delayed)
	{
	delayed = false;

	Addr vaddr = req->getVaddr() & ((static_cast<Addr>(1) << VADDR_BITS) - 1);
	SATP satp = tc->readMiscReg(MISCREG_SATP);

	TlbEntry *e = lookup(vaddr, satp.asid, mode, false);
	if (!e) {
	Fault fault = walker->start(tc, translation, req, mode);
	if (translation != nullptr \|\| fault != NoFault) {
	// This gets ignored in atomic mode.
	delayed = true;
	return fault;
	}
	e = lookup(vaddr, satp.asid, mode, false);
	assert(e != nullptr);
	}

	STATUS status = tc->readMiscReg(MISCREG_STATUS);
	PrivilegeMode pmode = getMemPriv(tc, mode);
	Fault fault = checkPermissions(status, pmode, vaddr, mode, e->pte);
	if (fault != NoFault) {
	// if we want to write and it isn't writable, do a page table walk
	// again to update the dirty flag.
	if (mode == TLB::Write && !e->pte.w) {
	DPRINTF(TLB, "Dirty bit not set, repeating PT walk\n");
	fault = walker->start(tc, translation, req, mode);
	if (translation != nullptr \|\| fault != NoFault) {
	delayed = true;
	return fault;
	}
	}
	if (fault != NoFault)
	return fault;
	}

	Addr paddr = e->paddr << PageShift \| (vaddr & mask(e->logBytes));
	DPRINTF(TLBVerbose, "translate(vpn=%#x, asid=%#x): %#x\n",
	vaddr, satp.asid, paddr);
	req->setPaddr(paddr);

	return NoFault;
	}

	PrivilegeMode
	TLB::getMemPriv(ThreadContext *tc, Mode mode)
	{
	STATUS status = (STATUS)tc->readMiscReg(MISCREG_STATUS);
	PrivilegeMode pmode = (PrivilegeMode)tc->readMiscReg(MISCREG_PRV);
	if (mode != Mode::Execute && status.mprv == 1)
	pmode = (PrivilegeMode)(RegVal)status.mpp;
	return pmode;
	}

	Fault
	TLB::translate(const RequestPtr &req, ThreadContext *tc,
	Translation *translation, Mode mode, bool &delayed)
	{
	delayed = false;

	if (FullSystem) {
	PrivilegeMode pmode = getMemPriv(tc, mode);
	SATP satp = tc->readMiscReg(MISCREG_SATP);
	if (pmode == PrivilegeMode::PRV_M \|\| satp.mode == AddrXlateMode::BARE)
	req->setFlags(Request::PHYSICAL);

	Fault fault;
	if (req->getFlags() & Request::PHYSICAL) {
	/**
	* we simply set the virtual address to physical address
	*/
	req->setPaddr(req->getVaddr());
	fault = NoFault;
	} else {
	fault = doTranslate(req, tc, translation, mode, delayed);
	}

	// according to the RISC-V tests, negative physical addresses trigger
	// an illegal address exception.
	// TODO where is that written in the manual?
	if (!delayed && fault == NoFault && bits(req->getPaddr(), 63)) {
	ExceptionCode code;
	if (mode == TLB::Read)
	code = ExceptionCode::LOAD_ACCESS;
	else if (mode == TLB::Write)
	code = ExceptionCode::STORE_ACCESS;
	else
	code = ExceptionCode::INST_ACCESS;
	fault = std::make_shared<AddressFault>(req->getVaddr(), code);
	}

	if (!delayed && fault == NoFault) {
	pma->check(req);
	}

	return fault;
	} else {
	// In the O3 CPU model, sometimes a memory access will be speculatively
	// executed along a branch that will end up not being taken where the
	// address is invalid. In that case, return a fault rather than trying
	// to translate it (which will cause a panic). Since RISC-V allows
	// unaligned memory accesses, this should only happen if the request's
	// length is long enough to wrap around from the end of the memory to
	// the start.
	assert(req->getSize() > 0);
	if (req->getVaddr() + req->getSize() - 1 < req->getVaddr())
	return std::make_shared<GenericPageTableFault>(req->getVaddr());

	Process * p = tc->getProcessPtr();

	Fault fault = p->pTable->translate(req);
	if (fault != NoFault)
	return fault;

	return NoFault;
	}
	}

	Fault
	TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode)
	{
	bool delayed;
	return translate(req, tc, nullptr, mode, delayed);
	}

	void
	TLB::translateTiming(const RequestPtr &req, ThreadContext *tc,
	Translation *translation, Mode mode)
	{
	bool delayed;
	assert(translation);
	Fault fault = translate(req, tc, translation, mode, delayed);
	if (!delayed)
	translation->finish(fault, req, tc, mode);
	else
	translation->markDelayed();
	}

	Fault
	TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode)
	{
	const Addr vaddr = req->getVaddr();
	Addr paddr = vaddr;

	if (FullSystem) {
	MMU mmu = static_cast<MMU >(tc->getMMUPtr());

	PrivilegeMode pmode = mmu->getMemPriv(tc, mode);
	SATP satp = tc->readMiscReg(MISCREG_SATP);
	if (pmode != PrivilegeMode::PRV_M &&
	satp.mode != AddrXlateMode::BARE) {
	Walker *walker = mmu->getDataWalker();
	unsigned logBytes;
	Fault fault = walker->startFunctional(
	tc, paddr, logBytes, mode);
	if (fault != NoFault)
	return fault;

	Addr masked_addr = vaddr & mask(logBytes);
	paddr \|= masked_addr;
	}
	}
	else {
	Process *process = tc->getProcessPtr();
	const auto *pte = process->pTable->lookup(vaddr);

	if (!pte && mode != Execute) {
	// Check if we just need to grow the stack.
	if (process->fixupFault(vaddr)) {
	// If we did, lookup the entry for the new page.
	pte = process->pTable->lookup(vaddr);
	}
	}

	if (!pte)
	return std::make_shared<GenericPageTableFault>(req->getVaddr());

	paddr = pte->paddr \| process->pTable->pageOffset(vaddr);
	}

	DPRINTF(TLB, "Translated (functional) %#x -> %#x.\n", vaddr, paddr);
	req->setPaddr(paddr);
	return NoFault;
	}

	Fault
	TLB::finalizePhysical(const RequestPtr &req,
	ThreadContext *tc, Mode mode) const
	{
	return NoFault;
	}

	void
	TLB::serialize(CheckpointOut &cp) const
	{
	// Only store the entries in use.
	uint32_t _size = size - freeList.size();
	SERIALIZE_SCALAR(_size);
	SERIALIZE_SCALAR(lruSeq);

	uint32_t _count = 0;
	for (uint32_t x = 0; x < size; x++) {
	if (tlb[x].trieHandle != NULL)
	tlb[x].serializeSection(cp, csprintf("Entry%d", _count++));
	}
	}

	void
	TLB::unserialize(CheckpointIn &cp)
	{
	// Do not allow to restore with a smaller tlb.
	uint32_t _size;
	UNSERIALIZE_SCALAR(_size);
	if (_size > size) {
	fatal("TLB size less than the one in checkpoint!");
	}

	UNSERIALIZE_SCALAR(lruSeq);

	for (uint32_t x = 0; x < _size; x++) {
	TlbEntry *newEntry = freeList.front();
	freeList.pop_front();

	newEntry->unserializeSection(cp, csprintf("Entry%d", x));
	Addr key = buildKey(newEntry->vaddr, newEntry->asid);
	newEntry->trieHandle = trie.insert(key,
	TlbEntryTrie::MaxBits - newEntry->logBytes, newEntry);
	}
	}

	TLB::TlbStats::TlbStats(Stats::Group *parent)
	: Stats::Group(parent),
	ADD_STAT(readHits, UNIT_COUNT, "read hits"),
	ADD_STAT(readMisses, UNIT_COUNT, "read misses"),
	ADD_STAT(readAccesses, UNIT_COUNT, "read accesses"),
	ADD_STAT(writeHits, UNIT_COUNT, "write hits"),
	ADD_STAT(writeMisses, UNIT_COUNT, "write misses"),
	ADD_STAT(writeAccesses, UNIT_COUNT, "write accesses"),
	ADD_STAT(hits, UNIT_COUNT, "Total TLB (read and write) hits",
	readHits + writeHits),
	ADD_STAT(misses, UNIT_COUNT, "Total TLB (read and write) misses",
	readMisses + writeMisses),
	ADD_STAT(accesses, UNIT_COUNT, "Total TLB (read and write) accesses",
	readAccesses + writeAccesses)
	{
	}

	Port *
	TLB::getTableWalkerPort()
	{
	return &walker->getPort("port");
	}