blob: a2df355614392cdb41bd7547cc7969b9003fc535 [file] [log] [blame]
/*
* Copyright (c) 2001-2005 The Regents of The University of Michigan
* Copyright (c) 2007 MIPS Technologies, Inc.
* Copyright (c) 2020 Barkhausen Institut
* Copyright (c) 2021 Huawei International
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met: redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer;
* redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution;
* neither the name of the copyright holders nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "arch/riscv/tlb.hh"
#include <string>
#include <vector>
#include "arch/riscv/faults.hh"
#include "arch/riscv/mmu.hh"
#include "arch/riscv/pagetable.hh"
#include "arch/riscv/pagetable_walker.hh"
#include "arch/riscv/pma_checker.hh"
#include "arch/riscv/pra_constants.hh"
#include "arch/riscv/utility.hh"
#include "base/inifile.hh"
#include "base/str.hh"
#include "base/trace.hh"
#include "cpu/thread_context.hh"
#include "debug/TLB.hh"
#include "debug/TLBVerbose.hh"
#include "mem/page_table.hh"
#include "params/RiscvTLB.hh"
#include "sim/full_system.hh"
#include "sim/process.hh"
#include "sim/system.hh"
using namespace RiscvISA;
///////////////////////////////////////////////////////////////////////
//
// RISC-V TLB
//
static Addr
buildKey(Addr vpn, uint16_t asid)
{
return (static_cast<Addr>(asid) << 48) | vpn;
}
TLB::TLB(const Params &p) :
BaseTLB(p), size(p.size), tlb(size),
lruSeq(0), stats(this), pma(p.pma_checker)
{
for (size_t x = 0; x < size; x++) {
tlb[x].trieHandle = NULL;
freeList.push_back(&tlb[x]);
}
walker = p.walker;
walker->setTLB(this);
}
Walker *
TLB::getWalker()
{
return walker;
}
void
TLB::evictLRU()
{
// Find the entry with the lowest (and hence least recently updated)
// sequence number.
size_t lru = 0;
for (size_t i = 1; i < size; i++) {
if (tlb[i].lruSeq < tlb[lru].lruSeq)
lru = i;
}
remove(lru);
}
TlbEntry *
TLB::lookup(Addr vpn, uint16_t asid, Mode mode, bool hidden)
{
TlbEntry *entry = trie.lookup(buildKey(vpn, asid));
if (!hidden) {
if (entry)
entry->lruSeq = nextSeq();
if (mode == Write)
stats.writeAccesses++;
else
stats.readAccesses++;
if (!entry) {
if (mode == Write)
stats.writeMisses++;
else
stats.readMisses++;
}
else {
if (mode == Write)
stats.writeHits++;
else
stats.readHits++;
}
DPRINTF(TLBVerbose, "lookup(vpn=%#x, asid=%#x): %s ppn %#x\n",
vpn, asid, entry ? "hit" : "miss", entry ? entry->paddr : 0);
}
return entry;
}
TlbEntry *
TLB::insert(Addr vpn, const TlbEntry &entry)
{
DPRINTF(TLB, "insert(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n",
vpn, entry.asid, entry.paddr, entry.pte, entry.size());
// If somebody beat us to it, just use that existing entry.
TlbEntry *newEntry = lookup(vpn, entry.asid, Mode::Read, true);
if (newEntry) {
// update PTE flags (maybe we set the dirty/writable flag)
newEntry->pte = entry.pte;
assert(newEntry->vaddr == vpn);
return newEntry;
}
if (freeList.empty())
evictLRU();
newEntry = freeList.front();
freeList.pop_front();
Addr key = buildKey(vpn, entry.asid);
*newEntry = entry;
newEntry->lruSeq = nextSeq();
newEntry->vaddr = vpn;
newEntry->trieHandle =
trie.insert(key, TlbEntryTrie::MaxBits - entry.logBytes, newEntry);
return newEntry;
}
void
TLB::demapPage(Addr vpn, uint64_t asid)
{
asid &= 0xFFFF;
if (vpn == 0 && asid == 0)
flushAll();
else {
DPRINTF(TLB, "flush(vpn=%#x, asid=%#x)\n", vpn, asid);
if (vpn != 0 && asid != 0) {
TlbEntry *newEntry = lookup(vpn, asid, Mode::Read, true);
if (newEntry)
remove(newEntry - tlb.data());
}
else {
for (size_t i = 0; i < size; i++) {
if (tlb[i].trieHandle) {
Addr mask = ~(tlb[i].size() - 1);
if ((vpn == 0 || (vpn & mask) == tlb[i].vaddr) &&
(asid == 0 || tlb[i].asid == asid))
remove(i);
}
}
}
}
}
void
TLB::flushAll()
{
DPRINTF(TLB, "flushAll()\n");
for (size_t i = 0; i < size; i++) {
if (tlb[i].trieHandle)
remove(i);
}
}
void
TLB::remove(size_t idx)
{
DPRINTF(TLB, "remove(vpn=%#x, asid=%#x): ppn=%#x pte=%#x size=%#x\n",
tlb[idx].vaddr, tlb[idx].asid, tlb[idx].paddr, tlb[idx].pte,
tlb[idx].size());
assert(tlb[idx].trieHandle);
trie.remove(tlb[idx].trieHandle);
tlb[idx].trieHandle = NULL;
freeList.push_back(&tlb[idx]);
}
Fault
TLB::checkPermissions(STATUS status, PrivilegeMode pmode, Addr vaddr,
Mode mode, PTESv39 pte)
{
Fault fault = NoFault;
if (mode == TLB::Read && !pte.r) {
DPRINTF(TLB, "PTE has no read perm, raising PF\n");
fault = createPagefault(vaddr, mode);
}
else if (mode == TLB::Write && !pte.w) {
DPRINTF(TLB, "PTE has no write perm, raising PF\n");
fault = createPagefault(vaddr, mode);
}
else if (mode == TLB::Execute && !pte.x) {
DPRINTF(TLB, "PTE has no exec perm, raising PF\n");
fault = createPagefault(vaddr, mode);
}
if (fault == NoFault) {
// check pte.u
if (pmode == PrivilegeMode::PRV_U && !pte.u) {
DPRINTF(TLB, "PTE is not user accessible, raising PF\n");
fault = createPagefault(vaddr, mode);
}
else if (pmode == PrivilegeMode::PRV_S && pte.u && status.sum == 0) {
DPRINTF(TLB, "PTE is only user accessible, raising PF\n");
fault = createPagefault(vaddr, mode);
}
}
return fault;
}
Fault
TLB::createPagefault(Addr vaddr, Mode mode)
{
ExceptionCode code;
if (mode == TLB::Read)
code = ExceptionCode::LOAD_PAGE;
else if (mode == TLB::Write)
code = ExceptionCode::STORE_PAGE;
else
code = ExceptionCode::INST_PAGE;
return std::make_shared<AddressFault>(vaddr, code);
}
Addr
TLB::translateWithTLB(Addr vaddr, uint16_t asid, Mode mode)
{
TlbEntry *e = lookup(vaddr, asid, mode, false);
assert(e != nullptr);
return e->paddr << PageShift | (vaddr & mask(e->logBytes));
}
Fault
TLB::doTranslate(const RequestPtr &req, ThreadContext *tc,
Translation *translation, Mode mode, bool &delayed)
{
delayed = false;
Addr vaddr = req->getVaddr() & ((static_cast<Addr>(1) << VADDR_BITS) - 1);
SATP satp = tc->readMiscReg(MISCREG_SATP);
TlbEntry *e = lookup(vaddr, satp.asid, mode, false);
if (!e) {
Fault fault = walker->start(tc, translation, req, mode);
if (translation != nullptr || fault != NoFault) {
// This gets ignored in atomic mode.
delayed = true;
return fault;
}
e = lookup(vaddr, satp.asid, mode, false);
assert(e != nullptr);
}
STATUS status = tc->readMiscReg(MISCREG_STATUS);
PrivilegeMode pmode = getMemPriv(tc, mode);
Fault fault = checkPermissions(status, pmode, vaddr, mode, e->pte);
if (fault != NoFault) {
// if we want to write and it isn't writable, do a page table walk
// again to update the dirty flag.
if (mode == TLB::Write && !e->pte.w) {
DPRINTF(TLB, "Dirty bit not set, repeating PT walk\n");
fault = walker->start(tc, translation, req, mode);
if (translation != nullptr || fault != NoFault) {
delayed = true;
return fault;
}
}
if (fault != NoFault)
return fault;
}
Addr paddr = e->paddr << PageShift | (vaddr & mask(e->logBytes));
DPRINTF(TLBVerbose, "translate(vpn=%#x, asid=%#x): %#x\n",
vaddr, satp.asid, paddr);
req->setPaddr(paddr);
return NoFault;
}
PrivilegeMode
TLB::getMemPriv(ThreadContext *tc, Mode mode)
{
STATUS status = (STATUS)tc->readMiscReg(MISCREG_STATUS);
PrivilegeMode pmode = (PrivilegeMode)tc->readMiscReg(MISCREG_PRV);
if (mode != Mode::Execute && status.mprv == 1)
pmode = (PrivilegeMode)(RegVal)status.mpp;
return pmode;
}
Fault
TLB::translate(const RequestPtr &req, ThreadContext *tc,
Translation *translation, Mode mode, bool &delayed)
{
delayed = false;
if (FullSystem) {
PrivilegeMode pmode = getMemPriv(tc, mode);
SATP satp = tc->readMiscReg(MISCREG_SATP);
if (pmode == PrivilegeMode::PRV_M || satp.mode == AddrXlateMode::BARE)
req->setFlags(Request::PHYSICAL);
Fault fault;
if (req->getFlags() & Request::PHYSICAL) {
/**
* we simply set the virtual address to physical address
*/
req->setPaddr(req->getVaddr());
fault = NoFault;
} else {
fault = doTranslate(req, tc, translation, mode, delayed);
}
// according to the RISC-V tests, negative physical addresses trigger
// an illegal address exception.
// TODO where is that written in the manual?
if (!delayed && fault == NoFault && bits(req->getPaddr(), 63)) {
ExceptionCode code;
if (mode == TLB::Read)
code = ExceptionCode::LOAD_ACCESS;
else if (mode == TLB::Write)
code = ExceptionCode::STORE_ACCESS;
else
code = ExceptionCode::INST_ACCESS;
fault = std::make_shared<AddressFault>(req->getVaddr(), code);
}
if (!delayed && fault == NoFault) {
pma->check(req);
}
return fault;
} else {
// In the O3 CPU model, sometimes a memory access will be speculatively
// executed along a branch that will end up not being taken where the
// address is invalid. In that case, return a fault rather than trying
// to translate it (which will cause a panic). Since RISC-V allows
// unaligned memory accesses, this should only happen if the request's
// length is long enough to wrap around from the end of the memory to
// the start.
assert(req->getSize() > 0);
if (req->getVaddr() + req->getSize() - 1 < req->getVaddr())
return std::make_shared<GenericPageTableFault>(req->getVaddr());
Process * p = tc->getProcessPtr();
Fault fault = p->pTable->translate(req);
if (fault != NoFault)
return fault;
return NoFault;
}
}
Fault
TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode)
{
bool delayed;
return translate(req, tc, nullptr, mode, delayed);
}
void
TLB::translateTiming(const RequestPtr &req, ThreadContext *tc,
Translation *translation, Mode mode)
{
bool delayed;
assert(translation);
Fault fault = translate(req, tc, translation, mode, delayed);
if (!delayed)
translation->finish(fault, req, tc, mode);
else
translation->markDelayed();
}
Fault
TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode)
{
const Addr vaddr = req->getVaddr();
Addr paddr = vaddr;
if (FullSystem) {
MMU *mmu = static_cast<MMU *>(tc->getMMUPtr());
PrivilegeMode pmode = mmu->getMemPriv(tc, mode);
SATP satp = tc->readMiscReg(MISCREG_SATP);
if (pmode != PrivilegeMode::PRV_M &&
satp.mode != AddrXlateMode::BARE) {
Walker *walker = mmu->getDataWalker();
unsigned logBytes;
Fault fault = walker->startFunctional(
tc, paddr, logBytes, mode);
if (fault != NoFault)
return fault;
Addr masked_addr = vaddr & mask(logBytes);
paddr |= masked_addr;
}
}
else {
Process *process = tc->getProcessPtr();
const auto *pte = process->pTable->lookup(vaddr);
if (!pte && mode != Execute) {
// Check if we just need to grow the stack.
if (process->fixupFault(vaddr)) {
// If we did, lookup the entry for the new page.
pte = process->pTable->lookup(vaddr);
}
}
if (!pte)
return std::make_shared<GenericPageTableFault>(req->getVaddr());
paddr = pte->paddr | process->pTable->pageOffset(vaddr);
}
DPRINTF(TLB, "Translated (functional) %#x -> %#x.\n", vaddr, paddr);
req->setPaddr(paddr);
return NoFault;
}
Fault
TLB::finalizePhysical(const RequestPtr &req,
ThreadContext *tc, Mode mode) const
{
return NoFault;
}
void
TLB::serialize(CheckpointOut &cp) const
{
// Only store the entries in use.
uint32_t _size = size - freeList.size();
SERIALIZE_SCALAR(_size);
SERIALIZE_SCALAR(lruSeq);
uint32_t _count = 0;
for (uint32_t x = 0; x < size; x++) {
if (tlb[x].trieHandle != NULL)
tlb[x].serializeSection(cp, csprintf("Entry%d", _count++));
}
}
void
TLB::unserialize(CheckpointIn &cp)
{
// Do not allow to restore with a smaller tlb.
uint32_t _size;
UNSERIALIZE_SCALAR(_size);
if (_size > size) {
fatal("TLB size less than the one in checkpoint!");
}
UNSERIALIZE_SCALAR(lruSeq);
for (uint32_t x = 0; x < _size; x++) {
TlbEntry *newEntry = freeList.front();
freeList.pop_front();
newEntry->unserializeSection(cp, csprintf("Entry%d", x));
Addr key = buildKey(newEntry->vaddr, newEntry->asid);
newEntry->trieHandle = trie.insert(key,
TlbEntryTrie::MaxBits - newEntry->logBytes, newEntry);
}
}
TLB::TlbStats::TlbStats(Stats::Group *parent)
: Stats::Group(parent),
ADD_STAT(readHits, UNIT_COUNT, "read hits"),
ADD_STAT(readMisses, UNIT_COUNT, "read misses"),
ADD_STAT(readAccesses, UNIT_COUNT, "read accesses"),
ADD_STAT(writeHits, UNIT_COUNT, "write hits"),
ADD_STAT(writeMisses, UNIT_COUNT, "write misses"),
ADD_STAT(writeAccesses, UNIT_COUNT, "write accesses"),
ADD_STAT(hits, UNIT_COUNT, "Total TLB (read and write) hits",
readHits + writeHits),
ADD_STAT(misses, UNIT_COUNT, "Total TLB (read and write) misses",
readMisses + writeMisses),
ADD_STAT(accesses, UNIT_COUNT, "Total TLB (read and write) accesses",
readAccesses + writeAccesses)
{
}
Port *
TLB::getTableWalkerPort()
{
return &walker->getPort("port");
}