blob: a7c3f1270d2b6d55417c7cfe7da817652b3787a9 [file] [log] [blame]
/*
* Copyright (c) 2010-2013, 2016-2021 Arm Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
* not be construed as granting a license to any other intellectual
* property including but not limited to intellectual property relating
* to a hardware implementation of the functionality of the software
* licensed hereunder. You may use the software subject to the license
* terms below provided that you ensure that this notice is replicated
* unmodified and in its entirety in all distributions of the software,
* modified or unmodified, in source code or in binary form.
*
* Copyright (c) 2001-2005 The Regents of The University of Michigan
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met: redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer;
* redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution;
* neither the name of the copyright holders nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "arch/arm/tlb.hh"
#include <memory>
#include <string>
#include <vector>
#include "arch/arm/table_walker.hh"
#include "arch/arm/tlbi_op.hh"
#include "arch/arm/utility.hh"
#include "base/trace.hh"
#include "cpu/thread_context.hh"
#include "debug/TLB.hh"
#include "debug/TLBVerbose.hh"
#include "params/ArmTLB.hh"
namespace gem5
{
using namespace ArmISA;
TLB::TLB(const ArmTLBParams &p)
: BaseTLB(p), table(new TlbEntry[p.size]), size(p.size),
isStage2(p.is_stage2),
_walkCache(false),
tableWalker(nullptr),
stats(*this), rangeMRU(1), vmid(0)
{
for (int lvl = LookupLevel::L0;
lvl < LookupLevel::Num_ArmLookupLevel; lvl++) {
auto it = std::find(
p.partial_levels.begin(),
p.partial_levels.end(),
lvl);
auto lookup_lvl = static_cast<LookupLevel>(lvl);
if (it != p.partial_levels.end()) {
// A partial entry from of the current LookupLevel can be
// cached within the TLB
partialLevels[lookup_lvl] = true;
// Make sure this is not the last level (complete translation)
if (lvl != LookupLevel::Num_ArmLookupLevel - 1) {
_walkCache = true;
}
} else {
partialLevels[lookup_lvl] = false;
}
}
}
TLB::~TLB()
{
delete[] table;
}
void
TLB::setTableWalker(TableWalker *table_walker)
{
tableWalker = table_walker;
tableWalker->setTlb(this);
}
TlbEntry*
TLB::match(const Lookup &lookup_data)
{
// Vector of TLB entry candidates.
// Only one of them will be assigned to retval and will
// be returned to the MMU (in case of a hit)
// The vector has one entry per lookup level as it stores
// both complete and partial matches
std::vector<std::pair<int, const TlbEntry*>> hits{
LookupLevel::Num_ArmLookupLevel, {0, nullptr}};
int x = 0;
while (x < size) {
if (table[x].match(lookup_data)) {
const TlbEntry &entry = table[x];
hits[entry.lookupLevel] = std::make_pair(x, &entry);
// This is a complete translation, no need to loop further
if (!entry.partial)
break;
}
++x;
}
// Loop over the list of TLB entries matching our translation
// request, starting from the highest lookup level (complete
// translation) and iterating backwards (using reverse iterators)
for (auto it = hits.rbegin(); it != hits.rend(); it++) {
const auto& [idx, entry] = *it;
if (!entry) {
// No match for the current LookupLevel
continue;
}
// Maintaining LRU array
// We only move the hit entry ahead when the position is higher
// than rangeMRU
if (idx > rangeMRU && !lookup_data.functional) {
TlbEntry tmp_entry = *entry;
for (int i = idx; i > 0; i--)
table[i] = table[i - 1];
table[0] = tmp_entry;
return &table[0];
} else {
return &table[idx];
}
}
return nullptr;
}
TlbEntry*
TLB::lookup(const Lookup &lookup_data)
{
const auto mode = lookup_data.mode;
TlbEntry *retval = match(lookup_data);
DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x hyp %d secure %d "
"ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d "
"el: %d\n",
lookup_data.va, lookup_data.asn, retval ? "hit" : "miss",
lookup_data.vmid, lookup_data.hyp, lookup_data.secure,
retval ? retval->pfn : 0, retval ? retval->size : 0,
retval ? retval->pAddr(lookup_data.va) : 0,
retval ? retval->ap : 0,
retval ? retval->ns : 0, retval ? retval->nstid : 0,
retval ? retval->global : 0, retval ? retval->asid : 0,
retval ? retval->el : 0);
// Updating stats if this was not a functional lookup
if (!lookup_data.functional) {
if (!retval) {
if (mode == BaseMMU::Execute) {
stats.instMisses++;
} else if (mode == BaseMMU::Write) {
stats.writeMisses++;
} else {
stats.readMisses++;
}
} else {
if (retval->partial) {
stats.partialHits++;
}
if (mode == BaseMMU::Execute) {
stats.instHits++;
} else if (mode == BaseMMU::Write) {
stats.writeHits++;
} else {
stats.readHits++;
}
}
}
return retval;
}
TlbEntry*
TLB::multiLookup(const Lookup &lookup_data)
{
TlbEntry* te = lookup(lookup_data);
if (te) {
checkPromotion(te, lookup_data.mode);
} else {
if (auto tlb = static_cast<TLB*>(nextLevel())) {
te = tlb->multiLookup(lookup_data);
if (te && !lookup_data.functional &&
(!te->partial || partialLevels[te->lookupLevel])) {
// Insert entry only if this is not a functional
// lookup and if the translation is complete (unless this
// TLB caches partial translations)
insert(*te);
}
}
}
return te;
}
void
TLB::checkPromotion(TlbEntry *entry, BaseMMU::Mode mode)
{
TypeTLB acc_type = (mode == BaseMMU::Execute) ?
TypeTLB::instruction : TypeTLB::data;
// Hitting an instruction TLB entry on a data access or
// a data TLB entry on an instruction access:
// promoting the entry to unified
if (!(entry->type & acc_type))
entry->type = TypeTLB::unified;
}
// insert a new TLB entry
void
TLB::insert(TlbEntry &entry)
{
DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x"
" asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d"
" ap:%#x domain:%#x ns:%d nstid:%d isHyp:%d\n", entry.pfn,
entry.size, entry.vpn, entry.asid, entry.vmid, entry.N,
entry.global, entry.valid, entry.nonCacheable, entry.xn,
entry.ap, static_cast<uint8_t>(entry.domain), entry.ns, entry.nstid,
entry.isHyp);
if (table[size - 1].valid)
DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x "
"size: %#x ap:%d ns:%d nstid:%d g:%d isHyp:%d el: %d\n",
table[size-1].vpn << table[size-1].N, table[size-1].asid,
table[size-1].vmid, table[size-1].pfn << table[size-1].N,
table[size-1].size, table[size-1].ap, table[size-1].ns,
table[size-1].nstid, table[size-1].global, table[size-1].isHyp,
table[size-1].el);
// inserting to MRU position and evicting the LRU one
for (int i = size - 1; i > 0; --i)
table[i] = table[i-1];
table[0] = entry;
stats.inserts++;
ppRefills->notify(1);
}
void
TLB::multiInsert(TlbEntry &entry)
{
// Insert a partial translation only if the TLB is configured
// as a walk cache
if (!entry.partial || partialLevels[entry.lookupLevel]) {
insert(entry);
}
if (auto next_level = static_cast<TLB*>(nextLevel())) {
next_level->multiInsert(entry);
}
}
void
TLB::printTlb() const
{
int x = 0;
TlbEntry *te;
DPRINTF(TLB, "Current TLB contents:\n");
while (x < size) {
te = &table[x];
if (te->valid)
DPRINTF(TLB, " * %s\n", te->print());
++x;
}
}
void
TLB::flushAll()
{
DPRINTF(TLB, "Flushing all TLB entries\n");
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
if (te->valid) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const TLBIALL& tlbi_op)
{
DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
(tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(
tlbi_op.targetEL, tlbi_op.inHost);
if (te->valid && tlbi_op.secureLookup == !te->nstid &&
(te->vmid == vmid || tlbi_op.el2Enabled) && el_match) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const ITLBIALL& tlbi_op)
{
DPRINTF(TLB, "Flushing all ITLB entries (%s lookup)\n",
(tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(
tlbi_op.targetEL, tlbi_op.inHost);
if (te->type & TypeTLB::instruction && te->valid &&
tlbi_op.secureLookup == !te->nstid &&
(te->vmid == vmid || tlbi_op.el2Enabled) && el_match) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const DTLBIALL& tlbi_op)
{
DPRINTF(TLB, "Flushing all DTLB entries (%s lookup)\n",
(tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(
tlbi_op.targetEL, tlbi_op.inHost);
if (te->type & TypeTLB::data && te->valid &&
tlbi_op.secureLookup == !te->nstid &&
(te->vmid == vmid || tlbi_op.el2Enabled) && el_match) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const TLBIALLEL &tlbi_op)
{
DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
(tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(
tlbi_op.targetEL, tlbi_op.inHost);
if (te->valid && tlbi_op.secureLookup == !te->nstid && el_match) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const TLBIVMALL &tlbi_op)
{
DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
(tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(
tlbi_op.targetEL, tlbi_op.inHost);
const bool vmid_match =
te->vmid == vmid ||
!tlbi_op.el2Enabled ||
(!tlbi_op.stage2Flush() && tlbi_op.inHost);
if (te->valid && tlbi_op.secureLookup == !te->nstid &&
el_match && vmid_match) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const TLBIALLN &tlbi_op)
{
bool hyp = tlbi_op.targetEL == EL2;
DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n",
(hyp ? "hyp" : "non-hyp"));
int x = 0;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(tlbi_op.targetEL, false);
if (te->valid && te->nstid && te->isHyp == hyp && el_match) {
DPRINTF(TLB, " - %s\n", te->print());
stats.flushedEntries++;
te->valid = false;
}
++x;
}
stats.flushTlb++;
}
void
TLB::flush(const TLBIMVA &tlbi_op)
{
DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x "
"(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
(tlbi_op.secureLookup ? "secure" : "non-secure"));
_flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified);
stats.flushTlbMvaAsid++;
}
void
TLB::flush(const ITLBIMVA &tlbi_op)
{
DPRINTF(TLB, "Flushing ITLB entries with mva: %#x, asid: %#x "
"(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
(tlbi_op.secureLookup ? "secure" : "non-secure"));
_flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::instruction);
stats.flushTlbMvaAsid++;
}
void
TLB::flush(const DTLBIMVA &tlbi_op)
{
DPRINTF(TLB, "Flushing DTLB entries with mva: %#x, asid: %#x "
"(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
(tlbi_op.secureLookup ? "secure" : "non-secure"));
_flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::data);
stats.flushTlbMvaAsid++;
}
void
TLB::flush(const TLBIASID &tlbi_op)
{
DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n",
tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0 ;
TlbEntry *te;
while (x < size) {
te = &table[x];
const bool el_match = te->checkELMatch(
tlbi_op.targetEL, tlbi_op.inHost);
const bool vmid_match =
te->vmid == vmid || !tlbi_op.el2Enabled || tlbi_op.inHost;
if (te->valid && te->asid == tlbi_op.asid &&
tlbi_op.secureLookup == !te->nstid &&
vmid_match && el_match) {
te->valid = false;
DPRINTF(TLB, " - %s\n", te->print());
stats.flushedEntries++;
}
++x;
}
stats.flushTlbAsid++;
}
void
TLB::flush(const ITLBIASID &tlbi_op)
{
DPRINTF(TLB, "Flushing ITLB entries with asid: %#x (%s lookup)\n",
tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0 ;
TlbEntry *te;
while (x < size) {
te = &table[x];
if (te->type & TypeTLB::instruction &&
te->valid && te->asid == tlbi_op.asid &&
tlbi_op.secureLookup == !te->nstid &&
(te->vmid == vmid || tlbi_op.el2Enabled) &&
te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) {
te->valid = false;
DPRINTF(TLB, " - %s\n", te->print());
stats.flushedEntries++;
}
++x;
}
stats.flushTlbAsid++;
}
void
TLB::flush(const DTLBIASID &tlbi_op)
{
DPRINTF(TLB, "Flushing DTLB entries with asid: %#x (%s lookup)\n",
tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));
int x = 0 ;
TlbEntry *te;
while (x < size) {
te = &table[x];
if (te->type & TypeTLB::data &&
te->valid && te->asid == tlbi_op.asid &&
tlbi_op.secureLookup == !te->nstid &&
(te->vmid == vmid || tlbi_op.el2Enabled) &&
te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) {
te->valid = false;
DPRINTF(TLB, " - %s\n", te->print());
stats.flushedEntries++;
}
++x;
}
stats.flushTlbAsid++;
}
void
TLB::flush(const TLBIMVAA &tlbi_op) {
DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n",
tlbi_op.addr,
(tlbi_op.secureLookup ? "secure" : "non-secure"));
_flushMva(tlbi_op.addr, 0xbeef, tlbi_op.secureLookup, true,
tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified);
stats.flushTlbMva++;
}
void
TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup,
bool ignore_asn, ExceptionLevel target_el, bool in_host,
TypeTLB entry_type)
{
TlbEntry *te;
Lookup lookup_data;
lookup_data.va = sext<56>(mva);
lookup_data.asn = asn;
lookup_data.ignoreAsn = ignore_asn;
lookup_data.vmid = vmid;
lookup_data.hyp = target_el == EL2;
lookup_data.secure = secure_lookup;
lookup_data.functional = true;
lookup_data.targetEL = target_el;
lookup_data.inHost = in_host;
lookup_data.mode = BaseMMU::Read;
te = lookup(lookup_data);
while (te != NULL) {
bool matching_type = (te->type & entry_type);
if (matching_type && secure_lookup == !te->nstid) {
DPRINTF(TLB, " - %s\n", te->print());
te->valid = false;
stats.flushedEntries++;
}
te = lookup(lookup_data);
}
}
void
TLB::takeOverFrom(BaseTLB *_otlb)
{
}
TLB::TlbStats::TlbStats(TLB &parent)
: statistics::Group(&parent), tlb(parent),
ADD_STAT(partialHits, statistics::units::Count::get(),
"partial translation hits"),
ADD_STAT(instHits, statistics::units::Count::get(), "Inst hits"),
ADD_STAT(instMisses, statistics::units::Count::get(), "Inst misses"),
ADD_STAT(readHits, statistics::units::Count::get(), "Read hits"),
ADD_STAT(readMisses, statistics::units::Count::get(), "Read misses"),
ADD_STAT(writeHits, statistics::units::Count::get(), "Write hits"),
ADD_STAT(writeMisses, statistics::units::Count::get(), "Write misses"),
ADD_STAT(inserts, statistics::units::Count::get(),
"Number of times an entry is inserted into the TLB"),
ADD_STAT(flushTlb, statistics::units::Count::get(),
"Number of times complete TLB was flushed"),
ADD_STAT(flushTlbMva, statistics::units::Count::get(),
"Number of times TLB was flushed by MVA"),
ADD_STAT(flushTlbMvaAsid, statistics::units::Count::get(),
"Number of times TLB was flushed by MVA & ASID"),
ADD_STAT(flushTlbAsid, statistics::units::Count::get(),
"Number of times TLB was flushed by ASID"),
ADD_STAT(flushedEntries, statistics::units::Count::get(),
"Number of entries that have been flushed from TLB"),
ADD_STAT(readAccesses, statistics::units::Count::get(), "Read accesses",
readHits + readMisses),
ADD_STAT(writeAccesses, statistics::units::Count::get(), "Write accesses",
writeHits + writeMisses),
ADD_STAT(instAccesses, statistics::units::Count::get(), "Inst accesses",
instHits + instMisses),
ADD_STAT(hits, statistics::units::Count::get(),
"Total TLB (inst and data) hits",
readHits + writeHits + instHits),
ADD_STAT(misses, statistics::units::Count::get(),
"Total TLB (inst and data) misses",
readMisses + writeMisses + instMisses),
ADD_STAT(accesses, statistics::units::Count::get(),
"Total TLB (inst and data) accesses",
readAccesses + writeAccesses + instAccesses)
{
// If this is a pure Data TLB, mark the instruction
// stats as nozero, so that they won't make it in
// into the final stats file
if (tlb.type() == TypeTLB::data) {
instHits.flags(statistics::nozero);
instMisses.flags(statistics::nozero);
instAccesses.flags(statistics::nozero);
}
// If this is a pure Instruction TLB, mark the data
// stats as nozero, so that they won't make it in
// into the final stats file
if (tlb.type() & TypeTLB::instruction) {
readHits.flags(statistics::nozero);
readMisses.flags(statistics::nozero);
writeHits.flags(statistics::nozero);
writeMisses.flags(statistics::nozero);
readAccesses.flags(statistics::nozero);
writeAccesses.flags(statistics::nozero);
}
partialHits.flags(statistics::nozero);
}
void
TLB::regProbePoints()
{
ppRefills.reset(new probing::PMU(getProbeManager(), "Refills"));
}
Port *
TLB::getTableWalkerPort()
{
return &tableWalker->getTableWalkerPort();
}
} // namespace gem5