src/arch/arm/tlb.cc - public/gem5 - Git at Google

 /*
  * Copyright (c) 2010-2013, 2016-2021 Arm Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
  * not be construed as granting a license to any other intellectual
  * property including but not limited to intellectual property relating
  * to a hardware implementation of the functionality of the software
  * licensed hereunder.  You may use the software subject to the license
  * terms below provided that you ensure that this notice is replicated
  * unmodified and in its entirety in all distributions of the software,
  * modified or unmodified, in source code or in binary form.
  *
  * Copyright (c) 2001-2005 The Regents of The University of Michigan
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met: redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer;
  * redistributions in binary form must reproduce the above copyright
  * notice, this list of conditions and the following disclaimer in the
  * documentation and/or other materials provided with the distribution;
  * neither the name of the copyright holders nor the names of its
  * contributors may be used to endorse or promote products derived from
  * this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 #include "arch/arm/tlb.hh"

 #include <memory>
 #include <string>
 #include <vector>

 #include "arch/arm/table_walker.hh"
 #include "arch/arm/tlbi_op.hh"
 #include "arch/arm/utility.hh"
 #include "base/trace.hh"
 #include "cpu/thread_context.hh"
 #include "debug/TLB.hh"
 #include "debug/TLBVerbose.hh"
 #include "params/ArmTLB.hh"

 namespace gem5
 {

 using namespace ArmISA;

 TLB::TLB(const ArmTLBParams &p)
     : BaseTLB(p), table(new TlbEntry[p.size]), size(p.size),
       isStage2(p.is_stage2),
       _walkCache(false),
       tableWalker(nullptr),
       stats(*this), rangeMRU(1), vmid(0)
 {
     for (int lvl = LookupLevel::L0;
          lvl < LookupLevel::Num_ArmLookupLevel; lvl++) {

         auto it = std::find(
             p.partial_levels.begin(),
             p.partial_levels.end(),
             lvl);

         auto lookup_lvl = static_cast<LookupLevel>(lvl);

         if (it != p.partial_levels.end()) {
             // A partial entry from of the current LookupLevel can be
             // cached within the TLB
             partialLevels[lookup_lvl] = true;

             // Make sure this is not the last level (complete translation)
             if (lvl != LookupLevel::Num_ArmLookupLevel - 1) {
                 _walkCache = true;
             }
         } else {
             partialLevels[lookup_lvl] = false;
         }
     }
 }

 TLB::~TLB()
 {
     delete[] table;
 }

 void
 TLB::setTableWalker(TableWalker *table_walker)
 {
     tableWalker = table_walker;
     tableWalker->setTlb(this);
 }

 TlbEntry*
 TLB::match(const Lookup &lookup_data)
 {
     // Vector of TLB entry candidates.
     // Only one of them will be assigned to retval and will
     // be returned to the MMU (in case of a hit)
     // The vector has one entry per lookup level as it stores
     // both complete and partial matches
     std::vector<std::pair<int, const TlbEntry*>> hits{
         LookupLevel::Num_ArmLookupLevel, {0, nullptr}};

     int x = 0;
     while (x < size) {
         if (table[x].match(lookup_data)) {
             const TlbEntry &entry = table[x];
             hits[entry.lookupLevel] = std::make_pair(x, &entry);

             // This is a complete translation, no need to loop further
             if (!entry.partial)
                 break;
         }
         ++x;
     }

     // Loop over the list of TLB entries matching our translation
     // request, starting from the highest lookup level (complete
     // translation) and iterating backwards (using reverse iterators)
     for (auto it = hits.rbegin(); it != hits.rend(); it++) {
         const auto& [idx, entry] = *it;
         if (!entry) {
             // No match for the current LookupLevel
             continue;
         }

         // Maintaining LRU array
         // We only move the hit entry ahead when the position is higher
         // than rangeMRU
         if (idx > rangeMRU && !lookup_data.functional) {
             TlbEntry tmp_entry = *entry;
             for (int i = idx; i > 0; i--)
                 table[i] = table[i - 1];
             table[0] = tmp_entry;
             return &table[0];
         } else {
             return &table[idx];
         }
     }

     return nullptr;
 }

 TlbEntry*
 TLB::lookup(const Lookup &lookup_data)
 {
     const auto mode = lookup_data.mode;

     TlbEntry *retval = match(lookup_data);

     DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x hyp %d secure %d "
             "ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d "
             "el: %d\n",
             lookup_data.va, lookup_data.asn, retval ? "hit" : "miss",
             lookup_data.vmid, lookup_data.hyp, lookup_data.secure,
             retval ? retval->pfn       : 0, retval ? retval->size  : 0,
             retval ? retval->pAddr(lookup_data.va) : 0,
             retval ? retval->ap        : 0,
             retval ? retval->ns        : 0, retval ? retval->nstid : 0,
             retval ? retval->global    : 0, retval ? retval->asid  : 0,
             retval ? retval->el        : 0);

     // Updating stats if this was not a functional lookup
     if (!lookup_data.functional) {
         if (!retval) {
             if (mode == BaseMMU::Execute) {
                 stats.instMisses++;
             } else if (mode == BaseMMU::Write) {
                 stats.writeMisses++;
             } else {
                 stats.readMisses++;
             }
         } else {
             if (retval->partial) {
                 stats.partialHits++;
             }

             if (mode == BaseMMU::Execute) {
                 stats.instHits++;
             } else if (mode == BaseMMU::Write) {
                stats.writeHits++;
             } else {
                 stats.readHits++;
             }
         }
     }

     return retval;
 }

 TlbEntry*
 TLB::multiLookup(const Lookup &lookup_data)
 {
     TlbEntry* te = lookup(lookup_data);

     if (te) {
         checkPromotion(te, lookup_data.mode);
     } else {
         if (auto tlb = static_cast<TLB*>(nextLevel())) {
             te = tlb->multiLookup(lookup_data);
             if (te && !lookup_data.functional &&
                 (!te->partial || partialLevels[te->lookupLevel])) {
                 // Insert entry only if this is not a functional
                 // lookup and if the translation is complete (unless this
                 // TLB caches partial translations)
                 insert(*te);
             }
         }
     }

     return te;
 }

 void
 TLB::checkPromotion(TlbEntry *entry, BaseMMU::Mode mode)
 {
     TypeTLB acc_type = (mode == BaseMMU::Execute) ?
        TypeTLB::instruction : TypeTLB::data;

     // Hitting an instruction TLB entry on a data access or
     // a data TLB entry on an instruction access:
     // promoting the entry to unified
     if (!(entry->type & acc_type))
        entry->type = TypeTLB::unified;
 }

 // insert a new TLB entry
 void
 TLB::insert(TlbEntry &entry)
 {
     DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x"
             " asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d"
             " ap:%#x domain:%#x ns:%d nstid:%d isHyp:%d\n", entry.pfn,
             entry.size, entry.vpn, entry.asid, entry.vmid, entry.N,
             entry.global, entry.valid, entry.nonCacheable, entry.xn,
             entry.ap, static_cast<uint8_t>(entry.domain), entry.ns, entry.nstid,
             entry.isHyp);

     if (table[size - 1].valid)
         DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x "
                 "size: %#x ap:%d ns:%d nstid:%d g:%d isHyp:%d el: %d\n",
                 table[size-1].vpn << table[size-1].N, table[size-1].asid,
                 table[size-1].vmid, table[size-1].pfn << table[size-1].N,
                 table[size-1].size, table[size-1].ap, table[size-1].ns,
                 table[size-1].nstid, table[size-1].global, table[size-1].isHyp,
                 table[size-1].el);

     // inserting to MRU position and evicting the LRU one
     for (int i = size - 1; i > 0; --i)
         table[i] = table[i-1];
     table[0] = entry;

     stats.inserts++;
     ppRefills->notify(1);
 }

 void
 TLB::multiInsert(TlbEntry &entry)
 {
     // Insert a partial translation only if the TLB is configured
     // as a walk cache
     if (!entry.partial || partialLevels[entry.lookupLevel]) {
         insert(entry);
     }

     if (auto next_level = static_cast<TLB*>(nextLevel())) {
         next_level->multiInsert(entry);
     }
 }

 void
 TLB::printTlb() const
 {
     int x = 0;
     TlbEntry *te;
     DPRINTF(TLB, "Current TLB contents:\n");
     while (x < size) {
         te = &table[x];
         if (te->valid)
             DPRINTF(TLB, " *  %s\n", te->print());
         ++x;
     }
 }

 void
 TLB::flushAll()
 {
     DPRINTF(TLB, "Flushing all TLB entries\n");
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];

         if (te->valid) {
             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const TLBIALL& tlbi_op)
 {
     DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];
         const bool el_match = te->checkELMatch(
             tlbi_op.targetEL, tlbi_op.inHost);
         if (te->valid && tlbi_op.secureLookup == !te->nstid &&
             (te->vmid == vmid || tlbi_op.el2Enabled) && el_match) {

             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const ITLBIALL& tlbi_op)
 {
     DPRINTF(TLB, "Flushing all ITLB entries (%s lookup)\n",
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];
         const bool el_match = te->checkELMatch(
             tlbi_op.targetEL, tlbi_op.inHost);
         if (te->type & TypeTLB::instruction && te->valid &&
             tlbi_op.secureLookup == !te->nstid &&
             (te->vmid == vmid || tlbi_op.el2Enabled) && el_match) {

             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const DTLBIALL& tlbi_op)
 {
     DPRINTF(TLB, "Flushing all DTLB entries (%s lookup)\n",
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];
         const bool el_match = te->checkELMatch(
             tlbi_op.targetEL, tlbi_op.inHost);
         if (te->type & TypeTLB::data && te->valid &&
             tlbi_op.secureLookup == !te->nstid &&
             (te->vmid == vmid || tlbi_op.el2Enabled) && el_match) {

             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const TLBIALLEL &tlbi_op)
 {
     DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];
         const bool el_match = te->checkELMatch(
             tlbi_op.targetEL, tlbi_op.inHost);
         if (te->valid && tlbi_op.secureLookup == !te->nstid && el_match) {

             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const TLBIVMALL &tlbi_op)
 {
     DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];
         const bool el_match = te->checkELMatch(
             tlbi_op.targetEL, tlbi_op.inHost);

         const bool vmid_match =
             te->vmid == vmid ||
             !tlbi_op.el2Enabled ||
             (!tlbi_op.stage2Flush() && tlbi_op.inHost);

         if (te->valid && tlbi_op.secureLookup == !te->nstid &&
             el_match && vmid_match) {

             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const TLBIALLN &tlbi_op)
 {
     bool hyp = tlbi_op.targetEL == EL2;

     DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n",
             (hyp ? "hyp" : "non-hyp"));
     int x = 0;
     TlbEntry *te;
     while (x < size) {
         te = &table[x];
         const bool el_match = te->checkELMatch(tlbi_op.targetEL, false);

         if (te->valid && te->nstid && te->isHyp == hyp && el_match) {

             DPRINTF(TLB, " -  %s\n", te->print());
             stats.flushedEntries++;
             te->valid = false;
         }
         ++x;
     }

     stats.flushTlb++;
 }

 void
 TLB::flush(const TLBIMVA &tlbi_op)
 {
     DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x "
             "(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     _flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
         tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified);
     stats.flushTlbMvaAsid++;
 }

 void
 TLB::flush(const ITLBIMVA &tlbi_op)
 {
     DPRINTF(TLB, "Flushing ITLB entries with mva: %#x, asid: %#x "
             "(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     _flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
         tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::instruction);
     stats.flushTlbMvaAsid++;
 }

 void
 TLB::flush(const DTLBIMVA &tlbi_op)
 {
     DPRINTF(TLB, "Flushing DTLB entries with mva: %#x, asid: %#x "
             "(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     _flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
         tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::data);
     stats.flushTlbMvaAsid++;
 }

 void
 TLB::flush(const TLBIASID &tlbi_op)
 {
     DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n",
             tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));

     int x = 0 ;
     TlbEntry *te;

     while (x < size) {
         te = &table[x];

         const bool el_match = te->checkELMatch(
             tlbi_op.targetEL, tlbi_op.inHost);

         const bool vmid_match =
             te->vmid == vmid || !tlbi_op.el2Enabled || tlbi_op.inHost;

         if (te->valid && te->asid == tlbi_op.asid &&
             tlbi_op.secureLookup == !te->nstid &&
             vmid_match && el_match) {

             te->valid = false;
             DPRINTF(TLB, " -  %s\n", te->print());
             stats.flushedEntries++;
         }
         ++x;
     }
     stats.flushTlbAsid++;
 }

 void
 TLB::flush(const ITLBIASID &tlbi_op)
 {
     DPRINTF(TLB, "Flushing ITLB entries with asid: %#x (%s lookup)\n",
             tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));

     int x = 0 ;
     TlbEntry *te;

     while (x < size) {
         te = &table[x];
         if (te->type & TypeTLB::instruction &&
             te->valid && te->asid == tlbi_op.asid &&
             tlbi_op.secureLookup == !te->nstid &&
             (te->vmid == vmid || tlbi_op.el2Enabled) &&
             te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) {

             te->valid = false;
             DPRINTF(TLB, " -  %s\n", te->print());
             stats.flushedEntries++;
         }
         ++x;
     }
     stats.flushTlbAsid++;
 }

 void
 TLB::flush(const DTLBIASID &tlbi_op)
 {
     DPRINTF(TLB, "Flushing DTLB entries with asid: %#x (%s lookup)\n",
             tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));

     int x = 0 ;
     TlbEntry *te;

     while (x < size) {
         te = &table[x];
         if (te->type & TypeTLB::data &&
             te->valid && te->asid == tlbi_op.asid &&
             tlbi_op.secureLookup == !te->nstid &&
             (te->vmid == vmid || tlbi_op.el2Enabled) &&
             te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) {

             te->valid = false;
             DPRINTF(TLB, " -  %s\n", te->print());
             stats.flushedEntries++;
         }
         ++x;
     }
     stats.flushTlbAsid++;
 }

 void
 TLB::flush(const TLBIMVAA &tlbi_op) {

     DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n",
             tlbi_op.addr,
             (tlbi_op.secureLookup ? "secure" : "non-secure"));
     _flushMva(tlbi_op.addr, 0xbeef, tlbi_op.secureLookup, true,
         tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified);
     stats.flushTlbMva++;
 }

 void
 TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup,
                bool ignore_asn, ExceptionLevel target_el, bool in_host,
                TypeTLB entry_type)
 {
     TlbEntry *te;
     Lookup lookup_data;

     lookup_data.va = sext<56>(mva);
     lookup_data.asn = asn;
     lookup_data.ignoreAsn = ignore_asn;
     lookup_data.vmid = vmid;
     lookup_data.hyp = target_el == EL2;
     lookup_data.secure = secure_lookup;
     lookup_data.functional = true;
     lookup_data.targetEL = target_el;
     lookup_data.inHost = in_host;
     lookup_data.mode = BaseMMU::Read;

     te = lookup(lookup_data);
     while (te != NULL) {
         bool matching_type = (te->type & entry_type);
         if (matching_type && secure_lookup == !te->nstid) {
             DPRINTF(TLB, " -  %s\n", te->print());
             te->valid = false;
             stats.flushedEntries++;
         }
         te = lookup(lookup_data);
     }
 }

 void
 TLB::takeOverFrom(BaseTLB *_otlb)
 {
 }

 TLB::TlbStats::TlbStats(TLB &parent)
   : statistics::Group(&parent), tlb(parent),
     ADD_STAT(partialHits, statistics::units::Count::get(),
              "partial translation hits"),
     ADD_STAT(instHits, statistics::units::Count::get(), "Inst hits"),
     ADD_STAT(instMisses, statistics::units::Count::get(), "Inst misses"),
     ADD_STAT(readHits, statistics::units::Count::get(), "Read hits"),
     ADD_STAT(readMisses, statistics::units::Count::get(),  "Read misses"),
     ADD_STAT(writeHits, statistics::units::Count::get(), "Write hits"),
     ADD_STAT(writeMisses, statistics::units::Count::get(), "Write misses"),
     ADD_STAT(inserts, statistics::units::Count::get(),
              "Number of times an entry is inserted into the TLB"),
     ADD_STAT(flushTlb, statistics::units::Count::get(),
              "Number of times complete TLB was flushed"),
     ADD_STAT(flushTlbMva, statistics::units::Count::get(),
              "Number of times TLB was flushed by MVA"),
     ADD_STAT(flushTlbMvaAsid, statistics::units::Count::get(),
              "Number of times TLB was flushed by MVA & ASID"),
     ADD_STAT(flushTlbAsid, statistics::units::Count::get(),
              "Number of times TLB was flushed by ASID"),
     ADD_STAT(flushedEntries, statistics::units::Count::get(),
              "Number of entries that have been flushed from TLB"),
     ADD_STAT(readAccesses, statistics::units::Count::get(), "Read accesses",
              readHits + readMisses),
     ADD_STAT(writeAccesses, statistics::units::Count::get(), "Write accesses",
              writeHits + writeMisses),
     ADD_STAT(instAccesses, statistics::units::Count::get(), "Inst accesses",
              instHits + instMisses),
     ADD_STAT(hits, statistics::units::Count::get(),
              "Total TLB (inst and data) hits",
              readHits + writeHits + instHits),
     ADD_STAT(misses, statistics::units::Count::get(),
              "Total TLB (inst and data) misses",
              readMisses + writeMisses + instMisses),
     ADD_STAT(accesses, statistics::units::Count::get(),
              "Total TLB (inst and data) accesses",
              readAccesses + writeAccesses + instAccesses)
 {
     // If this is a pure Data TLB, mark the instruction
     // stats as nozero, so that they won't make it in
     // into the final stats file
     if (tlb.type() == TypeTLB::data) {
         instHits.flags(statistics::nozero);
         instMisses.flags(statistics::nozero);

         instAccesses.flags(statistics::nozero);
     }

     // If this is a pure Instruction TLB, mark the data
     // stats as nozero, so that they won't make it in
     // into the final stats file
     if (tlb.type() & TypeTLB::instruction) {
         readHits.flags(statistics::nozero);
         readMisses.flags(statistics::nozero);
         writeHits.flags(statistics::nozero);
         writeMisses.flags(statistics::nozero);

         readAccesses.flags(statistics::nozero);
         writeAccesses.flags(statistics::nozero);
     }

     partialHits.flags(statistics::nozero);
 }

 void
 TLB::regProbePoints()
 {
     ppRefills.reset(new probing::PMU(getProbeManager(), "Refills"));
 }

 Port *
 TLB::getTableWalkerPort()
 {
     return &tableWalker->getTableWalkerPort();
 }

 } // namespace gem5
	/*
	* Copyright (c) 2010-2013, 2016-2021 Arm Limited
	* All rights reserved
	*
	* The license below extends only to copyright in the software and shall
	* not be construed as granting a license to any other intellectual
	* property including but not limited to intellectual property relating
	* to a hardware implementation of the functionality of the software
	* licensed hereunder. You may use the software subject to the license
	* terms below provided that you ensure that this notice is replicated
	* unmodified and in its entirety in all distributions of the software,
	* modified or unmodified, in source code or in binary form.
	*
	* Copyright (c) 2001-2005 The Regents of The University of Michigan
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are
	* met: redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer;
	* redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution;
	* neither the name of the copyright holders nor the names of its
	* contributors may be used to endorse or promote products derived from
	* this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include "arch/arm/tlb.hh"

	#include <memory>
	#include <string>
	#include <vector>

	#include "arch/arm/table_walker.hh"
	#include "arch/arm/tlbi_op.hh"
	#include "arch/arm/utility.hh"
	#include "base/trace.hh"
	#include "cpu/thread_context.hh"
	#include "debug/TLB.hh"
	#include "debug/TLBVerbose.hh"
	#include "params/ArmTLB.hh"

	namespace gem5
	{

	using namespace ArmISA;

	TLB::TLB(const ArmTLBParams &p)
	: BaseTLB(p), table(new TlbEntry[p.size]), size(p.size),
	isStage2(p.is_stage2),
	_walkCache(false),
	tableWalker(nullptr),
	stats(*this), rangeMRU(1), vmid(0)
	{
	for (int lvl = LookupLevel::L0;
	lvl < LookupLevel::Num_ArmLookupLevel; lvl++) {

	auto it = std::find(
	p.partial_levels.begin(),
	p.partial_levels.end(),
	lvl);

	auto lookup_lvl = static_cast<LookupLevel>(lvl);

	if (it != p.partial_levels.end()) {
	// A partial entry from of the current LookupLevel can be
	// cached within the TLB
	partialLevels[lookup_lvl] = true;

	// Make sure this is not the last level (complete translation)
	if (lvl != LookupLevel::Num_ArmLookupLevel - 1) {
	_walkCache = true;
	}
	} else {
	partialLevels[lookup_lvl] = false;
	}
	}
	}

	TLB::~TLB()
	{
	delete[] table;
	}

	void
	TLB::setTableWalker(TableWalker *table_walker)
	{
	tableWalker = table_walker;
	tableWalker->setTlb(this);
	}

	TlbEntry*
	TLB::match(const Lookup &lookup_data)
	{
	// Vector of TLB entry candidates.
	// Only one of them will be assigned to retval and will
	// be returned to the MMU (in case of a hit)
	// The vector has one entry per lookup level as it stores
	// both complete and partial matches
	std::vector<std::pair<int, const TlbEntry*>> hits{
	LookupLevel::Num_ArmLookupLevel, {0, nullptr}};

	int x = 0;
	while (x < size) {
	if (table[x].match(lookup_data)) {
	const TlbEntry &entry = table[x];
	hits[entry.lookupLevel] = std::make_pair(x, &entry);

	// This is a complete translation, no need to loop further
	if (!entry.partial)
	break;
	}
	++x;
	}

	// Loop over the list of TLB entries matching our translation
	// request, starting from the highest lookup level (complete
	// translation) and iterating backwards (using reverse iterators)
	for (auto it = hits.rbegin(); it != hits.rend(); it++) {
	const auto& [idx, entry] = *it;
	if (!entry) {
	// No match for the current LookupLevel
	continue;
	}

	// Maintaining LRU array
	// We only move the hit entry ahead when the position is higher
	// than rangeMRU
	if (idx > rangeMRU && !lookup_data.functional) {
	TlbEntry tmp_entry = *entry;
	for (int i = idx; i > 0; i--)
	table[i] = table[i - 1];
	table[0] = tmp_entry;
	return &table[0];
	} else {
	return &table[idx];
	}
	}

	return nullptr;
	}

	TlbEntry*
	TLB::lookup(const Lookup &lookup_data)
	{
	const auto mode = lookup_data.mode;

	TlbEntry *retval = match(lookup_data);

	DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x hyp %d secure %d "
	"ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d "
	"el: %d\n",
	lookup_data.va, lookup_data.asn, retval ? "hit" : "miss",
	lookup_data.vmid, lookup_data.hyp, lookup_data.secure,
	retval ? retval->pfn : 0, retval ? retval->size : 0,
	retval ? retval->pAddr(lookup_data.va) : 0,
	retval ? retval->ap : 0,
	retval ? retval->ns : 0, retval ? retval->nstid : 0,
	retval ? retval->global : 0, retval ? retval->asid : 0,
	retval ? retval->el : 0);

	// Updating stats if this was not a functional lookup
	if (!lookup_data.functional) {
	if (!retval) {
	if (mode == BaseMMU::Execute) {
	stats.instMisses++;
	} else if (mode == BaseMMU::Write) {
	stats.writeMisses++;
	} else {
	stats.readMisses++;
	}
	} else {
	if (retval->partial) {
	stats.partialHits++;
	}

	if (mode == BaseMMU::Execute) {
	stats.instHits++;
	} else if (mode == BaseMMU::Write) {
	stats.writeHits++;
	} else {
	stats.readHits++;
	}
	}
	}

	return retval;
	}

	TlbEntry*
	TLB::multiLookup(const Lookup &lookup_data)
	{
	TlbEntry* te = lookup(lookup_data);

	if (te) {
	checkPromotion(te, lookup_data.mode);
	} else {
	if (auto tlb = static_cast<TLB*>(nextLevel())) {
	te = tlb->multiLookup(lookup_data);
	if (te && !lookup_data.functional &&
	(!te->partial \|\| partialLevels[te->lookupLevel])) {
	// Insert entry only if this is not a functional
	// lookup and if the translation is complete (unless this
	// TLB caches partial translations)
	insert(*te);
	}
	}
	}

	return te;
	}

	void
	TLB::checkPromotion(TlbEntry *entry, BaseMMU::Mode mode)
	{
	TypeTLB acc_type = (mode == BaseMMU::Execute) ?
	TypeTLB::instruction : TypeTLB::data;

	// Hitting an instruction TLB entry on a data access or
	// a data TLB entry on an instruction access:
	// promoting the entry to unified
	if (!(entry->type & acc_type))
	entry->type = TypeTLB::unified;
	}

	// insert a new TLB entry
	void
	TLB::insert(TlbEntry &entry)
	{
	DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x"
	" asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d"
	" ap:%#x domain:%#x ns:%d nstid:%d isHyp:%d\n", entry.pfn,
	entry.size, entry.vpn, entry.asid, entry.vmid, entry.N,
	entry.global, entry.valid, entry.nonCacheable, entry.xn,
	entry.ap, static_cast<uint8_t>(entry.domain), entry.ns, entry.nstid,
	entry.isHyp);

	if (table[size - 1].valid)
	DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x "
	"size: %#x ap:%d ns:%d nstid:%d g:%d isHyp:%d el: %d\n",
	table[size-1].vpn << table[size-1].N, table[size-1].asid,
	table[size-1].vmid, table[size-1].pfn << table[size-1].N,
	table[size-1].size, table[size-1].ap, table[size-1].ns,
	table[size-1].nstid, table[size-1].global, table[size-1].isHyp,
	table[size-1].el);

	// inserting to MRU position and evicting the LRU one
	for (int i = size - 1; i > 0; --i)
	table[i] = table[i-1];
	table[0] = entry;

	stats.inserts++;
	ppRefills->notify(1);
	}

	void
	TLB::multiInsert(TlbEntry &entry)
	{
	// Insert a partial translation only if the TLB is configured
	// as a walk cache
	if (!entry.partial \|\| partialLevels[entry.lookupLevel]) {
	insert(entry);
	}

	if (auto next_level = static_cast<TLB*>(nextLevel())) {
	next_level->multiInsert(entry);
	}
	}

	void
	TLB::printTlb() const
	{
	int x = 0;
	TlbEntry *te;
	DPRINTF(TLB, "Current TLB contents:\n");
	while (x < size) {
	te = &table[x];
	if (te->valid)
	DPRINTF(TLB, " * %s\n", te->print());
	++x;
	}
	}

	void
	TLB::flushAll()
	{
	DPRINTF(TLB, "Flushing all TLB entries\n");
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];

	if (te->valid) {
	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const TLBIALL& tlbi_op)
	{
	DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];
	const bool el_match = te->checkELMatch(
	tlbi_op.targetEL, tlbi_op.inHost);
	if (te->valid && tlbi_op.secureLookup == !te->nstid &&
	(te->vmid == vmid \|\| tlbi_op.el2Enabled) && el_match) {

	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const ITLBIALL& tlbi_op)
	{
	DPRINTF(TLB, "Flushing all ITLB entries (%s lookup)\n",
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];
	const bool el_match = te->checkELMatch(
	tlbi_op.targetEL, tlbi_op.inHost);
	if (te->type & TypeTLB::instruction && te->valid &&
	tlbi_op.secureLookup == !te->nstid &&
	(te->vmid == vmid \|\| tlbi_op.el2Enabled) && el_match) {

	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const DTLBIALL& tlbi_op)
	{
	DPRINTF(TLB, "Flushing all DTLB entries (%s lookup)\n",
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];
	const bool el_match = te->checkELMatch(
	tlbi_op.targetEL, tlbi_op.inHost);
	if (te->type & TypeTLB::data && te->valid &&
	tlbi_op.secureLookup == !te->nstid &&
	(te->vmid == vmid \|\| tlbi_op.el2Enabled) && el_match) {

	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const TLBIALLEL &tlbi_op)
	{
	DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];
	const bool el_match = te->checkELMatch(
	tlbi_op.targetEL, tlbi_op.inHost);
	if (te->valid && tlbi_op.secureLookup == !te->nstid && el_match) {

	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const TLBIVMALL &tlbi_op)
	{
	DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];
	const bool el_match = te->checkELMatch(
	tlbi_op.targetEL, tlbi_op.inHost);

	const bool vmid_match =
	te->vmid == vmid \|\|
	!tlbi_op.el2Enabled \|\|
	(!tlbi_op.stage2Flush() && tlbi_op.inHost);

	if (te->valid && tlbi_op.secureLookup == !te->nstid &&
	el_match && vmid_match) {

	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const TLBIALLN &tlbi_op)
	{
	bool hyp = tlbi_op.targetEL == EL2;

	DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n",
	(hyp ? "hyp" : "non-hyp"));
	int x = 0;
	TlbEntry *te;
	while (x < size) {
	te = &table[x];
	const bool el_match = te->checkELMatch(tlbi_op.targetEL, false);

	if (te->valid && te->nstid && te->isHyp == hyp && el_match) {

	DPRINTF(TLB, " - %s\n", te->print());
	stats.flushedEntries++;
	te->valid = false;
	}
	++x;
	}

	stats.flushTlb++;
	}

	void
	TLB::flush(const TLBIMVA &tlbi_op)
	{
	DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x "
	"(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	_flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
	tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified);
	stats.flushTlbMvaAsid++;
	}

	void
	TLB::flush(const ITLBIMVA &tlbi_op)
	{
	DPRINTF(TLB, "Flushing ITLB entries with mva: %#x, asid: %#x "
	"(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	_flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
	tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::instruction);
	stats.flushTlbMvaAsid++;
	}

	void
	TLB::flush(const DTLBIMVA &tlbi_op)
	{
	DPRINTF(TLB, "Flushing DTLB entries with mva: %#x, asid: %#x "
	"(%s lookup)\n", tlbi_op.addr, tlbi_op.asid,
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	_flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false,
	tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::data);
	stats.flushTlbMvaAsid++;
	}

	void
	TLB::flush(const TLBIASID &tlbi_op)
	{
	DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n",
	tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));

	int x = 0 ;
	TlbEntry *te;

	while (x < size) {
	te = &table[x];

	const bool el_match = te->checkELMatch(
	tlbi_op.targetEL, tlbi_op.inHost);

	const bool vmid_match =
	te->vmid == vmid \|\| !tlbi_op.el2Enabled \|\| tlbi_op.inHost;

	if (te->valid && te->asid == tlbi_op.asid &&
	tlbi_op.secureLookup == !te->nstid &&
	vmid_match && el_match) {

	te->valid = false;
	DPRINTF(TLB, " - %s\n", te->print());
	stats.flushedEntries++;
	}
	++x;
	}
	stats.flushTlbAsid++;
	}

	void
	TLB::flush(const ITLBIASID &tlbi_op)
	{
	DPRINTF(TLB, "Flushing ITLB entries with asid: %#x (%s lookup)\n",
	tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));

	int x = 0 ;
	TlbEntry *te;

	while (x < size) {
	te = &table[x];
	if (te->type & TypeTLB::instruction &&
	te->valid && te->asid == tlbi_op.asid &&
	tlbi_op.secureLookup == !te->nstid &&
	(te->vmid == vmid \|\| tlbi_op.el2Enabled) &&
	te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) {

	te->valid = false;
	DPRINTF(TLB, " - %s\n", te->print());
	stats.flushedEntries++;
	}
	++x;
	}
	stats.flushTlbAsid++;
	}

	void
	TLB::flush(const DTLBIASID &tlbi_op)
	{
	DPRINTF(TLB, "Flushing DTLB entries with asid: %#x (%s lookup)\n",
	tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure"));

	int x = 0 ;
	TlbEntry *te;

	while (x < size) {
	te = &table[x];
	if (te->type & TypeTLB::data &&
	te->valid && te->asid == tlbi_op.asid &&
	tlbi_op.secureLookup == !te->nstid &&
	(te->vmid == vmid \|\| tlbi_op.el2Enabled) &&
	te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) {

	te->valid = false;
	DPRINTF(TLB, " - %s\n", te->print());
	stats.flushedEntries++;
	}
	++x;
	}
	stats.flushTlbAsid++;
	}

	void
	TLB::flush(const TLBIMVAA &tlbi_op) {

	DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n",
	tlbi_op.addr,
	(tlbi_op.secureLookup ? "secure" : "non-secure"));
	_flushMva(tlbi_op.addr, 0xbeef, tlbi_op.secureLookup, true,
	tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified);
	stats.flushTlbMva++;
	}

	void
	TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup,
	bool ignore_asn, ExceptionLevel target_el, bool in_host,
	TypeTLB entry_type)
	{
	TlbEntry *te;
	Lookup lookup_data;

	lookup_data.va = sext<56>(mva);
	lookup_data.asn = asn;
	lookup_data.ignoreAsn = ignore_asn;
	lookup_data.vmid = vmid;
	lookup_data.hyp = target_el == EL2;
	lookup_data.secure = secure_lookup;
	lookup_data.functional = true;
	lookup_data.targetEL = target_el;
	lookup_data.inHost = in_host;
	lookup_data.mode = BaseMMU::Read;

	te = lookup(lookup_data);
	while (te != NULL) {
	bool matching_type = (te->type & entry_type);
	if (matching_type && secure_lookup == !te->nstid) {
	DPRINTF(TLB, " - %s\n", te->print());
	te->valid = false;
	stats.flushedEntries++;
	}
	te = lookup(lookup_data);
	}
	}

	void
	TLB::takeOverFrom(BaseTLB *_otlb)
	{
	}

	TLB::TlbStats::TlbStats(TLB &parent)
	: statistics::Group(&parent), tlb(parent),
	ADD_STAT(partialHits, statistics::units::Count::get(),
	"partial translation hits"),
	ADD_STAT(instHits, statistics::units::Count::get(), "Inst hits"),
	ADD_STAT(instMisses, statistics::units::Count::get(), "Inst misses"),
	ADD_STAT(readHits, statistics::units::Count::get(), "Read hits"),
	ADD_STAT(readMisses, statistics::units::Count::get(), "Read misses"),
	ADD_STAT(writeHits, statistics::units::Count::get(), "Write hits"),
	ADD_STAT(writeMisses, statistics::units::Count::get(), "Write misses"),
	ADD_STAT(inserts, statistics::units::Count::get(),
	"Number of times an entry is inserted into the TLB"),
	ADD_STAT(flushTlb, statistics::units::Count::get(),
	"Number of times complete TLB was flushed"),
	ADD_STAT(flushTlbMva, statistics::units::Count::get(),
	"Number of times TLB was flushed by MVA"),
	ADD_STAT(flushTlbMvaAsid, statistics::units::Count::get(),
	"Number of times TLB was flushed by MVA & ASID"),
	ADD_STAT(flushTlbAsid, statistics::units::Count::get(),
	"Number of times TLB was flushed by ASID"),
	ADD_STAT(flushedEntries, statistics::units::Count::get(),
	"Number of entries that have been flushed from TLB"),
	ADD_STAT(readAccesses, statistics::units::Count::get(), "Read accesses",
	readHits + readMisses),
	ADD_STAT(writeAccesses, statistics::units::Count::get(), "Write accesses",
	writeHits + writeMisses),
	ADD_STAT(instAccesses, statistics::units::Count::get(), "Inst accesses",
	instHits + instMisses),
	ADD_STAT(hits, statistics::units::Count::get(),
	"Total TLB (inst and data) hits",
	readHits + writeHits + instHits),
	ADD_STAT(misses, statistics::units::Count::get(),
	"Total TLB (inst and data) misses",
	readMisses + writeMisses + instMisses),
	ADD_STAT(accesses, statistics::units::Count::get(),
	"Total TLB (inst and data) accesses",
	readAccesses + writeAccesses + instAccesses)
	{
	// If this is a pure Data TLB, mark the instruction
	// stats as nozero, so that they won't make it in
	// into the final stats file
	if (tlb.type() == TypeTLB::data) {
	instHits.flags(statistics::nozero);
	instMisses.flags(statistics::nozero);

	instAccesses.flags(statistics::nozero);
	}

	// If this is a pure Instruction TLB, mark the data
	// stats as nozero, so that they won't make it in
	// into the final stats file
	if (tlb.type() & TypeTLB::instruction) {
	readHits.flags(statistics::nozero);
	readMisses.flags(statistics::nozero);
	writeHits.flags(statistics::nozero);
	writeMisses.flags(statistics::nozero);

	readAccesses.flags(statistics::nozero);
	writeAccesses.flags(statistics::nozero);
	}

	partialHits.flags(statistics::nozero);
	}

	void
	TLB::regProbePoints()
	{
	ppRefills.reset(new probing::PMU(getProbeManager(), "Refills"));
	}

	Port *
	TLB::getTableWalkerPort()
	{
	return &tableWalker->getTableWalkerPort();
	}

	} // namespace gem5