<!DOCTYPE html>
<html>
<head>
	<meta charset="utf-8">
	<meta http-equiv="x-ua-compatible" content="ie=edge">
	<meta name="viewport" content="width=device-width, initial-scale=1">

	<title>gem5</title>

	<!-- SITE FAVICON -->
	<link rel="shortcut icon" type="image/gif" href="/assets/img/gem5ColorVert.gif"/>

	<link rel="canonical" href="http://localhost:4000/configuration/">
	<link href='https://fonts.googleapis.com/css?family=Open+Sans:400,300,700,800,600' rel='stylesheet' type='text/css'>
	<link href='https://fonts.googleapis.com/css?family=Muli:400,300' rel='stylesheet' type='text/css'>

	<!-- FAVICON -->
	<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css">

	<!-- BOOTSTRAP -->
	<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">

	<!-- CUSTOM CSS -->
	<link rel="stylesheet" href="/css/main.css">
</head>


<body>
	<nav class="navbar navbar-expand-md navbar-light bg-light">
  <a class="navbar-brand" href="/">
		<img src="/assets/img/gem5ColorLong.gif" alt="gem5" height=45px>
	</a>
  <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarNavDropdown" aria-controls="navbarNavDropdown" aria-expanded="false" aria-label="Toggle navigation">
    <span class="navbar-toggler-icon"></span>
  </button>
  <div class="collapse navbar-collapse" id="navbarNavDropdown">
    <ul class="navbar-nav ml-auto">
      <li class="nav-item ">
        <a class="nav-link" href="/">Home</a>
      </li>

			<li class="nav-item dropdown ">
				<a class="nav-link dropdown-toggle" href="/about" id="navbarDropdownMenuLink" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
					About
				</a>
				<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
          <a class="dropdown-item" href="/about">About</a>
          <a class="dropdown-item" href="/publications">Publications</a>
          <a class="dropdown-item" href="/governance">Governance</a>
				</div>
			</li>

			<li class="nav-item dropdown active">
				<a class="nav-link dropdown-toggle" href="#" id="navbarDropdownMenuLink" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
					Documentation
				</a>
				<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
					<!-- Pull navigation from _data/documentation.yml -->
					
            <a class="dropdown-item" href="/introduction">Introduction</a>
					
            <a class="dropdown-item" href="/building">Getting Started</a>
					
            <a class="dropdown-item" href="/environment">Modifying/Extending</a>
					
            <a class="dropdown-item" href="/MSIintro">Modeling Cache Coherence with Ruby</a>
					
				</div>
			</li>

      <li class="nav-item ">
        <a class="nav-link" href="/contributing">Contributing</a>
      </li>

			<li class="nav-item ">
        <a class="nav-link" href="/search">Search</a>
      </li>
    </ul>
  </div>
</nav>

	<main>
		<div class="sidenav-top">
  <img src="/assets/img/gem5ColorLong.gif" height="80">
  <div class="search">
    <form action="/search" method="get">
      <!-- <label for="search-box"><i class="fa fa-search"></i></label> -->
      <input type="text" name="query">
      <button type="submit" name="submit"><i class="fa fa-search"></i></button>
    </form>
  </div>
</div>
<div class="sidenav">
  <!-- Pull navigation from _data/documentation.yml -->
  
    <a class="item" href="/introduction" role="button" aria-expanded="false" aria-controls="collapseExample">
      Introduction
    </a>
    <div class="collapse " id="introduction">
      
    </div>
  
    <a class="item" data-toggle="collapse" href="#pt1" role="button" aria-expanded="false" aria-controls="collapseExample">
      Getting Started
    </a>
    <div class="collapse " id="pt1">
      
        <a class="subitem " href="/building">Building gem5</a>
      
        <a class="subitem " href="/simple_config">Creating a simple configuration script</a>
      
        <a class="subitem " href="/cache_config">Adding cache to configuration script</a>
      
        <a class="subitem " href="/gem5_stats">Understanding gem5 statistics and output</a>
      
        <a class="subitem " href="/example_configs">Using the default configuration scripts</a>
      
    </div>
  
    <a class="item" data-toggle="collapse" href="#pt2" role="button" aria-expanded="false" aria-controls="collapseExample">
      Modifying/Extending
    </a>
    <div class="collapse " id="pt2">
      
        <a class="subitem " href="/environment">Setting up your development environment</a>
      
        <a class="subitem " href="/helloobject">Creating a very simple SimObject</a>
      
        <a class="subitem " href="/debugging">Debugging gem5</a>
      
        <a class="subitem " href="/events">Event-driven programming</a>
      
        <a class="subitem " href="/parameters">Adding parameters to SimObjects and more events</a>
      
        <a class="subitem " href="/memoryobject">Creating SimObjects in the memory system</a>
      
        <a class="subitem " href="/simplecache">Creating a simple cache object</a>
      
    </div>
  
    <a class="item" data-toggle="collapse" href="#pt3" role="button" aria-expanded="false" aria-controls="collapseExample">
      Modeling Cache Coherence with Ruby
    </a>
    <div class="collapse show" id="pt3">
      
        <a class="subitem " href="/MSIintro">Introduction to Ruby</a>
      
        <a class="subitem " href="/cache-intro">MSI example cache protocol</a>
      
        <a class="subitem " href="/cache-declarations">Declaring a state machine</a>
      
        <a class="subitem " href="/cache-in-ports">In port code blocks</a>
      
        <a class="subitem " href="/cache-actions">Action code blocks</a>
      
        <a class="subitem " href="/cache-transitions">Transition code blocks</a>
      
        <a class="subitem " href="/directory">MSI Directory implementation</a>
      
        <a class="subitem " href="/MSIbuilding">Compiling a SLICC protocol</a>
      
        <a class="subitem active" href="/configuration">Configuring a simple Ruby system</a>
      
        <a class="subitem " href="/running">Running the simple Ruby system</a>
      
        <a class="subitem " href="/MSIdebugging">Debugging SLICC Protocols</a>
      
        <a class="subitem " href="/simple-MI_example">Configuring for a standard protocol</a>
      
    </div>
  
</div>

<div class="container" id="doc-container">
  <div class="edit"><a href="https://github.com/new-website/tree/master/_pages/documentation/part3/configuration.md">Edit this page</a></div>
  <dl>
  <dt>authors</dt>
  <dd>Jason Lowe-Power</dd>
</dl>

<h1 id="configuring-a-simple-ruby-system">Configuring a simple Ruby system</h1>

<p>First, create a new configuration directory in <code class="highlighter-rouge">configs/</code>. Just like all
gem5 configuration files, we will have a configuration run script. For
the run script, we can start with <code class="highlighter-rouge">simple.py</code> from
simple-config-chapter. Copy this file to <code class="highlighter-rouge">simple_ruby.py</code> in your new
directory.</p>

<p>We will make a couple of small changes to this file to use Ruby instead
of directly connecting the CPU to the memory controllers.</p>

<p>First, so we can test our <em>coherence</em> protocol, let’s use two CPUs.</p>

<p>``` {.sourceCode .python}
system.cpu = [TimingSimpleCPU(), TimingSimpleCPU()]</p>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>
Next, after the memory controllers have been instantiated, we are going
to create the cache system and set up all of the caches. Add the
following lines *after the CPU interrupts have been created, but before
instantiating the system*.

``` {.sourceCode .python}
system.caches = MyCacheSystem()
system.caches.setup(system, system.cpu, [system.mem_ctrl])
</code></pre></div></div>

<p>Like the classic cache example in cache-config-chapter, we are going to
create a second file that contains the cache configuration code. In this
file we are going to have a class called <code class="highlighter-rouge">MyCacheSystem</code> and we will
create a <code class="highlighter-rouge">setup</code> function that takes as parameters the CPUs in the
system and the memory controllers.</p>

<p>You can download the complete run script
here &lt;../../_static/scripts/part3/configs/simple_ruby.py&gt;</p>

<h2 id="cache-system-configuration">Cache system configuration</h2>

<p>Now, let’s create a file <code class="highlighter-rouge">msi_caches.py</code>. In this file, we will create
four classes: <code class="highlighter-rouge">MyCacheSystem</code> which will inherit from <code class="highlighter-rouge">RubySystem</code>,
<code class="highlighter-rouge">L1Cache</code> and <code class="highlighter-rouge">Directory</code> which will inherit from the SimObjects created
by SLICC from our two state machines, and <code class="highlighter-rouge">MyNetwork</code> which will inherit
from <code class="highlighter-rouge">SimpleNetwork</code>.</p>

<h3 id="l1-cache">L1 Cache</h3>

<p>Let’s start with the <code class="highlighter-rouge">L1Cache</code>. First, we will inherit from
<code class="highlighter-rouge">L1Cache_Controller</code> since we named our L1 cache “L1Cache” in the state
machine file. We also include a special class variable and class method
for tracking the “version number”. For each SLICC state machine, you
have to number them in ascending order from 0. Each machine of the same
type should have a unique version number. This is used to differentiate
the individual machines. (Hopefully, in the future this requirement will
be removed.)</p>

<p>``` {.sourceCode .python}
class L1Cache(L1Cache_Controller):</p>

<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>_version = 0
@classmethod
def versionCount(cls):
    cls._version += 1 # Use count for this particular type
    return cls._version - 1 ```
</code></pre></div></div>

<p>Next, we implement the constructor for the class.</p>

<p>``` {.sourceCode .python}
def <strong>init</strong>(self, system, ruby_system, cpu):
    super(L1Cache, self).<strong>init</strong>()</p>

<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>self.version = self.versionCount()
self.cacheMemory = RubyCache(size = '16kB',
                       assoc = 8,
                       start_index_bit = self.getBlockSizeBits(system))
self.clk_domain = cpu.clk_domain
self.send_evictions = self.sendEvicts(cpu)
self.ruby_system = ruby_system
self.connectQueues(ruby_system) ```
</code></pre></div></div>

<p>We need the CPUs in this function to grab the clock domain and system is
needed for the cache block size. Here, we set all of the parameters that
we named in the state machine file (e.g., <code class="highlighter-rouge">cacheMemory</code>). We will set
<code class="highlighter-rouge">sequencer</code> later. We also hardcode the size an associativity of the
cache. You could add command line parameters for these options, if it is
important to vary them at runtime.</p>

<p>Next, we implement a couple of helper functions. First, we need to
figure out how many bits of the address to use for indexing into the
cache, which is a simple log operation. We also need to decide whether
to send eviction notices to the CPU. Only if we are using the
out-of-order CPU and using x86 or ARM ISA should we forward evictions.</p>

<p>``` {.sourceCode .python}
def getBlockSizeBits(self, system):
    bits = int(math.log(system.cache_line_size, 2))
    if 2**bits != system.cache_line_size.value:
        panic(“Cache line size not a power of 2!”)
    return bits</p>

<p>def sendEvicts(self, cpu):
    “"”True if the CPU model or ISA requires sending evictions from caches
       to the CPU. Two scenarios warrant forwarding evictions to the CPU:
       1. The O3 model must keep the LSQ coherent with the caches
       2. The x86 mwait instruction is built on top of coherence
       3. The local exclusive monitor in ARM systems
    “””
    if type(cpu) is DerivO3CPU or \
       buildEnv[‘TARGET_ISA’] in (‘x86’, ‘arm’):
        return True
    return False</p>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>
Finally, we need to implement `connectQueues` to connect all of the
message buffers to the Ruby network. First, we create a message buffer
for the mandatory queue. Since this is an L1 cache and it will have a
sequencer, we need to instantiate this special message buffer. Next, we
instantiate a message buffer for each buffer in the controller. All of
the "to" buffers we must set the "master" to the network (i.e., the
buffer will send messages into the network), and all of the "from"
buffers we must set the "slave" to the network. These *names* are the
same as the gem5 ports, but *message buffers are not currently
implemented as gem5 ports*. In this protocol, we are assuming the
message buffers are ordered for simplicity.

``` {.sourceCode .python}
def connectQueues(self, ruby_system):
    self.mandatoryQueue = MessageBuffer()

    self.requestToDir = MessageBuffer(ordered = True)
    self.requestToDir.master = ruby_system.network.slave
    self.responseToDirOrSibling = MessageBuffer(ordered = True)
    self.responseToDirOrSibling.master = ruby_system.network.slave
    self.forwardFromDir = MessageBuffer(ordered = True)
    self.forwardFromDir.slave = ruby_system.network.master
    self.responseFromDirOrSibling = MessageBuffer(ordered = True)
    self.responseFromDirOrSibling.slave = ruby_system.network.master
</code></pre></div></div>

<h3 id="directory">Directory</h3>

<p>Now, we can similarly implement the directory. There are three
differences from the L1 cache. First, we need to set the address ranges
for the directory. Since each directory corresponds to a particular
memory controller for a subset of the address range (possibly), we need
to make sure the ranges match. The default address ranges for Ruby
controllers is <code class="highlighter-rouge">AllMemory</code>.</p>

<p>Next, we need to set the master port <code class="highlighter-rouge">memory</code>. This is the port that
sends messages when <code class="highlighter-rouge">queueMemoryRead/Write</code> is called in the SLICC code.
We set it the to the memory controller port. Similarly, in
<code class="highlighter-rouge">connectQueues</code> we need to instantiate the special message buffer
<code class="highlighter-rouge">responseFromMemory</code> like the <code class="highlighter-rouge">mandatoryQueue</code> in the L1 cache.</p>

<p>``` {.sourceCode .python}
class DirController(Directory_Controller):</p>

<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>_version = 0
@classmethod
def versionCount(cls):
    cls._version += 1 # Use count for this particular type
    return cls._version - 1

def __init__(self, ruby_system, ranges, mem_ctrls):
    """ranges are the memory ranges assigned to this controller.
    """
    if len(mem_ctrls) &gt; 1:
        panic("This cache system can only be connected to one mem ctrl")
    super(DirController, self).__init__()
    self.version = self.versionCount()
    self.addr_ranges = ranges
    self.ruby_system = ruby_system
    self.directory = RubyDirectoryMemory()
    # Connect this directory to the memory side.
    self.memory = mem_ctrls[0].port
    self.connectQueues(ruby_system)

def connectQueues(self, ruby_system):
    self.requestFromCache = MessageBuffer(ordered = True)
    self.requestFromCache.slave = ruby_system.network.master
    self.responseFromCache = MessageBuffer(ordered = True)
    self.responseFromCache.slave = ruby_system.network.master

    self.responseToCache = MessageBuffer(ordered = True)
    self.responseToCache.master = ruby_system.network.slave
    self.forwardToCache = MessageBuffer(ordered = True)
    self.forwardToCache.master = ruby_system.network.slave

    self.responseFromMemory = MessageBuffer() ```
</code></pre></div></div>

<h3 id="ruby-system">Ruby System</h3>

<p>Now, we can implement the Ruby system object. For this object, the
constructor is simple. It just checks the SCons variable <code class="highlighter-rouge">PROTOCOL</code> to
be sure that we are using the right configuration file for the protocol
that was compiled. We cannot create the controllers in the constructor
because they require a pointer to the this object. If we were to create
them in the constructor, there would be a circular dependence in the
SimObject hierarchy which will cause infinite recursion in when the
system in instantiated with <code class="highlighter-rouge">m5.instantiate</code>.</p>

<p>``` {.sourceCode .python}
class MyCacheSystem(RubySystem):</p>

<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>def __init__(self):
    if buildEnv['PROTOCOL'] != 'MSI':
        fatal("This system assumes MSI from learning gem5!")

    super(MyCacheSystem, self).__init__() ```
</code></pre></div></div>

<p>Instead of create the controllers in the constructor, we create a new
function to create all of the needed objects: <code class="highlighter-rouge">setup</code>. First, we create
the network. We will look at this object next. With the network, we need
to set the number of virtual networks in the system.</p>

<p>Next, we instantiate all of the controllers. Here, we use a single
global list of the controllers to make it easier to connect them to the
network later. However, for more complicated cache topologies, it can
make sense to use multiple lists of controllers. We create one L1 cache
for each CPU and one directory for the system.</p>

<p>Then, we instantiate all of the sequencers, one for each CPU. Each
sequencer needs a pointer to the instruction and data cache to simulate
the correct latency when initially accessing the cache. In more
complicated systems, you also have to create sequencers for other
objects like DMA controllers.</p>

<p>After creating the sequencers, we set the sequencer variable on each L1
cache controller.</p>

<p>Then, we connect all of the controllers to the network and call the
<code class="highlighter-rouge">setup_buffers</code> function on the network.</p>

<p>We then have to set the “port proxy” for both the Ruby system and the
<code class="highlighter-rouge">system</code> for making functional accesses (e.g., loading the binary in SE
mode).</p>

<p>Finally, we connect all of the CPUs to the ruby system. In this example,
we assume that there are only CPU sequencers so the first CPU is
connected to the first sequencer, and so on. We also have to connect the
TLBs and interrupt ports (if we are using x86).</p>

<p>``` {.sourceCode .python}
def setup(self, system, cpus, mem_ctrls):
    self.network = MyNetwork(self)</p>

<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>self.number_of_virtual_networks = 3
self.network.number_of_virtual_networks = 3

self.controllers = \
    [L1Cache(system, self, cpu) for cpu in cpus] + \
    [DirController(self, system.mem_ranges, mem_ctrls)]

self.sequencers = [RubySequencer(version = i,
                        # I/D cache is combined and grab from ctrl
                        icache = self.controllers[i].cacheMemory,
                        dcache = self.controllers[i].cacheMemory,
                        clk_domain = self.controllers[i].clk_domain,
                        ) for i in range(len(cpus))]

for i,c in enumerate(self.controllers[0:len(self.sequencers)]):
    c.sequencer = self.sequencers[i]

self.num_of_sequencers = len(self.sequencers)

self.network.connectControllers(self.controllers)
self.network.setup_buffers()

self.sys_port_proxy = RubyPortProxy()
system.system_port = self.sys_port_proxy.slave

for i,cpu in enumerate(cpus):
    cpu.icache_port = self.sequencers[i].slave
    cpu.dcache_port = self.sequencers[i].slave
    isa = buildEnv['TARGET_ISA']
    if isa == 'x86':
        cpu.interrupts[0].pio = self.sequencers[i].master
        cpu.interrupts[0].int_master = self.sequencers[i].slave
        cpu.interrupts[0].int_slave = self.sequencers[i].master
    if isa == 'x86' or isa == 'arm':
        cpu.itb.walker.port = self.sequencers[i].slave
        cpu.dtb.walker.port = self.sequencers[i].slave ```
</code></pre></div></div>

<h3 id="network">Network</h3>

<p>Finally, the last object we have to implement is the network. The
constructor is simple, but we need to declare an empty list for the list
of network interfaces (<code class="highlighter-rouge">netifs</code>).</p>

<p>Most of the code is in <code class="highlighter-rouge">connectControllers</code>. This function implements a
<em>very simple, unrealistic</em> point-to-point network. In other words, every
controller has a direct link to every other controller.</p>

<p>The Ruby network is made of three parts: routers that route data from
one router to another or to external controllers, external links that
link a controller to a router, and internal links that link two routers
together. First, we create a router for each controller. Then, we create
an external link from that router to the controller. Finally, we add all
of the “internal” links. Each router is connected to all other routers
to make the point-to-point network.</p>

<p>``` {.sourceCode .python}
class MyNetwork(SimpleNetwork):</p>

<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>def __init__(self, ruby_system):
    super(MyNetwork, self).__init__()
    self.netifs = []
    self.ruby_system = ruby_system

def connectControllers(self, controllers):
    self.routers = [Switch(router_id = i) for i in range(len(controllers))]

    self.ext_links = [SimpleExtLink(link_id=i, ext_node=c,
                                    int_node=self.routers[i])
                      for i, c in enumerate(controllers)]

    link_count = 0
    self.int_links = []
    for ri in self.routers:
        for rj in self.routers:
            if ri == rj: continue # Don't connect a router to itself!
            link_count += 1
            self.int_links.append(SimpleIntLink(link_id = link_count,
                                                src_node = ri,
                                                dst_node = rj)) ```
</code></pre></div></div>

<p>You can download the complete <code class="highlighter-rouge">msi_caches.py</code> file
here &lt;../../_static/scripts/part3/configs/msi_caches.py&gt;.</p>

  <br>

  <!-- RETRIVE PREVIOUS PAGE LINK -->
  
    
  <!-- RETRIEVE NEXT PAGE LINK -->
  
    
  <div class="navbuttons">
    
      <a href="/MSIbuilding"><button type="button" class="btn btn-outline-primary">PREVIOUS</button></a>
    

      <a href="/running"><button type="button" class="btn btn-outline-primary">NEXT</button></a>
    
  </div>
</div>

	</main>

	<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
	<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
	<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>

	<script>
	  // When the user scrolls down 20px from the top of the document, show the button
	  window.onscroll = function() {scrollFunction()};

	  function scrollFunction() {
	      if (document.body.scrollTop > 100 || document.documentElement.scrollTop > 20) {
	          document.getElementById("myBtn").style.display = "block";
	      } else {
	          document.getElementById("myBtn").style.display = "none";
	      }
	  }

	  // When the user clicks on the button, scroll to the top of the document
	  function topFunction() {
	      document.body.scrollTop = 0;
	      document.documentElement.scrollTop = 0;
	  }
	</script>

</body>

<footer class="page-footer">
	<div class="container">
		<div class="row">

			<div class="col-12 col-sm-4">
				<p><a href="/about">About</a></p>
				<p><a href="/publications">Publications</a></p>
				<p><a href="/contributing">Contributing</a></p>
				<p><a href="/governance">Governance</a></p>
			</div><br>

			<div class="col-12 col-sm-4">
				<p><a href="/introduction">Documentation</a></p>
				<p><a href="http://gem5.org/Documentation">Old Documentation</a></p>
				<p><a href="https://gem5.googlesource.com/public/gem5">Source</a></p>
			</div><br>

			<div class="col-12 col-sm-4">
				<p><a href="/search">Search</a></p>
				<p><a href="#">Mailing Lists</a></p>
				<p><a href="#">Source For This Site</a></p>
			</div>

		</div>
	</div>
</footer>


</html>