mem-ruby: Enable set size increase

Add NUMBER_BITS_PER_SET environment variable to control
the size of the bitmask in Set.hh (default=64).
Necessary for configs which require >64 instances of a given
machine type. This can be set in the build_opts file, e.g.
by adding the following line:
NUMBER_BITS_PER_SET = <number>

Change-Id: I314a3cadca8ce975fcf4a60d9022494751688e88
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18968
Reviewed-by: Tiago Mück <tiago.muck@arm.com>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Reviewed-by: Nikos Nikoleris <nikos.nikoleris@arm.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>
Tested-by: kokoro <noreply+kokoro@google.com>
diff --git a/SConstruct b/SConstruct
index 53b8c9d..28999a5 100755
--- a/SConstruct
+++ b/SConstruct
@@ -1013,14 +1013,17 @@
     EnumVariable('PROTOCOL', 'Coherence protocol for Ruby', 'None',
                   all_protocols),
     EnumVariable('BACKTRACE_IMPL', 'Post-mortem dump implementation',
-                 backtrace_impls[-1], backtrace_impls)
+                 backtrace_impls[-1], backtrace_impls),
+    ('NUMBER_BITS_PER_SET', 'Max elements in set (default 64)',
+                 64),
     )
 
 # These variables get exported to #defines in config/*.hh (see src/SConscript).
 export_vars += ['USE_FENV', 'SS_COMPATIBLE_FP', 'TARGET_ISA', 'TARGET_GPU_ISA',
                 'CP_ANNOTATE', 'USE_POSIX_CLOCK', 'USE_KVM', 'USE_TUNTAP',
                 'PROTOCOL', 'HAVE_PROTOBUF', 'HAVE_VALGRIND',
-                'HAVE_PERF_ATTR_EXCLUDE_HOST', 'USE_PNG']
+                'HAVE_PERF_ATTR_EXCLUDE_HOST', 'USE_PNG',
+                'NUMBER_BITS_PER_SET']
 
 ###################################################
 #
diff --git a/build_opts/X86_MESI_Two_Level b/build_opts/X86_MESI_Two_Level
index fc74d6f..eba850b 100644
--- a/build_opts/X86_MESI_Two_Level
+++ b/build_opts/X86_MESI_Two_Level
@@ -1,3 +1,4 @@
 TARGET_ISA = 'x86'
 CPU_MODELS = 'TimingSimpleCPU,O3CPU,AtomicSimpleCPU'
 PROTOCOL = 'MESI_Two_Level'
+NUMBER_BITS_PER_SET = '128'
diff --git a/src/mem/ruby/common/SConscript b/src/mem/ruby/common/SConscript
index a19268c..b97391c 100644
--- a/src/mem/ruby/common/SConscript
+++ b/src/mem/ruby/common/SConscript
@@ -33,6 +33,8 @@
 if env['PROTOCOL'] == 'None':
     Return()
 
+env.Append(CPPDEFINES={'NUMBER_BITS_PER_SET': env['NUMBER_BITS_PER_SET']})
+
 Source('Address.cc')
 Source('BoolVec.cc')
 Source('Consumer.cc')
diff --git a/src/mem/ruby/common/Set.hh b/src/mem/ruby/common/Set.hh
index cb01c96..aba38f5 100644
--- a/src/mem/ruby/common/Set.hh
+++ b/src/mem/ruby/common/Set.hh
@@ -39,13 +39,11 @@
 #include "base/logging.hh"
 #include "mem/ruby/common/TypeDefines.hh"
 
-// Change for systems with more than 64 controllers of a particular type.
-const int NUMBER_BITS_PER_SET = 64;
-
 class Set
 {
   private:
     // Number of bits in use in this set.
+    // can be defined in build_opts file (default=64).
     int m_nSize;
     std::bitset<NUMBER_BITS_PER_SET> bits;