| # Copyright (c) 2018 Advanced Micro Devices, Inc. |
| # All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are met: |
| # |
| # 1. Redistributions of source code must retain the above copyright notice, |
| # this list of conditions and the following disclaimer. |
| # |
| # 2. Redistributions in binary form must reproduce the above copyright notice, |
| # this list of conditions and the following disclaimer in the documentation |
| # and/or other materials provided with the distribution. |
| # |
| # 3. Neither the name of the copyright holder nor the names of its |
| # contributors may be used to endorse or promote products derived from this |
| # software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| # POSSIBILITY OF SUCH DAMAGE. |
| |
| import m5 |
| |
| import operator |
| from os import mkdir, makedirs, getpid, listdir, fsync |
| from os.path import join as joinpath |
| from os.path import isdir |
| from shutil import rmtree, copyfile |
| from m5.util.convert import toFrequency, toMemorySize |
| |
| def file_append(path, contents): |
| with open(joinpath(*path), 'a') as f: |
| f.write(str(contents)) |
| f.flush() |
| fsync(f.fileno()) |
| |
| def remake_dir(path): |
| if isdir(path): |
| rmtree(path) |
| makedirs(path) |
| |
| # This fakes out a dGPU setup so the runtime operates correctly. The spoofed |
| # system has a single dGPU and a single socket CPU. Note that more complex |
| # topologies (multi-GPU, multi-socket CPUs) need to have a different setup |
| # here or the runtime won't be able to issue Memcpies from one node to another. |
| # |
| # TODO: There is way too much hardcoded here. It doesn't effect anything in |
| # our current ROCm stack (1.6), but it is highly possible that it will in the |
| # future. We might need to scrub through this and extract the appropriate |
| # fields from the simulator in the future. |
| def createVegaTopology(options): |
| topology_dir = joinpath(m5.options.outdir, \ |
| 'fs/sys/devices/virtual/kfd/kfd/topology') |
| remake_dir(topology_dir) |
| |
| amdgpu_dir = joinpath(m5.options.outdir, \ |
| 'fs/sys/module/amdgpu/parameters') |
| remake_dir(amdgpu_dir) |
| |
| pci_ids_dir = joinpath(m5.options.outdir, \ |
| 'fs/usr/share/hwdata/') |
| remake_dir(pci_ids_dir) |
| |
| # Vega reported VM size in GB. Used to reserve an allocation from CPU |
| # to implement SVM (i.e. GPUVM64 pointers and X86 pointers agree) |
| file_append((amdgpu_dir, 'vm_size'), 256) |
| |
| # Ripped from real Vega platform to appease KMT version checks |
| file_append((topology_dir, 'generation_id'), 2) |
| |
| # Set up system properties. Regiter as ast-rocm server |
| sys_prop = 'platform_oem 35498446626881\n' + \ |
| 'platform_id 71791775140929\n' + \ |
| 'platform_rev 2\n' |
| file_append((topology_dir, 'system_properties'), sys_prop) |
| |
| # Populate the topology tree |
| # Our dGPU system is two nodes. Node 0 is a CPU and Node 1 is a dGPU |
| node_dir = joinpath(topology_dir, 'nodes/0') |
| remake_dir(node_dir) |
| |
| # Register as a CPU |
| file_append((node_dir, 'gpu_id'), 0) |
| file_append((node_dir, 'name'), '') |
| |
| # CPU links. Only thing that matters is we tell the runtime that GPU is |
| # connected through PCIe to CPU socket 0. |
| io_links = 1 |
| io_dir = joinpath(node_dir, 'io_links/0') |
| remake_dir(io_dir) |
| io_prop = 'type 2\n' + \ |
| 'version_major 0\n' + \ |
| 'version_minor 0\n' + \ |
| 'node_from 0\n' + \ |
| 'node_to 1\n' + \ |
| 'weight 20\n' + \ |
| 'min_latency 0\n' + \ |
| 'max_latency 0\n' + \ |
| 'min_bandwidth 0\n' + \ |
| 'max_bandwidth 0\n' + \ |
| 'recommended_transfer_size 0\n' + \ |
| 'flags 13\n' |
| file_append((io_dir, 'properties'), io_prop) |
| |
| # Populate CPU node properties |
| node_prop = 'cpu_cores_count %s\n' % options.num_cpus + \ |
| 'simd_count 0\n' + \ |
| 'mem_banks_count 1\n' + \ |
| 'caches_count 0\n' + \ |
| 'io_links_count %s\n' % io_links + \ |
| 'cpu_core_id_base 0\n' + \ |
| 'simd_id_base 0\n' + \ |
| 'max_waves_per_simd 0\n' + \ |
| 'lds_size_in_kb 0\n' + \ |
| 'gds_size_in_kb 0\n' + \ |
| 'wave_front_size 64\n' + \ |
| 'array_count 0\n' + \ |
| 'simd_arrays_per_engine 0\n' + \ |
| 'cu_per_simd_array 0\n' + \ |
| 'simd_per_cu 0\n' + \ |
| 'max_slots_scratch_cu 0\n' + \ |
| 'vendor_id 0\n' + \ |
| 'device_id 0\n' + \ |
| 'location_id 0\n' + \ |
| 'drm_render_minor 0\n' + \ |
| 'max_engine_clk_ccompute 3400\n' |
| |
| file_append((node_dir, 'properties'), node_prop) |
| |
| # CPU memory reporting |
| mem_dir = joinpath(node_dir, 'mem_banks/0') |
| remake_dir(mem_dir) |
| # Heap type value taken from real system, heap type values: |
| # https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/roc-4.0.x/include/hsakmttypes.h#L317 |
| mem_prop = 'heap_type 0\n' + \ |
| 'size_in_bytes 33704329216\n' + \ |
| 'flags 0\n' + \ |
| 'width 72\n' + \ |
| 'mem_clk_max 2400\n' |
| |
| file_append((mem_dir, 'properties'), mem_prop) |
| |
| # Build the GPU node |
| node_dir = joinpath(topology_dir, 'nodes/1') |
| remake_dir(node_dir) |
| |
| # Register as a Vega |
| file_append((node_dir, 'gpu_id'), 22124) |
| file_append((node_dir, 'name'), 'Vega\n') |
| |
| # Should be the same as the render driver filename (dri/renderD<drm_num>) |
| drm_num = 128 |
| |
| # 96 in real Vega |
| # Random comment for comparison purposes |
| caches = 0 |
| |
| # GPU links. Only thing that matters is we tell the runtime that GPU is |
| # connected through PCIe to CPU socket 0. |
| io_links = 1 |
| io_dir = joinpath(node_dir, 'io_links/0') |
| remake_dir(io_dir) |
| io_prop = 'type 2\n' + \ |
| 'version_major 0\n' + \ |
| 'version_minor 0\n' + \ |
| 'node_from 1\n' + \ |
| 'node_to 0\n' + \ |
| 'weight 20\n' + \ |
| 'min_latency 0\n' + \ |
| 'max_latency 0\n' + \ |
| 'min_bandwidth 0\n' + \ |
| 'max_bandwidth 0\n' + \ |
| 'recommended_transfer_size 0\n' + \ |
| 'flags 1\n' |
| file_append((io_dir, 'properties'), io_prop) |
| |
| # Populate GPU node properties |
| cu_scratch = options.simds_per_cu * options.wfs_per_simd |
| node_prop = 'cpu_cores_count 0\n' + \ |
| 'simd_count 256\n' + \ |
| 'mem_banks_count 1\n' + \ |
| 'caches_count %s\n' % caches + \ |
| 'io_links_count %s\n' % io_links + \ |
| 'cpu_core_id_base 0\n' + \ |
| 'simd_id_base 2147487744\n' + \ |
| 'max_waves_per_simd 10\n' + \ |
| 'lds_size_in_kb 64\n' + \ |
| 'gds_size_in_kb 0\n' + \ |
| 'wave_front_size 64\n' + \ |
| 'array_count 4\n' + \ |
| 'simd_arrays_per_engine 1\n' + \ |
| 'cu_per_simd_array 16\n' + \ |
| 'simd_per_cu 4\n' + \ |
| 'max_slots_scratch_cu %s\n' % cu_scratch + \ |
| 'vendor_id 4098\n' + \ |
| 'device_id 26720\n' + \ |
| 'location_id 1024\n' + \ |
| 'drm_render_minor %s\n' % drm_num + \ |
| 'hive_id 0\n' + \ |
| 'num_sdma_engines 2\n' + \ |
| 'num_sdma_xgmi_engines 0\n' + \ |
| 'max_engine_clk_fcompute 1500\n' + \ |
| 'local_mem_size 17163091968\n' + \ |
| 'fw_version 421\n' + \ |
| 'capability 238208\n' + \ |
| 'debug_prop 32768\n' + \ |
| 'sdma_fw_version 430\n' + \ |
| 'max_engine_clk_ccompute 3400\n' |
| |
| file_append((node_dir, 'properties'), node_prop) |
| |
| # Fiji HBM reporting |
| # TODO: Extract size, clk, and width from sim paramters |
| mem_dir = joinpath(node_dir, 'mem_banks/0') |
| remake_dir(mem_dir) |
| # Heap type value taken from real system, heap type values: |
| # https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/roc-4.0.x/include/hsakmttypes.h#L317 |
| mem_prop = 'heap_type 1\n' + \ |
| 'size_in_bytes 17163091968\n' + \ |
| 'flags 0\n' + \ |
| 'width 2048\n' + \ |
| 'mem_clk_max 945\n' |
| |
| file_append((mem_dir, 'properties'), mem_prop) |
| |
| # This fakes out a dGPU setup so the runtime correctly operations. The spoofed |
| # system has a single dGPU and a single socket CPU. Note that more complex |
| # topologies (multi-GPU, multi-socket CPUs) need to have a different setup |
| # here or the runtime won't be able to issue Memcpies from one node to another. |
| # |
| # TODO: There is way too much hardcoded here. It doesn't effect anything in |
| # our current ROCm stack (1.6), but it is highly possible that it will in the |
| # future. We might need to scrub through this and extract the appropriate |
| # fields from the simulator in the future. |
| def createFijiTopology(options): |
| topology_dir = joinpath(m5.options.outdir, \ |
| 'fs/sys/devices/virtual/kfd/kfd/topology') |
| remake_dir(topology_dir) |
| |
| amdgpu_dir = joinpath(m5.options.outdir, \ |
| 'fs/sys/module/amdgpu/parameters') |
| remake_dir(amdgpu_dir) |
| |
| # Fiji reported VM size in GB. Used to reserve an allocation from CPU |
| # to implement SVM (i.e. GPUVM64 pointers and X86 pointers agree) |
| file_append((amdgpu_dir, 'vm_size'), 256) |
| |
| # Ripped from real Fiji platform to appease KMT version checks |
| file_append((topology_dir, 'generation_id'), 2) |
| |
| # Set up system properties. Regiter as ast-rocm server |
| sys_prop = 'platform_oem 35498446626881\n' + \ |
| 'platform_id 71791775140929\n' + \ |
| 'platform_rev 2\n' |
| file_append((topology_dir, 'system_properties'), sys_prop) |
| |
| # Populate the topology tree |
| # Our dGPU system is two nodes. Node 0 is a CPU and Node 1 is a dGPU |
| node_dir = joinpath(topology_dir, 'nodes/0') |
| remake_dir(node_dir) |
| |
| # Register as a CPU |
| file_append((node_dir, 'gpu_id'), 0) |
| file_append((node_dir, 'name'), '') |
| |
| # CPU links. Only thing that matters is we tell the runtime that GPU is |
| # connected through PCIe to CPU socket 0. |
| io_links = 1 |
| io_dir = joinpath(node_dir, 'io_links/0') |
| remake_dir(io_dir) |
| io_prop = 'type 2\n' + \ |
| 'version_major 0\n' + \ |
| 'version_minor 0\n' + \ |
| 'node_from 0\n' + \ |
| 'node_to 1\n' + \ |
| 'weight 20\n' + \ |
| 'min_latency 0\n' + \ |
| 'max_latency 0\n' + \ |
| 'min_bandwidth 0\n' + \ |
| 'max_bandwidth 0\n' + \ |
| 'recommended_transfer_size 0\n' + \ |
| 'flags 13\n' |
| file_append((io_dir, 'properties'), io_prop) |
| |
| # Populate CPU node properties |
| node_prop = 'cpu_cores_count %s\n' % options.num_cpus + \ |
| 'simd_count 0\n' + \ |
| 'mem_banks_count 1\n' + \ |
| 'caches_count 0\n' + \ |
| 'io_links_count %s\n' % io_links + \ |
| 'cpu_core_id_base 0\n' + \ |
| 'simd_id_base 0\n' + \ |
| 'max_waves_per_simd 0\n' + \ |
| 'lds_size_in_kb 0\n' + \ |
| 'gds_size_in_kb 0\n' + \ |
| 'wave_front_size 64\n' + \ |
| 'array_count 0\n' + \ |
| 'simd_arrays_per_engine 0\n' + \ |
| 'cu_per_simd_array 0\n' + \ |
| 'simd_per_cu 0\n' + \ |
| 'max_slots_scratch_cu 0\n' + \ |
| 'vendor_id 0\n' + \ |
| 'device_id 0\n' + \ |
| 'location_id 0\n' + \ |
| 'drm_render_minor 0\n' + \ |
| 'max_engine_clk_ccompute 3400\n' |
| |
| file_append((node_dir, 'properties'), node_prop) |
| |
| # CPU memory reporting |
| mem_dir = joinpath(node_dir, 'mem_banks/0') |
| remake_dir(mem_dir) |
| # Heap type value taken from real system, heap type values: |
| # https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/roc-4.0.x/include/hsakmttypes.h#L317 |
| mem_prop = 'heap_type 0\n' + \ |
| 'size_in_bytes 33704329216\n' + \ |
| 'flags 0\n' + \ |
| 'width 72\n' + \ |
| 'mem_clk_max 2400\n' |
| |
| file_append((mem_dir, 'properties'), mem_prop) |
| |
| # Build the GPU node |
| node_dir = joinpath(topology_dir, 'nodes/1') |
| remake_dir(node_dir) |
| |
| # Register as a Fiji |
| file_append((node_dir, 'gpu_id'), 50156) |
| file_append((node_dir, 'name'), 'Fiji\n') |
| |
| # Should be the same as the render driver filename (dri/renderD<drm_num>) |
| drm_num = 128 |
| |
| # Real Fiji shows 96, but building that topology is complex and doesn't |
| # appear to be required for anything. |
| caches = 0 |
| |
| # GPU links. Only thing that matters is we tell the runtime that GPU is |
| # connected through PCIe to CPU socket 0. |
| io_links = 1 |
| io_dir = joinpath(node_dir, 'io_links/0') |
| remake_dir(io_dir) |
| io_prop = 'type 2\n' + \ |
| 'version_major 0\n' + \ |
| 'version_minor 0\n' + \ |
| 'node_from 1\n' + \ |
| 'node_to 0\n' + \ |
| 'weight 20\n' + \ |
| 'min_latency 0\n' + \ |
| 'max_latency 0\n' + \ |
| 'min_bandwidth 0\n' + \ |
| 'max_bandwidth 0\n' + \ |
| 'recommended_transfer_size 0\n' + \ |
| 'flags 1\n' |
| file_append((io_dir, 'properties'), io_prop) |
| |
| # Populate GPU node properties |
| node_prop = 'cpu_cores_count 0\n' + \ |
| 'simd_count %s\n' \ |
| % (options.num_compute_units * options.simds_per_cu) + \ |
| 'mem_banks_count 1\n' + \ |
| 'caches_count %s\n' % caches + \ |
| 'io_links_count %s\n' % io_links + \ |
| 'cpu_core_id_base 0\n' + \ |
| 'simd_id_base 2147487744\n' + \ |
| 'max_waves_per_simd %s\n' % options.wfs_per_simd + \ |
| 'lds_size_in_kb %s\n' % int(options.lds_size / 1024) + \ |
| 'gds_size_in_kb 0\n' + \ |
| 'wave_front_size %s\n' % options.wf_size + \ |
| 'array_count 4\n' + \ |
| 'simd_arrays_per_engine %s\n' % options.sa_per_complex + \ |
| 'cu_per_simd_array %s\n' % options.cu_per_sa + \ |
| 'simd_per_cu %s\n' % options.simds_per_cu + \ |
| 'max_slots_scratch_cu 32\n' + \ |
| 'vendor_id 4098\n' + \ |
| 'device_id 29440\n' + \ |
| 'location_id 512\n' + \ |
| 'drm_render_minor %s\n' % drm_num + \ |
| 'max_engine_clk_fcompute %s\n' \ |
| % int(toFrequency(options.gpu_clock) / 1e6) + \ |
| 'local_mem_size 4294967296\n' + \ |
| 'fw_version 730\n' + \ |
| 'capability 4736\n' + \ |
| 'max_engine_clk_ccompute %s\n' \ |
| % int(toFrequency(options.CPUClock) / 1e6) |
| |
| file_append((node_dir, 'properties'), node_prop) |
| |
| # Fiji HBM reporting |
| # TODO: Extract size, clk, and width from sim paramters |
| mem_dir = joinpath(node_dir, 'mem_banks/0') |
| remake_dir(mem_dir) |
| # Heap type value taken from real system, heap type values: |
| # https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/roc-4.0.x/include/hsakmttypes.h#L317 |
| mem_prop = 'heap_type 1\n' + \ |
| 'size_in_bytes 4294967296\n' + \ |
| 'flags 0\n' + \ |
| 'width 4096\n' + \ |
| 'mem_clk_max 500\n' |
| |
| file_append((mem_dir, 'properties'), mem_prop) |
| |
| |
| def createCarrizoTopology(options): |
| topology_dir = joinpath(m5.options.outdir, \ |
| 'fs/sys/devices/virtual/kfd/kfd/topology') |
| remake_dir(topology_dir) |
| |
| # Ripped from real Kaveri platform to appease kmt version checks |
| # Set up generation_id |
| file_append((topology_dir, 'generation_id'), 1) |
| |
| # Set up system properties |
| sys_prop = 'platform_oem 2314885673410447169\n' + \ |
| 'platform_id 35322352389441\n' + \ |
| 'platform_rev 1\n' |
| file_append((topology_dir, 'system_properties'), sys_prop) |
| |
| # Populate the topology tree |
| # TODO: Just the bare minimum to pass for now |
| node_dir = joinpath(topology_dir, 'nodes/0') |
| remake_dir(node_dir) |
| |
| # must show valid kaveri gpu id or massive meltdown |
| file_append((node_dir, 'gpu_id'), 2765) |
| |
| gfx_dict = { "gfx801": {"name": "Carrizo\n", "id": 39028}, |
| "gfx902": {"name": "Raven\n", "id": 5597}} |
| |
| # must have marketing name |
| file_append((node_dir, 'name'), gfx_dict[options.gfx_version]["name"]) |
| |
| mem_banks_cnt = 1 |
| |
| # Should be the same as the render driver filename (dri/renderD<drm_num>) |
| drm_num = 128 |
| |
| device_id = gfx_dict[options.gfx_version]["id"] |
| |
| # populate global node properties |
| # NOTE: SIMD count triggers a valid GPU agent creation |
| node_prop = 'cpu_cores_count %s\n' % options.num_cpus + \ |
| 'simd_count %s\n' \ |
| % (options.num_compute_units * options.simds_per_cu) + \ |
| 'mem_banks_count %s\n' % mem_banks_cnt + \ |
| 'caches_count 0\n' + \ |
| 'io_links_count 0\n' + \ |
| 'cpu_core_id_base 16\n' + \ |
| 'simd_id_base 2147483648\n' + \ |
| 'max_waves_per_simd %s\n' % options.wfs_per_simd + \ |
| 'lds_size_in_kb %s\n' % int(options.lds_size / 1024) + \ |
| 'gds_size_in_kb 0\n' + \ |
| 'wave_front_size %s\n' % options.wf_size + \ |
| 'array_count 1\n' + \ |
| 'simd_arrays_per_engine %s\n' % options.sa_per_complex + \ |
| 'cu_per_simd_array %s\n' % options.cu_per_sa + \ |
| 'simd_per_cu %s\n' % options.simds_per_cu + \ |
| 'max_slots_scratch_cu 32\n' + \ |
| 'vendor_id 4098\n' + \ |
| 'device_id %s\n' % device_id + \ |
| 'location_id 8\n' + \ |
| 'drm_render_minor %s\n' % drm_num + \ |
| 'max_engine_clk_fcompute %s\n' \ |
| % int(toFrequency(options.gpu_clock) / 1e6) + \ |
| 'local_mem_size 0\n' + \ |
| 'fw_version 699\n' + \ |
| 'capability 4738\n' + \ |
| 'max_engine_clk_ccompute %s\n' \ |
| % int(toFrequency(options.CPUClock) / 1e6) |
| |
| file_append((node_dir, 'properties'), node_prop) |
| |
| for i in range(mem_banks_cnt): |
| mem_dir = joinpath(node_dir, f'mem_banks/{i}') |
| remake_dir(mem_dir) |
| |
| # Heap type value taken from real system, heap type values: |
| # https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/roc-4.0.x/include/hsakmttypes.h#L317 |
| mem_prop = f'heap_type 0\n' + \ |
| f'size_in_bytes {toMemorySize(options.mem_size)}' + \ |
| f'flags 0\n' + \ |
| f'width 64\n' + \ |
| f'mem_clk_max 1600\n' |
| file_append((mem_dir, 'properties'), mem_prop) |
