| #! /bin/bash | |
| # | |
| # Copyright (c) 2015 ARM Limited | |
| # All rights reserved | |
| # | |
| # The license below extends only to copyright in the software and shall | |
| # not be construed as granting a license to any other intellectual | |
| # property including but not limited to intellectual property relating | |
| # to a hardware implementation of the functionality of the software | |
| # licensed hereunder. You may use the software subject to the license | |
| # terms below provided that you ensure that this notice is replicated | |
| # unmodified and in its entirety in all distributions of the software, | |
| # modified or unmodified, in source code or in binary form. | |
| # | |
| # Copyright (c) 2015 University of Illinois Urbana Champaign | |
| # All rights reserved | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions are | |
| # met: redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer; | |
| # redistributions in binary form must reproduce the above copyright | |
| # notice, this list of conditions and the following disclaimer in the | |
| # documentation and/or other materials provided with the distribution; | |
| # neither the name of the copyright holders nor the names of its | |
| # contributors may be used to endorse or promote products derived from | |
| # this software without specific prior written permission. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| # | |
| # Authors: Gabor Dozsa | |
| # Mohammad Alian | |
| # This is a wrapper script to run a dist gem5 simulations. | |
| # See the usage_func() below for hints on how to use it. Also, | |
| # there are some examples in the util/dist directory (e.g. | |
| # see util/dist/test-2nodes-AArch64.sh) | |
| # | |
| # | |
| # Allocated hosts/cores are assumed to be listed in the LSB_MCPU_HOSTS | |
| # environment variable (which is what LSF does by default). | |
| # E.g. LSB_MCPU_HOSTS=\"hname1 2 hname2 4\" means we have altogether 6 slots | |
| # allocated to launch the gem5 processes, 2 of them are on host hname1 | |
| # and 4 of them are on host hname2. | |
| # If LSB_MCPU_HOSTS environment variable is not defined then we launch all | |
| # processes on the localhost. | |
| # | |
| # Each gem5 process are passed in a unique rank ID [0..N-1] via the kernel | |
| # boot params. The total number of gem5 processes is also passed in. | |
| # These values can be used in the boot script to configure the MAC/IP | |
| # addresses - among other things (see util/dist/bootscript.rcS). | |
| # | |
| # Each gem5 process will create an m5out.$GEM5_RANK directory for | |
| # the usual output files. Furthermore, there will be a separate log file | |
| # for each ssh session (we use ssh to start gem5 processes) and one for | |
| # the server. These are called log.$GEM5_RANK and log.switch. | |
| # | |
| # print help | |
| usage_func () | |
| { | |
| echo "Usage:$0 [-debug] [-n nnodes] [-r rundir] [-c ckptdir] [-p port] [-sw switch] [--sw-args sw_args] [-fs fullsystem] [--fs-args fs_args] [--cf-args conf_args] [--m5-args m5_args] -x gem5_exe " | |
| echo " -debug : debug mode (start gem5 in gdb)" | |
| echo " nnodes : number of gem5 processes" | |
| echo " rundir : run simulation under this path. If not specified, current dir will be used" | |
| echo " ckptdir : dump/restore checkpoints to/from this path. If not specified, current dir will be used" | |
| echo " fullsystem: fullsystem config file" | |
| echo " fs_args : fullsystem config specific argument list: arg1 arg2 ..." | |
| echo " port : switch listen port" | |
| echo " switch : switch config file" | |
| echo " sw_args : switch config specific argument list: arg1 arg2 ..." | |
| echo " conf_args : common (for both fullsystem and switch) config argument list: arg1 arg2 ..." | |
| echo " gem5_exe : gem5 executable (full path required)" | |
| echo " m5_args : common m5 argument list (e.g. debug flags): arg1 arg2 ..." | |
| echo "Note: if no LSF slots allocation is found all proceses are launched on the localhost." | |
| } | |
| # Process (optional) command line options | |
| FS_ARGS=" " | |
| SW_ARGS=" " | |
| CF_ARGS=" " | |
| M5_ARGS=" " | |
| while (($# > 0)) | |
| do | |
| case "x$1" in | |
| x-debug) | |
| GEM5_DEBUG="-debug" | |
| shift 1 | |
| ;; | |
| x-n|x-nodes) | |
| NNODES=$2 | |
| shift 2 | |
| ;; | |
| x-r|x-rundir) | |
| RUN_DIR=$2 | |
| shift 2 | |
| ;; | |
| x-c|x-ckptdir) | |
| CKPT_DIR=$2 | |
| shift 2 | |
| ;; | |
| x-p|x-port) | |
| SW_PORT=$2 | |
| shift 2 | |
| ;; | |
| x-s|x-switch) | |
| SW_CONFIG=$2 | |
| shift 2 | |
| ;; | |
| x--sw-args) | |
| CUR_ARGS="SW_ARGS" | |
| shift 1 | |
| ;; | |
| x-f|x-fullsystem) | |
| FS_CONFIG=$2 | |
| shift 2 | |
| ;; | |
| x--fs-args) | |
| CUR_ARGS="FS_ARGS" | |
| shift 1 | |
| ;; | |
| x--cf-args) | |
| CUR_ARGS="CF_ARGS" | |
| shift 1 | |
| ;; | |
| x--m5-args) | |
| CUR_ARGS="M5_ARGS" | |
| shift 1 | |
| ;; | |
| x-x) | |
| GEM5_EXE=$2 | |
| shift 2 | |
| ;; | |
| x-*) | |
| [ -n "$CUR_ARGS" ] || { echo "Unexpected arg: $1"; usage_func; exit -1; } | |
| case "x$2" in | |
| x-*|x) | |
| eval $CUR_ARGS=\"${!CUR_ARGS} $1\" | |
| shift 1 | |
| ;; | |
| *) | |
| eval $CUR_ARGS=\"${!CUR_ARGS} $1 $2\" | |
| shift 2 | |
| ;; | |
| esac | |
| ;; | |
| *) | |
| echo "Unknown arg: $1" | |
| usage_func | |
| exit 1 | |
| ;; | |
| esac | |
| done | |
| # Default values to use (in case they are not defined as command line options) | |
| DEFAULT_FS_CONFIG=$M5_PATH/configs/example/fs.py | |
| DEFAULT_SW_CONFIG=$M5_PATH/configs/dist/sw.py | |
| DEFAULT_SW_PORT=2200 | |
| [ -z "$FS_CONFIG" ] && FS_CONFIG=$DEFAULT_FS_CONFIG | |
| [ -z "$SW_CONFIG" ] && SW_CONFIG=$DEFAULT_SW_CONFIG | |
| [ -z "$SW_PORT" ] && SW_PORT=$DEFAULT_SW_PORT | |
| [ -z "$NNODES" ] && NNODES=2 | |
| [ -z "$RUN_DIR" ] && RUN_DIR=$(pwd) | |
| [ -z "$CKPT_DIR" ] && CKPT_DIR=$(pwd) | |
| # Check if all the executables we need exist | |
| [ -f "$FS_CONFIG" ] || { echo "FS config ${FS_CONFIG} not found"; exit 1; } | |
| [ -f "$SW_CONFIG" ] || { echo "Switch config ${SW_CONFIG} not found"; exit 1; } | |
| [ -x "$GEM5_EXE" ] || { echo "Executable ${GEM5_EXE} not found"; exit 1; } | |
| # make sure that RUN_DIR exists | |
| mkdir -p $RUN_DIR > /dev/null 2>&1 | |
| declare -a SSH_PIDS | |
| declare -a HOSTS | |
| declare -a NCORES | |
| # Find out which cluster hosts/slots are allocated or | |
| # use localhost if there is no LSF allocation. | |
| # We assume that allocated slots are listed in the LSB_MCPU_HOSTS | |
| # environment variable in the form: | |
| # host1 nslots1 host2 nslots2 ... | |
| # (This is what LSF does by default.) | |
| NH=0 | |
| [ "x$LSB_MCPU_HOSTS" != "x" ] || LSB_MCPU_HOSTS="127.0.0.1 $NNODES" | |
| host="" | |
| for hc in $LSB_MCPU_HOSTS | |
| do | |
| if [ "x$host" == "x" ] | |
| then | |
| host=$hc | |
| HOSTS+=($hc) | |
| else | |
| NCORES+=($hc) | |
| ((NH+=hc)) | |
| host="" | |
| fi | |
| done | |
| ((NNODES==NH)) || { echo "(E) Number of cluster slots ($NH) and gem5 instances ($N) differ"; exit -1; } | |
| # function to clean up and abort if something goes wrong | |
| abort_func () | |
| { | |
| echo | |
| echo "KILLED $(date)" | |
| # Try to Kill the server first. That should trigger an exit for all connected | |
| # gem5 processes. | |
| [ "x$SW_PID" != "x" ] && kill $SW_PID 2>/dev/null | |
| sleep 20 | |
| # (try to) kill gem5 processes - just in case something went wrong with the | |
| # server triggered exit | |
| bname=$(basename $GEM5_EXE) | |
| killall -q -s SIGKILL $bname | |
| for h in ${HOSTS[@]} | |
| do | |
| ssh $h killall -q -s SIGKILL $bname | |
| done | |
| sleep 5 | |
| # kill the watchdog | |
| [ "x$WATCHDOG_PID" != "x" ] && kill $WATCHDOG_PID 2>/dev/null | |
| exit -1 | |
| } | |
| # We need a watchdog to trigger full clean up if a gem5 process dies | |
| watchdog_func () | |
| { | |
| while true | |
| do | |
| sleep 30 | |
| ((NDEAD=0)) | |
| for p in ${SSH_PIDS[*]} | |
| do | |
| kill -0 $p 2>/dev/null || ((NDEAD+=1)) | |
| done | |
| kill -0 $SW_PID || ((NDEAD+=1)) | |
| if ((NDEAD>0)) | |
| then | |
| # we may be in the middle of an orderly termination, | |
| # give it some time to complete before reporting abort | |
| sleep 60 | |
| echo -n "(I) (some) gem5 process(es) exited" | |
| abort_func | |
| fi | |
| done | |
| } | |
| # This function launches the gem5 processes. The only purpose is to enable | |
| # launching gem5 processes under gdb control for debugging | |
| start_func () | |
| { | |
| local N=$1 | |
| local HOST=$2 | |
| local ENV_ARGS=$3 | |
| shift 3 | |
| if [ "x$GEM5_DEBUG" != "x" ] | |
| then | |
| echo "DEBUG starting terminal..." | |
| MY_ARGS="$@" | |
| xterm -e "gdb --args $MY_ARGS" & | |
| else | |
| ssh $HOST $ENV_ARGS "$@" &> $RUN_DIR/log.$N & | |
| fi | |
| } | |
| # block till the gem5 process starts | |
| connected () | |
| { | |
| FILE=$1 | |
| STRING=$2 | |
| echo -n "waiting for $3 to start " | |
| while : ; | |
| do | |
| kill -0 $4 || { echo "Failed to start $3"; exit -1; } | |
| [[ -f "$FILE" ]] && \ | |
| grep -q "$STRING" "$FILE" && \ | |
| echo -e "\nnode #$3 started" && \ | |
| break | |
| sleep 2 | |
| echo -n "." | |
| done | |
| } | |
| # Trigger full clean up in case we are being killed by external signal | |
| trap 'abort_func' INT TERM | |
| # env args to be passed explicitly to gem5 processes started via ssh | |
| ENV_ARGS="LD_LIBRARY_PATH=$LD_LIBRARY_PATH M5_PATH=$M5_PATH" | |
| #cleanup log files before starting gem5 processes | |
| rm $RUN_DIR/log.switch > /dev/null 2>&1 | |
| # make sure that CKPT_DIR exists | |
| mkdir -p $CKPT_DIR/m5out.switch > /dev/null 2>&1 | |
| # launch switch gem5 | |
| SW_HOST=${HOSTS[0]} | |
| echo "launch switch gem5 process on $SW_HOST ..." | |
| start_func "switch" $SW_HOST "$ENV_ARGS" $GEM5_EXE -d $RUN_DIR/m5out.switch \ | |
| $M5_ARGS \ | |
| $SW_CONFIG \ | |
| $SW_ARGS \ | |
| $CF_ARGS \ | |
| --checkpoint-dir=$CKPT_DIR/m5out.switch \ | |
| --is-switch \ | |
| --dist-size=$NNODES \ | |
| --dist-server-port=$SW_PORT | |
| SW_PID=$! | |
| # block here till switch process starts | |
| connected $RUN_DIR/log.switch "tcp_iface listening on port" "switch" $SW_PID | |
| LINE=$(grep -r "tcp_iface listening on port" $RUN_DIR/log.switch) | |
| IFS=' ' read -ra ADDR <<< "$LINE" | |
| # actual port that switch is listening on may be different | |
| # from what we specified if the port was busy | |
| SW_PORT=${ADDR[5]} | |
| # Now launch all the gem5 processes with ssh. | |
| echo "START $(date)" | |
| n=0 | |
| for ((i=0; i < ${#HOSTS[@]}; i++)) | |
| do | |
| h=${HOSTS[$i]} | |
| for ((j=0; j < ${NCORES[i]}; j++)) | |
| do | |
| #cleanup log files before starting gem5 processes | |
| rm $RUN_DIR/log.$n > /dev/null 2>&1 | |
| # make sure that CKPT_DIR exists | |
| mkdir -p $CKPT_DIR/m5out.$n > /dev/null 2>&1 | |
| echo "starting gem5 on $h ..." | |
| start_func $n $h "$ENV_ARGS" $GEM5_EXE -d $RUN_DIR/m5out.$n \ | |
| $M5_ARGS \ | |
| $FS_CONFIG \ | |
| $FS_ARGS \ | |
| $CF_ARGS \ | |
| --checkpoint-dir=$CKPT_DIR/m5out.$n \ | |
| --dist \ | |
| --dist-rank=$n \ | |
| --dist-size=$NNODES \ | |
| --dist-server-name=${HOSTS[0]} \ | |
| --dist-server-port=$SW_PORT | |
| SSH_PIDS[$n]=$! | |
| ((n+=1)) | |
| done | |
| done | |
| # Wait here if it is a debug session | |
| [ "x$GEM5_DEBUG" == "x" ] || { echo "DEBUG session"; wait $SW_PID; exit -1; } | |
| # start watchdog to trigger complete abort (after a grace period) if any | |
| # gem5 process dies | |
| watchdog_func & | |
| WATCHDOG_PID=$! | |
| # wait for exit statuses | |
| ((NFAIL=0)) | |
| for p in ${SSH_PIDS[*]} | |
| do | |
| wait $p || ((NFAIL+=1)) | |
| done | |
| wait $SW_PID || ((NFAIL+=1)) | |
| # all done, let's terminate the watchdog | |
| kill $WATCHDOG_PID 2>/dev/null | |
| if ((NFAIL==0)) | |
| then | |
| echo "EXIT $(date)" | |
| else | |
| echo "ABORT $(date)" | |
| fi |