time-lazy-isolator.sh

#!/bin/bash

set -e
set -x
set -o pipefail

date

exename="isolator"
timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
TIMING_HOME=$(pwd)
TIMING_HOST=$(hostname)
TIMING_DATE=$(date "+%Y-%m-%d %H:%M")
TIME_SINCE_EPOCH=$(date +%s)
TIMING_PLATFORM=$(uname)
TIMING_ARCH=$(uname -m)
TIMING_REPO="illinois-ceesd/timing.git"
TIMING_BRANCH="main"
TIMING_ENV_NAME="${exename}.lazy.timing.env"
MIRGE_BRANCH="production"
DRIVER_REPO="illinois-ceesd/drivers_y2-isolator"
DRIVER_BRANCH="update-to-y3"
DRIVER_NAME="y2-isolator"
SUMMARY_FILE_ROOT="${exename}_lazy"
YAML_FILE_NAME="${exename}-timings.yaml"
BATCH_OUTPUT_FILE="${SUMMARY_FILE_ROOT}_${timestamp}.out"
LOGDIR="${exename}_logs"
EXEOPTS="--lazy --log"
SQL_PATH="./log_data"

# -- Install conda env, dependencies and MIRGE-Com via *emirge*
# --- remove old run if it exists
if [ -f "INSTALL_MIRGECOM" ]
then
    if [ -d "emirge" ]
    then
        echo "Removing old timing run."
        mv -f emirge emirge.old
        rm -rf emirge.old &
    fi

    # --- grab emirge and install MIRGE-Com 
    git clone https://github.com/illinois-ceesd/emirge.git
    cd emirge
    ./install.sh --branch=${MIRGE_BRANCH} --env-name=${TIMING_ENV_NAME}
    cd ../
fi
rm -f INSTALL_MIRGECOM
# -- Activate the env we just created above
export EMIRGE_HOME="${TIMING_HOME}/emirge"
source ${EMIRGE_HOME}/config/activate_env.sh

cd emirge/mirgecom
# cd mirgecom

# -- Grab and merge the branch with the case-dependent features
MIRGE_HASH=$(git rev-parse origin/${MIRGE_BRANCH})
Y1_HASH=$MIRGE_HASH

# --- Grab the case driver repo
rm -Rf ${DRIVER_NAME}
git clone -b ${DRIVER_BRANCH} https://github.com/${DRIVER_REPO} ${DRIVER_NAME}
cd ${DRIVER_NAME}/timing_run
DRIVER_HASH=$(git rev-parse HEAD)

cat <<EOF > timing_params.yaml
nviz: 100
nrestart: 100
nhealth: 100
nstatus: 100
current_dt: 1.0e-8
t_final: 2.e-7
order:  1
alpha_sc: 0.5
s0_sc: -5.0
kappa_sc: 0.5
integrator: rk4
health_pres_min: 1700
health_pres_max: 280000
EOF

# --- Get an MD5Sum for the untracked timing driver
DRIVER_MD5SUM="None"
if command -v md5sum &> /dev/null
then 
    DRIVER_MD5SUM=$(md5sum ./${exename}.py | cut -d " " -f 1)
else
    echo "Warning: No md5sum command found. Skipping  md5sum for untracked driver."
fi

# -- Run the case (platform-dependent)
printf "Running on Host: ${TIMING_HOST}\n"
date
GPU_ARCH="Unknown"
case $TIMING_HOST in

    # --- Run the timing test in a batch job on Lassen@LLC
    lassen*)
        echo "Resolved Host: Lassen"
        TIMING_HOST="Lassen"
        GPU_ARCH="GV100GL"
        BATCH_SCRIPT_NAME="${exename}_timing_job.sh"
        rm -f ${BATCH_SCRIPT_NAME}
        rm -f timing-run-done
        # ---- Generate a batch script for running the timing job
        cat <<EOF > ${BATCH_SCRIPT_NAME}
#!/bin/bash

#BSUB -nnodes 1
#BSUB -G uiuc
#BSUB -W 60
#BSUB -q pbatch

printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n"

source "${EMIRGE_HOME}/config/activate_env.sh"
export PYOPENCL_CTX="port:tesla"
export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch"
rm -rf \$XDG_CACHE_HOME
rm -f timing-run-done
which python
conda env list
env
env | grep LSB_MCPU_HOSTS

jsrun -g 1 -a 1 -n 1 python -O -u -m mpi4py ./${exename}.py -i timing_params.yaml ${EXEOPTS}
touch timing-run-done

EOF
        chmod +x ${BATCH_SCRIPT_NAME}
        # ---- Submit the batch script and wait for the job to finish
        bsub ${BATCH_SCRIPT_NAME}
        # ---- Wait 10 minutes right off the bat (the job is at least 10 min)
        sleep 600
        iwait=0
        while [ ! -f ./timing-run-done ]; do 
            iwait=$((iwait+1))
            if [ "$iwait" -gt 360 ]; then # give up after 1 hour
                printf "Timed out waiting on batch job.\n"
                exit 1 # skip the rest of the script
            fi
            sleep 20
        done
        sleep 30  # give the batch system time to spew its junk into the log
        ;;

    # --- Run the timing test on an unknown/generic machine 
    *)
        printf "Host: Unknown\n"
        PYOPENCL_TEST=port:pthread python -O -m mpi4py ./${exename}.py -i timing_params.yaml ${EXEOPTS}
        ;;
esac

date
# -- Process the results of the timing run
RUN_LOG_FILE="${SQL_PATH}/${exename}-rank0.sqlite"
if [[ -f "${RUN_LOG_FILE}" ]]; then

    rm -f ${YAML_FILE_NAME}
    SUMMARY_FILE_NAME="${SQL_PATH}/${SUMMARY_FILE_ROOT}_${timestamp}.sqlite"
    rm -f ${SUMMARY_FILE_NAME}

    # --- Pull the timings out of the sqlite files generated by logging
    runalyzer-gather ${SUMMARY_FILE_NAME} ${RUN_LOG_FILE}
    CL_DEVICE=$(sqlite3 ${SUMMARY_FILE_NAME} 'SELECT cl_device_name FROM runs')
    STARTUP_TIME=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(q("select $t_init.max").fetchall()[0][0])' | grep -v INFO)
    FIRST_STEP=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[0:1]))' | grep -v INFO)
    FIRST_10_STEPS=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[0:10]))' | grep -v INFO)
    SECOND_10_STEPS=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[10:19]))' | grep -v INFO)
    MAX_PYTHON_MEM_USAGE=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(max(p[0] for p in q("select $memory_usage_python.max").fetchall()))' | grep -v INFO)
    MAX_GPU_MEM_USAGE=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(max(p[0] for p in q("select $memory_usage_gpu.max").fetchall()))' | grep -v INFO)

    # --- Create a YAML-compatible text snippet with the timing info
    printf "run_date: ${TIMING_DATE}\nrun_host: ${TIMING_HOST}\n" > ${YAML_FILE_NAME}
    printf "cl_device: ${CL_DEVICE}\n" >> ${YAML_FILE_NAME}
    printf "run_epoch: ${TIME_SINCE_EPOCH}\nrun_platform: ${TIMING_PLATFORM}\n" >> ${YAML_FILE_NAME}
    printf "run_arch: ${TIMING_ARCH}\ngpu_arch: ${GPU_ARCH}\n" >> ${YAML_FILE_NAME}
    printf "mirge_version: ${MIRGE_HASH}\ny1_version: ${Y1_HASH}\n" >> ${YAML_FILE_NAME}
    printf "driver_version: ${DRIVER_HASH}\ndriver_md5sum: ${DRIVER_MD5SUM}\n" >> ${YAML_FILE_NAME}
    printf "time_startup: ${STARTUP_TIME}\ntime_first_step: ${FIRST_STEP}\n" >> ${YAML_FILE_NAME}
    printf "time_first_10: ${FIRST_10_STEPS}\ntime_second_10: ${SECOND_10_STEPS}\n" >> ${YAML_FILE_NAME}
    printf "max_python_mem_usage: ${MAX_PYTHON_MEM_USAGE}\n" >> ${YAML_FILE_NAME}
    printf "max_gpu_mem_usage: ${MAX_GPU_MEM_USAGE}\n---\n" >> ${YAML_FILE_NAME}

    # Users should set special keys for using git over
    # ssh for security concerns. This snippet will use
    # a pre-arranged ssh key if the user provides one
    # and indicates it with the TESTING_SSH_KEY environment
    # variable.
    # ===== To create a key:
    # - Run ssh-keygen:
    # $ ssh-keygen
    # [enter a <keyname> when prompted]
    # - Put the key(s) in a /secure/filesystem/location:
    # $ mv <keyname>* /secure/filesystem/location
    # - Add the key to GIT:
    # $ [browse to] https://github.com/illinois-ceesd/timing/settings/keys/new
    # $ Choose (New SSH key)
    # $ Paste in the contents of /secure/filesystem/location/<keyname>.pub
    # - Set the ENV variable before using this script:
    # $ export TESTING_SSH_KEY=/secure/filesystem/location/<keyname>
    if [ ! -z "${TESTING_SSH_KEY}" ]; then
        eval $(ssh-agent)
        trap "kill $SSH_AGENT_PID" EXIT
        ssh-add ${TESTING_SSH_KEY}
    fi

    # --- Update the timing data in the repo
    # ---- First, clone the timing repo
    git clone -b ${TIMING_BRANCH} git@github.com:${TIMING_REPO}
    # ---- Create the timing file if it does not exist
    if [[ ! -f timing/${YAML_FILE_NAME} ]]; then 
        touch timing/${YAML_FILE_NAME}
        (cd timing && git add ${YAML_FILE_NAME})
    fi
    # ---- Update the timing file with the current test data
    cat ${YAML_FILE_NAME} >> timing/${YAML_FILE_NAME}
    mkdir -p timing/${LOGDIR}
    cp ${SUMMARY_FILE_NAME} timing/${LOGDIR}
    cat *.out > timing/${LOGDIR}/${BATCH_OUTPUT_FILE}
    cd timing
    git add ${LOGDIR}/
    # ---- Commit the new data to the repo
    (git commit -am "Automatic commit: ${TIMING_HOST} ${TIMING_DATE}" && git push)
    cd ../
else
    printf "Timing run did not produce the expected sqlite file: ${RUN_LOG_FILE}\n"
    exit 1
fi

date