time-flame1d.sh

#!/bin/bash

set -e
set -x
set -o pipefail

date

exename="flame1d"
TIMING_HOME=$(pwd)
TIMING_HOST=$(hostname)
TIMING_DATE=$(date "+%Y-%m-%d %H:%M")
TIME_SINCE_EPOCH=$(date +%s)
TIMING_PLATFORM=$(uname)
TIMING_ARCH=$(uname -m)
TIMING_REPO="illinois-ceesd/timing.git"
TIMING_BRANCH="y1-production"
TIMING_ENV_NAME="${exename}.timing.env"
MIRGE_BRANCH="y1-production"
DRIVER_REPO="anderson2981/CEESD-Y1_Flame1D.git"
DRIVER_BRANCH="main"
DRIVER_NAME="y1-production-flame"

# -- Install conda env, dependencies and MIRGE-Com via *emirge*
# --- remove old run if it exists
if [ -d "emirge" ]
then
    echo "Removing old timing run."
    mv -f emirge emirge.old
    rm -rf emirge.old &
fi

# --- grab emirge and install MIRGE-Com 
git clone https://github.com/illinois-ceesd/emirge.git
cd emirge
./install.sh --branch=${MIRGE_BRANCH} --env-name=${TIMING_ENV_NAME}

# -- Activate the env we just created above
export EMIRGE_HOME="${TIMING_HOME}/emirge"
source ${EMIRGE_HOME}/config/activate_env.sh

cd mirgecom

# -- Grab and merge the branch with the case-dependent features
Y1_HASH=$(git rev-parse origin/${MIRGE_BRANCH})
MIRGE_HASH=$(git rev-parse origin/main)

# --- Grab the case driver repo
rm -Rf ${DRIVER_NAME}
git clone -b ${DRIVER_BRANCH} https://github.com/${DRIVER_REPO} ${DRIVER_NAME}
cd ${DRIVER_NAME}/timing_run
DRIVER_HASH=$(git rev-parse HEAD)

cat <<EOF > timing_params.yaml
nviz: 100
nrestart: 100
current_dt: 5e-9
t_final: 1.e-7
order:  1
alpha_sc: 0.5
s0_sc: -5.0
kappa_sc: 0.5
EOF

# --- Get an MD5Sum for the untracked timing driver
DRIVER_MD5SUM="None"
if command -v md5sum &> /dev/null
then 
    DRIVER_MD5SUM=$(md5sum ./${exename}.py | cut -d " " -f 1)
else
    echo "Warning: No md5sum command found. Skipping  md5sum for untracked driver."
fi

# -- Run the case (platform-dependent)
printf "Running on Host: ${TIMING_HOST}\n"
date
GPU_ARCH="Unknown"
case $TIMING_HOST in

    # --- Run the timing test in a batch job on Lassen@LLC
    lassen*)
        echo "Resolved Host: Lassen"
        TIMING_HOST="Lassen"
        GPU_ARCH="GV100GL"
        BATCH_SCRIPT_NAME="${exename}_timing_job.sh"
        rm -f ${BATCH_SCRIPT_NAME}
        rm -f timing-run-done
        # ---- Generate a batch script for running the timing job
        cat <<EOF > ${BATCH_SCRIPT_NAME}
#!/bin/bash

#BSUB -nnodes 1
#BSUB -G uiuc
#BSUB -W 30
#BSUB -q pdebug

printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n"

source "${EMIRGE_HOME}/config/activate_env.sh"
export PYOPENCL_CTX="port:tesla"
export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch"
rm -rf \$XDG_CACHE_HOME
rm -f timing-run-done
which python
conda env list
jsrun -g 1 -a 1 -n 1 python -O -u -m mpi4py ./${exename}.py -i timing_params.yaml
touch timing-run-done

EOF
        chmod +x ${BATCH_SCRIPT_NAME}
        # ---- Submit the batch script and wait for the job to finish
        bsub ${BATCH_SCRIPT_NAME}
        # ---- Wait 10 minutes right off the bat (the job is at least 10 min)
        sleep 300
        iwait=0
        while [ ! -f ./timing-run-done ]; do 
            iwait=$((iwait+1))
            if [ "$iwait" -gt 360 ]; then # give up after 1 hour
                printf "Timed out waiting on batch job.\n"
                exit 1 # skip the rest of the script
            fi
            sleep 10
        done
        ;;

    # --- Run the timing test on an unknown/generic machine 
    *)
        printf "Host: Unknown\n"
        PYOPENCL_TEST=port:pthread python -O -m mpi4py ./${exename}.py -i timing_params.yaml
        ;;
esac

date
timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
# -- Process the results of the timing run
RUN_LOG_FILE="${exename}-rank0.sqlite"
if [[ -f "${RUN_LOG_FILE}" ]]; then

    SUMMARY_FILE_NAME="${exename}_${timestamp}.sqlite"
    YAML_FILE_NAME="${exename}_timings.yaml"
    rm -f ${YAML_FILE_NAME}
    rm -f ${SUMMARY_FILE_NAME}

    # --- Pull the timings out of the sqlite files generated by logging
    runalyzer-gather ${SUMMARY_FILE_NAME} ${RUN_LOG_FILE}
    CL_DEVICE=$(sqlite3 ${SUMMARY_FILE_NAME} 'SELECT cl_device_name FROM runs')
    STARTUP_TIME=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(q("select $t_init.max").fetchall()[0][0])' | grep -v INFO)
    FIRST_STEP=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[0:1]))' | grep -v INFO)
    FIRST_10_STEPS=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[0:10]))' | grep -v INFO)
    SECOND_10_STEPS=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[10:19]))' | grep -v INFO)

    # --- Create a YAML-compatible text snippet with the timing info
    printf "run_date: ${TIMING_DATE}\nrun_host: ${TIMING_HOST}\n" > ${YAML_FILE_NAME}
    printf "cl_device: ${CL_DEVICE}\n" >> ${YAML_FILE_NAME}
    printf "run_epoch: ${TIME_SINCE_EPOCH}\nrun_platform: ${TIMING_PLATFORM}\n" >> ${YAML_FILE_NAME}
    printf "run_arch: ${TIMING_ARCH}\ngpu_arch: ${GPU_ARCH}\n" >> ${YAML_FILE_NAME}
    printf "mirge_version: ${MIRGE_HASH}\ny1_version: ${Y1_HASH}\n" >> ${YAML_FILE_NAME}
    printf "driver_version: ${DRIVER_HASH}\ndriver_md5sum: ${DRIVER_MD5SUM}\n" >> ${YAML_FILE_NAME}
    printf "time_startup: ${STARTUP_TIME}\ntime_first_step: ${FIRST_STEP}\n" >> ${YAML_FILE_NAME}
    printf "time_first_10: ${FIRST_10_STEPS}\ntime_second_10: ${SECOND_10_STEPS}\n---\n" >> ${YAML_FILE_NAME}

    # Users should set special keys for using git over
    # ssh for security concerns. This snippet will use
    # a pre-arranged ssh key if the user provides one
    # and indicates it with the TESTING_SSH_KEY environment
    # variable.
    # ===== To create a key:
    # - Run ssh-keygen:
    # $ ssh-keygen
    # [enter a <keyname> when prompted]
    # - Put the key(s) in a /secure/filesystem/location:
    # $ mv <keyname>* /secure/filesystem/location
    # - Add the key to GIT:
    # $ [browse to] https://github.com/illinois-ceesd/timing/settings/keys/new
    # $ Choose (New SSH key)
    # $ Paste in the contents of /secure/filesystem/location/<keyname>.pub
    # - Set the ENV variable before using this script:
    # $ export TESTING_SSH_KEY=/secure/filesystem/location/<keyname>
    if [ ! -z "${TESTING_SSH_KEY}" ]; then
        eval $(ssh-agent)
        trap "kill $SSH_AGENT_PID" EXIT
        ssh-add ${TESTING_SSH_KEY}
    fi

    # --- Update the timing data in the repo
    # ---- First, clone the timing repo
    git clone -b ${TIMING_BRANCH} git@github.com:${TIMING_REPO}
    # ---- Create the timing file if it does not exist
    if [[ ! -f timing/${exename}-timings.yaml ]]; then 
        touch timing/${exename}-timings.yaml
        (cd timing && git add ${exename}-timings.yaml)
    fi
    # ---- Update the timing file with the current test data
    cat ${YAML_FILE_NAME} >> timing/${exename}-timings.yaml
    mkdir -p timing/${exename}_logs
    cp ${SUMMARY_FILE_NAME} timing/${exename}_logs
    cd timing
    git add ${exename}_logs/${SUMMARY_FILE_NAME}
    # ---- Commit the new data to the repo
    (git commit -am "Automatic commit: ${TIMING_HOST} ${TIMING_DATE}" && git push)
    cd ../
else
    printf "Timing run did not produce the expected sqlite file: ${RUN_LOG_FILE}\n"
    exit 1
fi

date