-
Notifications
You must be signed in to change notification settings - Fork 2
/
generate_mirge_batch_script.sh
executable file
·158 lines (137 loc) · 4.26 KB
/
generate_mirge_batch_script.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#!/bin/bash
nnodes="1"
time_limit="120"
command_script=""
finish_filename="mirge-batch-finished"
output_script_name="mirge-batch-script.sh"
queue_name="pdebug"
emirge_home="."
execute_batch_script=""
stdio_filename="mirge-batch-script-out.txt"
while getopts n:c:t:o:q:e:x:d: flag
do
case "${flag}" in
n) nnodes=${OPTARG};;
c) command_script=${OPTARG};;
t) time_limit=${OPTARG};;
o) output_script_name=${OPTARG};;
q) queue_name=${OPTARG};;
x) execute_batch_script=${OPTARG};;
e) emirge_home=${OPTARG};;
d) stdio_filename=${OPTARG};;
esac
done
echo "Number of Nodes: $nnodes";
echo "Time Limit: $time_limit";
echo "Command script: $command_script";
echo "Output script: $output_script_name";
echo "Queue: ${queue_name}";
echo "Emirge home: ${emirge_home}";
echo "Stdio filename: ${stdio_filename}"
BATCH_SCRIPT_NAME="${output_script_name}"
TIMING_HOST=$(hostname)
rm -f ${finish_filename}
return_code=0
case $TIMING_HOST in
# --- Run the timing test in a batch job on Lassen@LLC
lassen*)
echo "Resolved Host: Lassen"
TIMING_HOST="Lassen"
# GPU_ARCH="GV100GL"
# BATCH_SCRIPT_NAME="${exename}_timing_job.sh"
rm -f ${BATCH_SCRIPT_NAME}
# ---- Generate a batch script for running a job
cat <<EOF > ${BATCH_SCRIPT_NAME}
#!/bin/bash
#BSUB -nnodes ${nnodes}
#BSUB -G uiuc
#BSUB -W ${time_limit}
#BSUB -q ${queue_name}
#BSUB -o ${stdio_filename}
printf "Running with EMIRGE_HOME=${emirge_home}\n"
source ${emirge_home}/config/activate_env.sh
export MIRGE_HOME="${emirge_home}/mirgecom"
export PYOPENCL_CTX="port:tesla"
export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch"
export POCL_CACHE_DIR_ROOT="/tmp/$USER/pocl-cache"
unset CONDA_CACHE_DISABLE
rm -rf \$XDG_CACHE_HOME
rm -f ${finish_filename}
which python
conda env list
env
env | grep LSB_MCPU_HOSTS
date
# --- User's commands go here ----
# example:
# jsrun -g 1 -a 1 -n 16 bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -u -m mpi4py ./${exename}.py -i run_params.yaml --log --lazy'
return_code=0
if [[ ! -z "${command_script}" ]]; then
. ${command_script}
return_code=\$?
fi
# --------------------------------
date
touch ${finish_filename}
return \$return_code
EOF
chmod +x ${BATCH_SCRIPT_NAME}
# ---- Submit the batch script and wait for the job to finish
if [[ ! -z "${execute_batch_script}" ]]; then
bsub ${BATCH_SCRIPT_NAME}
# ---- Wait 10 minutes right off the bat (the job is at least 10 min)
sleep 600
iwait=0
while [ ! -f "./${finish_filename}" ]; do
iwait=$((iwait+1))
if [ "$iwait" -gt 360 ]; then # give up after 1 hour
printf "Timed out waiting on batch job.\n"
exit 1 # skip the rest of the script
fi
sleep 20
done
sleep 30 # give the batch system time to spew its junk into the log
fi
;;
# --- Run the timing test on an unknown/generic machine
*)
printf "Host: Unknown\n"
rm -f ${BATCH_SCRIPT_NAME}
# ---- Generate a batch script for running a job
cat <<EOF > ${BATCH_SCRIPT_NAME}
#!/bin/bash
printf "Running with EMIRGE_HOME=${emirge_home}\n"
source "${emirge_home}/config/activate_env.sh"
export PYOPENCL_CTX="port:pthread"
export PYOPENCL_TEST="port:pthread"
export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch"
export POCL_CACHE_DIR_ROOT="/tmp/$USER/pocl-cache"
unset CONDA_CACHE_DISABLE
rm -rf \$XDG_CACHE_HOME
rm -f ${finish_filename}
which python
conda env list
env
date
return_code=0
# --- User's commands go here ----
# jsrun -g 1 -a 1 -n 16 bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -u -m mpi4py ./${exename}.py -i run_params.yaml --log --lazy'
if [[ -z "${command_script}" ]]; then
. ${command_script}
return_code=\$?
fi
# --------------------------------
date
touch ${finish_filename}
return \$return_code
EOF
chmod +x ${BATCH_SCRIPT_NAME}
# PYOPENCL_TEST=port:pthread python -O -m mpi4py ./${exename}.py -i timing_params.yaml ${EXEOPTS}
if [[ ! -z "${execute_batch_script}" ]]; then
. ${BATCH_SCRIPT_NAME} >& ${stdio_filename}
return_code=$?
fi
;;
esac
return $return_code
date