Add run-larnd-sim

Author: mjkramer

run-larnd-sim/.gitignore

@@ -0,0 +1,5 @@
+/larnd-sim
+/venv
+/liblockfile
+/logs
+/input

run-larnd-sim/do_submit.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+mkdir -p logs/"$ARCUBE_OUT_NAME"
+
+sbatch -o logs/"$ARCUBE_OUT_NAME"/slurm-%j.out "$@" slurm_job.sh

run-larnd-sim/install_larnd_sim.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+module load cudatoolkit         # 11.7
+module load python              # 3.9-anaconda-2021.11
+
+rm -rf larnd-sim venv
+
+python -m venv venv/
+source venv/bin/activate
+
+pip install -U pip wheel setuptools
+
+# https://docs.nersc.gov/development/languages/python/using-python-perlmutter/#installing-with-pip
+( git clone https://github.com/DUNE/larnd-sim.git
+  cd larnd-sim || exit
+  # HACK: Replace cupy with cupy-cuda11x
+  mv setup.py setup.py.orig
+  sed 's/cupy/cupy-cuda11x/' setup.py.orig > setup.py
+  pip install .
+  mv setup.py.orig setup.py )
+
+# The job needs dotlockfile
+( git clone https://github.com/miquels/liblockfile.git
+  cd liblockfile || exit
+  ./configure
+  make -j4 )

run-larnd-sim/load_larnd_sim.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# source me
+
+module load cudatoolkit         # 11.7
+module load python              # 3.9-anaconda-2021.11
+
+source venv/bin/activate

run-larnd-sim/monitor.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+while true; do
+    top ibn1
+    # top -u $UID -b -n 1
+    nvidia-smi
+    printf "\n-oOo-\n"
+    sleep 60
+done

run-larnd-sim/perf_tests/submit/submit_perftest_noCherry.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export ARCUBE_OUT_NAME=NuMI_RHC_128tasks
+
+./do_submit.sh "$@"

run-larnd-sim/perf_tests/submit/submit_perftest_noCherry_withRock.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export ARCUBE_OUT_NAME=NuMI_RHC_withRock_128tasks
+
+./do_submit.sh "$@"

run-larnd-sim/prep_job.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+outName=$1; shift
+outDir=../run-edep-sim/output/"$outName"
+inDir=input/"$outName"
+
+mkdir -p "$inDir"
+
+ls $(realpath "$outDir"/EDEPSIM_H5)/*.h5 > "$inDir"/input.list

run-larnd-sim/slurm_job.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#SBATCH --account=dune_g
+#SBATCH --qos=regular
+#SBATCH --constraint=gpu
+#SBATCH --time=2:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=4
+#SBATCH --gpus-per-task=1
+#SBATCH --cpus-per-task=32
+
+# https://docs.nersc.gov/systems/perlmutter/running-jobs/#1-node-4-tasks-4-gpus-1-gpu-visible-to-each-task
+export SLURM_CPU_BIND="cores"
+
+# NOTE: Beginning with 22.05, srun will not inherit the --cpus-per-task value
+# requested by salloc or sbatch. It must be requested again with the call to
+# srun or set with the SRUN_CPUS_PER_TASK environment variable if desired for
+# the task(s). (WTF?!??!)
+
+export SRUN_CPUS_PER_TASK="$SLURM_CPUS_PER_TASK"
+
+logDir=logs/"$ARCUBE_OUT_NAME"/"$SLURM_JOBID"
+mkdir -p "$logDir"
+
+echo JOB-START "$(date)"
+
+srun -o "$logDir"/output-%j.%t.txt ./slurm_task.sh
+
+echo JOB-END "$(date)"

run-larnd-sim/slurm_task.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# HACK: This will not wait for other tasks on the node to complete
+if [[ "$SLURM_LOCALID" == 0 ]]; then
+    monitorFile=monitor-$SLURM_JOBID.$SLURM_NODEID.txt
+    ./monitor.sh >logs/"$ARCUBE_OUT_NAME"/"$SLURM_JOBID"/"$monitorFile" &
+fi
+
+# Just in case, to avoid a rush for the lockfile
+sleep $((RANDOM % 20))
+
+source venv/bin/activate
+
+outDir=../run-edep-sim/output/${ARCUBE_OUT_NAME}/LARNDSIM
+mkdir -p "$outDir"
+
+inFile=input/${ARCUBE_OUT_NAME}/input.list
+
+dotlockfile=liblockfile/dotlockfile
+
+while read -r input; do
+    outName=$(basename "$input" .EDEPSIM.h5)
+
+    [[ -e "$outDir/$outName.lock" ]] && continue
+    [[ -e "$outDir/$outName.time" ]] && continue
+    $dotlockfile -r 0 "$outDir/$outName.lock" || continue
+
+    echo FILE-START "$(date)"
+
+    /usr/bin/time -f "%P %M %E" -o "$outDir/$outName.time" \
+        simulate_pixels.py --input_filename "$input" \
+        --output_filename "$outDir/$outName.LARNDSIM.h5" \
+        --detector_properties larnd-sim/larndsim/detector_properties/2x2.yaml \
+        --pixel_layout larnd-sim/larndsim/pixel_layouts/multi_tile_layout-2.3.16.yaml \
+        --response_file larnd-sim/larndsim/bin/response_44.npy \
+        --light_lut_filename larnd-sim/larndsim/bin/lightLUT.npz \
+        --light_det_noise_filename larnd-sim/larndsim/bin/light_noise-2x2-example.npy
+
+    echo FILE-END "$(date)"
+
+    rm -f "$outDir/$outName.lock"
+done < "$inFile"