From a9d60d84d1cd53955786dc119f1e698f4db48098 Mon Sep 17 00:00:00 2001
From: Hirokazu Kiyomaru <h.kiyomaru@gmail.com>
Date: Fri, 23 Aug 2024 13:49:57 +0900
Subject: [PATCH 1/2] [wip] automate evaluation using llm-jp-eval v1.3.1

---
 .../llm-jp-eval-v1.3.1/scripts/cron.sh        | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100755 evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh

diff --git a/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh b/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh
new file mode 100755
index 0000000..1b5382c
--- /dev/null
+++ b/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# This script is intended to be run as a cron job to evaluate new checkpoints.
+# Usage:
+#   bash cron.sh <checkpoint_glob_pattern> <log_file>
+
+set -eux
+
+# Get the path to the evaluation script
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+SCRIPT_PATH="${SCRIPT_DIR}/run_llm-jp-eval.sh"
+
+# Arguments
+CHECKPOINT_GLOB_PATTERN=$1
+LOG_FILE=$2
+
+# Create log file if it doesn't exist
+touch "$LOG_FILE"
+
+for checkpoint_path in $CHECKPOINT_GLOB_PATTERN; do
+    # Skip if the checkpoint has already been evaluated
+    if grep -qx $checkpoint_path $LOG_FILE; then
+        continue
+    fi
+    
+    # Evaluate the checkpoint
+    echo "Evaluating checkpoint $checkpoint_path"
+    wandb_run_name=$(basename $checkpoint_path)
+    sbatch $SCRIPT_PATH $checkpoint_path $wandb_run_name
+    
+    # Log the checkpoint path
+    echo $checkpoint_path >> $LOG_FILE
+done

From 5a13c84fe55d2ebc6bc9224e894aaed53262d063 Mon Sep 17 00:00:00 2001
From: Hirokazu Kiyomaru <h.kiyomaru@gmail.com>
Date: Fri, 23 Aug 2024 14:21:21 +0900
Subject: [PATCH 2/2] [wip] log checkpoint path before evaluation

---
 evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh b/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh
index 1b5382c..9257983 100755
--- a/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh
+++ b/evaluation/installers/llm-jp-eval-v1.3.1/scripts/cron.sh
@@ -23,11 +23,12 @@ for checkpoint_path in $CHECKPOINT_GLOB_PATTERN; do
         continue
     fi
     
+    # Log the checkpoint path
+    # NOTE: This is done before evaluation to avoid re-evaluating the same checkpoint
+    echo $checkpoint_path >> $LOG_FILE
+
     # Evaluate the checkpoint
     echo "Evaluating checkpoint $checkpoint_path"
     wandb_run_name=$(basename $checkpoint_path)
     sbatch $SCRIPT_PATH $checkpoint_path $wandb_run_name
-    
-    # Log the checkpoint path
-    echo $checkpoint_path >> $LOG_FILE
 done