check checkpoint 1.3B

scripts/mdx/ckpt/hf_megatron_llama2_1.3b.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# python virtualenv
+cd /model/llmjp0/nii-geniac-megatron/Megatron-LM
+source .env/bin/activate
+
+# distributed settings
+TENSOR_PARALLEL_SIZE=1
+PIPELINE_PARALLEL_SIZE=1
+
+# model config
+MEGATRON_CHECKPOINT_DIR=/model/checkpoints_1.3b_mcore/CC_v2_code20K_en40K_ja60K_ver2.2/
+
+HF_CHECKPOINT_DIR=/model/checkpoints_1.3b_hf/CC_v2_code20K_en40K_ja60K_ver2.2/
+MEGATRON_CHECKPOINT_DIR=/model/checkpoints_1.3b_mcore/CC_v2_code20K_en40K_ja60K_ver2.2/
+
+mkdir -p ${MEGATRON_CHECKPOINT_DIR}
+
+# tokenizer config
+TOKENIZER_MODEL=/model/llmjp0/llm-jp-tokenizer/models/ver2.2/code20K_en40K_ja60K.ver2.2.model
+
+# convert
+python tools/checkpoint/convert.py \
+  --model-type GPT \
+  --loader llama2_hf \
+  --saver mcore \
+  --target-tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \
+  --target-pipeline-parallel-size ${PIPELINE_PARALLEL_SIZE} \
+  --load-dir ${HF_CHECKPOINT_DIR} \
+  --save-dir ${MEGATRON_CHECKPOINT_DIR} \
+  --tokenizer-model ${TOKENIZER_MODEL} \
+  --bf16 \
+  --saver-transformer-impl "transformer_engine"

scripts/mdx/ckpt/is_same.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+# python virtualenv
+source .env/bin/activate
+
+python scripts/abci/megatron_to_hf/check.py \
+  --base-hf-model-path /model/checkpoints_1.3b_hf/CC_v2_code20K_en40K_ja60K_ver2.2/ \
+  --converted-hf-model-path /model/checkpoints_1.3b_hf/megatron_to_hf/CC_v2_code20K_en40K_ja60K_ver2.2/ \

scripts/mdx/ckpt/megatron_hf_llama2_1.3b.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# python virtualenv
+cd /model/llmjp0/nii-geniac-megatron/Megatron-LM
+source .env/bin/activate
+
+# distributed settings
+TENSOR_PARALLEL_SIZE=1
+PIPELINE_PARALLEL_SIZE=1
+
+# model config
+MEGATRON_CHECKPOINT_DIR=/model/checkpoints_1.3b_mcore/CC_v2_code20K_en40K_ja60K_ver2.2/
+HF_CHECKPOINT_DIR=/model/checkpoints_1.3b_hf/megatron_to_hf/CC_v2_code20K_en40K_ja60K_ver2.2/
+
+mkdir -p ${MEGATRON_CHECKPOINT_DIR}
+
+# tokenizer config
+TOKENIZER_MODEL_DIR=/model/checkpoints_1.3b_hf/megatron_to_hf/CC_v2_code20K_en40K_ja60K_ver2.2/
+
+# convert
+python tools/checkpoint/convert.py \
+  --model-type GPT \
+  --loader mcore \
+  --saver llama2_hf \
+  --load-dir ${MEGATRON_CHECKPOINT_DIR} \
+  --save-dir ${HF_CHECKPOINT_DIR} \
+  --true-vocab-size 96867 \
+  --hf-tokenizer-path ${TOKENIZER_MODEL_DIR} \
+  --save-dtype bfloat16 \
+  --loader-transformer-impl "transformer_engine" \
+  --megatron-path /model/llmjp0/nii-geniac-megatron/Megatron-LM