minor changes

CERC-AAI · Apr 4, 2024 · 81f5899 · 81f5899
1 parent 5971218
commit 81f5899
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 14 deletions.
diff --git a/megatron/neox_arguments/neox_args.py b/megatron/neox_arguments/neox_args.py
@@ -688,11 +688,6 @@ class NeoXArgsOther(NeoXArgsTemplate):
     Include sequence id as attention bias.
     """
 
-    concat_data: bool = True
-    """
-    Concatenate sequences for training.
-    """
-
 
 @dataclass
 class NeoXArgsTokenizer(NeoXArgsTemplate):

diff --git a/tools/convert_hf_to_sequential.py b/tools/convert_hf_to_sequential.py
@@ -35,28 +35,28 @@
 Example usage: (Converts the 70M Pythia model to NeoX format)
 ================================================================
 OMPI_COMM_WORLD_RANK=0 CUDA_VISIBLE_DEVICES=0 python tools/convert_hf_to_sequential.py \
-    --hf-model-name pythia-1b \
+    --hf-model-name pythia-1.4b \
     --revision 143000 \
-    --output-dir /p/fastdata/mmlaion/hummingbird/checkpoints/1_neox \
-    --cache-dir /p/fastdata/mmlaion/hummingbird/checkpoints/pythia1B \
-    --config configs/1B.yml configs/hummingbird_streaming.yml \
+    --output-dir /p/fastdata/mmlaion/hummingbird/checkpoints/1_4_final \
+    --cache-dir /p/fastdata/mmlaion/hummingbird/checkpoints/pythia_final \
+    --config configs/pythia/1-4B.yml configs/hummingbird_streaming.yml \
     --test
 
 
 For multi-gpu support we must initialize deepspeed:
 NOTE: This requires manually changing the arguments below.
 ================================================================
 CUDA_VISIBLE_DEVICES=0,1,2,3 python ./deepy.py tools/convert_hf_to_sequential.py \
-    -d configs 1B.yml hummingbird_streaming.yml
+    -d configs pythia/1-4B.yml hummingbird_streaming.yml
 """
 
 MULTI_GPU_ARGS = " ".join(
     [
-        "--hf-model-name pythia-1b",
+        "--hf-model-name pythia-1.4b",
         "--revision 143000",
-        "--output-dir /p/fastdata/mmlaion/hummingbird/checkpoints/1_neox",
-        "--cache-dir /p/fastdata/mmlaion/hummingbird/checkpoints/pythia1B",
-        "--config configs/1B.yml configs/hummingbird_streaming.yml",
+        "--output-dir /p/fastdata/mmlaion/hummingbird/checkpoints/1_4_final",
+        "--cache-dir /p/fastdata/mmlaion/hummingbird/checkpoints/pythia_final",
+        "--config configs/pythia/1-4B.yml configs/hummingbird_streaming.yml",
         "--test",
     ]
 )