summit_setup.yml

# Suggested data paths when using GPT-NeoX locally
{
  "train-data-paths": [
    "/gpfs/alpine/csc499/proj-shared/LAION-5B/laion2B-en/{00000..23925}.tar",
    "/gpfs/alpine/csc499/proj-shared/LAION-400m-webdataset/data/{00000..40000}.tar"
    ],
  "train-data-weights": [
    1.0,
    1.0,
  ],
  "valid-data-paths": ["/gpfs/alpine/csc499/proj-shared/LAION-400m-webdataset/data/{40001..41000}.tar",],
  "valid-data-weights": [
    1.0,
  ], 
  "test-data-paths": ["/gpfs/alpine/csc499/proj-shared/LAION-400m-webdataset/data/{41000..41455}.tar",], 
  "test-data-weights": [
    1.0,
  ],


  "vocab-file": "./data/20B_tokenizer.json",

  
  "tokenizer-type": "HFTokenizer",
  "vocab-file": "/gpfs/alpine/csc499/scratch/kshitijkg/gpt-neox/data/20B_tokenizer.json",

  "save": "/gpfs/alpine/csc499/scratch/kshitijkg/magma/gpt-neox/checkpoints/magma_test64",
  # "load": "/gpfs/alpine/csc499/scratch/kshitijkg/neox_converted_magma/mp1_pp1/pythia",
  "load": "/gpfs/alpine/csc499/scratch/kshitijkg/magma/gpt-neox/checkpoints/magma_test64",
  "load_clip": "/gpfs/alpine/csc499/proj-shared/magma/models/clip",
  "checkpoint_validation_with_forward_pass": False,

  "tensorboard-dir": "/gpfs/alpine/csc499/scratch/kshitijkg/magma/gpt-neox/tensorboard/magma_test64",
  "log-dir": "/gpfs/alpine/csc499/scratch/kshitijkg/magma/gpt-neox/logs/magma_test64",
  "use_wandb": False,
  "wandb_host": "https://api.wandb.ai",
  "wandb_project": "magma_test64",

  "launcher": "jsrun",
  "deepspeed_jsrun": true,
  # "finetune": true,
  "num_workers": 0,
}