[huggingface_pytorch] Training - update for Transformers to 4.41.2 PyTorch 2.2

dlc_developer_config.toml

@@ -1,6 +1,6 @@
 [dev]
 # Set to "huggingface", for example, if you are a huggingface developer. Default is ""
-partner_developer = ""
+partner_developer = "huggingface"
 # Please only set it to true if you are preparing an EI related PR
 # Do remember to revert it back to false before merging any PR (including EI dedicated PR)
 ei_mode = false
@@ -34,11 +34,11 @@ deep_canary_mode = false
 [build]
 # Add in frameworks you would like to build. By default, builds are disabled unless you specify building an image.
 # available frameworks - ["autogluon", "huggingface_tensorflow", "huggingface_pytorch", "huggingface_tensorflow_trcomp", "huggingface_pytorch_trcomp", "pytorch_trcomp", "tensorflow", "mxnet", "pytorch", "stabilityai_pytorch"]
-build_frameworks = []
+build_frameworks = ["huggingface_pytorch"]
 
 # By default we build both training and inference containers. Set true/false values to determine which to build.
 build_training = true
-build_inference = true
+build_inference = false
 
 # Set do_build to "false" to skip builds and test the latest image built by this PR
 # Note: at least one build is required to set do_build to "false"

huggingface/pytorch/training/buildspec-2-1.yml

@@ -0,0 +1,32 @@
+account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
+region: &REGION <set-$REGION-in-environment>
+base_framework: &BASE_FRAMEWORK pytorch
+framework: &FRAMEWORK !join [ "huggingface_", *BASE_FRAMEWORK]
+version: &VERSION 2.1.0
+short_version: &SHORT_VERSION "2.1"
+contributor: huggingface
+arch_type: x86
+
+repository_info:
+  training_repository: &TRAINING_REPOSITORY
+    image_type: &TRAINING_IMAGE_TYPE training
+    root: !join [ "huggingface/", *BASE_FRAMEWORK, "/", *TRAINING_IMAGE_TYPE ]
+    repository_name: &REPOSITORY_NAME !join ["pr", "-", "huggingface", "-", *BASE_FRAMEWORK, "-", *TRAINING_IMAGE_TYPE]
+    repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/, *REPOSITORY_NAME ]
+
+images:
+  BuildHuggingFacePytorchGpuPy310Cu121TrainingDockerImage:
+    <<: *TRAINING_REPOSITORY
+    build: &HUGGINGFACE_PYTORCH_GPU_TRAINING_PY3 false
+    image_size_baseline: &IMAGE_SIZE_BASELINE 21500
+    device_type: &DEVICE_TYPE gpu
+    python_version: &DOCKER_PYTHON_VERSION py3
+    tag_python_version: &TAG_PYTHON_VERSION py310
+    cuda_version: &CUDA_VERSION cu121
+    os_version: &OS_VERSION ubuntu20.04
+    transformers_version: &TRANSFORMERS_VERSION 4.36.0
+    datasets_version: &DATASETS_VERSION 2.15.0
+    tag: !join [ *VERSION, '-', 'transformers', *TRANSFORMERS_VERSION, '-', *DEVICE_TYPE, '-', *TAG_PYTHON_VERSION, '-',
+                 *CUDA_VERSION, '-', *OS_VERSION ]
+    docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /,
+                         *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]

huggingface/pytorch/training/buildspec.yml

@@ -2,8 +2,8 @@ account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
 region: &REGION <set-$REGION-in-environment>
 base_framework: &BASE_FRAMEWORK pytorch
 framework: &FRAMEWORK !join [ "huggingface_", *BASE_FRAMEWORK]
-version: &VERSION 2.1.0
-short_version: &SHORT_VERSION "2.1"
+version: &VERSION 2.2.0
+short_version: &SHORT_VERSION "2.2"
 contributor: huggingface
 arch_type: x86
 
@@ -24,8 +24,8 @@ images:
     tag_python_version: &TAG_PYTHON_VERSION py310
     cuda_version: &CUDA_VERSION cu121
     os_version: &OS_VERSION ubuntu20.04
-    transformers_version: &TRANSFORMERS_VERSION 4.36.0
-    datasets_version: &DATASETS_VERSION 2.18.0
+    transformers_version: &TRANSFORMERS_VERSION 4.41.2
+    datasets_version: &DATASETS_VERSION 2.19.2
     tag: !join [ *VERSION, '-', 'transformers', *TRANSFORMERS_VERSION, '-', *DEVICE_TYPE, '-', *TAG_PYTHON_VERSION, '-',
                  *CUDA_VERSION, '-', *OS_VERSION ]
     docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /,

huggingface/pytorch/training/docker/2.2/py3/cu121/Dockerfile.gpu

@@ -0,0 +1,58 @@
+# https://github.com/aws/deep-learning-containers/blob/master/available_images.md
+# refer to the above page to pull latest Pytorch image
+
+# docker image region us-west-2
+FROM 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.2.0-gpu-py310-cu121-ubuntu20.04-sagemaker
+
+LABEL maintainer="Amazon AI"
+LABEL dlc_major_version="1"
+
+# version args
+ARG TRANSFORMERS_VERSION
+ARG DATASETS_VERSION
+ARG DIFFUSERS_VERSION=0.28.2
+ARG EVALUATE_VERSION=0.4.2
+ARG ACCELERATE_VERSION=0.31.0
+ARG TRL_VERSION=0.9.4
+ARG PEFT_VERSION=0.11.1
+ARG FLASH_ATTN_VERSION=2.5.8
+ARG MULTIPROCESS_VERSION=0.70.16
+ARG DILL_VERSION=0.3.8
+ARG PYTHON=python3
+
+# TODO: Remove when the base image is updated
+RUN pip install --upgrade pip \
+ && pip uninstall -y transformer-engine flash-attn pyarrow cryptography \
+ && pip install --no-cache-dir -U pyarrow cryptography pyopenssl Pillow
+
+# install Hugging Face libraries and its dependencies
+RUN pip install --no-cache-dir \
+	transformers[sklearn,sentencepiece,audio,vision]==${TRANSFORMERS_VERSION} \
+	datasets==${DATASETS_VERSION} \
+	diffusers==${DIFFUSERS_VERSION} \
+	Jinja2 \
+	tensorboard \
+	bitsandbytes \
+	evaluate==${EVALUATE_VERSION} \
+	accelerate==${ACCELERATE_VERSION} \
+	trl==${TRL_VERSION} \
+	peft==${PEFT_VERSION} \
+	flash-attn==${FLASH_ATTN_VERSION} \
+	multiprocess==${MULTIPROCESS_VERSION} \
+	dill==${DILL_VERSION}
+
+RUN apt-get update \
+ # TODO: Remove upgrade statements once packages are updated in base image
+ && apt-get -y upgrade --only-upgrade systemd openssl cryptsetup \
+ && apt-get install -y git-lfs \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN HOME_DIR=/root \
+ && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
+ && unzip -o ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \
+ && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \
+ && chmod +x /usr/local/bin/testOSSCompliance \
+ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \
+ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \
+ && rm -rf ${HOME_DIR}/oss_compliance*

test/sagemaker_tests/huggingface_pytorch/training/integration/local/test_single_machine_training.py

@@ -32,7 +32,7 @@ def test_distilbert_base(
     hyperparameters = {
         "max_steps": 5,
         "train_batch_size": 4,
-        "model_name": "distilbert-base-uncased",
+        "model_name": "distilbert/distilbert-base-uncased",
     }
 
     estimator = HuggingFace(
@@ -65,7 +65,7 @@ def test_distilbert_base_torch_compiled(
     hyperparameters = {
         "max_steps": 5,
         "train_batch_size": 4,
-        "model_name": "distilbert-base-uncased",
+        "model_name": "distilbert/distilbert-base-uncased",
     }
 
     estimator = HuggingFace(

test/sagemaker_tests/huggingface_pytorch/training/integration/sagemaker/test_smdp.py

@@ -40,7 +40,10 @@
     "dataset_name": "squad",
     "do_train": True,
     "do_eval": True,
-    "fp16": True,
+    # Q: Does the base pt image setup the cuda environment correctly? ValueError: FP16 Mixed precision
+    # training with AMP or APEX (`--fp16`) and FP16 half precision evaluation (`--fp16_full_eval`) can
+    # only be used on CUDA or MLU devices or NPU devices or certain XPU devices (with IPEX).
+    "fp16": False,
     "per_device_train_batch_size": 1,
     "per_device_eval_batch_size": 1,
     "num_train_epochs": 1,

test/sagemaker_tests/huggingface_pytorch/training/integration/sagemaker/test_trcomp.py

@@ -33,7 +33,7 @@
 
 
 hyperparameters = {
-    "model_name_or_path": "bert-large-uncased-whole-word-masking",
+    "model_name_or_path": "google-bert/bert-large-uncased-whole-word-masking-finetuned-squad",
     "dataset_name": "squad",
     "do_train": True,
     "do_eval": True,

test/sagemaker_tests/huggingface_pytorch/training/resources/scripts/train.py

@@ -49,7 +49,7 @@
 
     # tokenizer helper function
     def tokenize(batch):
-        return tokenizer(batch["text"], padding="max_length", truncation=True)
+        return tokenizer(batch["text"], padding="max_length", max_length=512, truncation=True)
 
     # load dataset
     train_dataset, test_dataset = load_dataset("imdb", split=["train", "test"])

test/sagemaker_tests/huggingface_pytorch/training/resources/scripts/train_torch_compiled.py

@@ -52,7 +52,7 @@
 
     # tokenizer helper function
     def tokenize(batch):
-        return tokenizer(batch["text"], padding="max_length", truncation=True)
+        return tokenizer(batch["text"], padding="max_length", max_length=512, truncation=True)
 
     # load dataset
     train_dataset, test_dataset = load_dataset("imdb", split=["train", "test"])