diff --git a/aws-examples/README.md b/aws-examples/README.md new file mode 100644 index 000000000..355b30ace --- /dev/null +++ b/aws-examples/README.md @@ -0,0 +1,7 @@ +# AWS Examples with 🤗 Optimum Neuron + +These example scripts are used in conjunction with posts on the [AWS Machine Learning Blog](https://aws.amazon.com/blogs/machine-learning/). + +## Current Examples + +- End-to-end Fine-tuning and Deployment of Mistral (`mistral-e2e`) \ No newline at end of file diff --git a/aws-examples/mistral-e2e/chat.py b/aws-examples/mistral-e2e/chat.py new file mode 100644 index 000000000..c470f96e5 --- /dev/null +++ b/aws-examples/mistral-e2e/chat.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team and Amazon Web Services, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This is a simple command-line chat application that has contextual memory. +""" + +from transformers import AutoTokenizer +from optimum.neuron import NeuronModelForCausalLM + +# Load the model compiled for AWS Neuron +model = NeuronModelForCausalLM.from_pretrained("./mistral_neuron", local_files_only=True) + +# Load the tokenizer +tokenizer = AutoTokenizer.from_pretrained("./mistral_neuron") +tokenizer.pad_token_id = tokenizer.eos_token_id + +def format_chat_prompt(message, history, max_tokens): + """Formats an entire chat history to enable contextual memory.""" + chat = [] + + # Add each former interaction to the chat list with alternating roles, user and assistant. + for interaction in history: + chat.append({"role": "user", "content": interaction[0]}) + chat.append({"role": "assistant", "content": interaction[1]}) + + # Add the new (user) message to the chat flow. + chat.append({"role": "user", "content": message}) + + # Apply the chat template to each chat message and ensure we do not exceed max_tokens + for i in range(0, len(chat), 2): + # apply the chat message to every pair of messages from user and assistant up to i + prompt = tokenizer.apply_chat_template(chat[i:], tokenize=False) + + # Validate that our response does not exceed max_tokens. + # If it does, the for loop truncates the first message from the prompt until we are under max_tokens. + # This way, we never pass more than the alloted max_tokens to the model. + tokens = tokenizer(prompt) + if len(tokens.input_ids) <= max_tokens: + return prompt + + # If we've exceeded max_tokens, raise SystemError. + # This shouldn't be reached unless something goes wrong, such as the + # initial message and subsequent response exceeding the token limit. + raise SystemError + +def chat(history, max_tokens): + """This function runs recursively to take user input, making the chat bot functional.""" + # Take input from user + message = input("Enter input: ") + + # Stop the program if the user types "quit" + if message == "quit": + return + + # Tokenize the formatted prompt + inputs = tokenizer(format_chat_prompt(message, history, max_tokens), return_tensors="pt") + + # Do inference to generate a response + outputs = model.generate( + **inputs, + max_new_tokens=512, + do_sample=True, + temperature=0.9, + top_k=50, + top_p=0.9 + ) + + # Decode the response to a string, and remove the prompt. + response = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True) + + # Print the response + print(response) + + # Add the message and response to history + history.append([message, response]) + + # Repeat + chat(history, max_tokens) + +if __name__ == "__main__": + # Define an empty history and max number of tokens + history = [] + max_tokens = 4096 + + chat(history, max_tokens) diff --git a/aws-examples/mistral-e2e/compile.py b/aws-examples/mistral-e2e/compile.py new file mode 100644 index 000000000..06da6d693 --- /dev/null +++ b/aws-examples/mistral-e2e/compile.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team and Amazon Web Services, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script compiles a model to be usable as an Optimum Neuron NeuronModelForCausalLM. +""" + +from optimum.neuron import NeuronModelForCausalLM + +# num_cores is the number of neuron cores. Find this with the command neuron-ls +compiler_args = {"num_cores": 12, "auto_cast_type": 'bf16'} +input_shapes = {"batch_size": 1, "sequence_length": 4096} + +# Compiles an Optimum Neuron model from the previously trained (uncompiled) model +model = NeuronModelForCausalLM.from_pretrained( + "mistral_trained", + export=True, + **compiler_args, + **input_shapes +) + +# Saves the compiled model to the directory mistral_neuron +model.save_pretrained("mistral_neuron") diff --git a/aws-examples/mistral-e2e/dataset.py b/aws-examples/mistral-e2e/dataset.py new file mode 100644 index 000000000..0a63d1387 --- /dev/null +++ b/aws-examples/mistral-e2e/dataset.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team and Amazon Web Services, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script formats the gsm8k dataset for use by the training script (run_clm.py) in this directory. +""" + +from datasets import DatasetDict, load_dataset + +def format(sample): + sample['text'] = f"[INST] {sample['question']} [/INST]\n\n{sample['answer']}" + return sample + +# Downloads the gsm8k dataset directly from Hugging Face. +dataset = load_dataset("gsm8k", "main") + +# We need to split the dataset into a training, and validation set. +# Note gsm8k has 'test', we rename to 'validation' for our training script. +train = dataset['train'] +validation = dataset['test'] + +# Map the format function on all elements of the training and validation splits. +# Also removes the question and answer columns we no longer need. +train = train.map(format, remove_columns=list(train.features)) +validation = validation.map(format, remove_columns=list(validation.features)) + +# Create a new DatasetDict with our train and validation splits. +dataset = DatasetDict({"train": train, "validation": validation}) + +dataset.save_to_disk('dataset_formatted') diff --git a/aws-examples/mistral-e2e/hfon.yaml b/aws-examples/mistral-e2e/hfon.yaml new file mode 100644 index 000000000..5df6aef15 --- /dev/null +++ b/aws-examples/mistral-e2e/hfon.yaml @@ -0,0 +1,135 @@ +# This is an AWS CloudFormation template to deploy the environment used in this example. Must be deployed in us-east-1. +AWSTemplateFormatVersion: '2010-09-09' +Description: Creates an EC2 trn1.32xlarge and inf2.24xlarge instance, and an S3 bucket, for end-to-end training and deployment of LLMs. + +# Selectable parameters in the AWS CloudFormation deployment +Parameters: + VPC: + Type: AWS::EC2::VPC::Id + Description: The VPC ID where the resources will be deployed. + Subnet: + Type: AWS::EC2::Subnet::Id + Description: The public subnet ID where the EC2 instances will be deployed. + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + Description: The key pair to use for SSH access to the EC2 instances. + +Resources: + # S3 Bucket + S3Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub aws-hfon-${AWS::AccountId} + + # EC2 trn1.32xlarge Instance + trn1Instance: + Type: AWS::EC2::Instance + Properties: + ImageId: ami-0ce2d16d374f959dd + InstanceType: trn1.32xlarge + KeyName: !Ref KeyPair + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - !Ref InstanceSecurityGroup + SubnetId: !Ref Subnet + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: 512 + VolumeType: gp3 + UserData: + Fn::Base64: !Sub | + #!/bin/bash + yum update -y + yum install -y aws-cli + su - ubuntu + echo "export S3_BUCKET=s3://${S3Bucket}" >> /home/ubuntu/.bashrc + + # EC2 inf2.24xlarge Instance + inf2Instance: + Type: AWS::EC2::Instance + # Wait until the trn1 instance was successfully provisioned + DependsOn: trn1Instance + Properties: + ImageId: ami-0ce2d16d374f959dd + InstanceType: inf2.24xlarge + KeyName: !Ref KeyPair + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - !Ref InstanceSecurityGroup + SubnetId: !Ref Subnet + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: 256 + VolumeType: gp3 + UserData: + Fn::Base64: !Sub | + #!/bin/bash + yum update -y + yum install -y aws-cli + su - ubuntu + echo "export S3_BUCKET=s3://${S3Bucket}" >> /home/ubuntu/.bashrc + + # Security Group + InstanceSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Allows SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VPC + + # Instance Role + InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: ec2-hfon-s3-bucket-access + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:ListBucket + Resource: + - !GetAtt S3Bucket.Arn + - !Sub "${S3Bucket.Arn}/*" + + # IAM Instance Profile + InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref InstanceRole + +Outputs: + S3BucketName: + Description: The name of the S3 bucket + Value: !Ref S3Bucket + trn1InstanceId: + Description: The ID of the trn1.32xlarge EC2 instance + Value: !Ref trn1Instance + inf2InstanceId: + Description: The ID of the inf2.24xlarge EC2 instance + Value: !Ref inf2Instance diff --git a/aws-examples/mistral-e2e/run_clm.py b/aws-examples/mistral-e2e/run_clm.py new file mode 100644 index 000000000..c940d7fd1 --- /dev/null +++ b/aws-examples/mistral-e2e/run_clm.py @@ -0,0 +1,688 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. + +Here is the full list of checkpoints on the hub that can be fine-tuned by this script: +https://huggingface.co/models?filter=text-generation +""" +# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments. + +import logging +import math +import os +import sys +import warnings +from dataclasses import dataclass, field +from itertools import chain +from typing import Optional + +import datasets +import evaluate +import torch +import transformers +from datasets import load_dataset, load_from_disk +from transformers import ( + CONFIG_MAPPING, + MODEL_FOR_CAUSAL_LM_MAPPING, + AutoConfig, + AutoModelForCausalLM, + AutoTokenizer, + default_data_collator, + is_torch_tpu_available, + set_seed, +) +from transformers.testing_utils import CaptureLogger +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import check_min_version, send_example_telemetry +from transformers.utils.versions import require_version + +from optimum.neuron import NeuronHfArgumentParser as HfArgumentParser +from optimum.neuron import NeuronTrainer as Trainer +from optimum.neuron import NeuronTrainingArguments as TrainingArguments +from optimum.neuron.distributed import lazy_load_for_parallelism + + +# Will error if the minimal version of Transformers is not installed. Remove at your own risks. +check_min_version("4.35.0") + +require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") + +logger = logging.getLogger(__name__) + + +MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys()) +MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The model checkpoint for weights initialization. Don't set if you want to train a model from scratch." + ) + }, + ) + model_type: Optional[str] = field( + default=None, + metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)}, + ) + config_overrides: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override some existing default config settings when a model is trained from scratch. Example: " + "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index" + ) + }, + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + token: str = field( + default=None, + metadata={ + "help": ( + "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token " + "generated when running `huggingface-cli login` (stored in `~/.huggingface`)." + ) + }, + ) + use_auth_token: bool = field( + default=None, + metadata={ + "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead." + }, + ) + trust_remote_code: bool = field( + default=False, + metadata={ + "help": ( + "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option" + "should only be set to `True` for repositories you trust and in which you have read the code, as it will " + "execute code present on the Hub on your local machine." + ) + }, + ) + torch_dtype: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the " + "dtype will be automatically derived from the model's weights." + ), + "choices": ["auto", "bfloat16", "float16", "float32"], + }, + ) + low_cpu_mem_usage: bool = field( + default=False, + metadata={ + "help": ( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. " + "set True will benefit LLM loading time and RAM consumption." + ) + }, + ) + + def __post_init__(self): + if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): + raise ValueError( + "--config_overrides can't be used in combination with --config_name or --model_name_or_path" + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."}) + validation_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"}) + block_size: Optional[int] = field( + default=None, + metadata={ + "help": ( + "Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." + ) + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + validation_split_percentage: Optional[int] = field( + default=5, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + keep_linebreaks: bool = field( + default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."} + ) + + def __post_init__(self): + if self.streaming: + require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`") + + if self.dataset_name is None and self.train_file is None and self.validation_file is None: + raise ValueError("Need either a dataset name or a training/validation file.") + else: + if self.train_file is not None: + extension = self.train_file.split(".")[-1] + assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file." + if self.validation_file is not None: + extension = self.validation_file.split(".")[-1] + assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file." + + +def main(): + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + if model_args.use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead.", + FutureWarning, + ) + if model_args.token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + model_args.token = model_args.use_auth_token + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_clm", model_args, data_args) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, " + + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}" + ) + logger.info(f"Training/evaluation parameters {training_args}") + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Set seed before initializing model. + set_seed(training_args.seed) + + # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below) + # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/ + # (the dataset will be downloaded automatically from the datasets Hub). + # + # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called + # 'text' is found. You can easily tweak this behavior (see below). + # + # In distributed training, the load_dataset function guarantee that only one local process can concurrently + # download the dataset. + if data_args.dataset_name is not None: + try: + # Check if there's a dataset on disk + raw_datasets = load_from_disk( + data_args.dataset_name, + ) + if "validation" not in raw_datasets.keys(): + raw_datasets["validation"] = load_from_disk( + data_args.dataset_name, + split=f"train[:{data_args.validation_split_percentage}%]" + ) + raw_datasets["train"] = load_from_disk( + data_args.dataset_name, + split=f"train[{data_args.validation_split_percentage}%:]" + ) + except: + # Downloading and loading a dataset from the hub. + raw_datasets = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + cache_dir=model_args.cache_dir, + token=model_args.token, + streaming=data_args.streaming, + ) + if "validation" not in raw_datasets.keys(): + raw_datasets["validation"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=f"train[:{data_args.validation_split_percentage}%]", + cache_dir=model_args.cache_dir, + token=model_args.token, + streaming=data_args.streaming, + ) + raw_datasets["train"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=f"train[{data_args.validation_split_percentage}%:]", + cache_dir=model_args.cache_dir, + token=model_args.token, + streaming=data_args.streaming, + ) + else: + data_files = {} + dataset_args = {} + if data_args.train_file is not None: + data_files["train"] = data_args.train_file + if data_args.validation_file is not None: + data_files["validation"] = data_args.validation_file + extension = ( + data_args.train_file.split(".")[-1] + if data_args.train_file is not None + else data_args.validation_file.split(".")[-1] + ) + if extension == "txt": + extension = "text" + dataset_args["keep_linebreaks"] = data_args.keep_linebreaks + raw_datasets = load_dataset( + extension, + data_files=data_files, + cache_dir=model_args.cache_dir, + token=model_args.token, + **dataset_args, + ) + # If no validation data is there, validation_split_percentage will be used to divide the dataset. + if "validation" not in raw_datasets.keys(): + raw_datasets["validation"] = load_dataset( + extension, + data_files=data_files, + split=f"train[:{data_args.validation_split_percentage}%]", + cache_dir=model_args.cache_dir, + token=model_args.token, + **dataset_args, + ) + raw_datasets["train"] = load_dataset( + extension, + data_files=data_files, + split=f"train[{data_args.validation_split_percentage}%:]", + cache_dir=model_args.cache_dir, + token=model_args.token, + **dataset_args, + ) + + # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # Load pretrained model and tokenizer + # + # Distributed training: + # The .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. + + config_kwargs = { + "cache_dir": model_args.cache_dir, + "revision": model_args.model_revision, + "token": model_args.token, + "trust_remote_code": model_args.trust_remote_code, + } + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + if model_args.config_overrides is not None: + logger.info(f"Overriding config: {model_args.config_overrides}") + config.update_from_string(model_args.config_overrides) + logger.info(f"New config: {config}") + + tokenizer_kwargs = { + "cache_dir": model_args.cache_dir, + "use_fast": model_args.use_fast_tokenizer, + "revision": model_args.model_revision, + "token": model_args.token, + "trust_remote_code": model_args.trust_remote_code, + } + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs) + elif model_args.model_name_or_path: + tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script. " + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + + if model_args.model_name_or_path: + torch_dtype = ( + model_args.torch_dtype + if model_args.torch_dtype in ["auto", None] + else getattr(torch, model_args.torch_dtype) + ) + with lazy_load_for_parallelism( + tensor_parallel_size=training_args.tensor_parallel_size, + pipeline_parallel_size=training_args.pipeline_parallel_size, + ): + model = AutoModelForCausalLM.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + token=model_args.token, + trust_remote_code=model_args.trust_remote_code, + torch_dtype=torch_dtype, + low_cpu_mem_usage=model_args.low_cpu_mem_usage, + ) + + else: + with lazy_load_for_parallelism( + tensor_parallel_size=training_args.tensor_parallel_size, + pipeline_parallel_size=training_args.pipeline_parallel_size, + ): + model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code) + + n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values()) + logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params") + + # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch + # on a small vocab and want a smaller embedding size, remove this test. + # embedding_size = model.get_input_embeddings().weight.shape[0] + # if len(tokenizer) > embedding_size: + # model.resize_token_embeddings(len(tokenizer)) + + # Preprocessing the datasets. + # First we tokenize all the texts. + if training_args.do_train: + column_names = list(raw_datasets["train"].features) + else: + column_names = list(raw_datasets["validation"].features) + text_column_name = "text" if "text" in column_names else column_names[0] + + # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function + tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base") + + def tokenize_function(examples): + with CaptureLogger(tok_logger) as cl: + output = tokenizer(examples[text_column_name]) + # clm input could be much much longer than block_size + if "Token indices sequence length is longer than the" in cl.out: + tok_logger.warning( + "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits" + " before being passed to the model." + ) + return output + + with training_args.main_process_first(desc="dataset map tokenization"): + if not data_args.streaming: + tokenized_datasets = raw_datasets.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on dataset", + ) + else: + tokenized_datasets = raw_datasets.map( + tokenize_function, + batched=True, + remove_columns=column_names, + ) + + if data_args.block_size is None: + block_size = tokenizer.model_max_length + if block_size > config.max_position_embeddings: + logger.warning( + f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " + f"Using block_size={min(1024, config.max_position_embeddings)} instead. You can change that default value by passing --block_size xxx." + ) + block_size = min(1024, config.max_position_embeddings) + else: + if data_args.block_size > tokenizer.model_max_length: + logger.warning( + f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model " + f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}." + ) + block_size = min(data_args.block_size, tokenizer.model_max_length) + + # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, and if the total_length < block_size we exclude this batch and return an empty dict. + # We could add padding if the model supported it instead of this drop, you can customize this part to your needs. + total_length = (total_length // block_size) * block_size + # Split by chunks of max_len. + result = { + k: [t[i : i + block_size] for i in range(0, total_length, block_size)] + for k, t in concatenated_examples.items() + } + result["labels"] = result["input_ids"].copy() + return result + + # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder + # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower + # to preprocess. + # + # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: + # https://huggingface.co/docs/datasets/process#map + + with training_args.main_process_first(desc="grouping texts together"): + if not data_args.streaming: + lm_datasets = tokenized_datasets.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=not data_args.overwrite_cache, + desc=f"Grouping texts in chunks of {block_size}", + ) + else: + lm_datasets = tokenized_datasets.map( + group_texts, + batched=True, + ) + + if training_args.do_train: + if "train" not in tokenized_datasets: + raise ValueError("--do_train requires a train dataset") + train_dataset = lm_datasets["train"] + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + + if training_args.do_eval: + if "validation" not in tokenized_datasets: + raise ValueError("--do_eval requires a validation dataset") + eval_dataset = lm_datasets["validation"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + + def preprocess_logits_for_metrics(logits, labels): + if isinstance(logits, tuple): + # Depending on the model and config, logits may contain extra tensors, + # like past_key_values, but logits always come first + logits = logits[0] + return logits.argmax(dim=-1) + + metric = evaluate.load("accuracy") + + def compute_metrics(eval_preds): + preds, labels = eval_preds + # preds have the same shape as the labels, after the argmax(-1) has been calculated + # by preprocess_logits_for_metrics but we need to shift the labels + labels = labels[:, 1:].reshape(-1) + preds = preds[:, :-1].reshape(-1) + return metric.compute(predictions=preds, references=labels) + + # Initialize our Trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + tokenizer=tokenizer, + # Data collator will default to DataCollatorWithPadding, so we change it. + data_collator=default_data_collator, + compute_metrics=compute_metrics if training_args.do_eval and not is_torch_tpu_available() else None, + preprocess_logits_for_metrics=( + preprocess_logits_for_metrics if training_args.do_eval and not is_torch_tpu_available() else None + ), + ) + + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() # Saves the tokenizer too for easy upload + + metrics = train_result.metrics + + max_train_samples = ( + data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset) + ) + metrics["train_samples"] = min(max_train_samples, len(train_dataset)) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + if training_args.do_eval: + logger.info("*** Evaluate ***") + + metrics = trainer.evaluate() + + max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) + metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + try: + perplexity = math.exp(metrics["eval_loss"]) + except OverflowError: + perplexity = float("inf") + metrics["perplexity"] = perplexity + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"} + if data_args.dataset_name is not None: + kwargs["dataset_tags"] = data_args.dataset_name + if data_args.dataset_config_name is not None: + kwargs["dataset_args"] = data_args.dataset_config_name + kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}" + else: + kwargs["dataset"] = data_args.dataset_name + + if training_args.push_to_hub: + trainer.push_to_hub(**kwargs) + else: + trainer.create_model_card(**kwargs) + + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +if __name__ == "__main__": + main() diff --git a/aws-examples/mistral-e2e/test.py b/aws-examples/mistral-e2e/test.py new file mode 100644 index 000000000..54437cb42 --- /dev/null +++ b/aws-examples/mistral-e2e/test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team and Amazon Web Services, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script demonstrates the results of fine-tuning on the gsm8k dataset by +generating a response to a grade school math question taken from the dataset. +""" + +from transformers import AutoTokenizer +from optimum.neuron import NeuronModelForCausalLM + +# Load the compiled model. +model = NeuronModelForCausalLM.from_pretrained("./mistral_neuron", local_files_only=True) + +# Load the tokenizer +tokenizer = AutoTokenizer.from_pretrained("./mistral_neuron") +tokenizer.pad_token_id = tokenizer.eos_token_id + +# Set a message to send to the model for inferencing +message = f"[INST] Two girls each got 1/6 of the 24 liters of water. Then a boy got 6 liters of water. How many liters of water were left? [/INST]\n\n" +tokenized_message = tokenizer(message, return_tensors="pt") + +# Do the inferencing +outputs = model.generate( + **tokenized_message, + max_new_tokens=512, # How many tokens the model can generate in the response + do_sample=True, # Use sampling or greedy decoding + temperature=0.9, # The value used to modulate next token probabilities + top_k=50, # The number of highest probability vocabulary tokens to keep for top-k-filtering + top_p=0.9 # If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to top_p or higher are kept for generation. +) + +# Decode the output from a tensor array to text. +answer = tokenizer.decode(outputs[0][len(tokenized_message[0]):], skip_special_tokens=True) + +print(answer) diff --git a/aws-examples/mistral-e2e/train.sh b/aws-examples/mistral-e2e/train.sh new file mode 100644 index 000000000..05117f168 --- /dev/null +++ b/aws-examples/mistral-e2e/train.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -ex + +# In PT2.1, functionalization is needed to close 3% convergence gap compared to PT1.13 for ZeRO1 +export XLA_DISABLE_FUNCTIONALIZATION=1 + +export NEURON_FUSE_SOFTMAX=1 +export NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS=3 +# Limit memory allocation to prevent crashes +export MALLOC_ARENA_MAX=64 +export NEURON_CC_FLAGS="--model-type=transformer --distribution-strategy=llm-training --enable-saturate-infinity --cache_dir=/home/ubuntu/cache_dir_neuron/" + +# Distributed configs +PROCESSES_PER_NODE=32 +WORLD_SIZE=1 +DISTRIBUTED_ARGS="--nproc_per_node $PROCESSES_PER_NODE" +LOG_PATH=logs + +# Create the log path +mkdir -p $LOG_PATH +echo $DISTRIBUTED_ARGS + +# Parallelism configuration +GBS=512 +NUM_EPOCHS=10 +TP_DEGREE=8 +PP_DEGREE=1 +DP=$(($PROCESSES_PER_NODE * $WORLD_SIZE / $TP_DEGREE / $PP_DEGREE)) +BS=1 +GRADIENT_ACCUMULATION_STEPS=1 +BLOCK_SIZE=2048 +LOGGING_STEPS=1 +MODEL_NAME="mistralai/Mistral-7B-Instruct-v0.3" +OUTPUT_DIR="mistral_trained" + +MAX_STEPS=-1 + +# Our script will first look in the working directory for a dataset matching the name, or download it from the Hugging Face hub +DATASET_NAME="dataset_formatted" + +XLA_USE_BF16=1 torchrun $DISTRIBUTED_ARGS examples/run_clm.py \ + --model_name_or_path $MODEL_NAME \ + --num_train_epochs $NUM_EPOCHS \ + --dataset_name $DATASET_NAME \ + --do_train \ + --learning_rate 8e-6 \ + --warmup_steps 30 \ + --max_steps $MAX_STEPS \ + --per_device_train_batch_size $BS \ + --per_device_eval_batch_size $BS \ + --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS \ + --gradient_checkpointing \ + --block_size $BLOCK_SIZE \ + --bf16 \ + --zero_1 false \ + --tensor_parallel_size $TP_DEGREE \ + --pipeline_parallel_size $PP_DEGREE \ + --logging_steps $LOGGING_STEPS \ + --save_total_limit 1 \ + --output_dir $OUTPUT_DIR \ + --overwrite_output_dir diff --git a/infrastructure/ami/scripts/install-huggingface-libraries.sh b/infrastructure/ami/scripts/install-huggingface-libraries.sh index c9825ddec..61a460be0 100644 --- a/infrastructure/ami/scripts/install-huggingface-libraries.sh +++ b/infrastructure/ami/scripts/install-huggingface-libraries.sh @@ -27,11 +27,12 @@ cd optimum-neuron pip install ".[neuronx, diffusers, sentence-transformers]" cd .. -mkdir /home/ubuntu/huggingface-neuron-samples/ /home/ubuntu/huggingface-neuron-notebooks/ +mkdir /home/ubuntu/huggingface-neuron-samples/ /home/ubuntu/huggingface-neuron-notebooks/ /home/ubuntu/aws-examples/ mv optimum-neuron/examples/* /home/ubuntu/huggingface-neuron-samples/ mv optimum-neuron/notebooks/* /home/ubuntu/huggingface-neuron-notebooks/ +mv optimum-neuron/aws-examples/* /home/ubuntu/aws-examples/ rm -rf optimum-neuron -chmod -R 777 /home/ubuntu/huggingface-neuron-samples /home/ubuntu/huggingface-neuron-notebooks +chmod -R 777 /home/ubuntu/huggingface-neuron-samples /home/ubuntu/huggingface-neuron-notebooks /home/ubuntu/aws-examples/ echo "Step: validate-imports-of-huggingface-libraries" -bash -c 'python -c "import transformers;import datasets;import accelerate;import evaluate;import tensorboard; import torch;from optimum.neuron import pipeline"' \ No newline at end of file +bash -c 'python -c "import transformers;import datasets;import accelerate;import evaluate;import tensorboard; import torch;from optimum.neuron import pipeline"'