From 4c943738275d3b633db63b1e8e4adda259fc820f Mon Sep 17 00:00:00 2001 From: LokeZhou Date: Tue, 28 Nov 2023 07:35:50 +0000 Subject: [PATCH 1/2] processors support aistudio --- paddlemix/models/blip2/base_model.py | 7 +- paddlemix/models/blip2/utils.py | 10 +-- .../processors/image_processing_utils.py | 14 ++++ paddlemix/processors/processing_utils.py | 73 +++++++++++++++++++ requirements.txt | 2 +- 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/paddlemix/models/blip2/base_model.py b/paddlemix/models/blip2/base_model.py index c22d8e196..9b9a71dd5 100644 --- a/paddlemix/models/blip2/base_model.py +++ b/paddlemix/models/blip2/base_model.py @@ -118,6 +118,7 @@ def init_tokenizer(cls, tokenizer_name="bert-base-uncased"): @classmethod def refine_state_dict(self, model, state_dict): from paddlemix.models.blip2.eva_vit import interpolate_pos_embed + interpolate_pos_embed(model, state_dict) def get_expected_keys(self, model_state_dict, name=None): @@ -203,6 +204,7 @@ def from_pretrained( subfolder = kwargs.pop("subfolder", "") variant = kwargs.pop("variant", None) use_safetensors = kwargs.pop("use_safetensors", None if is_safetensors_available() else False) + from_aistudio = kwargs.get("from_aistudio", False) low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", False) convert_from_torch = kwargs.pop("convert_from_torch", None) @@ -269,6 +271,7 @@ def from_pretrained( cache_dir=cache_dir, subfolder=subfolder, from_hf_hub=from_hf_hub, + from_aistudio=from_aistudio, config=config, convert_from_torch=convert_from_torch, use_safetensors=use_safetensors, @@ -322,7 +325,9 @@ def from_pretrained( init_args = config["init_args"] or () with ContextManagers(init_contexts): model = cls(config, *init_args, **model_kwargs) - cls.refine_state_dict(model, state_dict) + if state_dict is not None: + cls.refine_state_dict(model, state_dict) + if use_keep_in_fp32_modules: # low_cpu_mem_usage = True keep_in_fp32_modules = model._keep_in_fp32_modules diff --git a/paddlemix/models/blip2/utils.py b/paddlemix/models/blip2/utils.py index 0c30b4142..459ccfaef 100644 --- a/paddlemix/models/blip2/utils.py +++ b/paddlemix/models/blip2/utils.py @@ -100,16 +100,16 @@ def load_real_time_tokens(): return tokens -def create_tokenizer(text_model_name_or_path): +def create_tokenizer(text_model_name_or_path, **kwags): if "opt" in text_model_name_or_path: - tokenizer_class = AutoTokenizer.from_pretrained(text_model_name_or_path, use_fast=False) + tokenizer_class = AutoTokenizer.from_pretrained(text_model_name_or_path, use_fast=False, **kwags) elif "t5" in text_model_name_or_path: - tokenizer_class = T5Tokenizer.from_pretrained(text_model_name_or_path, use_fast=False) + tokenizer_class = T5Tokenizer.from_pretrained(text_model_name_or_path, use_fast=False, **kwags) elif "llama" in text_model_name_or_path: - tokenizer_class = LlamaTokenizer.from_pretrained(text_model_name_or_path) + tokenizer_class = LlamaTokenizer.from_pretrained(text_model_name_or_path, **kwags) tokenizer_class.pad_token = tokenizer_class.eos_token elif "bloom" in text_model_name_or_path: - tokenizer_class = BloomTokenizer.from_pretrained(text_model_name_or_path) + tokenizer_class = BloomTokenizer.from_pretrained(text_model_name_or_path, **kwags) tokenizer_class.pad_token = tokenizer_class.eos_token else: raise NotImplementedError diff --git a/paddlemix/processors/image_processing_utils.py b/paddlemix/processors/image_processing_utils.py index 0c8a0913d..f9dbf5841 100644 --- a/paddlemix/processors/image_processing_utils.py +++ b/paddlemix/processors/image_processing_utils.py @@ -40,6 +40,14 @@ ) from paddlemix.utils.log import logger +try: + from paddlenlp.transformers.aistudio_utils import aistudio_download +except: + logger.warning("aistudio_download not import, if you want to use , require paddlenlp develop") + aistudio_download = None + pass + + IMAGE_PROCESSOR_NAME = "image_preprocessor_config.json" TEXT_PROCESSOR_NAME = "text_processor_config.json" @@ -272,6 +280,7 @@ def get_image_processor_dict( """ cache_dir = kwargs.pop("cache_dir", None) from_hf_hub = kwargs.pop("from_hf_hub", False) + from_aistudio = kwargs.get("from_aistudio", False) subfolder = kwargs.pop("subfolder", None) cache_dir = resolve_cache_dir(pretrained_model_name_or_path, from_hf_hub, cache_dir) @@ -292,6 +301,11 @@ def get_image_processor_dict( library_name="PaddleNLP", library_version=__version__, ) + elif from_aistudio and aistudio_download is not None: + image_processor_file = IMAGE_PROCESSOR_NAME + resolved_image_processor_file = aistudio_download( + repo_id=pretrained_model_name_or_path, filename=image_processor_file + ) else: # Assuming from community-contributed pretrained models image_processor_file = "/".join( diff --git a/paddlemix/processors/processing_utils.py b/paddlemix/processors/processing_utils.py index 0ba43a999..bda4e3550 100644 --- a/paddlemix/processors/processing_utils.py +++ b/paddlemix/processors/processing_utils.py @@ -44,6 +44,7 @@ logger.warning("aistudio_download not import, if you want to use , require paddlenlp develop") aistudio_download = None pass +import aistudio_sdk PROCESSOR_CONFIG_MAPPING = { "image": "image_preprocessor_config.json", @@ -228,6 +229,69 @@ def save_to_hf_hub( create_pr=create_pr, ) + def save_to_aistudio( + self, + repo_id, + private=True, + license="Apache License 2.0", + exist_ok=True, + safe_serialization=True, + subfolder=None, + merge_tensor_parallel=False, + **kwargs + ): + """ + Uploads all elements of this model to a new AiStudio Hub repository. + Args: + repo_id (str): Repository name for your model/tokenizer in the Hub. + token (str): Your token for the Hub. + private (bool, optional): Whether the model/tokenizer is set to private. Defaults to True. + license (str): The license of your model/tokenizer. Defaults to: "Apache License 2.0". + exist_ok (bool, optional): Whether to override existing repository. Defaults to: True. + safe_serialization (bool, optional): Whether to save the model in safe serialization way. Defaults to: True. + subfolder (str, optional): Push to a subfolder of the repo instead of the root + merge_tensor_parallel (bool): Whether to merge the tensor parallel weights. Defaults to False. + """ + + res = aistudio_sdk.hub.create_repo(repo_id=repo_id, private=private, license=license, **kwargs) + if "error_code" in res: + if res["error_code"] == 10003 and exist_ok: + logger.info( + f"Repo {repo_id} already exists, it will override files with the same name. To avoid this, please set exist_ok=False" + ) + else: + logger.error( + f"Failed to create repo {repo_id}, error_code: {res['error_code']}, error_msg: {res['error_msg']}" + ) + else: + logger.info(f"Successfully created repo {repo_id}") + + with tempfile.TemporaryDirectory() as root_dir: + if subfolder is not None: + save_dir = os.path.join(root_dir, subfolder) + else: + save_dir = root_dir + + # save model + self.save_pretrained(save_dir) + + # Upload model and return + logger.info(f"Pushing to the {repo_id}. This might take a while") + for filename in os.listdir(save_dir): + path_in_repo = os.path.join(subfolder, filename) if subfolder is not None else filename + res = aistudio_sdk.hub.upload( + repo_id=repo_id, + path_or_fileobj=os.path.join(save_dir, filename), + path_in_repo=path_in_repo, + **kwargs, + ) + if "error_code" in res: + logger.error( + f"Failed to upload {filename}, error_code: {res['error_code']}, error_msg: {res['error_msg']}" + ) + else: + logger.info(f"{filename}: {res['message']}") + @classmethod def get_processor_dict( cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs @@ -273,6 +337,15 @@ def get_processor_dict( ) elif from_aistudio and aistudio_download is not None: processor_file = PROCESSOR_CONFIG_MAPPING[cls.input_type] + if subfolder is not None: + processor_file = os.path.join(subfolder, processor_file) + + pretrained_model_name_or_path_list = pretrained_model_name_or_path.split("/") + if len(pretrained_model_name_or_path_list) > 2: + pretrained_model_name_or_path = os.path.join( + pretrained_model_name_or_path_list[0], pretrained_model_name_or_path_list[1] + ) + resolved_processor_file = aistudio_download(repo_id=pretrained_model_name_or_path, filename=processor_file) else: # Assuming from community-contributed pretrained models diff --git a/requirements.txt b/requirements.txt index 195b3c4c1..88faef1c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ pycocoevalcap ftfy regex einops>=0.6.1 - +aistudio-sdk>=0.1.3 From e6ae2f67156c23056efc6f3c1cfa6f112035274a Mon Sep 17 00:00:00 2001 From: LokeZhou Date: Tue, 28 Nov 2023 09:23:04 +0000 Subject: [PATCH 2/2] add from_aistudio up and load unit test --- tests/processors/__init__.py | 13 ++++ tests/processors/test_from_aistudio.py | 91 ++++++++++++++++++++++++++ tests/testing_utils.py | 1 + 3 files changed, 105 insertions(+) create mode 100644 tests/processors/__init__.py create mode 100644 tests/processors/test_from_aistudio.py diff --git a/tests/processors/__init__.py b/tests/processors/__init__.py new file mode 100644 index 000000000..595add0ae --- /dev/null +++ b/tests/processors/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/processors/test_from_aistudio.py b/tests/processors/test_from_aistudio.py new file mode 100644 index 000000000..441b8290c --- /dev/null +++ b/tests/processors/test_from_aistudio.py @@ -0,0 +1,91 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) + +from paddlemix.models.groundingdino.modeling import GroundingDinoModel +from paddlemix.processors.groundingdino_processing import ( + GroudingDinoImageProcessor, + GroudingDinoTextProcessor, +) +from tests.testing_utils import ai_studio_token, slow + +repo_id = "aistudio/groundingdino-swint-ogc" +bos_model_name = "GroundingDino/groundingdino-swint-ogc" + + +class FromAiStudioTester: + def __init__(self): + self.bos_model_name = bos_model_name + self.token = ai_studio_token + self.repo_id = repo_id + + def prepare_from_bos(self): + model = GroundingDinoModel.from_pretrained(self.bos_model_name) + image_processor = GroudingDinoImageProcessor.from_pretrained(self.bos_model_name) + text_processor = GroudingDinoTextProcessor.from_pretrained(self.bos_model_name) + + return model, image_processor, text_processor + + +class AIStudioUpTester(unittest.TestCase): + def setUp(self): + self.tester = FromAiStudioTester() + self.model, self.image_processor, self.text_processor = self.tester.prepare_from_bos() + + @slow + def test_model_up_aistusio(self): + self.model.save_to_aistudio( + repo_id=self.tester.repo_id, + token=self.tester.token, + private=True, + license="Apache License 2.0", + exist_ok=True, + safe_serialization=True, + ) + + def test_processor_up_aistusio(self): + self.image_processor.save_to_aistudio( + repo_id=self.tester.repo_id, + token=self.tester.token, + private=True, + license="Apache License 2.0", + exist_ok=True, + ) + self.text_processor.save_to_aistudio( + repo_id=self.tester.repo_id, + token=self.tester.token, + private=True, + license="Apache License 2.0", + exist_ok=True, + ) + + +class AIStudioLoadTester(unittest.TestCase): + def setUp(self): + self.tester = FromAiStudioTester() + + @slow + def test_model_load_aistusio(self): + GroundingDinoModel.from_pretrained(self.tester.repo_id, from_aistudio=True) + + def test_processor_load_aistusio(self): + GroudingDinoTextProcessor.from_pretrained(self.tester.repo_id, from_aistudio=True) + + def image_processor_load_aistusio(self): + GroudingDinoImageProcessor.from_pretrained(self.tester.repo_id, from_aistudio=True) diff --git a/tests/testing_utils.py b/tests/testing_utils.py index dbf16a003..7fe940d46 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -44,6 +44,7 @@ def get_bool_from_env(key, default_value=False): _run_slow_test = get_bool_from_env("RUN_SLOW_TEST") +ai_studio_token = os.getenv("AI_STUDIO_TOKEN") def slow(test):