From c16a604feae02c702b578e7d10481c3a92342045 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 5 Dec 2023 18:02:27 +0100 Subject: [PATCH 1/4] add 80% for cpu --- src/accelerate/utils/modeling.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py index e792140ac8c..00e61dd59f3 100644 --- a/src/accelerate/utils/modeling.py +++ b/src/accelerate/utils/modeling.py @@ -664,7 +664,8 @@ def get_max_layer_size( def get_max_memory(max_memory: Optional[Dict[Union[int, str], Union[int, str]]] = None): """ - Get the maximum memory available if nothing is passed, converts string to int otherwise. + Get the maximum memory available if nothing is passed. + Otherwise, we convert string to int and we allocate 80% of the cpu memory if cpu is not passed in max_memory. """ import psutil @@ -692,6 +693,12 @@ def get_max_memory(max_memory: Optional[Dict[Union[int, str], Union[int, str]]] for key in max_memory: if isinstance(max_memory[key], str): max_memory[key] = convert_file_size_to_int(max_memory[key]) + if "cpu" not in max_memory: + max_memory["cpu"] = psutil.virtual_memory().available * 0.8 + logger.info( + "We will use 80% of the memory on cpu for storing the model." + "You can set `max_memory` in to a higher value to use more memory (at your own risk)." + ) # Need to sort the device by type to make sure that we allocate the gpu first. # As gpu/xpu are represented by int, we need to sort them first. From 2505923cf9ebc738960f56f01e3d1e3cbd2df5fb Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 5 Dec 2023 18:16:43 +0100 Subject: [PATCH 2/4] style --- src/accelerate/utils/modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py index 00e61dd59f3..8fedb402cb8 100644 --- a/src/accelerate/utils/modeling.py +++ b/src/accelerate/utils/modeling.py @@ -664,8 +664,8 @@ def get_max_layer_size( def get_max_memory(max_memory: Optional[Dict[Union[int, str], Union[int, str]]] = None): """ - Get the maximum memory available if nothing is passed. - Otherwise, we convert string to int and we allocate 80% of the cpu memory if cpu is not passed in max_memory. + Get the maximum memory available if nothing is passed. Otherwise, we convert string to int and we allocate 80% of + the cpu memory if cpu is not passed in max_memory. """ import psutil From 728ccfb3fe0076899034758afa77f60c07cac147 Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Tue, 5 Dec 2023 12:17:32 -0500 Subject: [PATCH 3/4] Update src/accelerate/utils/modeling.py Co-authored-by: Zach Mueller --- src/accelerate/utils/modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py index 8fedb402cb8..42f1fed6e2b 100644 --- a/src/accelerate/utils/modeling.py +++ b/src/accelerate/utils/modeling.py @@ -697,7 +697,7 @@ def get_max_memory(max_memory: Optional[Dict[Union[int, str], Union[int, str]]] max_memory["cpu"] = psutil.virtual_memory().available * 0.8 logger.info( "We will use 80% of the memory on cpu for storing the model." - "You can set `max_memory` in to a higher value to use more memory (at your own risk)." + "You can set `max_memory["cpu"]` in to a higher value to use more memory (at your own risk)." ) # Need to sort the device by type to make sure that we allocate the gpu first. From 67bc22805527892899729fe5834dbab440c122ca Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Fri, 8 Dec 2023 09:41:21 -0500 Subject: [PATCH 4/4] Update src/accelerate/utils/modeling.py Co-authored-by: Zach Mueller --- src/accelerate/utils/modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py index 42f1fed6e2b..acfa8bdd0f2 100644 --- a/src/accelerate/utils/modeling.py +++ b/src/accelerate/utils/modeling.py @@ -697,7 +697,7 @@ def get_max_memory(max_memory: Optional[Dict[Union[int, str], Union[int, str]]] max_memory["cpu"] = psutil.virtual_memory().available * 0.8 logger.info( "We will use 80% of the memory on cpu for storing the model." - "You can set `max_memory["cpu"]` in to a higher value to use more memory (at your own risk)." + "You can set `max_memory['cpu']` in to a higher value to use more memory (at your own risk)." ) # Need to sort the device by type to make sure that we allocate the gpu first.