From c57d8dbc850e37ea271dfef9669986a0b0b32ae0 Mon Sep 17 00:00:00 2001 From: jafetcc02 <114709102+jafetcc02@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:12:33 -0600 Subject: [PATCH 1/3] feature/slots Used slots in classes LightCleaner and DataFrametoSQL to optimize the use of memory --- pydbsmgr/fast_upload.py | 3 ++- pydbsmgr/lightest.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pydbsmgr/fast_upload.py b/pydbsmgr/fast_upload.py index ee6f4d1..e5f2ec0 100644 --- a/pydbsmgr/fast_upload.py +++ b/pydbsmgr/fast_upload.py @@ -11,10 +11,10 @@ from pydbsmgr.utils.tools import ColumnsCheck - class DataFrameToSQL(ColumnsCheck): """Allows you to create a table from a dataframe""" + __slots__ = ['_connection_string','_con', '_cur' ] def __init__(self, connection_string: str) -> None: """Set the connection with the database""" self._connection_string = connection_string @@ -165,6 +165,7 @@ def __init__(self, connection_string: str) -> None: """Establish the connection to the database using the `DataFrameToSQL` class.""" super().__init__(connection_string) + def execute( self, df: DataFrame, diff --git a/pydbsmgr/lightest.py b/pydbsmgr/lightest.py index 788da95..0da4344 100644 --- a/pydbsmgr/lightest.py +++ b/pydbsmgr/lightest.py @@ -1,5 +1,6 @@ import concurrent.futures from functools import partial +from functools import cached_property from pydbsmgr.main import * from pydbsmgr.utils.tools import coerce_datetime, most_repeated_item @@ -42,6 +43,9 @@ def process_dates(x: str, format_type: str, auxiliary_type: str) -> str: class LightCleaner: """Performs a light cleaning on the table""" + #Increase memory efficiency + __slots__ = ['df', 'dict_dtypes'] + def __init__(self, df_: DataFrame): self.df = df_.copy() self.dict_dtypes = dict(zip(["float", "int", "str"], ["float64", "int64", "object"])) From 99b6d28bc0252852e77b15c323c9ef005be93940 Mon Sep 17 00:00:00 2001 From: jafetcc02 <114709102+jafetcc02@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:15:58 -0600 Subject: [PATCH 2/3] Format with black --- pydbsmgr/fast_upload.py | 5 +++-- pydbsmgr/lightest.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pydbsmgr/fast_upload.py b/pydbsmgr/fast_upload.py index e5f2ec0..35cc6c3 100644 --- a/pydbsmgr/fast_upload.py +++ b/pydbsmgr/fast_upload.py @@ -11,10 +11,12 @@ from pydbsmgr.utils.tools import ColumnsCheck + class DataFrameToSQL(ColumnsCheck): """Allows you to create a table from a dataframe""" - __slots__ = ['_connection_string','_con', '_cur' ] + __slots__ = ["_connection_string", "_con", "_cur"] + def __init__(self, connection_string: str) -> None: """Set the connection with the database""" self._connection_string = connection_string @@ -165,7 +167,6 @@ def __init__(self, connection_string: str) -> None: """Establish the connection to the database using the `DataFrameToSQL` class.""" super().__init__(connection_string) - def execute( self, df: DataFrame, diff --git a/pydbsmgr/lightest.py b/pydbsmgr/lightest.py index 0da4344..052313e 100644 --- a/pydbsmgr/lightest.py +++ b/pydbsmgr/lightest.py @@ -43,8 +43,8 @@ def process_dates(x: str, format_type: str, auxiliary_type: str) -> str: class LightCleaner: """Performs a light cleaning on the table""" - #Increase memory efficiency - __slots__ = ['df', 'dict_dtypes'] + # Increase memory efficiency + __slots__ = ["df", "dict_dtypes"] def __init__(self, df_: DataFrame): self.df = df_.copy() From 0460ab133c832a5174576926f4401b38763d4cfe Mon Sep 17 00:00:00 2001 From: jzsmoreno <42299052+jzsmoreno@users.noreply.github.com> Date: Thu, 18 Jan 2024 09:51:45 -0600 Subject: [PATCH 3/3] Update `lightest.py` * reformatted with `isort` --- pydbsmgr/lightest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pydbsmgr/lightest.py b/pydbsmgr/lightest.py index 052313e..b4ae04e 100644 --- a/pydbsmgr/lightest.py +++ b/pydbsmgr/lightest.py @@ -1,6 +1,5 @@ import concurrent.futures -from functools import partial -from functools import cached_property +from functools import cached_property, partial from pydbsmgr.main import * from pydbsmgr.utils.tools import coerce_datetime, most_repeated_item