From 3550ef2c8095b7bb5405126291d6294d1ee0e0c9 Mon Sep 17 00:00:00 2001 From: jzsmoreno <42299052+jzsmoreno@users.noreply.github.com> Date: Tue, 31 Oct 2023 17:12:35 -0600 Subject: [PATCH] Update `schema_config.py` * Add `pyarrow.lib.ArrowInvalid` exception --- merge_by_lev/schema_config.py | 9 +++++---- setup.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/merge_by_lev/schema_config.py b/merge_by_lev/schema_config.py index 451d187..a4e57da 100644 --- a/merge_by_lev/schema_config.py +++ b/merge_by_lev/schema_config.py @@ -1,12 +1,13 @@ import json +import pyarrow import pyarrow as pa import yaml +from pyarrow import Table +from pydbsmgr.lightest import * from pydbsmgr.main import * from pydbsmgr.main import DataFrame -from pydbsmgr.lightest import * from pydbsmgr.utils.azure_sdk import * -from pyarrow import Table class StandardColumns: @@ -217,7 +218,7 @@ def recursive_correction(df_: DataFrame, input_string: str) -> Table: try: data_handler = DataSchema(df_) return pa.Table.from_pandas(df_, schema=data_handler.get_schema()) - except pa.lib.ArrowTypeError as e: + except (pa.lib.ArrowTypeError, pyarrow.lib.ArrowInvalid) as e: iteration_match = re.search(pattern, (str(e).split(","))[-1]) iteration_column_name = iteration_match.group(1) if column_name != iteration_column_name: @@ -307,7 +308,7 @@ def get_table(self) -> Table: try: record_batch = pa.RecordBatch.from_pandas(self.df) table = pa.Table.from_pandas(self.df) - except pa.lib.ArrowTypeError as e: + except (pa.lib.ArrowTypeError, pyarrow.lib.ArrowInvalid) as e: warning_type = "UserWarning" msg = "It was not possible to create the table\n" msg += "Error: {%s}" % e diff --git a/setup.py b/setup.py index 84455d3..98e7cc2 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="merge_by_lev", - version="0.2.4", + version="0.2.5", author="J. A. Moreno-Guerra", author_email="jzs.gm27@gmail.com", description="Testing installation of Package",