Update tools.py

* Added files_not_loaded logic

pydbsmgr/fast_upload.py

@@ -13,7 +13,16 @@ class DataFrameToSQL:
     """Allows you to create a table from a dataframe"""
 
     sql_types = ["FLOAT", "BIGINT", "INT", "DATETIME", "VARCHAR(MAX)", "BIGINT", "INT", "BIT"]
-    pandas_types = ["float64", "int64", "int32", "datetime64[ns]", "object", "Int64", "Int32", "bool"]
+    pandas_types = [
+        "float64",
+        "int64",
+        "int32",
+        "datetime64[ns]",
+        "object",
+        "Int64",
+        "Int32",
+        "bool",
+    ]
     datatype_dict = dict(zip(pandas_types, sql_types))
 
     def __init__(self, connection_string: str) -> None:

pydbsmgr/utils/tools.py

@@ -89,6 +89,7 @@ def write_pyarrow(
     ) -> None:
         """Write pyarrow table as `parquet` format"""
         format_type = "parquet"
+        files_not_loaded = []
         for table, blob_name in zip(pytables, names):
             if table != None:
                 buf = pa.BufferOutputStream()
@@ -98,6 +99,10 @@ def write_pyarrow(
                 self._container_client.upload_blob(
                     name=blob_path_name + "." + format_type, data=parquet_data, overwrite=overwrite
                 )
+            else:
+                files_not_loaded.append(blob_name)
+        if len(files_not_loaded) > 0:
+            return files_not_loaded
 
     def write_parquet(
         self,
@@ -110,6 +115,7 @@ def write_parquet(
         """Write dataframes as `parquet` format by converting them first into `bytes`"""
         files = []
         format_type = "parquet"
+        files_not_loaded = []
         for data, blob_name in zip(dfs, names):
             if data != None:
                 table = pa.Table.from_pandas(data)
@@ -119,10 +125,16 @@ def write_parquet(
                 blob_path_name = directory_name + "/" + blob_name
                 if upload:
                     self._container_client.upload_blob(
-                        name=blob_path_name + "." + format_type, data=parquet_data, overwrite=overwrite
+                        name=blob_path_name + "." + format_type,
+                        data=parquet_data,
+                        overwrite=overwrite,
                     )
                 else:
                     files.append(parquet_data)
+            else:
+                files_not_loaded.append(blob_name)
+        if len(files_not_loaded) > 0:
+            return files_not_loaded
 
         if not upload:
             return files

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pydbsmgr",
-    version="0.4.8",
+    version="0.4.9",
     author="J. A. Moreno-Guerra",
     author_email="[email protected]",
     description="Testing installation of Package",