Skip to content

Commit

Permalink
Update tools.py
Browse files Browse the repository at this point in the history
* Improve `ControllerFeatures` class
  • Loading branch information
jzsmoreno committed Sep 28, 2023
1 parent bf6b8e9 commit a9e4823
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 19 deletions.
38 changes: 20 additions & 18 deletions pydbsmgr/utils/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,14 @@ def write_pyarrow(
"""Write pyarrow table as `parquet` format"""
format_type = "parquet"
for table, blob_name in zip(pytables, names):
buf = pa.BufferOutputStream()
pq.write_table(table, buf)
parquet_data = buf.getvalue().to_pybytes()
blob_path_name = directory_name + "/" + blob_name
self._container_client.upload_blob(
name=blob_path_name + "." + format_type, data=parquet_data, overwrite=overwrite
)
if table != None:
buf = pa.BufferOutputStream()
pq.write_table(table, buf)
parquet_data = buf.getvalue().to_pybytes()
blob_path_name = directory_name + "/" + blob_name
self._container_client.upload_blob(
name=blob_path_name + "." + format_type, data=parquet_data, overwrite=overwrite
)

def write_parquet(
self,
Expand All @@ -110,17 +111,18 @@ def write_parquet(
files = []
format_type = "parquet"
for data, blob_name in zip(dfs, names):
table = pa.Table.from_pandas(data)
buf = pa.BufferOutputStream()
pq.write_table(table, buf)
parquet_data = buf.getvalue().to_pybytes()
blob_path_name = directory_name + "/" + blob_name
if upload:
self._container_client.upload_blob(
name=blob_path_name + "." + format_type, data=parquet_data, overwrite=overwrite
)
else:
files.append(parquet_data)
if data != None:
table = pa.Table.from_pandas(data)
buf = pa.BufferOutputStream()
pq.write_table(table, buf)
parquet_data = buf.getvalue().to_pybytes()
blob_path_name = directory_name + "/" + blob_name
if upload:
self._container_client.upload_blob(
name=blob_path_name + "." + format_type, data=parquet_data, overwrite=overwrite
)
else:
files.append(parquet_data)

if not upload:
return files
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

setuptools.setup(
name="pydbsmgr",
version="0.4.7",
version="0.4.8",
author="J. A. Moreno-Guerra",
author_email="[email protected]",
description="Testing installation of Package",
Expand Down

0 comments on commit a9e4823

Please sign in to comment.