diff --git a/CHANGES.md b/CHANGES.md index 4d6234a3..bf86e9a7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Release History +## 0.21.5 (2024-mm-dd) + +- FIX: Allow to load numpy pickles stored in S3 buckets + ## 0.21.4 (2024-10-08) - DEPS: Don't force using geopandas 1.0.0, 0.14.4 should be enough. diff --git a/eoreader/products/optical/dimap_v2_product.py b/eoreader/products/optical/dimap_v2_product.py index 5d8fc3e0..9645eced 100644 --- a/eoreader/products/optical/dimap_v2_product.py +++ b/eoreader/products/optical/dimap_v2_product.py @@ -671,7 +671,7 @@ def _manage_invalid_pixels( np.save(str(mask_path), nodata) else: - nodata = np.load(str(mask_path)) + nodata = utils.load_np(mask_path, self._tmp_output) except InvalidProductError: pass @@ -923,7 +923,7 @@ def _open_clouds( ) np.save(str(cld_path), cld_arr) else: - cld_arr = np.load(str(cld_path)) + cld_arr = utils.load_np(cld_path, self._tmp_output) # Rasterize gives a 2D array, we want a 3D array cld_arr = np.expand_dims(cld_arr, axis=0) @@ -1124,7 +1124,7 @@ def _load_nodata( ) np.save(str(nodata_path), nodata) else: - nodata = np.load(str(nodata_path)) + nodata = utils.load_np(nodata_path, self._tmp_output) return nodata diff --git a/eoreader/products/optical/s3_olci_product.py b/eoreader/products/optical/s3_olci_product.py index 2eca5922..c61c5477 100644 --- a/eoreader/products/optical/s3_olci_product.py +++ b/eoreader/products/optical/s3_olci_product.py @@ -596,7 +596,7 @@ def _rad_2_refl( else: # Open rad_2_refl_coeff (resampled to band_arr size) - rad_2_refl_coeff = np.load(str(rad_2_refl_path)) + rad_2_refl_coeff = utils.load_np(rad_2_refl_path, self._tmp_output) return band_arr * rad_2_refl_coeff diff --git a/eoreader/products/optical/s3_slstr_product.py b/eoreader/products/optical/s3_slstr_product.py index 9e982768..4dbbe2a3 100644 --- a/eoreader/products/optical/s3_slstr_product.py +++ b/eoreader/products/optical/s3_slstr_product.py @@ -797,10 +797,7 @@ def _rad_2_refl( else: # Open rad_2_refl_coeff (resampled to band_arr size) - if path.is_cloud_path(rad_2_refl_path): - rad_2_refl_path = rad_2_refl_path.fspath - - rad_2_refl_coeff = np.load(str(rad_2_refl_path)) + rad_2_refl_coeff = utils.load_np(rad_2_refl_path, self._tmp_output) return band_arr * rad_2_refl_coeff @@ -886,10 +883,7 @@ def _compute_sza_img_grid(self, suffix) -> np.ndarray: else: # Open sza_img (resampled to band_arr size) - if path.is_cloud_path(sza_img_path): - sza_img_path = sza_img_path.fspath - - sza_img = np.load(str(sza_img_path)) + sza_img = utils.load_np(sza_img_path, self._tmp_output) return sza_img diff --git a/eoreader/utils.py b/eoreader/utils.py index 09864702..23978e88 100644 --- a/eoreader/utils.py +++ b/eoreader/utils.py @@ -492,3 +492,19 @@ def get_dim_img_path(dim_path: AnyPathStrType, img_name: str = "*") -> list: return path.get_file_in_dir( dim_path, img_name, extension="img", exact_name=True, get_list=True ) + + +def load_np(path_to_load: AnyPathStrType, output: AnyPathStrType) -> np.ndarray: + """ + Load numpy pickles, with a handling of cloud-stored files. + + Args: + path_to_load (AnyPathStrType): Pickle path + output (AnyPathStrType): Where to download the pickle if it's stored on the cloud + + Returns: + np.ndarray: Numpy array + """ + if path.is_cloud_path(path_to_load): + path_to_load = path_to_load.download_to(output) + return np.load(str(path_to_load))