FIX: Allow to load numpy pickles stored in S3 buckets

CHANGES.md

@@ -1,5 +1,9 @@
 # Release History
 
+## 0.21.5 (2024-mm-dd)
+
+- FIX: Allow to load numpy pickles stored in S3 buckets
+
 ## 0.21.4 (2024-10-08)
 
 - DEPS: Don't force using geopandas 1.0.0, 0.14.4 should be enough.

eoreader/products/optical/dimap_v2_product.py

@@ -671,7 +671,7 @@ def _manage_invalid_pixels(
 
                 np.save(str(mask_path), nodata)
             else:
-                nodata = np.load(str(mask_path))
+                nodata = utils.load_np(mask_path, self._tmp_output)
         except InvalidProductError:
             pass
 
@@ -923,7 +923,7 @@ def _open_clouds(
                         )
                         np.save(str(cld_path), cld_arr)
                     else:
-                        cld_arr = np.load(str(cld_path))
+                        cld_arr = utils.load_np(cld_path, self._tmp_output)
 
                     # Rasterize gives a 2D array, we want a 3D array
                     cld_arr = np.expand_dims(cld_arr, axis=0)
@@ -1124,7 +1124,7 @@ def _load_nodata(
                 )
                 np.save(str(nodata_path), nodata)
             else:
-                nodata = np.load(str(nodata_path))
+                nodata = utils.load_np(nodata_path, self._tmp_output)
 
             return nodata
 

eoreader/products/optical/s3_olci_product.py

@@ -596,7 +596,7 @@ def _rad_2_refl(
 
         else:
             # Open rad_2_refl_coeff (resampled to band_arr size)
-            rad_2_refl_coeff = np.load(str(rad_2_refl_path))
+            rad_2_refl_coeff = utils.load_np(rad_2_refl_path, self._tmp_output)
 
         return band_arr * rad_2_refl_coeff
 

eoreader/products/optical/s3_slstr_product.py

@@ -797,10 +797,7 @@ def _rad_2_refl(
 
         else:
             # Open rad_2_refl_coeff (resampled to band_arr size)
-            if path.is_cloud_path(rad_2_refl_path):
-                rad_2_refl_path = rad_2_refl_path.fspath
-
-            rad_2_refl_coeff = np.load(str(rad_2_refl_path))
+            rad_2_refl_coeff = utils.load_np(rad_2_refl_path, self._tmp_output)
 
         return band_arr * rad_2_refl_coeff
 
@@ -886,10 +883,7 @@ def _compute_sza_img_grid(self, suffix) -> np.ndarray:
 
         else:
             # Open sza_img (resampled to band_arr size)
-            if path.is_cloud_path(sza_img_path):
-                sza_img_path = sza_img_path.fspath
-
-            sza_img = np.load(str(sza_img_path))
+            sza_img = utils.load_np(sza_img_path, self._tmp_output)
 
         return sza_img
 

eoreader/utils.py

@@ -492,3 +492,19 @@ def get_dim_img_path(dim_path: AnyPathStrType, img_name: str = "*") -> list:
     return path.get_file_in_dir(
         dim_path, img_name, extension="img", exact_name=True, get_list=True
     )
+
+
+def load_np(path_to_load: AnyPathStrType, output: AnyPathStrType) -> np.ndarray:
+    """
+    Load numpy pickles, with a handling of cloud-stored files.
+
+    Args:
+        path_to_load (AnyPathStrType): Pickle path
+        output (AnyPathStrType): Where to download the pickle if it's stored on the cloud
+
+    Returns:
+        np.ndarray: Numpy array
+    """
+    if path.is_cloud_path(path_to_load):
+        path_to_load = path_to_load.download_to(output)
+    return np.load(str(path_to_load))