From 42175e8b06003471e743b596422fe7849d506b99 Mon Sep 17 00:00:00 2001 From: BRAUN REMI Date: Tue, 13 Aug 2024 17:08:54 +0200 Subject: [PATCH] ENH: Allow the process of Sentinel-1 COGs (provided by the Copernicus DataSpace) for SNAP >= 10 #172 --- CHANGES.md | 1 + .../test_all_sat_end_to_end_on_disk.py | 8 +-- docs/sar.md | 2 + eoreader/products/sar/s1_product.py | 54 ++++++++----------- eoreader/products/sar/sar_product.py | 2 +- eoreader/utils.py | 4 ++ 6 files changed, 35 insertions(+), 36 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4a2a5f2a..2107616e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ## 0.21.3 (2024-mm-dd) +- **ENH: Allow the process of Sentinel-1 COGs (provided by the Copernicus DataSpace) for SNAP >= 10** ([#172](https://github.com/sertit/eoreader/issues/172)) - ENH: Add a `BandType` alias for any types that could be a band: a string, a `BandNames` or any of its children: Spectral, SAR, DEM or Cloud band names - FIX: Get better window name (if available) when writing bands on disk (in tmp folder) - OPTIM: Use default (and optimized) predictor in `rasters.write` if SNAP is version 10 or higher ([#173](https://github.com/sertit/eoreader/issues/173)) diff --git a/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py b/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py index 55ae6744..11bc3013 100644 --- a/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py +++ b/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py @@ -63,6 +63,7 @@ "MERIT_Hydrologically_Adjusted_Elevations", "MERIT_DEM.vrt", ] +WRITE_ON_DISK = False def set_dem(dem_path): @@ -198,9 +199,10 @@ def _test_core( LOGGER.info(f"Product name: {prod.name}") with tempfile.TemporaryDirectory() as tmp_dir: - # output = os.path.join( - # "/mnt", "ds2_db3", "CI", "eoreader", "DATA", "OUTPUT_ON_DISK_CLEAN" - # ) + if WRITE_ON_DISK: + tmp_dir = os.path.join( + "/mnt", "ds2_db3", "CI", "eoreader", "DATA", "OUTPUT" + ) output = tmp_dir is_zip = "_ZIP" if prod.is_archived else "" prod.output = os.path.join(output, f"{prod.condensed_name}{is_zip}") diff --git a/docs/sar.md b/docs/sar.md index e6fce9f1..63fea4cf 100644 --- a/docs/sar.md +++ b/docs/sar.md @@ -67,6 +67,8 @@ the [Copernicus Emergency Management Service](https://emergency.copernicus.eu/). The constellations that can be used during CEMS activations are (as of 09/2021): ![cems_constellations](https://www.esa.int/var/esa/storage/images/esa_multimedia/images/2021/09/copernicus_contributing_missions_overview/23461131-1-eng-GB/Copernicus_Contributing_Missions_overview_pillars.jpg) +[Sentinel-1 COGS](https://dataspace.copernicus.eu/news/2023-6-6-sentinel-1-grd-cloud-optimized-geotiff-now-available) provided by the Copernicus Dataspace are handled if you are using SNAP 10 or higher. + ## SAR Bands ```{warning} diff --git a/eoreader/products/sar/s1_product.py b/eoreader/products/sar/s1_product.py index 3b818b99..c79608db 100644 --- a/eoreader/products/sar/s1_product.py +++ b/eoreader/products/sar/s1_product.py @@ -17,9 +17,6 @@ """ Sentinel-1 products """ import logging import os -import re -import tempfile -import zipfile from datetime import datetime from enum import unique from typing import Union @@ -200,27 +197,28 @@ def _pre_init(self, **kwargs) -> None: # Its original filename is its name self._use_filename = True - # Check if COG in name - if "_COG" in self.filename: - raise NotImplementedError( - "These S1 COG products are not yet handled by SNAP. " - "EOReader will handle them when this issue is fixed. " - "See https://forum.step.esa.int/t/handle-sentinel-1-cog-collection/40840. " - "Please use the classical format instead." - ) - # Zipped and SNAP can process its archive self.needs_extraction = False # Pre init done by the super class super()._pre_init(**kwargs) + # Check if COG in name + if ( + "_COG" in self.filename + and self._need_snap + and not self._has_snap_10_or_higher() + ): + raise NotImplementedError( + "S1 COG products are only handled by SNAP 10.0 or higher. " + "Please upgrade your software to process this product." + ) + def _post_init(self, **kwargs) -> None: """ Function used to post_init the products (setting product-type, band names and so on) """ - # Post init done by the super class super()._post_init(**kwargs) @@ -244,39 +242,31 @@ def wgs84_extent(self) -> gpd.GeoDataFrame: gpd.GeoDataFrame: WGS84 extent as a gpd.GeoDataFrame """ - tmp_dir = tempfile.TemporaryDirectory() - try: # Open the map-overlay file if self.is_archived: - # We need to extract the file here as we need a proper file - with zipfile.ZipFile(self.path, "r") as zip_ds: - filenames = [f.filename for f in zip_ds.filelist] - regex = re.compile(".*preview.*map-overlay.kml") - preview_overlay = zip_ds.extract( - list(filter(regex.match, filenames))[0], tmp_dir.name - ) + extent_wgs84 = vectors.read( + self.path, archive_regex=".*preview.*map-overlay.kml" + ) else: preview_overlay = self.path.joinpath("preview", "map-overlay.kml") - if os.path.isfile(preview_overlay): - # Open the KML file - extent_wgs84 = vectors.read(preview_overlay) - if extent_wgs84.empty: + if os.path.isfile(preview_overlay): + # Open the KML file + extent_wgs84 = vectors.read(preview_overlay) + else: raise InvalidProductError( - f"Cannot determine the WGS84 extent of {self.name}" + f"Impossible to find the map-overlay.kml in {self.path}" ) - else: + + if extent_wgs84.empty: raise InvalidProductError( - f"Impossible to find the map-overlay.kml in {self.path}" + f"Cannot determine the WGS84 extent of {self.name}" ) except Exception as ex: raise InvalidProductError(ex) from ex - finally: - tmp_dir.cleanup() - return extent_wgs84 def _set_product_type(self) -> None: diff --git a/eoreader/products/sar/sar_product.py b/eoreader/products/sar/sar_product.py index 2465d6dd..f719b189 100644 --- a/eoreader/products/sar/sar_product.py +++ b/eoreader/products/sar/sar_product.py @@ -268,7 +268,7 @@ def _has_snap_10_or_higher(self) -> bool: # TODO: To be removed with sertit >= 1.42 from packaging.version import Version - misc.compare(self.get_snap_version(), Version("10.0.0"), ">=") + return misc.compare(self.get_snap_version(), Version("10.0.0"), ">=") def _get_predictor(self) -> int: """ diff --git a/eoreader/utils.py b/eoreader/utils.py index 2836849d..921d1c40 100644 --- a/eoreader/utils.py +++ b/eoreader/utils.py @@ -229,6 +229,10 @@ def write(xds: xr.DataArray, filepath: AnyPathStrType, **kwargs) -> None: filepath (AnyPathStrType): Path where to save it (directories should be existing) **kwargs: Overloading metadata, ie :code:`nodata=255` or :code:`dtype=np.uint8` """ + # TODO: remove this with sertit > 1.41.0 + # Prune empty kwargs to avoid throwing GDAL warnings/errors + kwargs = {k: v for k, v in kwargs.items() if v is not None} + lock = None if use_dask(): from distributed import Lock, get_client