diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e1a4d40..93f01e32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,61 +117,61 @@ pytest_end_to_end: - if: $CI_COMMIT_TAG when: always -pytest_end_to_end310: - image: $EO_CONTAINER:geo_310 - stage: test - variables: - # Use Dask for python 3.10 - EOREADER_USE_DASK: "1" - before_script: - - python -m pip install --upgrade pip - - pip install --ignore-installed PyYAML - - pip install pytest coverage pytest-cov pylint - - pip install -e . - script: - - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG --capture=tee-sys - artifacts: - paths: - - ${CI_PROJECT_DIR}/cov_e2e.html - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - tags: - - sertit - - linux - - high_memory - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "END_TO_END"' - when: always - - if: $CI_COMMIT_TAG - when: always - needs: [ "pytest_end_to_end" ] - -pytest_end_to_end311: - image: $EO_CONTAINER:geo_311 - stage: test - variables: - # Use Dask for python 3.11 - EOREADER_USE_DASK: "1" - before_script: - - python -m pip install --upgrade pip - - pip install --ignore-installed PyYAML - - pip install pytest coverage pytest-cov pylint - - pip install -e . - script: - - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG --capture=tee-sys - artifacts: - paths: - - ${CI_PROJECT_DIR}/cov_e2e.html - coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' - tags: - - sertit - - linux - - high_memory - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "END_TO_END"' - when: always - - if: $CI_COMMIT_TAG - when: always - needs: [ "pytest_end_to_end310" ] +#pytest_end_to_end310: +# image: $EO_CONTAINER:geo_310 +# stage: test +# variables: +# # Use Dask for python 3.10 +# EOREADER_USE_DASK: "1" +# before_script: +# - python -m pip install --upgrade pip +# - pip install --ignore-installed PyYAML +# - pip install pytest coverage pytest-cov pylint +# - pip install -e . +# script: +# - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG --capture=tee-sys +# artifacts: +# paths: +# - ${CI_PROJECT_DIR}/cov_e2e.html +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# tags: +# - sertit +# - linux +# - high_memory +# rules: +# - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "END_TO_END"' +# when: always +# - if: $CI_COMMIT_TAG +# when: always +# needs: [ "pytest_end_to_end" ] +# +#pytest_end_to_end311: +# image: $EO_CONTAINER:geo_311 +# stage: test +# variables: +# # Use Dask for python 3.11 +# EOREADER_USE_DASK: "1" +# before_script: +# - python -m pip install --upgrade pip +# - pip install --ignore-installed PyYAML +# - pip install pytest coverage pytest-cov pylint +# - pip install -e . +# script: +# - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov.html --cov=eoreader --cov-config=.coveragerc CI/SCRIPTS --log-cli-level DEBUG --capture=tee-sys +# artifacts: +# paths: +# - ${CI_PROJECT_DIR}/cov_e2e.html +# coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' +# tags: +# - sertit +# - linux +# - high_memory +# rules: +# - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_NAME == "END_TO_END"' +# when: always +# - if: $CI_COMMIT_TAG +# when: always +# needs: [ "pytest_end_to_end310" ] #pytest_end_to_end312: # image: $EO_CONTAINER:geo_312 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83bec319..76394dbb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,12 +20,12 @@ repos: - id: check-merge-conflict - repo: 'https://github.com/PyCQA/flake8' - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 - repo: 'https://github.com/psf/black' - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black diff --git a/CHANGES.md b/CHANGES.md index 0d6dabdf..8bfeeee6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,28 @@ # Release History -## 0.21.2 (2024-mm-dd) +## 0.21.3 (2024-mm-dd) + +- FIX: Get better window name (if available) when writing bands on disk (in tmp folder) + +## 0.21.2 (2024-07-30) - ENH: `to_str` and `to_band`: add a `as_list` argument defaulting to `True`. When set as False, return a str from `to_str` and a band from `to_band` ([#138](https://github.com/sertit/eoreader/issues/138)). Thanks @jsetty! +- FIX: `Sentinel-2` product with `StopIteration` error ([#142](https://github.com/sertit/eoreader/issues/142)) +- FIX: Fix error in looking for bands in `Sentinel-2 L1C` archived products ([#168](https://github.com/sertit/eoreader/issues/168)) - FIX: Fix issue with geocoding with unzipped `Sentinel-3 OLCI` product ([#137](https://github.com/sertit/eoreader/issues/137)) -- FIX: Fix iceye product when extent file (*.kml) not found ([#135](https://github.com/sertit/eoreader/pull/135)) -- FIX: Add missing `pystac[validation]` in setup.py -- CI: Fix S3 endpoint management with `sertit==1.37.0` +- FIX: In `SPOT` products, METADATA.DIM and IMAGERY.TIF must be at the root of the product ([#145](https://github.com/sertit/eoreader/issues/145)) +- FIX: Fix `Maxar` product with `QB02` satellite ID ([#140](https://github.com/sertit/eoreader/issues/140)) +- FIX: Fix `ICEYE` product when extent file (*.kml) not found ([#135](https://github.com/sertit/eoreader/pull/135)) +- FIX: Handle `RCM` and `RS2` products that doesn't bundle their extent in a KML file ([#155](https://github.com/sertit/eoreader/issues/155)) +- FIX: Handle wrongly recognized `Planet` products because of the recursive nested mtd in the Reader ([#169](https://github.com/sertit/eoreader/issues/169)) +- FIX: Fix an unknown `Planet` bug that just appeared (`'...Path' has no len()`) +- FIX: Force the loading of `DimapV1` bands in `float32` +- FIX: Handle the case where `fiona` isn't installed anymore (with `geopandas 1.0`) +- FIX: Don't make `pystac` a mandatory requirement +- OPTIM: Search correctly nested metadata in the Reader (without accidentally using a recursive glob) +- CI: Fix S3 endpoint management with `sertit>=1.37` +- CI: Remove for now end-to-end tests with Python 3.11 and 3.10. +- INSTALL: Remove `pystac[validation]` (as it is an optional dependency) from setup.py, and create a `stac` extra feature. ## 0.21.1 (2024-04-03) diff --git a/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py b/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py index 8bb8b234..55ae6744 100644 --- a/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py +++ b/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py @@ -7,7 +7,7 @@ import xarray as xr from lxml import etree -from sertit import AnyPath, ci, path +from sertit import AnyPath, ci, path, types from CI.scripts_utils import ( CI_EOREADER_S3, @@ -166,8 +166,7 @@ def _test_core( with xr.set_options(warn_for_unclosed_files=debug): # DATA paths - if not isinstance(prod_dirs, list): - prod_dirs = [prod_dirs] + prod_dirs = types.make_iterable(prod_dirs) pattern_paths = [] for prod_dir in prod_dirs: diff --git a/docs/sar.md b/docs/sar.md index f5adcdd4..e6fce9f1 100644 --- a/docs/sar.md +++ b/docs/sar.md @@ -43,8 +43,8 @@ mostly because SNAP doesn't handle them. | `RADARSAT Constellation Mission` | GRC, GCC, GCD | ⚠ | | `RADARSAT Constellation Mission` | GRD | ✅ | | `RADARSAT-2` | SLC | ✅ | -| `RADARSAT-2` | SGX, SCN, SCW,
SCF, SCS, SSG, SPG | ⚠ | -| `RADARSAT-2` | SGF | ✅ | +| `RADARSAT-2` | SGF, SGX, SSG | ✅ | +| `RADARSAT-2` | SCN, SCF, SCW, SCS, SPG | ⚠ | | `Sentinel-1` | SLC | ✅ | | `Sentinel-1` | GRD | ✅ | | `Sentinel-1` | RTC ([ASF](https://hyp3-docs.asf.alaska.edu/guides/rtc_product_guide/#readme-file) and [MPC](https://planetarycomputer.microsoft.com/dataset/sentinel-1-rtc)) | ✅ | diff --git a/environment-doc.yml b/environment-doc.yml index 6384f67b..53fe054f 100644 --- a/environment-doc.yml +++ b/environment-doc.yml @@ -5,5 +5,4 @@ dependencies: # everything under this, installed by conda - python=3.9 - pip - pip: # everything under this, installed by pip - - --extra-index-url ${WINDOWS_WHEELS} # Use windows wheels on Windows - -r ./requirements-doc.txt diff --git a/eoreader/__meta__.py b/eoreader/__meta__.py index 75d864e8..f99f28bf 100644 --- a/eoreader/__meta__.py +++ b/eoreader/__meta__.py @@ -17,7 +17,7 @@ """ **EOReader** library """ -__version__ = "0.21.1" +__version__ = "0.21.2" __title__ = "eoreader" __description__ = ( "Remote-sensing opensource python library reading optical and SAR constellations, " diff --git a/eoreader/bands/__init__.py b/eoreader/bands/__init__.py index f04b8a87..77d2288e 100644 --- a/eoreader/bands/__init__.py +++ b/eoreader/bands/__init__.py @@ -416,6 +416,7 @@ def to_band( list: converted values """ + from sertit import types def convert_to_band(tc) -> BandNames: band_or_idx = None @@ -460,14 +461,14 @@ def convert_to_band(tc) -> BandNames: if as_list: band_list = [] - if not isinstance(to_convert, list): - to_convert = [to_convert] + to_convert = types.make_iterable(to_convert) + for tc in to_convert: tc_band = convert_to_band(tc=tc) band_list.append(tc_band) return band_list else: - if isinstance(to_convert, list): + if types.is_iterable(to_convert): raise _ite(f"Set as_list=True(default) for list arguments") return convert_to_band(to_convert) @@ -491,9 +492,10 @@ def to_str( Returns: list: str bands """ + from sertit import types + if as_list: - if not isinstance(to_convert, list): - to_convert = [to_convert] + to_convert = types.make_iterable(to_convert) bands_str = [] for tc in to_convert: @@ -508,7 +510,7 @@ def to_str( bands_str.append(band_str) return bands_str else: - if isinstance(to_convert, list): + if types.is_iterable(to_convert): raise _ite(f"Set as_list=True(default) for list arguments") try: band_str = tc.name diff --git a/eoreader/bands/band_names.py b/eoreader/bands/band_names.py index 7c06be19..6fec4239 100644 --- a/eoreader/bands/band_names.py +++ b/eoreader/bands/band_names.py @@ -1,12 +1,12 @@ from typing import Union -from sertit.misc import ListEnum +from sertit import misc, types from eoreader.exceptions import InvalidTypeError from eoreader.stac import StacCommonNames -class BandNames(ListEnum): +class BandNames(misc.ListEnum): """Super class for band names, **do not use it**.""" @classmethod @@ -25,8 +25,8 @@ def from_list(cls, name_list: Union[list, str]) -> list: Returns: list: List of enums """ - if not isinstance(name_list, list): - name_list = [name_list] + name_list = types.make_iterable(name_list) + try: band_names = [cls(name) for name in name_list] except ValueError as ex: diff --git a/eoreader/products/custom_product.py b/eoreader/products/custom_product.py index deda4781..0804a671 100644 --- a/eoreader/products/custom_product.py +++ b/eoreader/products/custom_product.py @@ -28,7 +28,7 @@ from lxml.builder import E from rasterio import crs from rasterio.enums import Resampling -from sertit import logs, misc, path, rasters +from sertit import logs, misc, path, rasters, types from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType @@ -423,8 +423,7 @@ def _load_bands( return {} # Get band paths - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) if pixel_size is None and size is not None: pixel_size = self._pixel_size_from_img_size(size) diff --git a/eoreader/products/optical/gs2_product.py b/eoreader/products/optical/gs2_product.py index aa0e0bfa..060de209 100644 --- a/eoreader/products/optical/gs2_product.py +++ b/eoreader/products/optical/gs2_product.py @@ -405,14 +405,13 @@ def get_curr_band_mtd(root_mtd): # Compute band in radiance band_arr = bias + band_arr * gain + band_arr = self._toa_rad_to_toa_refl(band_arr, band, e_sun, sun_earth_dist) - # To float32 - if band_arr.dtype != np.float32: - band_arr = band_arr.astype(np.float32) + # To float32 + if band_arr.dtype != np.float32: + band_arr = band_arr.astype(np.float32) - return self._toa_rad_to_toa_refl(band_arr, band, e_sun, sun_earth_dist) - else: - return band_arr + return band_arr @cache def _read_mtd(self) -> (etree._Element, dict): diff --git a/eoreader/products/optical/hls_product.py b/eoreader/products/optical/hls_product.py index 9dabab6f..fb0b9ae2 100644 --- a/eoreader/products/optical/hls_product.py +++ b/eoreader/products/optical/hls_product.py @@ -31,7 +31,7 @@ import xarray as xr from lxml import etree from rasterio.enums import Resampling -from sertit import path, rasters, rasters_rio, xml +from sertit import path, rasters, rasters_rio, types, xml from sertit.misc import ListEnum from sertit.types import AnyPathType @@ -873,8 +873,7 @@ def _load_bands( return {} # Get band paths - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) if pixel_size is None and size is not None: pixel_size = self._pixel_size_from_img_size(size) diff --git a/eoreader/products/optical/landsat_product.py b/eoreader/products/optical/landsat_product.py index f92ec239..59c2c3b8 100644 --- a/eoreader/products/optical/landsat_product.py +++ b/eoreader/products/optical/landsat_product.py @@ -28,9 +28,8 @@ import xarray as xr from lxml import etree from lxml.builder import E -from pystac import Item from rasterio.enums import Resampling -from sertit import AnyPath, path, rasters, rasters_rio +from sertit import AnyPath, path, rasters, rasters_rio, types from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType @@ -1007,11 +1006,16 @@ def _read_mtd(self, force_pd=False) -> (etree._Element, dict): # FOR COLLECTION 1 AND 2 tar_ds = None try: - mtd_path = next(self.path.glob(f"**/*{mtd_name}")) - except ValueError: - mtd_path = next(self.path.glob(f"*{mtd_name}")) - - if not mtd_path.is_file(): + try: + mtd_path = next(self.path.glob(f"**/*{mtd_name}")) + except ValueError: + mtd_path = next(self.path.glob(f"*{mtd_name}")) + + if not mtd_path.is_file(): + raise InvalidProductError( + f"No metadata file found in {self.name} !" + ) + except StopIteration: raise InvalidProductError( f"No metadata file found in {self.name} !" ) @@ -1426,8 +1430,7 @@ def _load_bands( return {} # Get band paths - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) if pixel_size is None and size is not None: pixel_size = self._pixel_size_from_img_size(size) @@ -1860,16 +1863,8 @@ def __init__( super_kwargs = kwargs.copy() # Get STAC Item - self.item = None + self.item = self._set_item(product_path, **super_kwargs) """ STAC Item of the product """ - self.item = super_kwargs.pop("item", None) - if self.item is None: - try: - self.item = Item.from_file(product_path) - except TypeError: - raise InvalidProductError( - "You should either fill 'product_path' or 'item'." - ) if not self._is_mpc(): self.default_clients = [ diff --git a/eoreader/products/optical/maxar_product.py b/eoreader/products/optical/maxar_product.py index 9aceb60d..3495fc0f 100644 --- a/eoreader/products/optical/maxar_product.py +++ b/eoreader/products/optical/maxar_product.py @@ -123,7 +123,7 @@ NIR: GainOffset(gain=1.0, offset=0.0), NARROW_NIR: GainOffset(gain=1.0, offset=0.0), }, # 2017v0 - Constellation.QB: { + Constellation.QB02: { PAN: GainOffset(gain=0.870, offset=-1.491), BLUE: GainOffset(gain=1.105, offset=-2.820), GREEN: GainOffset(gain=1.071, offset=-3.338), @@ -191,7 +191,7 @@ NIR: 937.8, NARROW_NIR: 937.8, }, - Constellation.QB: { + Constellation.QB02: { PAN: 1370.92, BLUE: 1949.59, GREEN: 1823.64, @@ -432,7 +432,7 @@ def _set_instrument(self) -> None: elif self.constellation == Constellation.WV04: # SpaceView-110 camera self.instrument = "SpaceView-110 camera" - elif self.constellation == Constellation.QB: + elif self.constellation == Constellation.QB02: # Ball Global Imaging System 2000 self.instrument = "BGIS-2000" elif self.constellation == Constellation.GE01: @@ -575,7 +575,7 @@ def _get_spectral_bands(self) -> dict: }, ), } - elif self.constellation == Constellation.QB: + elif self.constellation == Constellation.QB02: spectral_bands = { "pan": SpectralBand( eoreader_name=PAN, diff --git a/eoreader/products/optical/optical_product.py b/eoreader/products/optical/optical_product.py index 19c66274..5896c7dc 100644 --- a/eoreader/products/optical/optical_product.py +++ b/eoreader/products/optical/optical_product.py @@ -656,11 +656,18 @@ def _get_clean_band_path( # Radiometric processing rad_proc = "" if kwargs.get(TO_REFLECTANCE, True) else "_as_is" - # Window + # Window name window = kwargs.get("window") - win_suffix = ( - f"win{files.hash_file_content(str(window))}_" if window is not None else "" - ) + + win_suffix = "" + if window is not None: + if path.is_path(window): + win_suffix = path.get_filename(window) + elif isinstance(window, gpd.GeoDataFrame): + win_suffix = window.attrs.get("name") + + if not win_suffix: + win_suffix = f"win{files.hash_file_content(str(window))}_" return self._get_band_folder(writable).joinpath( f"{self.condensed_name}_{band.name}_{res_str.replace('.', '-')}_{win_suffix}{cleaning_method.value}{rad_proc}.tif", diff --git a/eoreader/products/optical/pla_product.py b/eoreader/products/optical/pla_product.py index 5d5927d4..3ab87324 100644 --- a/eoreader/products/optical/pla_product.py +++ b/eoreader/products/optical/pla_product.py @@ -472,6 +472,9 @@ def _get_stack_path(self, as_list: bool = False) -> Union[str, list]: "Analytic", "tif", invalid_lookahead="udm", as_list=as_list ) + if as_list and len(stack_path) < 1: + raise InvalidProductError("This is not a Planet Product.") + return stack_path def _to_reflectance( diff --git a/eoreader/products/optical/planet_product.py b/eoreader/products/optical/planet_product.py index 371a5d71..66d73e47 100644 --- a/eoreader/products/optical/planet_product.py +++ b/eoreader/products/optical/planet_product.py @@ -31,7 +31,7 @@ import xarray as xr from lxml import etree from rasterio.enums import Resampling -from sertit import path, rasters, strings +from sertit import path, rasters, strings, types from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType @@ -1004,8 +1004,8 @@ def _get_path( Union[list, str]: Paths(s) """ - if invalid_lookahead is not None and not isinstance(invalid_lookahead, list): - invalid_lookahead = [invalid_lookahead] + if invalid_lookahead is not None: + invalid_lookahead = types.make_iterable(invalid_lookahead) ok_paths = [] try: diff --git a/eoreader/products/optical/s2_e84_product.py b/eoreader/products/optical/s2_e84_product.py index 679cf536..ca8c7c9f 100644 --- a/eoreader/products/optical/s2_e84_product.py +++ b/eoreader/products/optical/s2_e84_product.py @@ -25,9 +25,8 @@ import numpy as np import xarray as xr from lxml import etree -from pystac import Item from rasterio.enums import Resampling -from sertit import AnyPath, files, path, rasters_rio +from sertit import AnyPath, files, path, rasters_rio, types from sertit.files import CustomDecoder from sertit.types import AnyPathStrType, AnyPathType @@ -311,8 +310,7 @@ def _load_bands( return {} # Get band paths - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) if pixel_size is None and size is not None: pixel_size = self._pixel_size_from_img_size(size) @@ -795,16 +793,7 @@ def __init__( super_kwargs = kwargs.copy() # Get STAC Item - self.item = None - """ STAC Item of the product """ - self.item = super_kwargs.pop("item", None) - if self.item is None: - try: - self.item = Item.from_file(product_path) - except TypeError: - raise InvalidProductError( - "You should either fill 'product_path' or 'item'." - ) + self.item = self._set_item(product_path, **super_kwargs) if not self._is_mpc(): self.default_clients = [self.get_e84_client(), self.get_sinergise_client()] diff --git a/eoreader/products/optical/s2_mpc_product.py b/eoreader/products/optical/s2_mpc_product.py index 98ff67b4..8f26d82b 100644 --- a/eoreader/products/optical/s2_mpc_product.py +++ b/eoreader/products/optical/s2_mpc_product.py @@ -21,13 +21,11 @@ import numpy as np import xarray as xr from lxml import etree -from pystac import Item from sertit import AnyPath from sertit.types import AnyPathStrType, AnyPathType from eoreader import EOREADER_NAME from eoreader.bands import BandNames -from eoreader.exceptions import InvalidProductError from eoreader.products import S2E84Product from eoreader.products.optical.optical_product import RawUnits from eoreader.products.stac_product import StacProduct @@ -54,16 +52,8 @@ def __init__( super_kwargs = kwargs.copy() # Get STAC Item - self.item = None + self.item = self._set_item(product_path, **super_kwargs) """ STAC Item of the product """ - self.item = super_kwargs.pop("item", None) - if self.item is None: - try: - self.item = Item.from_file(product_path) - except TypeError: - raise InvalidProductError( - "You should either fill 'product_path' or 'item'." - ) self.default_clients = [] self.clients = super_kwargs.pop("client", self.default_clients) diff --git a/eoreader/products/optical/s2_product.py b/eoreader/products/optical/s2_product.py index 55d70a0b..38ca6bc5 100644 --- a/eoreader/products/optical/s2_product.py +++ b/eoreader/products/optical/s2_product.py @@ -32,13 +32,11 @@ import rasterio import xarray as xr from affine import Affine -from fiona.errors import DriverError from lxml import etree -from pystac import Item from rasterio import errors, features, transform from rasterio.crs import CRS from rasterio.enums import Resampling -from sertit import AnyPath, files, geometry, path, rasters, vectors +from sertit import AnyPath, files, geometry, path, rasters, types, vectors from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType from shapely.geometry import box @@ -493,7 +491,7 @@ def _get_name_constellation_specific(self) -> str: next(self.path.glob("**/tileInfo.json")), print_file=False ) name = tile_info["productName"] - except json.JSONDecodeError: + except (json.JSONDecodeError, StopIteration): raise InvalidProductError( f"Corrupted metadata and bad filename for {self.path}! " f"Impossible to process this product." @@ -537,7 +535,7 @@ def _get_res_band_folder(self, band_list: list, pixel_size: float = None) -> dic dict: Dictionary containing the folder path for each queried band """ if pixel_size is not None: - if isinstance(pixel_size, (list, tuple)): + if types.is_iterable(pixel_size): pixel_size = pixel_size[0] # Open the band directory names @@ -566,26 +564,31 @@ def _get_res_band_folder(self, band_list: list, pixel_size: float = None) -> dic dir_name = band_dir if self.is_archived: - # Open the zip file - with zipfile.ZipFile(self.path, "r") as zip_ds: - # Get the band folder (use dirname is the first of the list is a band) - band_path = [ - os.path.dirname(f.filename) - for f in zip_ds.filelist - if dir_name in f.filename - ][0] - - # Workaround for a bug involving some bad archives - if band_path.startswith("/"): - band_path = band_path[1:] - s2_bands_folder[band] = band_path + # Get the band folder (use dirname is the first of the list is a band) + band_path = os.path.dirname( + path.get_archived_rio_path( + self.path, f"{self._get_image_folder()}.*{dir_name}" + ) + ) + + # Workaround for a bug involving some bad archives + if band_path.startswith("/"): + band_path = band_path[1:] + + # Workaround for PEPS Sentinel-2 archives with incomplete manifest (without any directory) + if band_path.endswith(".jp2"): + band_path = os.path.dirname(band_path) + else: + band_path = os.path.basename(band_path) + + s2_bands_folder[band] = band_path else: # Search for the name of the folder into the S2 products try: s2_bands_folder[band] = next( self.path.glob(f"{self._get_image_folder()}/{dir_name}") ) - except IndexError: + except (IndexError, StopIteration): s2_bands_folder[band] = self.path for band in band_list: @@ -897,7 +900,7 @@ def _open_mask_lt_4_0( # Read vector try: mask = vectors.read(mask_path, crs=self.crs()) - except DriverError: + except vectors.DataSourceError: LOGGER.warning(f"Corrupted mask: {mask_path}. Returning an empty one.") mask = gpd.GeoDataFrame(geometry=[], crs=self.crs()) @@ -1724,16 +1727,8 @@ def __init__( super_kwargs = kwargs.copy() # Get STAC Item - self.item = None + self.item = self._set_item(product_path, **super_kwargs) """ STAC Item of the product """ - self.item = super_kwargs.pop("item", None) - if self.item is None: - try: - self.item = Item.from_file(product_path) - except TypeError: - raise InvalidProductError( - "You should either fill 'product_path' or 'item'." - ) if not self._is_mpc(): self.default_clients = [ diff --git a/eoreader/products/optical/s2_theia_product.py b/eoreader/products/optical/s2_theia_product.py index 53a4bbe3..dd0b2779 100644 --- a/eoreader/products/optical/s2_theia_product.py +++ b/eoreader/products/optical/s2_theia_product.py @@ -29,7 +29,7 @@ import xarray as xr from lxml import etree from rasterio.enums import Resampling -from sertit import geometry, path, rasters +from sertit import geometry, path, rasters, types from sertit.types import AnyPathStrType, AnyPathType from eoreader import DATETIME_FMT, EOREADER_NAME, cache, utils @@ -820,8 +820,7 @@ def _create_mask( xr.DataArray: Mask masked array """ - if not isinstance(bit_ids, list): - bit_ids = [bit_ids] + bit_ids = types.make_iterable(bit_ids) conds = rasters.read_bit_array(bit_array.astype(np.uint8), bit_ids) cond = reduce(lambda x, y: x | y, conds) # Use every condition (bitwise or) diff --git a/eoreader/products/optical/s3_olci_product.py b/eoreader/products/optical/s3_olci_product.py index 534cd7fd..ed217e06 100644 --- a/eoreader/products/optical/s3_olci_product.py +++ b/eoreader/products/optical/s3_olci_product.py @@ -478,7 +478,12 @@ def get_raw_band_paths(self, **kwargs) -> dict: if self.is_archived: raw_path = path.get_archived_path(self.path, f".*{filename}") else: - raw_path = next(self.path.glob(f"*{filename}")) + try: + raw_path = next(self.path.glob(f"*{filename}")) + except StopIteration: + raise FileNotFoundError( + f"Non existing file {filename} in {self.path}" + ) raw_band_paths[band] = raw_path diff --git a/eoreader/products/optical/s3_product.py b/eoreader/products/optical/s3_product.py index 398512fe..f2231df2 100644 --- a/eoreader/products/optical/s3_product.py +++ b/eoreader/products/optical/s3_product.py @@ -41,7 +41,7 @@ from rasterio import crs as riocrs from rasterio.enums import Resampling from rasterio.errors import NotGeoreferencedWarning -from sertit import path, vectors, xml +from sertit import path, types, vectors, xml from sertit.misc import ListEnum from sertit.rasters import MAX_CORES from sertit.types import AnyPathStrType, AnyPathType @@ -558,8 +558,7 @@ def _load_bands( return {} # Get band paths - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) if pixel_size is None and size is not None: pixel_size = self._pixel_size_from_img_size(size) diff --git a/eoreader/products/optical/s3_slstr_product.py b/eoreader/products/optical/s3_slstr_product.py index d9ab7f06..9e982768 100644 --- a/eoreader/products/optical/s3_slstr_product.py +++ b/eoreader/products/optical/s3_slstr_product.py @@ -31,7 +31,7 @@ import xarray as xr from rasterio import features from rasterio.enums import Resampling -from sertit import path, rasters +from sertit import path, rasters, types from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType @@ -531,7 +531,10 @@ def _get_raw_band_path(self, band: BandNames, **kwargs) -> AnyPathType: if self.is_archived: raw_path = path.get_archived_path(self.path, f".*{filename}*") else: - raw_path = next(self.path.glob(f"*{filename}*")) + try: + raw_path = next(self.path.glob(f"*{filename}*")) + except StopIteration: + raise FileNotFoundError(f"Non existing file {filename} in {self.path}") return raw_path @@ -1095,8 +1098,7 @@ def _create_mask( xr.DataArray: Mask masked array """ - if not isinstance(bit_ids, list): - bit_ids = [bit_ids] + bit_ids = types.make_iterable(bit_ids) conds = rasters.read_bit_array(bit_array, bit_ids) cond = reduce(lambda x, y: x | y, conds) # Use every condition (bitwise or) diff --git a/eoreader/products/optical/spot45_product.py b/eoreader/products/optical/spot45_product.py index 2939fcb7..e031f4bf 100644 --- a/eoreader/products/optical/spot45_product.py +++ b/eoreader/products/optical/spot45_product.py @@ -233,6 +233,8 @@ def _set_band_combi(self) -> None: """ root, _ = self.read_mtd() band_combi = root.findtext(".//SPECTRAL_PROCESSING") + if not band_combi: + raise InvalidProductError("SPECTRAL_PROCESSING not found in metadata!") if self.constellation == Constellation.SPOT4: self.band_combi = Spot4BandCombination.from_value(band_combi) else: diff --git a/eoreader/products/optical/sv1_product.py b/eoreader/products/optical/sv1_product.py index 1e8ea7a4..baace26f 100644 --- a/eoreader/products/optical/sv1_product.py +++ b/eoreader/products/optical/sv1_product.py @@ -309,7 +309,10 @@ def footprint(self) -> gpd.GeoDataFrame: if self.is_archived: footprint = vectors.read(self.path, archive_regex=r".*\.shp") else: - footprint = vectors.read(next(self.path.glob("*.shp"))) + try: + footprint = vectors.read(next(self.path.glob("*.shp"))) + except StopIteration: + raise FileNotFoundError(f"Non existing file *.shp in {self.path}") return footprint.to_crs(self.crs()) diff --git a/eoreader/products/optical/vhr_product.py b/eoreader/products/optical/vhr_product.py index 78318430..f004dc5f 100644 --- a/eoreader/products/optical/vhr_product.py +++ b/eoreader/products/optical/vhr_product.py @@ -515,8 +515,8 @@ def _get_path(self, filename: str = "", extension: str = "") -> AnyPathType: else: prod_path = next(self.path.glob(f"{filename}*.{extension}")) - except (FileNotFoundError, IndexError): - LOGGER.warning( + except (FileNotFoundError, IndexError, StopIteration): + raise InvalidProductError( f"No file corresponding to *{filename}*.{extension} found in {self.path}" ) diff --git a/eoreader/products/optical/vis1_product.py b/eoreader/products/optical/vis1_product.py index ba3f0fa7..27d0b3b8 100644 --- a/eoreader/products/optical/vis1_product.py +++ b/eoreader/products/optical/vis1_product.py @@ -346,11 +346,13 @@ def _to_reflectance( if original_dtype == "uint16": band_arr /= 100.0 + band_arr = self._toa_rad_to_toa_refl(band_arr, band, _VIS1_E0[band]) + # To float32 if band_arr.dtype != np.float32: band_arr = band_arr.astype(np.float32) - return self._toa_rad_to_toa_refl(band_arr, band, _VIS1_E0[band]) + return band_arr @cache def _read_mtd(self) -> (etree._Element, dict): diff --git a/eoreader/products/product.py b/eoreader/products/product.py index 58a7d8ea..d5fd717c 100644 --- a/eoreader/products/product.py +++ b/eoreader/products/product.py @@ -43,7 +43,7 @@ from rasterio.crs import CRS from rasterio.enums import Resampling from rasterio.vrt import WarpedVRT -from sertit import AnyPath, files, logs, misc, path, rasters, strings, xml +from sertit import AnyPath, files, logs, misc, path, rasters, strings, types, xml from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType @@ -873,7 +873,7 @@ def load( ) pixel_size = kwargs.pop("resolution") - if (isinstance(bands, list) and ("GREEN1" in bands or GREEN1 in bands)) or ( + if (types.is_iterable(bands) and ("GREEN1" in bands or GREEN1 in bands)) or ( "GREEN1" == bands or GREEN1 == bands ): logs.deprecation_warning( @@ -1239,9 +1239,7 @@ def has_bands(self, bands: Union[list, BandNames, str]) -> bool: Returns: bool: True if the products has the specified band """ - if not isinstance(bands, list): - bands = [bands] - + bands = types.make_iterable(bands) return all([self.has_band(band) for band in set(bands)]) @abstractmethod @@ -1715,8 +1713,7 @@ def _update_attrs( # Are we sure of that ? xarr.attrs = {} - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) long_name = to_str(bands) xr_name = "_".join(long_name) attr_name = " ".join(long_name) @@ -1887,7 +1884,7 @@ def _res_to_str(res): return f"{abs(res):.2f}m".replace(".", "-") if pixel_size: - if isinstance(pixel_size, (tuple, list)): + if types.is_iterable(pixel_size): res_x = _res_to_str(pixel_size[0]) res_y = _res_to_str(pixel_size[1]) if res_x == res_y: @@ -2032,8 +2029,7 @@ def to_band(self, raw_bands: Union[list, BandNames, str, int]) -> list: Returns: list: Mapped bands """ - if not isinstance(raw_bands, list): - raw_bands = [raw_bands] + raw_bands = types.make_iterable(raw_bands) bands = [] for raw_band in raw_bands: diff --git a/eoreader/products/sar/capella_product.py b/eoreader/products/sar/capella_product.py index 19f2121b..eabe8fec 100644 --- a/eoreader/products/sar/capella_product.py +++ b/eoreader/products/sar/capella_product.py @@ -206,12 +206,22 @@ def _post_init(self, **kwargs) -> None: (setting product-type, band names and so on) """ # Private attributes - self.snap_filename = str(next(self.path.glob("*CAPELLA*.json")).name) + try: + self.snap_filename = str(next(self.path.glob("*CAPELLA*.json")).name) + except StopIteration: + raise FileNotFoundError(f"Non existing file *CAPELLA*.json in {self.path}") try: self._raw_band_regex = str(next(self.path.glob(f"{self.name}.tif")).name) except StopIteration: # For SICD and SIDD - self._raw_band_regex = str(next(self.path.glob(f"{self.name}.ntf")).name) + try: + self._raw_band_regex = str( + next(self.path.glob(f"{self.name}.ntf")).name + ) + except StopIteration: + raise FileNotFoundError( + f"Non existing file {self.name}.tif or {self.name}.ntf in {self.path}" + ) # Post init done by the super class super()._post_init(**kwargs) @@ -235,7 +245,12 @@ def wgs84_extent(self) -> gpd.GeoDataFrame: gpd.GeoDataFrame: WGS84 extent as a gpd.GeoDataFrame """ if self._has_stac_mtd: - mtd_file = next(self.path.glob(f"{self.name}.json")) + try: + mtd_file = next(self.path.glob(f"{self.name}.json")) + except StopIteration: + raise FileNotFoundError( + f"Non existing file {self.name}.json in {self.path}" + ) extent = vectors.read(mtd_file) else: extent = None diff --git a/eoreader/products/sar/cosmo_product.py b/eoreader/products/sar/cosmo_product.py index 1fa6b368..487fd72d 100644 --- a/eoreader/products/sar/cosmo_product.py +++ b/eoreader/products/sar/cosmo_product.py @@ -93,7 +93,7 @@ def __init__( try: product_path = AnyPath(product_path) self._img_path = next(product_path.glob("*.h5")) - except IndexError as ex: + except (IndexError, StopIteration) as ex: raise InvalidProductError( f"Image file (*.h5) not found in {product_path}" ) from ex diff --git a/eoreader/products/sar/csg_product.py b/eoreader/products/sar/csg_product.py index 789aeb4d..579e51bb 100644 --- a/eoreader/products/sar/csg_product.py +++ b/eoreader/products/sar/csg_product.py @@ -218,7 +218,7 @@ def _post_init(self, **kwargs) -> None: """ # Calibration fails with CSG data LOGGER.debug( - "SNAP Error: Calibration currently fails for CSG data. Removing this step." + "SNAP Error: Calibration is useless for CSG data. Removing this step." ) self._calibrate = False diff --git a/eoreader/products/sar/iceye_product.py b/eoreader/products/sar/iceye_product.py index bfe54e8a..0a7defe9 100644 --- a/eoreader/products/sar/iceye_product.py +++ b/eoreader/products/sar/iceye_product.py @@ -149,12 +149,17 @@ def _post_init(self, **kwargs) -> None: (setting product-type, band names and so on) """ # Private attributes - if self._use_slc: - self.snap_filename = str(next(self.path.glob("*ICEYE*SLC*.xml")).name) - self._raw_band_regex = "*ICEYE*SLC*.h5" - else: - self.snap_filename = str(next(self.path.glob("*ICEYE*GRD*.xml")).name) - self._raw_band_regex = "*ICEYE*GRD*.tif" + try: + if self._use_slc: + self.snap_filename = str(next(self.path.glob("*ICEYE*SLC*.xml")).name) + self._raw_band_regex = "*ICEYE*SLC*.h5" + else: + self.snap_filename = str(next(self.path.glob("*ICEYE*GRD*.xml")).name) + self._raw_band_regex = "*ICEYE*GRD*.tif" + except StopIteration: + raise FileNotFoundError( + f"Non existing file *ICEYE*SLC*.xml or *ICEYE*GRD*.xml in {self.path}" + ) # Post init done by the super class super()._post_init(**kwargs) diff --git a/eoreader/products/sar/rcm_product.py b/eoreader/products/sar/rcm_product.py index bc4f4567..e3b76ef8 100644 --- a/eoreader/products/sar/rcm_product.py +++ b/eoreader/products/sar/rcm_product.py @@ -25,10 +25,12 @@ from typing import Union import geopandas as gpd +import rasterio from lxml import etree -from sertit import vectors +from sertit import geometry, vectors from sertit.misc import ListEnum from sertit.vectors import WGS84 +from shapely import Polygon from eoreader import DATETIME_FMT, EOREADER_NAME, cache from eoreader.exceptions import InvalidProductError, InvalidTypeError @@ -205,14 +207,40 @@ def wgs84_extent(self) -> gpd.GeoDataFrame: try: extent_file = next(self.path.joinpath("preview").glob("*mapOverlay.kml")) product_kml = vectors.read(extent_file) - except IndexError as ex: - raise InvalidProductError( - f"Extent file (product.kml) not found in {self.path}" - ) from ex - extent_wgs84 = product_kml[ - product_kml.Name == "Polygon Outline" - ].envelope.to_crs(WGS84) + extent_wgs84 = product_kml[product_kml.Name == "Polygon Outline"].envelope + + except (IndexError, StopIteration) as ex: + # Some RCM products don't have any mapOverlay.kml file as it is not a mandatory file! + with rasterio.open( + self.get_raw_band_paths()[self.get_default_band()] + ) as ds: + if ds.crs is not None: + extent_wgs84 = gpd.GeoDataFrame( + geometry=[geometry.from_bounds_to_polygon(*ds.bounds)], + crs=ds.crs, + ) + elif ds.gcps is not None: + gcps, crs = ds.gcps + corners = geometry.from_bounds_to_polygon( + *ds.bounds + ).exterior.coords + extent_poly = Polygon( + [ + rasterio.transform.from_gcps(gcps) * corner + for corner in corners + ] + ) + extent_wgs84 = gpd.GeoDataFrame(geometry=[extent_poly], crs=crs) + else: + raise InvalidProductError( + f"Extent file (preview/mapOverlay.kml) not found in {self.path}. " + "Default band isn't georeferenced and have no GCPs. " + "It is therefore impossible to determine quickly the extent of this product. " + "Please write an issue on GitHub!" + ) from ex + + extent_wgs84 = extent_wgs84.to_crs(WGS84) return gpd.GeoDataFrame(geometry=extent_wgs84.geometry, crs=extent_wgs84.crs) @@ -228,7 +256,7 @@ def _set_product_type(self) -> None: """Set products type""" # Get MTD XML file root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Open identifier prod_type = root.findtext(f".//{namespace}productType") @@ -252,7 +280,7 @@ def _set_product_type(self) -> None: if self.product_type != RcmProductType.GRD: LOGGER.warning( - "Other products type than SGF has not been tested for %s data. " + "Other products type than GRD has not been tested for %s data. " "Use it at your own risks !", self.constellation.value, ) @@ -263,7 +291,7 @@ def _set_sensor_mode(self) -> None: """ # Get metadata root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Get sensor mode # WARNING: this word may differ from the Enum !!! (no docs available) @@ -303,7 +331,7 @@ def get_datetime(self, as_datetime: bool = False) -> Union[str, datetime]: """ # Get MTD XML file root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Open identifier acq_date = root.findtext(f".//{namespace}rawDataStartTime") @@ -426,7 +454,7 @@ def get_orbit_direction(self) -> OrbitDirection: """ # Get MTD XML file root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Get the orbit direction try: diff --git a/eoreader/products/sar/rs2_product.py b/eoreader/products/sar/rs2_product.py index 32f9fd30..47bf17fc 100644 --- a/eoreader/products/sar/rs2_product.py +++ b/eoreader/products/sar/rs2_product.py @@ -25,10 +25,12 @@ from typing import Union import geopandas as gpd +import rasterio from lxml import etree -from sertit import path, vectors +from sertit import geometry, path, vectors from sertit.misc import ListEnum from sertit.vectors import WGS84 +from shapely import Polygon from eoreader import DATETIME_FMT, EOREADER_NAME, cache from eoreader.exceptions import InvalidProductError, InvalidTypeError @@ -214,6 +216,7 @@ def _set_pixel_size(self) -> None: Rs2ProductType.SCW, Rs2ProductType.SCF, Rs2ProductType.SCS, + Rs2ProductType.SGF, ]: def_pixel_size = 50.0 def_res = 100.0 @@ -379,7 +382,7 @@ def _pre_init(self, **kwargs) -> None: # SNAP can process non-complex archive root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Open identifier prod_type = root.findtext(f".//{namespace}productType") @@ -427,14 +430,40 @@ def wgs84_extent(self) -> gpd.GeoDataFrame: else: extent_file = next(self.path.glob("*product.kml")) product_kml = vectors.read(extent_file) - except IndexError as ex: - raise InvalidProductError( - f"Extent file (product.kml) not found in {self.path}" - ) from ex - extent_wgs84 = product_kml[ - product_kml.Name == "Polygon Outline" - ].envelope.to_crs(WGS84) + extent_wgs84 = product_kml[product_kml.Name == "Polygon Outline"].envelope + except (IndexError, StopIteration) as ex: + # Some RS2 products don't have any product.kml file as it is not a mandatory file! + with rasterio.open( + self.get_raw_band_paths()[self.get_default_band()] + ) as ds: + if ds.crs is not None: + extent_wgs84 = gpd.GeoDataFrame( + geometry=[geometry.from_bounds_to_polygon(*ds.bounds)], + crs=ds.crs, + ) + elif ds.gcps is not None: + gcps, crs = ds.gcps + corners = geometry.from_bounds_to_polygon( + *ds.bounds + ).exterior.coords + extent_poly = Polygon( + [ + rasterio.transform.from_gcps(gcps) * corner + for corner in corners + ] + ) + extent_wgs84 = gpd.GeoDataFrame(geometry=[extent_poly], crs=crs) + else: + raise InvalidProductError( + f"Extent file (product.kml) not found in {self.path}. " + "Default band isn't georeferenced and have no GCPs. " + "It is therefore impossible to determine quickly the extent of this product. " + "Please write an issue on GitHub!" + ) from ex + + # Just to be sure + extent_wgs84 = extent_wgs84.to_crs(WGS84) return gpd.GeoDataFrame(geometry=extent_wgs84.geometry, crs=extent_wgs84.crs) @@ -445,9 +474,14 @@ def _set_product_type(self) -> None: else: self.sar_prod_type = SarProductType.GDRG - if self.product_type not in [Rs2ProductType.SGF, Rs2ProductType.SLC]: + if self.product_type not in [ + Rs2ProductType.SGF, + Rs2ProductType.SGX, + Rs2ProductType.SSG, + Rs2ProductType.SLC, + ]: LOGGER.warning( - "Other product types than SGF or SLC haven't been tested for %s data. " + "Other product types than SGF, SGX, SSG or SLC haven't been tested for %s data. " "Use it at your own risks !", self.constellation.value, ) @@ -458,7 +492,7 @@ def _set_sensor_mode(self) -> None: """ # Get metadata root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Get sensor mode # WARNING: this word may differ from the Enum !!! (no docs available) @@ -499,7 +533,7 @@ def get_datetime(self, as_datetime: bool = False) -> Union[str, datetime]: if self.datetime is None: # Get MTD XML file root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Open identifier acq_date = root.findtext(f".//{namespace}rawDataStartTime") @@ -596,7 +630,7 @@ def get_orbit_direction(self) -> OrbitDirection: """ # Get MTD XML file root, nsmap = self.read_mtd() - namespace = nsmap[None] + namespace = nsmap.get(None, "") # Get the orbit direction try: diff --git a/eoreader/products/sar/s1_rtc_asf_product.py b/eoreader/products/sar/s1_rtc_asf_product.py index ac39732b..ab194510 100644 --- a/eoreader/products/sar/s1_rtc_asf_product.py +++ b/eoreader/products/sar/s1_rtc_asf_product.py @@ -165,7 +165,10 @@ def footprint(self) -> gpd.GeoDataFrame: if self.is_archived: footprint = vectors.read(self.path, archive_regex=r".*\.shp") else: - footprint = vectors.read(next(self.path.glob("*.shp"))) + try: + footprint = vectors.read(next(self.path.glob("*.shp"))) + except StopIteration: + raise FileNotFoundError(f"Non existing file *.shp in {self.path}") return footprint diff --git a/eoreader/products/sar/s1_rtc_mpc_product.py b/eoreader/products/sar/s1_rtc_mpc_product.py index 9b8c354c..d7c9cc1b 100644 --- a/eoreader/products/sar/s1_rtc_mpc_product.py +++ b/eoreader/products/sar/s1_rtc_mpc_product.py @@ -24,7 +24,6 @@ from typing import Union from lxml import etree -from pystac import Item from sertit import AnyPath, path, xml from sertit.types import AnyPathStrType @@ -61,16 +60,8 @@ def __init__( super_kwargs = kwargs.copy() # Get STAC Item - self.item = None + self.item = self._set_item(product_path, **super_kwargs) """ STAC Item of the product """ - self.item = super_kwargs.pop("item", None) - if self.item is None: - try: - self.item = Item.from_file(product_path) - except TypeError: - raise InvalidProductError( - "You should either fill 'product_path' or 'item'." - ) # Nothing here works for MPC self.default_clients = [] diff --git a/eoreader/products/sar/saocom_product.py b/eoreader/products/sar/saocom_product.py index 593931f4..b0352857 100644 --- a/eoreader/products/sar/saocom_product.py +++ b/eoreader/products/sar/saocom_product.py @@ -242,7 +242,7 @@ def wgs84_extent(self) -> gpd.GeoDataFrame: # else: # try: # extent_file = next(self.path.glob("**/Images/*.kml")) - # except IndexError as ex: + # except (IndexError, StopIteration) as ex: # raise InvalidProductError( # f"Extent file (products.kml) not found in {self.path}" # ) from ex @@ -286,7 +286,10 @@ def get_raw_band_paths(self, **kwargs) -> dict: dict: Dictionary containing the path of every band existing in the raw products """ extended_fmt = _ExtendedFormatter() - cuss_file = next(self.path.glob("*.zip")) + try: + cuss_file = next(self.path.glob("*.zip")) + except StopIteration: + raise FileNotFoundError(f"Non existing file *.zip in {self.path}") band_paths = {} for band in SarBandNames.speckle_list(): band_regex = extended_fmt.format(self._raw_band_regex, band.value) diff --git a/eoreader/products/sar/sar_product.py b/eoreader/products/sar/sar_product.py index 81a04eb5..9d2a42e2 100644 --- a/eoreader/products/sar/sar_product.py +++ b/eoreader/products/sar/sar_product.py @@ -30,7 +30,7 @@ import xarray as xr from rasterio import crs from rasterio.enums import Resampling -from sertit import AnyPath, misc, path, rasters, snap, strings +from sertit import AnyPath, misc, path, rasters, snap, strings, types from sertit.misc import ListEnum from sertit.types import AnyPathStrType, AnyPathType @@ -633,8 +633,7 @@ def _load_bands( return {} # Get band paths - if not isinstance(bands, list): - bands = [bands] + bands = types.make_iterable(bands) if pixel_size is None and size is not None: pixel_size = self._pixel_size_from_img_size(size) diff --git a/eoreader/products/sar/tsx_product.py b/eoreader/products/sar/tsx_product.py index 2581dd77..b00b0c07 100644 --- a/eoreader/products/sar/tsx_product.py +++ b/eoreader/products/sar/tsx_product.py @@ -379,7 +379,7 @@ def wgs84_extent(self) -> gpd.GeoDataFrame: # Open extent KML file try: extent_file = next(self.path.glob("**/*SUPPORT/GEARTH_POLY.kml")) - except IndexError as ex: + except (IndexError, StopIteration) as ex: raise InvalidProductError( f"Extent file (products.kml) not found in {self.path}" ) from ex diff --git a/eoreader/products/stac_product.py b/eoreader/products/stac_product.py index 13b8d47f..3382c7ff 100644 --- a/eoreader/products/stac_product.py +++ b/eoreader/products/stac_product.py @@ -24,12 +24,19 @@ from lxml import etree from rasterio import crs from sertit import geometry, path, rasters, vectors +from sertit.types import AnyPathStrType from eoreader import EOREADER_NAME, cache +from eoreader.exceptions import InvalidProductError from eoreader.products.product import Product from eoreader.stac import PROJ_EPSG from eoreader.utils import simplify +try: + from pystac import Item +except ModuleNotFoundError: + from typing import Any as Item + LOGGER = logging.getLogger(EOREADER_NAME) @@ -45,6 +52,32 @@ class StacProduct(Product): clients = None default_clients = None + def _set_item(self, product_path: AnyPathStrType, **kwargs) -> Item: + """ + Set the STAC Item as member + + Args: + product_path (AnyPathStrType): Product path + **kwargs: Other argumlents + """ + item = kwargs.pop("item", None) + if item is None: + try: + import pystac + + item = pystac.Item.from_file(product_path) + except ModuleNotFoundError: + raise InvalidProductError( + "You should install 'pystac' to use STAC Products." + ) + + except TypeError: + raise InvalidProductError( + "You should either fill 'product_path' or 'item'." + ) + + return item + @cache def extent(self) -> gpd.GeoDataFrame: """ diff --git a/eoreader/reader.py b/eoreader/reader.py index ee5d80c2..419b5b33 100644 --- a/eoreader/reader.py +++ b/eoreader/reader.py @@ -25,16 +25,24 @@ from typing import Union from zipfile import BadZipFile -import pystac import validators -from pystac import Item -from sertit import AnyPath, path, strings +from sertit import AnyPath, path, strings, types from sertit.misc import ListEnum from sertit.types import AnyPathStrType from eoreader import EOREADER_NAME from eoreader.exceptions import InvalidProductError +try: + import pystac + from pystac import Item + + PYSTAC_INSTALLED = True +except ModuleNotFoundError: + from typing import Any as Item + + PYSTAC_INSTALLED = False + LOGGER = logging.getLogger(EOREADER_NAME) @@ -183,7 +191,7 @@ class Constellation(ListEnum): HLS = "HLS" """Harmonized Landsat-Sentinel""" - QB = "QuickBird" + QB02 = "QuickBird" """QuickBird""" GE01 = "GeoEye-1" @@ -275,7 +283,7 @@ class Constellation(ListEnum): Constellation.SPOT5: r"SP05_HRG_(HM_|J__|T__|X__|TX__|HMX)__\d_\d{8}T\d{6}_\d{8}T\d{6}_.*", Constellation.VIS1: r"VIS1_(PAN|BUN|PSH|MS4)_.+_\d{2}-\d", Constellation.RCM: r"RCM\d_OK\d+_PK\d+_\d_.{4,}_\d{8}_\d{6}(_(HH|VV|VH|HV|RV|RH)){1,4}_(SLC|GRC|GRD|GCC|GCD)", - Constellation.QB: r"\d{12}_\d{2}_P\d{3}_(MUL|PAN|PSH|MOS)", + Constellation.QB02: r"\d{12}_\d{2}_P\d{3}_(MUL|PAN|PSH|MOS)", Constellation.GE01: r"\d{12}_\d{2}_P\d{3}_(MUL|PAN|PSH|MOS)", Constellation.WV01: r"\d{12}_\d{2}_P\d{3}_(MUL|PAN|PSH|MOS)", Constellation.WV02: r"\d{12}_\d{2}_P\d{3}_(MUL|PAN|PSH|MOS)", @@ -301,7 +309,7 @@ class Constellation(ListEnum): # File that can be found at any level (product/**/file) "regex": r".*s1[ab]-(iw|ew|sm|wv|s\d)\d*-(raw|slc|grd|ocn)-[hv]{2}-\d{8}t\d{6}-\d{8}t\d{6}-\d{6}-\w{6}-\d{3}(-cog|)\.xml", }, - Constellation.S2: {"nested": 3, "regex": r"MTD_TL.xml"}, + Constellation.S2: {"nested": 2, "regex": r"MTD_TL.xml"}, Constellation.S2_E84: rf"{CONSTELLATION_REGEX[Constellation.S2_E84]}\.json", Constellation.S2_THEIA: rf"{CONSTELLATION_REGEX[Constellation.S2_THEIA]}_MTD_ALL\.xml", Constellation.S3_OLCI: r"Oa\d{2}_(radiance|reflectance).nc", @@ -347,7 +355,7 @@ class Constellation(ListEnum): r"\d+_[RHV]{2}\.tif", ], }, - Constellation.QB: r"\d{2}\w{3}\d{8}-.{4}(_R\dC\d|)-\d{12}_\d{2}_P\d{3}.TIL", + Constellation.QB02: r"\d{2}\w{3}\d{8}-.{4}(_R\dC\d|)-\d{12}_\d{2}_P\d{3}.TIL", Constellation.GE01: r"\d{2}\w{3}\d{8}-.{4}(_R\dC\d|)-\d{12}_\d{2}_P\d{3}.TIL", Constellation.WV01: r"\d{2}\w{3}\d{8}-.{4}(_R\dC\d|)-\d{12}_\d{2}_P\d{3}.TIL", Constellation.WV02: r"\d{2}\w{3}\d{8}-.{4}(_R\dC\d|)-\d{12}_\d{2}_P\d{3}.TIL", @@ -360,27 +368,18 @@ class Constellation(ListEnum): Constellation.SV1: r"SV1-0[1-4]_\d{8}_L(1B|2A)\d{10}_\d{13}_\d{2}-(MUX|PSH)\.xml", Constellation.HLS: rf"{CONSTELLATION_REGEX[Constellation.HLS]}\.Fmask\.tif", Constellation.GS2: rf"{CONSTELLATION_REGEX[Constellation.GS2]}\.dim", - Constellation.SPOT45: { - "nested": -1, # File that can be found at any level (product/**/file) - "regex": [ - r"METADATA\.DIM", # Too generic name, check also a band - r"IMAGERY\.TIF", - ], - }, - Constellation.SPOT4: { - "nested": -1, # File that can be found at any level (product/**/file) - "regex": [ - r"METADATA\.DIM", # Too generic name, check also a band - r"IMAGERY\.TIF", - ], - }, - Constellation.SPOT5: { - "nested": -1, # File that can be found at any level (product/**/file) - "regex": [ - r"METADATA\.DIM", # Too generic name, check also a band - r"IMAGERY\.TIF", - ], - }, + Constellation.SPOT45: [ + r"METADATA\.DIM", # Too generic name, check also a band + r"IMAGERY\.TIF", + ], + Constellation.SPOT4: [ + r"METADATA\.DIM", # Too generic name, check also a band + r"IMAGERY\.TIF", + ], + Constellation.SPOT5: [ + r"METADATA\.DIM", # Too generic name, check also a band + r"IMAGERY\.TIF", + ], Constellation.S2_SIN: { "nested": 1, # File that can be found at any level (product/**/file) "regex": [ @@ -441,7 +440,7 @@ def _compile_(regex_str: str): return re.compile(f"{prefix}{regex_str}{suffix}") # Case folder is not enough to identify the products (i.e. COSMO Skymed) - if isinstance(regex, list): + if types.is_iterable(regex): comp = [_compile_(regex) for regex in regex] else: comp = [_compile_(regex)] @@ -521,16 +520,32 @@ def open( Returns: Product: EOReader's product """ - # If an URL is given, it must point to an URL translatable to a STAC Item + prod = None + # If a URL is given, it must point to a URL translatable to a STAC Item if validators.url(product_path): - try: - product_path = pystac.Item.from_file(product_path) - except Exception: - raise InvalidProductError( - f"Cannot convert your URL ({product_path}) to a STAC Item." + if PYSTAC_INSTALLED: + try: + product_path = Item.from_file(product_path) + is_stac = True + except Exception: + raise InvalidProductError( + f"Cannot convert your URL ({product_path}) to a STAC Item." + ) + else: + raise ModuleNotFoundError( + "You should install 'pystac' to use STAC Products." ) + # Check path (first check URL as they are also strings) + elif path.is_path(product_path): + is_stac = False + else: + # Check STAC Item + if PYSTAC_INSTALLED: + is_stac = isinstance(product_path, pystac.Item) + else: + is_stac = False - if isinstance(product_path, Item): + if is_stac: prod = self._open_stac_item(product_path, output_path, remove_tmp, **kwargs) else: # If not an Item, it should be a path to somewhere @@ -816,7 +831,7 @@ def valid_mtd( ) else: nested_wildcard = "/".join(["*" for _ in range(nested)]) - prod_files = list(product_path.glob(f"*{nested_wildcard}/*.*")) + prod_files = list(product_path.glob(f"{nested_wildcard}/*.*")) # Archive else: @@ -922,7 +937,7 @@ def create_product( constellation = None # All product names are the same, so assess it with MTD # Maxar-like constellations elif constellation in [ - Constellation.QB, + Constellation.QB02, Constellation.GE01, Constellation.WV01, Constellation.WV02, diff --git a/eoreader/stac/stac_extensions.py b/eoreader/stac/stac_extensions.py index e8ae9d11..e77e198e 100644 --- a/eoreader/stac/stac_extensions.py +++ b/eoreader/stac/stac_extensions.py @@ -29,6 +29,7 @@ from rasterio.crs import CRS from eoreader import cache +from eoreader.exceptions import InvalidProductError from eoreader.stac import StacCommonNames, stac_utils from eoreader.stac._stac_keywords import ( DESCRIPTION, @@ -77,7 +78,7 @@ def __init__(self, prod, **kwargs): try: if prod._has_cloud_cover: self.cloud_cover = prod.get_cloud_cover() - except AttributeError: + except (AttributeError, InvalidProductError): pass self.bands = prod.bands @@ -156,7 +157,7 @@ def add_to_item(self, item) -> None: center_wavelength = stac_utils.to_float(band.center_wavelength) solar_illumination = stac_utils.to_float(band.solar_illumination) full_width_half_max = stac_utils.to_float(band.full_width_half_max) - except AttributeError: + except (AttributeError, InvalidProductError): center_wavelength = None solar_illumination = None full_width_half_max = None @@ -337,7 +338,7 @@ def __init__(self, prod, **kwargs): # Convert from numpy dtype (which are not JSON serializable) to standard dtype self.sun_az = stac_utils.to_float(sun_az) self.sun_el = stac_utils.to_float(sun_el) - except AttributeError: + except (AttributeError, InvalidProductError): self.sun_az = None self.sun_el = None @@ -348,7 +349,7 @@ def __init__(self, prod, **kwargs): self.view_az = stac_utils.to_float(view_az) self.off_nadir = stac_utils.to_float(off_nadir) self.incidence_angle = stac_utils.to_float(incidence_angle) - except AttributeError: + except (AttributeError, InvalidProductError): self.view_az = None self.off_nadir = None self.incidence_angle = None diff --git a/requirements-doc.txt b/requirements-doc.txt index 9dd82a2c..ad0eee25 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,8 +1,8 @@ -r requirements.txt # Doc -sphinx<6.0.0 # workaround ipython # workaround +sphinx sphinx-book-theme sphinx-copybutton myst-nb @@ -20,6 +20,3 @@ eodag folium -e . - -# workaround for sphinx v6 -# jinja2==3.0.3 diff --git a/requirements.txt b/requirements.txt index e9ff7ca8..ab208dcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ geopandas>=0.11.0 spyndex>=0.3.0 # SERTIT libs -sertit[full]>=1.33.0 +sertit[full]>=1.41.0 # Optimizations dask[complete]>=2021.10.0 @@ -47,6 +47,7 @@ methodtools matplotlib # MPC, AWS and STAC +# /!\ Should not be mandatory requirements! pystac[validation] stac-asset planetary_computer diff --git a/setup.py b/setup.py index 61ca0772..889eda7a 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "xarray>=0.18.0", "rioxarray>=0.10.0", "geopandas>=0.11.0", - "sertit[full]>=1.30.0", + "sertit[full]>=1.41.0", "spyndex>=0.3.0", "pyresample", "zarr", @@ -41,8 +41,14 @@ "validators", "methodtools", "dicttoxml", - "pystac[validation]", ], + extras_require={ + "stac": [ + "pystac[validation]", + "stac-asset", + "planetary_computer", + ] + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers",