Merge pull request #126 from sertit/0.21.0

0.21.0
sertit · Dec 22, 2023 · e582287 · e582287
2 parents fcad089 + 129728a
commit e582287
Show file tree

Hide file tree

Showing 116 changed files with 17,894 additions and 32,807 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -102,7 +102,7 @@ pytest_end_to_end:
     - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_WEEKLY --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html
@@ -129,7 +129,7 @@ pytest_end_to_end310:
     - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html
@@ -157,7 +157,7 @@ pytest_end_to_end311:
     - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html
@@ -185,7 +185,7 @@ pytest_end_to_end312:
     - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_SNAP/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,8 +2,9 @@ exclude: |
   (?x)^(docs/.*)$ |
   .*\.md
 fail_fast: false
+ci:
+    autoupdate_schedule: monthly
 repos:
-
   - repo: https://github.com/pre-commit/pre-commit-hooks.git
     rev: v4.5.0
     hooks:

diff --git a/CHANGES.md b/CHANGES.md
@@ -4,11 +4,23 @@
 
 - **BREAKING CHANGES: Rename `utils.stack_dict` to `utils.stack` since we are stacking datasets and not dict anymore.**
 - **BREAKING CHANGES: Band ID for Sentinel-3 OLCI are now int instead of band names (i.e. `7` instead of `Oa07`. The names don't change).**
-- **ENH: Allow to use bands IDs, names and common name added to mapped names when trying to load a spectral band. ([#111](https://github.com/sertit/eoreader/issues/111)**
-- CI: Update pre-commit hooks
+- **ENH: Allow to use bands IDs, names and common name added to mapped names when trying to load a spectral band. ([#111](https://github.com/sertit/eoreader/issues/111))**
+- **ENH: Manage Sentinel-2 as formatted on the cloud (Element84 or Sinergise's way). ([#104](https://github.com/sertit/eoreader/issues/104))**
+- **ENH: Handle Python 3.12. ([#113](https://github.com/sertit/eoreader/issues/113))**
+- **ENH: Guard against S1 COG format, not yet handled by SNAP.**
+- **ENH: Calibration step for `Capella` products now exists in ESA SNAP. Add it in pre-processing.**
+- **ENH: Handling of Sentinel-1 [ASF](https://hyp3-docs.asf.alaska.edu/guides/rtc_product_guide/#readme-file) and [MPC](https://planetarycomputer.microsoft.com/dataset/sentinel-1-rtc) RTC products. ([#112](https://github.com/sertit/eoreader/issues/112), [#118](https://github.com/sertit/eoreader/issues/118))**
+- **ENH: Handling of Sentinel-1 SM products.**
+- FIX: Fix jpg, png... quicklooks management when plotting
+- FIX: Fix an `xarray` issue when trying to compute percentiles when stacking bands
 - DEPS: Remove as many mention as possible to `cloudpathlib`
+- DEPS: Update minimum versions of some libraries
+- DOC: Add example about the output management (in `base` notebook, [#117](https://github.com/sertit/eoreader/issues/117))
 - GITHUB: Update bug template
 - CI: Enabling pre-commit.ci and dependabot bots
+- CI: Update pre-commit hooks
+- CI: Revamping `test_satellites`
+- CI: Some refactoring and speed-ups
 
 ## 0.20.4 (2023-09-26)
 
@@ -21,6 +33,7 @@
 
 ### Bug Fixes
 
+- FIX: Use `auto` as default dask chunk (instead of `2048`)
 - FIX: Minor fix in RasterioError handling when reading bands 
 - FIX: Fix Landsat L2 surface reflectance and temperature computation ([#99](https://github.com/sertit/eoreader/issues/99))
 - FIX: Fixing TIR temperature conversion for Landsat-7

diff --git a/CI/SCRIPTS/test_bands.py b/CI/SCRIPTS/test_bands.py
@@ -1,8 +1,9 @@
 """ Script testing EOReader bands """
-from CI.SCRIPTS.scripts_utils import READER, opt_path
+from CI.scripts_utils import READER, opt_path, s3_env
 from eoreader.bands import BLUE, YELLOW
 
 
+@s3_env
 def test_bands_s3_olci():
     """Test EOReader bands for Sentinel-3 OLCI"""
     prod_path = opt_path().joinpath(
@@ -14,6 +15,7 @@ def test_bands_s3_olci():
     assert list(set(prod.to_band(["Oa07", YELLOW, "YELLOW"]))) == [YELLOW]
 
 
+@s3_env
 def test_bands_l8():
     """Test EOReader bands for Landsat-8"""
     prod_path = opt_path().joinpath("LC08_L1GT_023030_20200518_20200527_01_T2")

diff --git a/CI/SCRIPTS/test_custom.py b/CI/SCRIPTS/test_custom.py
@@ -1,34 +1,18 @@
+""" Testing custom stacks. """
 import logging
 import os
 
-import numpy as np
 import pytest
 from rasterio.windows import Window
 from sertit import ci
 
+from CI.scripts_utils import READER, dask_env, get_db_dir, others_path, s3_env
 from eoreader import EOREADER_NAME
-from eoreader.bands import (
-    BLUE,
-    CA,
-    CLOUDS,
-    GREEN,
-    HH,
-    HILLSHADE,
-    NDVI,
-    NIR,
-    RED,
-    RH,
-    SWIR_1,
-    VV,
-    VV_DSPK,
-    YELLOW,
-)
+from eoreader.bands import BLUE, CA, GREEN, HILLSHADE, NDVI, NIR, RED, SWIR_1
 from eoreader.env_vars import DEM_PATH
-from eoreader.exceptions import InvalidProductError, InvalidTypeError
+from eoreader.exceptions import InvalidTypeError
 from eoreader.products import SensorType
 
-from .scripts_utils import READER, dask_env, get_db_dir, others_path, s3_env
-
 LOGGER = logging.getLogger(EOREADER_NAME)
 
 ci.reduce_verbosity()
@@ -155,133 +139,6 @@ def test_custom_optical():
         )
 
 
-def test_custom_sar():
-    # SAR
-    sar_stack = others_path() / "20210827T162210_ICEYE_SC_GRD_STK.tif"
-
-    # Load with all info
-    prod_sar = READER.open(
-        sar_stack,
-        custom=True,
-        sensor_type=SensorType.SAR,
-        name="20210827T162210_ICEYE_SC_GRD",
-        datetime="20210827T162210",
-        constellation="ICEYE",
-        instrument="SAR X-band",
-        pixel_size=6.0,
-        product_type="GRD",
-        band_map={VV: 1, VV_DSPK: 2},
-    )
-    LOGGER.info(prod_sar)
-    extent_sar = prod_sar.extent()
-    footprint_sar = prod_sar.footprint()
-    crs_sar = prod_sar.crs()
-    stack_sar = prod_sar.stack([VV, VV_DSPK], prod_sar.pixel_size * 10)
-
-    # Errors
-    with pytest.raises(AssertionError):
-        prod_sar.load(NDVI)
-
-    # Check attributes
-    assert stack_sar.attrs["long_name"] == "VV VV_DSPK"
-    assert stack_sar.attrs["constellation"] == "ICEYE"
-    assert stack_sar.attrs["constellation_id"] == "ICEYE"
-    assert stack_sar.attrs["product_type"] == "GRD"
-    assert stack_sar.attrs["instrument"] == "SAR X-band"
-    assert stack_sar.attrs["acquisition_date"] == "20210827T162210"
-    assert stack_sar.attrs["condensed_name"] == "20210827T162210_ICEYE_GRD"
-    assert stack_sar.attrs["product_path"] == str(sar_stack)
-
-    # MIX
-    sar_stack = others_path() / "20210827T162210_ICEYE_SC_GRD_STK.tif"
-    prod_wtf = READER.open(
-        sar_stack,
-        custom=True,
-        sensor_type=SensorType.SAR,
-        band_map={HH: 1, RH: 2},
-        name=None,
-        product_type=None,
-        instrument=None,
-        datetime=None,
-        pixel_size=6.0,
-    )
-    LOGGER.info(prod_wtf)
-    extent_wtf = prod_wtf.extent()
-    footprint_wtf = prod_wtf.footprint()
-    crs_wtf = prod_wtf.crs()
-    stack_wtf = prod_wtf.stack([HH, RH], prod_wtf.pixel_size * 10)
-
-    ci.assert_geom_equal(extent_sar, extent_wtf)
-    ci.assert_geom_equal(footprint_sar, footprint_wtf)
-    assert crs_sar == crs_wtf
-    assert prod_wtf.name is not None
-    assert prod_wtf.product_type is not None
-    assert prod_wtf.instrument is not None
-
-    np.testing.assert_array_equal(stack_sar.data, stack_wtf.data)
-
-
-def test_custom_wgs84():
-    # WGS84
-    wgs84_stack = others_path() / "SPOT6_WGS84.tif"
-    prod_wgs84 = READER.open(
-        wgs84_stack,
-        custom=True,
-        sensor_type=SensorType.OPTICAL,
-        name="SPOT6_WGS84",
-        datetime="20181218T090308",
-        constellation="SPOT6",
-        pixel_size=1.5 * 15,
-        instrument="NAOMI",
-        product_type="ORT",
-        band_map={RED: 1, GREEN: 2, BLUE: 3, NIR: 4},
-    )
-    LOGGER.info(prod_wgs84)
-
-    # Check geometries -> assert projected
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.extent()  # noqa
-
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.footprint()  # noqa
-
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.crs()  # noqa
-
-    # Read mtd
-    root, nsp = prod_wgs84.read_mtd()
-    assert nsp == {}
-    assert root.findtext("name") == "SPOT6_WGS84"
-    assert root.findtext("datetime") == "2018-12-18T09:03:08"
-    assert root.findtext("sensor_type") == "Optical"
-    assert root.findtext("constellation") == "Spot-6"
-    assert root.findtext("pixel_size") == str(1.5 * 15)
-    assert root.findtext("product_type") == "ORT"
-    assert root.findtext("band_map") == "{'BLUE': 3, 'GREEN': 2, 'RED': 1, 'NIR': 4}"
-    assert root.findtext("sun_azimuth") == "None"
-    assert root.findtext("sun_zenith") == "None"
-    assert root.findtext("orbit_direction") == "None"
-    assert root.findtext("cloud_cover") == "None"
-    assert root.findtext("instrument") == "NAOMI"
-
-    # Band paths
-    assert prod_wgs84.get_existing_bands() == [BLUE, GREEN, RED, NIR]
-    assert prod_wgs84.get_default_band() == BLUE
-    for key, ppath in prod_wgs84.get_existing_band_paths().items():
-        assert key in [BLUE, GREEN, RED, NIR]
-        assert str(ppath) == str(wgs84_stack)
-
-    # Load without a list and nothing
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.load(BLUE, size=[3863, 1049])[BLUE]  # noqa
-
-    # Try non-available clouds and bands
-    assert len(prod_wgs84.load([])) == 0
-    assert not prod_wgs84.has_bands(CLOUDS)
-    with pytest.raises(AssertionError):
-        prod_wgs84.load(CLOUDS, YELLOW)
-
-
 def test_custom_invalid():
     # Invalid tests
     opt_stack = others_path() / "20200310T030415_WV02_Ortho_BGRN_STK.tif"

diff --git a/CI/SCRIPTS/test_end_to_end.py b/CI/SCRIPTS/test_end_to_end.py
@@ -1,4 +1,4 @@
-""" Script testing EOReader satellites in a push routine """
+""" Script testing EOReader satellites in an end-to-end manner. """
 import logging
 import os
 import shutil
@@ -9,7 +9,7 @@
 import xarray as xr
 from sertit import AnyPath, ci, path
 
-from CI.SCRIPTS.scripts_utils import (
+from CI.scripts_utils import (
     CI_EOREADER_S3,
     READER,
     dask_env,
@@ -153,7 +153,7 @@ def _test_core(
 
         for pattern_path in pattern_paths:
             LOGGER.info(
-                "%s on drive %s (CI_EOREADER_S3: %s)",
+                f"%s on drive %s ({CI_EOREADER_S3}: %s)",
                 pattern_path.name,
                 pattern_path.drive,
                 os.getenv(CI_EOREADER_S3),
@@ -217,14 +217,15 @@ def _test_core(
             prod.clear()
 
 
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="Not enough memory to reproject on Windows runner",
-)
-@dask_env
-def test_spot6():
-    """Function testing the support of SPOT-6 constellation"""
-    _test_core_optical("*IMG_SPOT6*")
+# Test this only weekly now.
+# @pytest.mark.skipif(
+#     sys.platform == "win32",
+#     reason="Not enough memory to reproject on Windows runner",
+# )
+# @dask_env
+# def test_spot6():
+#     """Function testing the support of SPOT-6 constellation"""
+#     _test_core_optical("*IMG_SPOT6*")
 
 
 @pytest.mark.skipif(

diff --git a/CI/SCRIPTS/test_index.py b/CI/SCRIPTS/test_index.py
@@ -6,23 +6,9 @@
 import numpy as np
 from sertit import ci, rasters
 
+from CI.scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env
 from eoreader import EOREADER_NAME
-from eoreader.bands import (
-    BAI,
-    BAIM,
-    BAIS2,
-    EVI,
-    NBR,
-    NDMI,
-    NDVI,
-    SAVI,
-    VARI,
-    AWEInsh,
-    AWEIsh,
-    get_eoreader_indices,
-)
-
-from .scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env
+from eoreader.bands import BAI, NBR, NDVI
 
 LOGGER = logging.getLogger(EOREADER_NAME)
 
@@ -44,25 +30,9 @@ def test_index():
     with tempfile.TemporaryDirectory() as tmp_dir:
         # tmp_dir = os.path.join("/mnt", "ds2_db3", "CI", "eoreader", "DATA", "INDEX")
         prod.output = os.path.join(tmp_dir, prod.condensed_name)
+        idx_list = [NDVI, NBR, BAI]
 
-        # Load every index
-        spyndex_list = [
-            NDVI,
-            NDMI,
-            NBR,
-            BAI,
-            AWEIsh,
-            AWEInsh,
-            BAIM,
-            BAIS2,
-            EVI,
-            SAVI,
-            VARI,
-        ]
-        LOGGER.info(f"Load selected indices (EOReader's + {spyndex_list})")
-        idx_list = [
-            idx for idx in spyndex_list + get_eoreader_indices() if prod.has_band(idx)
-        ]
+        LOGGER.info(f"Load selected indices (EOReader's + {idx_list})")
         idx = prod.load(idx_list, pixel_size=RES)
 
         for idx_name, idx_arr in idx.items():