Big internal refactoring + speeding up the every day CI + fix issues with STAC products

Author: remi-braun

.gitlab-ci.yml

@@ -102,7 +102,7 @@ pytest_end_to_end:
     - pip install pytest coverage pytest-cov pylint
     - pip install -e .
   script:
-    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py --log-cli-level DEBUG --capture=tee-sys
+    - python -m pytest -v --durations=0 --cov-report term --cov-report html:${CI_PROJECT_DIR}/cov_e2e.html --cov=eoreader  --cov-config=.coveragerc CI/SCRIPTS_WEEKLY --log-cli-level DEBUG --capture=tee-sys
   artifacts:
     paths:
       - ${CI_PROJECT_DIR}/cov_e2e.html

CHANGES.md

@@ -20,7 +20,7 @@
 - CI: Enabling pre-commit.ci and dependabot bots
 - CI: Update pre-commit hooks
 - CI: Revamping `test_satellites`
-- CI: Some refactoring
+- CI: Some refactoring and speed-ups
 
 ## 0.20.4 (2023-09-26)
 

CI/SCRIPTS/test_bands.py

@@ -1,8 +1,9 @@
 """ Script testing EOReader bands """
-from CI.SCRIPTS.scripts_utils import READER, opt_path
+from CI.scripts_utils import READER, opt_path, s3_env
 from eoreader.bands import BLUE, YELLOW
 
 
+@s3_env
 def test_bands_s3_olci():
     """Test EOReader bands for Sentinel-3 OLCI"""
     prod_path = opt_path().joinpath(
@@ -14,6 +15,7 @@ def test_bands_s3_olci():
     assert list(set(prod.to_band(["Oa07", YELLOW, "YELLOW"]))) == [YELLOW]
 
 
+@s3_env
 def test_bands_l8():
     """Test EOReader bands for Landsat-8"""
     prod_path = opt_path().joinpath("LC08_L1GT_023030_20200518_20200527_01_T2")

CI/SCRIPTS/test_custom.py

@@ -2,34 +2,17 @@
 import logging
 import os
 
-import numpy as np
 import pytest
 from rasterio.windows import Window
 from sertit import ci
 
+from CI.scripts_utils import READER, dask_env, get_db_dir, others_path, s3_env
 from eoreader import EOREADER_NAME
-from eoreader.bands import (
-    BLUE,
-    CA,
-    CLOUDS,
-    GREEN,
-    HH,
-    HILLSHADE,
-    NDVI,
-    NIR,
-    RED,
-    RH,
-    SWIR_1,
-    VV,
-    VV_DSPK,
-    YELLOW,
-)
+from eoreader.bands import BLUE, CA, GREEN, HILLSHADE, NDVI, NIR, RED, SWIR_1
 from eoreader.env_vars import DEM_PATH
-from eoreader.exceptions import InvalidProductError, InvalidTypeError
+from eoreader.exceptions import InvalidTypeError
 from eoreader.products import SensorType
 
-from .scripts_utils import READER, dask_env, get_db_dir, others_path, s3_env
-
 LOGGER = logging.getLogger(EOREADER_NAME)
 
 ci.reduce_verbosity()
@@ -156,133 +139,6 @@ def test_custom_optical():
         )
 
 
-def test_custom_sar():
-    # SAR
-    sar_stack = others_path() / "20210827T162210_ICEYE_SC_GRD_STK.tif"
-
-    # Load with all info
-    prod_sar = READER.open(
-        sar_stack,
-        custom=True,
-        sensor_type=SensorType.SAR,
-        name="20210827T162210_ICEYE_SC_GRD",
-        datetime="20210827T162210",
-        constellation="ICEYE",
-        instrument="SAR X-band",
-        pixel_size=6.0,
-        product_type="GRD",
-        band_map={VV: 1, VV_DSPK: 2},
-    )
-    LOGGER.info(prod_sar)
-    extent_sar = prod_sar.extent()
-    footprint_sar = prod_sar.footprint()
-    crs_sar = prod_sar.crs()
-    stack_sar = prod_sar.stack([VV, VV_DSPK], prod_sar.pixel_size * 10)
-
-    # Errors
-    with pytest.raises(AssertionError):
-        prod_sar.load(NDVI)
-
-    # Check attributes
-    assert stack_sar.attrs["long_name"] == "VV VV_DSPK"
-    assert stack_sar.attrs["constellation"] == "ICEYE"
-    assert stack_sar.attrs["constellation_id"] == "ICEYE"
-    assert stack_sar.attrs["product_type"] == "GRD"
-    assert stack_sar.attrs["instrument"] == "SAR X-band"
-    assert stack_sar.attrs["acquisition_date"] == "20210827T162210"
-    assert stack_sar.attrs["condensed_name"] == "20210827T162210_ICEYE_GRD"
-    assert stack_sar.attrs["product_path"] == str(sar_stack)
-
-    # MIX
-    sar_stack = others_path() / "20210827T162210_ICEYE_SC_GRD_STK.tif"
-    prod_wtf = READER.open(
-        sar_stack,
-        custom=True,
-        sensor_type=SensorType.SAR,
-        band_map={HH: 1, RH: 2},
-        name=None,
-        product_type=None,
-        instrument=None,
-        datetime=None,
-        pixel_size=6.0,
-    )
-    LOGGER.info(prod_wtf)
-    extent_wtf = prod_wtf.extent()
-    footprint_wtf = prod_wtf.footprint()
-    crs_wtf = prod_wtf.crs()
-    stack_wtf = prod_wtf.stack([HH, RH], prod_wtf.pixel_size * 10)
-
-    ci.assert_geom_equal(extent_sar, extent_wtf)
-    ci.assert_geom_equal(footprint_sar, footprint_wtf)
-    assert crs_sar == crs_wtf
-    assert prod_wtf.name is not None
-    assert prod_wtf.product_type is not None
-    assert prod_wtf.instrument is not None
-
-    np.testing.assert_array_equal(stack_sar.data, stack_wtf.data)
-
-
-def test_custom_wgs84():
-    # WGS84
-    wgs84_stack = others_path() / "SPOT6_WGS84.tif"
-    prod_wgs84 = READER.open(
-        wgs84_stack,
-        custom=True,
-        sensor_type=SensorType.OPTICAL,
-        name="SPOT6_WGS84",
-        datetime="20181218T090308",
-        constellation="SPOT6",
-        pixel_size=1.5 * 15,
-        instrument="NAOMI",
-        product_type="ORT",
-        band_map={RED: 1, GREEN: 2, BLUE: 3, NIR: 4},
-    )
-    LOGGER.info(prod_wgs84)
-
-    # Check geometries -> assert projected
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.extent()  # noqa
-
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.footprint()  # noqa
-
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.crs()  # noqa
-
-    # Read mtd
-    root, nsp = prod_wgs84.read_mtd()
-    assert nsp == {}
-    assert root.findtext("name") == "SPOT6_WGS84"
-    assert root.findtext("datetime") == "2018-12-18T09:03:08"
-    assert root.findtext("sensor_type") == "Optical"
-    assert root.findtext("constellation") == "Spot-6"
-    assert root.findtext("pixel_size") == str(1.5 * 15)
-    assert root.findtext("product_type") == "ORT"
-    assert root.findtext("band_map") == "{'BLUE': 3, 'GREEN': 2, 'RED': 1, 'NIR': 4}"
-    assert root.findtext("sun_azimuth") == "None"
-    assert root.findtext("sun_zenith") == "None"
-    assert root.findtext("orbit_direction") == "None"
-    assert root.findtext("cloud_cover") == "None"
-    assert root.findtext("instrument") == "NAOMI"
-
-    # Band paths
-    assert prod_wgs84.get_existing_bands() == [BLUE, GREEN, RED, NIR]
-    assert prod_wgs84.get_default_band() == BLUE
-    for key, ppath in prod_wgs84.get_existing_band_paths().items():
-        assert key in [BLUE, GREEN, RED, NIR]
-        assert str(ppath) == str(wgs84_stack)
-
-    # Load without a list and nothing
-    with pytest.raises(InvalidProductError):
-        prod_wgs84.load(BLUE, size=[3863, 1049])[BLUE]  # noqa
-
-    # Try non-available clouds and bands
-    assert len(prod_wgs84.load([])) == 0
-    assert not prod_wgs84.has_bands(CLOUDS)
-    with pytest.raises(AssertionError):
-        prod_wgs84.load(CLOUDS, YELLOW)
-
-
 def test_custom_invalid():
     # Invalid tests
     opt_stack = others_path() / "20200310T030415_WV02_Ortho_BGRN_STK.tif"

CI/SCRIPTS/test_end_to_end.py

@@ -9,7 +9,7 @@
 import xarray as xr
 from sertit import AnyPath, ci, path
 
-from CI.SCRIPTS.scripts_utils import (
+from CI.scripts_utils import (
     CI_EOREADER_S3,
     READER,
     dask_env,
@@ -153,7 +153,7 @@ def _test_core(
 
         for pattern_path in pattern_paths:
             LOGGER.info(
-                "%s on drive %s (CI_EOREADER_S3: %s)",
+                f"%s on drive %s ({CI_EOREADER_S3}: %s)",
                 pattern_path.name,
                 pattern_path.drive,
                 os.getenv(CI_EOREADER_S3),

CI/SCRIPTS/test_index.py

@@ -6,23 +6,9 @@
 import numpy as np
 from sertit import ci, rasters
 
+from CI.scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env
 from eoreader import EOREADER_NAME
-from eoreader.bands import (
-    BAI,
-    BAIM,
-    BAIS2,
-    EVI,
-    NBR,
-    NDMI,
-    NDVI,
-    SAVI,
-    VARI,
-    AWEInsh,
-    AWEIsh,
-    get_eoreader_indices,
-)
-
-from .scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env
+from eoreader.bands import BAI, NBR, NDVI
 
 LOGGER = logging.getLogger(EOREADER_NAME)
 
@@ -44,25 +30,9 @@ def test_index():
     with tempfile.TemporaryDirectory() as tmp_dir:
         # tmp_dir = os.path.join("/mnt", "ds2_db3", "CI", "eoreader", "DATA", "INDEX")
         prod.output = os.path.join(tmp_dir, prod.condensed_name)
+        idx_list = [NDVI, NBR, BAI]
 
-        # Load every index
-        spyndex_list = [
-            NDVI,
-            NDMI,
-            NBR,
-            BAI,
-            AWEIsh,
-            AWEInsh,
-            BAIM,
-            BAIS2,
-            EVI,
-            SAVI,
-            VARI,
-        ]
-        LOGGER.info(f"Load selected indices (EOReader's + {spyndex_list})")
-        idx_list = [
-            idx for idx in spyndex_list + get_eoreader_indices() if prod.has_band(idx)
-        ]
+        LOGGER.info(f"Load selected indices (EOReader's + {idx_list})")
         idx = prod.load(idx_list, pixel_size=RES)
 
         for idx_name, idx_arr in idx.items():

CI/SCRIPTS/test_others.py

@@ -11,6 +11,16 @@
 from rasterio.windows import Window
 from sertit import AnyPath, ci, path, unistra
 
+from CI.scripts_utils import (
+    READER,
+    dask_env,
+    get_db_dir,
+    get_db_dir_on_disk,
+    opt_path,
+    others_path,
+    s3_env,
+    sar_path,
+)
 from eoreader import utils
 from eoreader.bands import (
     AFRI_1_6,
@@ -53,17 +63,6 @@
 from eoreader.products import SensorType
 from eoreader.reader import Constellation
 
-from .scripts_utils import (
-    READER,
-    dask_env,
-    get_db_dir,
-    get_db_dir_on_disk,
-    opt_path,
-    others_path,
-    s3_env,
-    sar_path,
-)
-
 ci.reduce_verbosity()
 
 
@@ -294,7 +293,6 @@ def test_dems_https():
     xr.testing.assert_equal(dem_local[DEM], dem_remote[DEM])
 
 
-@s3_env
 def test_dems_S3():
     # Get paths
     prod_path = opt_path().joinpath("LC08_L1GT_023030_20200518_20200527_01_T2")
@@ -311,24 +309,24 @@ def test_dems_S3():
     local_path = str(get_db_dir_on_disk().joinpath(*dem_sub_dir_path))
 
     # ON S3
-    unistra.define_s3_client()
-    s3_path = str(AnyPath("s3://sertit-geodatastore").joinpath(*dem_sub_dir_path))
-
-    # Loading same DEM from two different sources (one hosted locally and the other hosted on S3 compatible storage)
-    with tempenv.TemporaryEnvironment({DEM_PATH: local_path}):  # Local DEM
-        dem_local = prod.load(
-            [DEM],
-            pixel_size=30,
-            window=Window(col_off=0, row_off=0, width=100, height=100),
-        )  # Loading same DEM from two different sources (one hosted locally and the other hosted on S3 compatible storage)
-    with tempenv.TemporaryEnvironment({DEM_PATH: s3_path}):  # S3 DEM
-        dem_s3 = prod.load(
-            [DEM],
-            pixel_size=30,
-            window=Window(col_off=0, row_off=0, width=100, height=100),
-        )
+    with unistra.unistra_s3():
+        s3_path = str(AnyPath("s3://sertit-geodatastore").joinpath(*dem_sub_dir_path))
+
+        # Loading same DEM from two different sources (one hosted locally and the other hosted on S3 compatible storage)
+        with tempenv.TemporaryEnvironment({DEM_PATH: local_path}):  # Local DEM
+            dem_local = prod.load(
+                [DEM],
+                pixel_size=30,
+                window=Window(col_off=0, row_off=0, width=100, height=100),
+            )  # Loading same DEM from two different sources (one hosted locally and the other hosted on S3 compatible storage)
+        with tempenv.TemporaryEnvironment({DEM_PATH: s3_path}):  # S3 DEM
+            dem_s3 = prod.load(
+                [DEM],
+                pixel_size=30,
+                window=Window(col_off=0, row_off=0, width=100, height=100),
+            )
 
-    xr.testing.assert_equal(dem_local[DEM], dem_s3[DEM])
+        xr.testing.assert_equal(dem_local[DEM], dem_s3[DEM])
 
 
 def test_bands():