From 41a2472b639b4e74892dfa705277b1ad396de254 Mon Sep 17 00:00:00 2001 From: BRAUN REMI Date: Tue, 3 Dec 2024 14:02:26 +0100 Subject: [PATCH] CI: Remove useless verbosity in CI --- CHANGES.md | 1 + CI/SCRIPTS/test_bands.py | 4 +++- CI/SCRIPTS/test_custom.py | 12 ++++++++++-- CI/SCRIPTS/test_end_to_end.py | 5 +++-- CI/SCRIPTS/test_index.py | 11 +++++++++-- CI/SCRIPTS/test_others.py | 5 +++-- CI/SCRIPTS/test_satellites.py | 4 ++-- CI/SCRIPTS/test_stac_items.py | 7 +++---- .../test_all_sat_end_to_end_on_disk.py | 6 +++--- CI/SCRIPTS_WEEKLY/test_broken_s2.py | 5 ++--- CI/SCRIPTS_WEEKLY/test_custom.py | 4 ++-- CI/SCRIPTS_WEEKLY/test_index.py | 12 ++++++++++-- CI/SCRIPTS_WEEKLY/test_satellites.py | 4 ++-- CI/SCRIPTS_WEEKLY/test_stac_creation.py | 6 +++--- CI/SCRIPTS_WEEKLY/test_stac_items.py | 7 +++---- CI/scripts_utils.py | 19 ++++++++++++++++++- 16 files changed, 77 insertions(+), 35 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 96c4f77f..3ecebede 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ - FIX: Don't throw an error in case of missing cloud coverage, only a warning and set the cloud coverage to 0 [#159](https://github.com/sertit/eoreader/issues/159) - FIX: Use the sun elevation angle rather than the sun zenith angle for STAC [#158](https://github.com/sertit/eoreader/issues/158) - FIX: Create comparison operators for `BandNames`, removing the `xarray RuntimeWarning` about `sort order is undefined for incomparable objects`. +- CI: Remove useless verbosity in CI - DOC: Update `conf.py` (remove useless hunks and set Sphinx 7 as base) - DOC: Added the [PAZ product guide](https://earth.esa.int/eogateway/documents/20142/37627/PAZ-Image-Products-Guide.pdf) to the PAZ Product documentation instead of the TerraSAR-X one - by @guillemc23 diff --git a/CI/SCRIPTS/test_bands.py b/CI/SCRIPTS/test_bands.py index 53b14eb5..fe44e554 100644 --- a/CI/SCRIPTS/test_bands.py +++ b/CI/SCRIPTS/test_bands.py @@ -1,8 +1,10 @@ """ Script testing EOReader bands """ -from CI.scripts_utils import READER, opt_path, s3_env +from CI.scripts_utils import READER, opt_path, reduce_verbosity, s3_env from eoreader.bands import BLUE, YELLOW +reduce_verbosity() + @s3_env def test_bands_s3_olci(): diff --git a/CI/SCRIPTS/test_custom.py b/CI/SCRIPTS/test_custom.py index d24bab04..865ae315 100644 --- a/CI/SCRIPTS/test_custom.py +++ b/CI/SCRIPTS/test_custom.py @@ -7,7 +7,14 @@ from rasterio.windows import Window from sertit import ci -from CI.scripts_utils import READER, dask_env, get_db_dir, others_path, s3_env +from CI.scripts_utils import ( + READER, + dask_env, + get_db_dir, + others_path, + reduce_verbosity, + s3_env, +) from eoreader import EOREADER_NAME from eoreader.bands import BLUE, CA, GREEN, HILLSHADE, NDVI, NIR, RED, SWIR_1 from eoreader.env_vars import DEM_PATH @@ -16,7 +23,8 @@ LOGGER = logging.getLogger(EOREADER_NAME) -ci.reduce_verbosity() + +reduce_verbosity() @s3_env diff --git a/CI/SCRIPTS/test_end_to_end.py b/CI/SCRIPTS/test_end_to_end.py index 7d9f6ed0..77dd8b74 100644 --- a/CI/SCRIPTS/test_end_to_end.py +++ b/CI/SCRIPTS/test_end_to_end.py @@ -8,7 +8,7 @@ import pytest import xarray as xr -from sertit import AnyPath, ci, path +from sertit import AnyPath, path from CI.scripts_utils import ( CI_EOREADER_S3, @@ -18,6 +18,7 @@ get_db_dir, get_db_dir_on_disk, opt_path, + reduce_verbosity, ) from eoreader import EOREADER_NAME from eoreader.bands import ( @@ -46,7 +47,7 @@ from eoreader.products.product import Product, SensorType from eoreader.reader import CheckMethod -ci.reduce_verbosity() +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/SCRIPTS/test_index.py b/CI/SCRIPTS/test_index.py index 472bf9c6..405355f5 100644 --- a/CI/SCRIPTS/test_index.py +++ b/CI/SCRIPTS/test_index.py @@ -7,7 +7,14 @@ import numpy as np from sertit import ci, rasters -from CI.scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env +from CI.scripts_utils import ( + READER, + dask_env, + get_ci_data_dir, + opt_path, + reduce_verbosity, + s3_env, +) from eoreader import EOREADER_NAME from eoreader.bands import BAI, NBR, NDVI @@ -15,7 +22,7 @@ RES = 2000.0 # 2000 meters -ci.reduce_verbosity() +reduce_verbosity() @s3_env diff --git a/CI/SCRIPTS/test_others.py b/CI/SCRIPTS/test_others.py index 33f926bc..877c5315 100644 --- a/CI/SCRIPTS/test_others.py +++ b/CI/SCRIPTS/test_others.py @@ -10,7 +10,7 @@ import tempenv import xarray as xr from rasterio.windows import Window -from sertit import AnyPath, ci, path, unistra +from sertit import AnyPath, path, unistra from CI.scripts_utils import ( READER, @@ -19,6 +19,7 @@ get_db_dir_on_disk, opt_path, others_path, + reduce_verbosity, s3_env, sar_path, ) @@ -64,7 +65,7 @@ from eoreader.products import SensorType from eoreader.reader import Constellation -ci.reduce_verbosity() +reduce_verbosity() @pytest.mark.xfail diff --git a/CI/SCRIPTS/test_satellites.py b/CI/SCRIPTS/test_satellites.py index 016c5b67..a740cd10 100644 --- a/CI/SCRIPTS/test_satellites.py +++ b/CI/SCRIPTS/test_satellites.py @@ -20,6 +20,7 @@ get_db_dir, get_db_dir_on_disk, opt_path, + reduce_verbosity, s3_env, sar_path, ) @@ -49,8 +50,7 @@ from eoreader.products import Product, SensorType, SlstrRadAdjust from eoreader.reader import CheckMethod -ci.reduce_verbosity(["dicttoxml"]) -logging.getLogger("rasterio._env").setLevel(logging.ERROR) +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/SCRIPTS/test_stac_items.py b/CI/SCRIPTS/test_stac_items.py index 87f7005d..fd723d1a 100644 --- a/CI/SCRIPTS/test_stac_items.py +++ b/CI/SCRIPTS/test_stac_items.py @@ -5,16 +5,15 @@ import tempfile import xarray as xr -from sertit import ci, s3 +from sertit import s3 from tempenv import tempenv -from CI.scripts_utils import READER, compare +from CI.scripts_utils import READER, compare, reduce_verbosity from eoreader import EOREADER_NAME from eoreader.products import Product from eoreader.reader import Constellation -ci.reduce_verbosity(["dicttoxml"]) -logging.getLogger("rasterio._env").setLevel(logging.ERROR) +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py b/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py index 11bc3013..1d4e8d3e 100644 --- a/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py +++ b/CI/SCRIPTS_WEEKLY/test_all_sat_end_to_end_on_disk.py @@ -7,7 +7,7 @@ import xarray as xr from lxml import etree -from sertit import AnyPath, ci, path, types +from sertit import AnyPath, path, types from CI.scripts_utils import ( CI_EOREADER_S3, @@ -17,6 +17,7 @@ get_db_dir, get_db_dir_on_disk, opt_path, + reduce_verbosity, ) from eoreader import EOREADER_NAME from eoreader.bands import ( @@ -53,8 +54,7 @@ from eoreader.products.product import Product, SensorType from eoreader.reader import CheckMethod -ci.reduce_verbosity(["dicttoxml"]) -logging.getLogger("rasterio._env").setLevel(logging.ERROR) +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/SCRIPTS_WEEKLY/test_broken_s2.py b/CI/SCRIPTS_WEEKLY/test_broken_s2.py index eb6cd3e0..f86b75c3 100644 --- a/CI/SCRIPTS_WEEKLY/test_broken_s2.py +++ b/CI/SCRIPTS_WEEKLY/test_broken_s2.py @@ -3,16 +3,15 @@ import logging import pytest -from sertit import ci -from CI.scripts_utils import READER, broken_s2_path, dask_env, s3_env +from CI.scripts_utils import READER, broken_s2_path, dask_env, reduce_verbosity, s3_env from eoreader import EOREADER_NAME from eoreader.bands import NIR, RED from eoreader.exceptions import InvalidProductError LOGGER = logging.getLogger(EOREADER_NAME) -ci.reduce_verbosity() +reduce_verbosity() @s3_env diff --git a/CI/SCRIPTS_WEEKLY/test_custom.py b/CI/SCRIPTS_WEEKLY/test_custom.py index 2e278d30..e76e12f9 100644 --- a/CI/SCRIPTS_WEEKLY/test_custom.py +++ b/CI/SCRIPTS_WEEKLY/test_custom.py @@ -7,7 +7,7 @@ from sertit import ci from CI.SCRIPTS.test_custom import test_custom_invalid, test_custom_optical -from CI.scripts_utils import READER, dask_env, others_path, s3_env +from CI.scripts_utils import READER, dask_env, others_path, reduce_verbosity, s3_env from eoreader import EOREADER_NAME from eoreader.bands import ( BLUE, @@ -27,7 +27,7 @@ LOGGER = logging.getLogger(EOREADER_NAME) -ci.reduce_verbosity() +reduce_verbosity() @s3_env diff --git a/CI/SCRIPTS_WEEKLY/test_index.py b/CI/SCRIPTS_WEEKLY/test_index.py index 77945c30..bb859247 100644 --- a/CI/SCRIPTS_WEEKLY/test_index.py +++ b/CI/SCRIPTS_WEEKLY/test_index.py @@ -7,7 +7,14 @@ import numpy as np from sertit import ci, rasters -from CI.scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env +from CI.scripts_utils import ( + READER, + dask_env, + get_ci_data_dir, + opt_path, + reduce_verbosity, + s3_env, +) from eoreader import EOREADER_NAME from eoreader.bands import ( BAI, @@ -28,7 +35,8 @@ RES = 2000.0 # 2000 meters -ci.reduce_verbosity() + +reduce_verbosity() @s3_env diff --git a/CI/SCRIPTS_WEEKLY/test_satellites.py b/CI/SCRIPTS_WEEKLY/test_satellites.py index cc05f279..6914b8ce 100644 --- a/CI/SCRIPTS_WEEKLY/test_satellites.py +++ b/CI/SCRIPTS_WEEKLY/test_satellites.py @@ -20,6 +20,7 @@ get_ci_data_dir, get_db_dir_on_disk, opt_path, + reduce_verbosity, s3_env, sar_path, ) @@ -47,8 +48,7 @@ from eoreader.products.product import OrbitDirection from eoreader.reader import Constellation -ci.reduce_verbosity(["dicttoxml"]) -logging.getLogger("rasterio._env").setLevel(logging.ERROR) +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/SCRIPTS_WEEKLY/test_stac_creation.py b/CI/SCRIPTS_WEEKLY/test_stac_creation.py index 66ff8405..b5d8a5bf 100644 --- a/CI/SCRIPTS_WEEKLY/test_stac_creation.py +++ b/CI/SCRIPTS_WEEKLY/test_stac_creation.py @@ -9,7 +9,7 @@ import pystac import pytest import xarray as xr -from sertit import ci, path +from sertit import path from sertit.vectors import WGS84 from shapely.geometry import mapping @@ -19,6 +19,7 @@ compare, dask_env, opt_path, + reduce_verbosity, s3_env, sar_path, ) @@ -66,8 +67,7 @@ VIEW_SUN_ELEVATION, ) -ci.reduce_verbosity(["dicttoxml"]) -logging.getLogger("rasterio._env").setLevel(logging.ERROR) +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/SCRIPTS_WEEKLY/test_stac_items.py b/CI/SCRIPTS_WEEKLY/test_stac_items.py index 5c698450..eb28cf19 100644 --- a/CI/SCRIPTS_WEEKLY/test_stac_items.py +++ b/CI/SCRIPTS_WEEKLY/test_stac_items.py @@ -7,16 +7,15 @@ import xarray as xr from botocore.exceptions import ClientError from rasterio.windows import Window -from sertit import ci, s3 +from sertit import s3 from tempenv import tempenv -from CI.scripts_utils import READER, compare +from CI.scripts_utils import READER, compare, reduce_verbosity from eoreader import EOREADER_NAME from eoreader.products import Product from eoreader.reader import Constellation -ci.reduce_verbosity(["dicttoxml"]) -logging.getLogger("rasterio._env").setLevel(logging.ERROR) +reduce_verbosity() LOGGER = logging.getLogger(EOREADER_NAME) diff --git a/CI/scripts_utils.py b/CI/scripts_utils.py index 584d0862..ab922ffa 100644 --- a/CI/scripts_utils.py +++ b/CI/scripts_utils.py @@ -2,11 +2,13 @@ import logging import os +import warnings from functools import wraps from typing import Callable +import rasterio import tempenv -from sertit import AnyPath, s3, unistra +from sertit import AnyPath, ci, s3, unistra from sertit.types import AnyPathType from sertit.unistra import get_db2_path, get_db3_path, get_geodatastore @@ -166,3 +168,18 @@ def compare(to_be_checked, ref, topic): assert str(to_be_checked).startswith("No") and str(to_be_checked).endswith( "available" ), f"Non equal {topic}: ref={ref} != to_be_checked={to_be_checked}" + + +def reduce_verbosity(): + # Ignore warning + warnings.filterwarnings("ignore", category=rasterio.errors.NotGeoreferencedWarning) + + # Reduce verbosity to warning + ci.reduce_verbosity(["dicttoxml", "pyogrio"]) + + # Errors + logging.getLogger("rasterio._env").setLevel(logging.ERROR) + logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR) + + # Critical + logging.getLogger("distributed.worker").setLevel(logging.CRITICAL)