From 51af1fe20369f3e86e5b7418c0d15b6ac5e1820b Mon Sep 17 00:00:00 2001 From: Jens Verrydt Date: Thu, 31 Oct 2024 11:58:27 +0100 Subject: [PATCH 1/6] feat: extended pystac client to support aggregations stac-api extension calls [https://github.com/stac-api-extensions/aggregation] --- tests/fixtures/catalog.json | 17 ++++ tests/test_advanced_pystac_client.py | 119 +++++++++++++++++++++++++++ titiler/pystac/__init__.py | 7 ++ titiler/pystac/advanced_client.py | 87 ++++++++++++++++++++ titiler/stacapi/factory.py | 19 ++++- 5 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 tests/test_advanced_pystac_client.py create mode 100644 titiler/pystac/__init__.py create mode 100644 titiler/pystac/advanced_client.py diff --git a/tests/fixtures/catalog.json b/tests/fixtures/catalog.json index 292a6e6..6d3c3a4 100644 --- a/tests/fixtures/catalog.json +++ b/tests/fixtures/catalog.json @@ -321,6 +321,23 @@ "rel": "self", "type": "application/json", "href": "https://stac.endpoint.io/collections" + }, + { + "rel": "data", + "type": "application/json", + "href": "https://stac.endpoint.io/collections" + }, + { + "rel": "aggregate", + "type": "application/json", + "title": "Aggregate", + "href": "https://stac.endpoint.io/aggregate" + }, + { + "rel": "aggregations", + "type": "application/json", + "title": "Aggregations", + "href": "https://stac.endpoint.io/aggregations" } ] } diff --git a/tests/test_advanced_pystac_client.py b/tests/test_advanced_pystac_client.py new file mode 100644 index 0000000..29af8b5 --- /dev/null +++ b/tests/test_advanced_pystac_client.py @@ -0,0 +1,119 @@ +"""Test Advanced PySTAC client.""" +import json +import os +from unittest.mock import MagicMock, patch + +import pytest + +from titiler.pystac import AdvancedClient + +catalog_json = os.path.join(os.path.dirname(__file__), "fixtures", "catalog.json") + + +@pytest.fixture +def mock_stac_io(): + """STAC IO mock""" + return MagicMock() + + +@pytest.fixture +def client(mock_stac_io): + """STAC client mock""" + client = AdvancedClient(id="pystac-client", description="pystac-client") + + with open(catalog_json, "r") as f: + catalog = json.loads(f.read()) + client.open = MagicMock() + client.open.return_value = catalog + client._collections_href = MagicMock() + client._collections_href.return_value = "http://example.com/collections" + + client._stac_io = mock_stac_io + return client + + +def test_get_supported_aggregations(client, mock_stac_io): + """Test supported STAC aggregation methods""" + mock_stac_io.read_json.return_value = { + "aggregations": [{"name": "aggregation1"}, {"name": "aggregation2"}] + } + supported_aggregations = client.get_supported_aggregations() + assert supported_aggregations == ["aggregation1", "aggregation2"] + + +@patch( + "titiler.pystac.advanced_client.AdvancedClient.get_supported_aggregations", + return_value=["datetime_frequency"], +) +def test_get_aggregation_unsupported(supported_aggregations, client): + """Test handling of unsupported aggregation types""" + collection_id = "sentinel-2-l2a" + aggregation = "unsupported-aggregation" + + with pytest.warns( + UserWarning, match="Aggregation type unsupported-aggregation is not supported" + ): + aggregation_data = client.get_aggregation(collection_id, aggregation) + assert aggregation_data == [] + + +@patch( + "titiler.pystac.advanced_client.AdvancedClient.get_supported_aggregations", + return_value=["datetime_frequency"], +) +def test_get_aggregation(supported_aggregations, client, mock_stac_io): + """Test handling aggregation response""" + collection_id = "sentinel-2-l2a" + aggregation = "datetime_frequency" + aggregation_params = {"datetime_frequency_interval": "day"} + + mock_stac_io.read_json.return_value = { + "aggregations": [ + { + "name": "datetime_frequency", + "buckets": [ + { + "key": "2023-12-11T00:00:00.000Z", + "data_type": "frequency_distribution", + "frequency": 1, + "to": None, + "from": None, + } + ], + }, + { + "name": "unusable_aggregation", + "buckets": [ + { + "key": "2023-12-11T00:00:00.000Z", + } + ], + }, + ] + } + + aggregation_data = client.get_aggregation( + collection_id, aggregation, aggregation_params + ) + assert aggregation_data[0]["key"] == "2023-12-11T00:00:00.000Z" + assert aggregation_data[0]["data_type"] == "frequency_distribution" + assert aggregation_data[0]["frequency"] == 1 + assert len(aggregation_data) == 1 + + +@patch( + "titiler.pystac.advanced_client.AdvancedClient.get_supported_aggregations", + return_value=["datetime_frequency"], +) +def test_get_aggregation_no_response(supported_aggregations, client, mock_stac_io): + """Test handling of no aggregation response""" + collection_id = "sentinel-2-l2a" + aggregation = "datetime_frequency" + aggregation_params = {"datetime_frequency_interval": "day"} + + mock_stac_io.read_json.return_value = [] + + aggregation_data = client.get_aggregation( + collection_id, aggregation, aggregation_params + ) + assert aggregation_data == [] diff --git a/titiler/pystac/__init__.py b/titiler/pystac/__init__.py new file mode 100644 index 0000000..f398c4b --- /dev/null +++ b/titiler/pystac/__init__.py @@ -0,0 +1,7 @@ +"""titiler.pystac""" + +__all__ = [ + "AdvancedClient", +] + +from titiler.pystac.advanced_client import AdvancedClient diff --git a/titiler/pystac/advanced_client.py b/titiler/pystac/advanced_client.py new file mode 100644 index 0000000..24f77f5 --- /dev/null +++ b/titiler/pystac/advanced_client.py @@ -0,0 +1,87 @@ +""" +This module provides an advanced client for interacting with STAC (SpatioTemporal Asset Catalog) APIs. + +The `AdvancedClient` class extends the basic functionality of the `pystac.Client` to include +methods for retrieving and aggregating data from STAC collections. +""" + +import warnings +from typing import Optional +from urllib.parse import urlencode + +import pystac +from pystac_client import Client + + +class AdvancedClient(Client): + """AdvancedClient extends the basic functionality of the pystac.Client class.""" + + def get_aggregation( + self, + collection_id: str, + aggregation: str, + aggregation_params: Optional[dict] = None, + ) -> list[dict]: + """Perform an aggregation on a STAC collection. + + Args: + collection_id (str): The ID of the collection to aggregate. + aggregation (str): The aggregation type to perform. + aggregation_params (Optional[dict], optional): Additional parameters for the aggregation. Defaults to None. + Returns: + List[str]: The aggregation response. + """ + assert self._stac_io is not None + + if aggregation not in self.get_supported_aggregations(): + warnings.warn( + f"Aggregation type {aggregation} is not supported", stacklevel=1 + ) + return [] + + # Construct the URL for aggregation + url = ( + self._collections_href(collection_id) + + f"/aggregate?aggregations={aggregation}" + ) + if aggregation_params: + params = urlencode(aggregation_params) + url += f"&{params}" + + aggregation_response = self._stac_io.read_json(url) + + if not aggregation_response: + return [] + + aggregation_data = [] + for agg in aggregation_response["aggregations"]: + if agg["name"] == aggregation: + aggregation_data = agg["buckets"] + + return aggregation_data + + def get_supported_aggregations(self) -> list[str]: + """Get the supported aggregation types. + + Returns: + List[str]: The supported aggregations. + """ + response = self._stac_io.read_json(self.get_aggregations_link()) + aggregations = response.get("aggregations", []) + return [agg["name"] for agg in aggregations] + + def get_aggregations_link(self) -> Optional[pystac.Link]: + """Returns this client's aggregations link. + + Returns: + Optional[pystac.Link]: The aggregations link, or None if there is not one found. + """ + return next( + ( + link + for link in self.links + if link.rel == "aggregations" + and link.media_type == pystac.MediaType.JSON + ), + None, + ) diff --git a/titiler/stacapi/factory.py b/titiler/stacapi/factory.py index 854c25f..d9a569e 100644 --- a/titiler/stacapi/factory.py +++ b/titiler/stacapi/factory.py @@ -19,7 +19,6 @@ from morecantile import tms as morecantile_tms from morecantile.defaults import TileMatrixSets from pydantic import conint -from pystac_client import Client from pystac_client.stac_api_io import StacApiIO from rasterio.transform import xy as rowcol_to_coords from rasterio.warp import transform as transform_points @@ -45,6 +44,7 @@ from titiler.core.resources.responses import GeoJSONResponse, XMLResponse from titiler.core.utils import render_image from titiler.mosaic.factory import PixelSelectionParams +from titiler.pystac import AdvancedClient from titiler.stacapi.backend import STACAPIBackend from titiler.stacapi.dependencies import APIParams, STACApiParams, STACSearchParams from titiler.stacapi.models import FeatureInfo, LayerDict @@ -568,7 +568,7 @@ def get_layer_from_collections( # noqa: C901 ), headers=headers, ) - catalog = Client.open(url, stac_io=stac_api_io) + catalog = AdvancedClient.open(url, stac_io=stac_api_io) layers: Dict[str, LayerDict] = {} for collection in catalog.get_collections(): @@ -580,6 +580,7 @@ def get_layer_from_collections( # noqa: C901 tilematrixsets = render.pop("tilematrixsets", None) output_format = render.pop("format", None) + aggregation = render.pop("aggregation", None) _ = render.pop("minmax_zoom", None) # Not Used _ = render.pop("title", None) # Not Used @@ -643,6 +644,20 @@ def get_layer_from_collections( # noqa: C901 "values" ] ] + elif aggregation and aggregation["name"] == "datetime_frequency": + datetime_aggregation = catalog.get_aggregation( + collection_id=collection.id, + aggregation="datetime_frequency", + aggregation_params=aggregation["params"], + ) + layer["time"] = [ + python_datetime.datetime.strptime( + t["key"], + "%Y-%m-%dT%H:%M:%S.000Z", + ).strftime("%Y-%m-%d") + for t in datetime_aggregation + if t["frequency"] > 0 + ] elif intervals := temporal_extent.intervals: start_date = intervals[0][0] end_date = ( From 6a819e49d716bc96d29416be9ac8fbd5f9be7bc1 Mon Sep 17 00:00:00 2001 From: Jens Verrydt Date: Thu, 31 Oct 2024 13:06:51 +0100 Subject: [PATCH 2/6] fix: optional typing --- titiler/pystac/advanced_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/titiler/pystac/advanced_client.py b/titiler/pystac/advanced_client.py index 24f77f5..e3fc99b 100644 --- a/titiler/pystac/advanced_client.py +++ b/titiler/pystac/advanced_client.py @@ -6,7 +6,7 @@ """ import warnings -from typing import Optional +from typing import Dict, List, Optional from urllib.parse import urlencode import pystac @@ -20,8 +20,8 @@ def get_aggregation( self, collection_id: str, aggregation: str, - aggregation_params: Optional[dict] = None, - ) -> list[dict]: + aggregation_params: Optional[Dict] = None, + ) -> List[Dict]: """Perform an aggregation on a STAC collection. Args: From b4278e505b247a6fd0e4dcfd0eead22e7d62560e Mon Sep 17 00:00:00 2001 From: Jens Verrydt Date: Thu, 31 Oct 2024 13:14:29 +0100 Subject: [PATCH 3/6] fix: renamed pystac advanced_client --- tests/test_advanced_pystac_client.py | 10 +++++----- titiler/pystac/__init__.py | 4 ++-- titiler/pystac/advanced_client.py | 8 ++++---- titiler/stacapi/factory.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_advanced_pystac_client.py b/tests/test_advanced_pystac_client.py index 29af8b5..adb7d76 100644 --- a/tests/test_advanced_pystac_client.py +++ b/tests/test_advanced_pystac_client.py @@ -5,7 +5,7 @@ import pytest -from titiler.pystac import AdvancedClient +from titiler.pystac import Client catalog_json = os.path.join(os.path.dirname(__file__), "fixtures", "catalog.json") @@ -19,7 +19,7 @@ def mock_stac_io(): @pytest.fixture def client(mock_stac_io): """STAC client mock""" - client = AdvancedClient(id="pystac-client", description="pystac-client") + client = Client(id="pystac-client", description="pystac-client") with open(catalog_json, "r") as f: catalog = json.loads(f.read()) @@ -42,7 +42,7 @@ def test_get_supported_aggregations(client, mock_stac_io): @patch( - "titiler.pystac.advanced_client.AdvancedClient.get_supported_aggregations", + "titiler.pystac.advanced_client.Client.get_supported_aggregations", return_value=["datetime_frequency"], ) def test_get_aggregation_unsupported(supported_aggregations, client): @@ -58,7 +58,7 @@ def test_get_aggregation_unsupported(supported_aggregations, client): @patch( - "titiler.pystac.advanced_client.AdvancedClient.get_supported_aggregations", + "titiler.pystac.advanced_client.Client.get_supported_aggregations", return_value=["datetime_frequency"], ) def test_get_aggregation(supported_aggregations, client, mock_stac_io): @@ -102,7 +102,7 @@ def test_get_aggregation(supported_aggregations, client, mock_stac_io): @patch( - "titiler.pystac.advanced_client.AdvancedClient.get_supported_aggregations", + "titiler.pystac.advanced_client.Client.get_supported_aggregations", return_value=["datetime_frequency"], ) def test_get_aggregation_no_response(supported_aggregations, client, mock_stac_io): diff --git a/titiler/pystac/__init__.py b/titiler/pystac/__init__.py index f398c4b..3feb0d0 100644 --- a/titiler/pystac/__init__.py +++ b/titiler/pystac/__init__.py @@ -1,7 +1,7 @@ """titiler.pystac""" __all__ = [ - "AdvancedClient", + "Client", ] -from titiler.pystac.advanced_client import AdvancedClient +from titiler.pystac.advanced_client import Client diff --git a/titiler/pystac/advanced_client.py b/titiler/pystac/advanced_client.py index e3fc99b..ee6bf1a 100644 --- a/titiler/pystac/advanced_client.py +++ b/titiler/pystac/advanced_client.py @@ -1,7 +1,7 @@ """ This module provides an advanced client for interacting with STAC (SpatioTemporal Asset Catalog) APIs. -The `AdvancedClient` class extends the basic functionality of the `pystac.Client` to include +The `Client` class extends the basic functionality of the `pystac.Client` to include methods for retrieving and aggregating data from STAC collections. """ @@ -10,11 +10,11 @@ from urllib.parse import urlencode import pystac -from pystac_client import Client +import pystac_client -class AdvancedClient(Client): - """AdvancedClient extends the basic functionality of the pystac.Client class.""" +class Client(pystac_client.Client): + """Client extends the basic functionality of the pystac.Client class.""" def get_aggregation( self, diff --git a/titiler/stacapi/factory.py b/titiler/stacapi/factory.py index d9a569e..a46ba9f 100644 --- a/titiler/stacapi/factory.py +++ b/titiler/stacapi/factory.py @@ -44,7 +44,7 @@ from titiler.core.resources.responses import GeoJSONResponse, XMLResponse from titiler.core.utils import render_image from titiler.mosaic.factory import PixelSelectionParams -from titiler.pystac import AdvancedClient +from titiler.pystac import Client from titiler.stacapi.backend import STACAPIBackend from titiler.stacapi.dependencies import APIParams, STACApiParams, STACSearchParams from titiler.stacapi.models import FeatureInfo, LayerDict @@ -568,7 +568,7 @@ def get_layer_from_collections( # noqa: C901 ), headers=headers, ) - catalog = AdvancedClient.open(url, stac_io=stac_api_io) + catalog = Client.open(url, stac_io=stac_api_io) layers: Dict[str, LayerDict] = {} for collection in catalog.get_collections(): From 1f95b97e2eaf38c85994e986d46d898ecd5b617c Mon Sep 17 00:00:00 2001 From: Jens Verrydt Date: Thu, 31 Oct 2024 13:20:44 +0100 Subject: [PATCH 4/6] fix: optional typing --- titiler/pystac/advanced_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titiler/pystac/advanced_client.py b/titiler/pystac/advanced_client.py index ee6bf1a..4a37903 100644 --- a/titiler/pystac/advanced_client.py +++ b/titiler/pystac/advanced_client.py @@ -60,7 +60,7 @@ def get_aggregation( return aggregation_data - def get_supported_aggregations(self) -> list[str]: + def get_supported_aggregations(self) -> List[str]: """Get the supported aggregation types. Returns: From ac773ac65ee1b47696b9e2b1233f214ef9a1bb6a Mon Sep 17 00:00:00 2001 From: Jens Verrydt Date: Thu, 31 Oct 2024 17:03:32 +0100 Subject: [PATCH 5/6] chore: move advanced stac client to submodule inside titiler.stacapi and removed _stac_io assertion --- tests/test_advanced_pystac_client.py | 8 ++++---- titiler/pystac/__init__.py | 7 ------- titiler/stacapi/factory.py | 2 +- titiler/stacapi/pystac/__init__.py | 7 +++++++ titiler/{ => stacapi}/pystac/advanced_client.py | 2 -- 5 files changed, 12 insertions(+), 14 deletions(-) delete mode 100644 titiler/pystac/__init__.py create mode 100644 titiler/stacapi/pystac/__init__.py rename titiler/{ => stacapi}/pystac/advanced_client.py (98%) diff --git a/tests/test_advanced_pystac_client.py b/tests/test_advanced_pystac_client.py index adb7d76..d2dc74b 100644 --- a/tests/test_advanced_pystac_client.py +++ b/tests/test_advanced_pystac_client.py @@ -5,7 +5,7 @@ import pytest -from titiler.pystac import Client +from titiler.stacapi.pystac import Client catalog_json = os.path.join(os.path.dirname(__file__), "fixtures", "catalog.json") @@ -42,7 +42,7 @@ def test_get_supported_aggregations(client, mock_stac_io): @patch( - "titiler.pystac.advanced_client.Client.get_supported_aggregations", + "titiler.stacapi.pystac.advanced_client.Client.get_supported_aggregations", return_value=["datetime_frequency"], ) def test_get_aggregation_unsupported(supported_aggregations, client): @@ -58,7 +58,7 @@ def test_get_aggregation_unsupported(supported_aggregations, client): @patch( - "titiler.pystac.advanced_client.Client.get_supported_aggregations", + "titiler.stacapi.pystac.advanced_client.Client.get_supported_aggregations", return_value=["datetime_frequency"], ) def test_get_aggregation(supported_aggregations, client, mock_stac_io): @@ -102,7 +102,7 @@ def test_get_aggregation(supported_aggregations, client, mock_stac_io): @patch( - "titiler.pystac.advanced_client.Client.get_supported_aggregations", + "titiler.stacapi.pystac.advanced_client.Client.get_supported_aggregations", return_value=["datetime_frequency"], ) def test_get_aggregation_no_response(supported_aggregations, client, mock_stac_io): diff --git a/titiler/pystac/__init__.py b/titiler/pystac/__init__.py deleted file mode 100644 index 3feb0d0..0000000 --- a/titiler/pystac/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""titiler.pystac""" - -__all__ = [ - "Client", -] - -from titiler.pystac.advanced_client import Client diff --git a/titiler/stacapi/factory.py b/titiler/stacapi/factory.py index a46ba9f..e97bbc1 100644 --- a/titiler/stacapi/factory.py +++ b/titiler/stacapi/factory.py @@ -44,10 +44,10 @@ from titiler.core.resources.responses import GeoJSONResponse, XMLResponse from titiler.core.utils import render_image from titiler.mosaic.factory import PixelSelectionParams -from titiler.pystac import Client from titiler.stacapi.backend import STACAPIBackend from titiler.stacapi.dependencies import APIParams, STACApiParams, STACSearchParams from titiler.stacapi.models import FeatureInfo, LayerDict +from titiler.stacapi.pystac import Client from titiler.stacapi.settings import CacheSettings, RetrySettings from titiler.stacapi.utils import _tms_limits diff --git a/titiler/stacapi/pystac/__init__.py b/titiler/stacapi/pystac/__init__.py new file mode 100644 index 0000000..0b5003f --- /dev/null +++ b/titiler/stacapi/pystac/__init__.py @@ -0,0 +1,7 @@ +"""titiler.pystac""" + +__all__ = [ + "Client", +] + +from titiler.stacapi.pystac.advanced_client import Client diff --git a/titiler/pystac/advanced_client.py b/titiler/stacapi/pystac/advanced_client.py similarity index 98% rename from titiler/pystac/advanced_client.py rename to titiler/stacapi/pystac/advanced_client.py index 4a37903..f02bd26 100644 --- a/titiler/pystac/advanced_client.py +++ b/titiler/stacapi/pystac/advanced_client.py @@ -31,8 +31,6 @@ def get_aggregation( Returns: List[str]: The aggregation response. """ - assert self._stac_io is not None - if aggregation not in self.get_supported_aggregations(): warnings.warn( f"Aggregation type {aggregation} is not supported", stacklevel=1 From a9097b973599c26fb069208da96aad1b277fd54d Mon Sep 17 00:00:00 2001 From: Vincent Sarago Date: Mon, 18 Nov 2024 22:01:31 +0000 Subject: [PATCH 6/6] Update titiler/stacapi/pystac/__init__.py --- titiler/stacapi/pystac/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titiler/stacapi/pystac/__init__.py b/titiler/stacapi/pystac/__init__.py index 0b5003f..3733c66 100644 --- a/titiler/stacapi/pystac/__init__.py +++ b/titiler/stacapi/pystac/__init__.py @@ -4,4 +4,4 @@ "Client", ] -from titiler.stacapi.pystac.advanced_client import Client +from .advanced_client import Client