diff --git a/docs/stac.rst b/docs/stac.rst index 4f4f828cc..6edcb9130 100644 --- a/docs/stac.rst +++ b/docs/stac.rst @@ -41,6 +41,14 @@ STAC support will render links as follows: * links that are enclosures will be encoded as STAC assets (in ``assets``) * all other links remain as record links (in ``links``) +Transactions +^^^^^^^^^^^^ + +STAC Transactions are supported as per the following STAC API specifications: + +* `STAC API - Transaction Extension Specification`_. +* `STAC API - Collection Transaction Extension`_. + Request Examples ---------------- @@ -106,3 +114,5 @@ Request Examples http://localhost:8000/stac/collections/metadata:main/items/{itemId} .. _`SpatioTemporal Asset Catalog API version v1.0.0`: https://github.com/radiantearth/stac-api-spec +.. _`STAC API - Transaction Extension Specification`: https://github.com/stac-api-extensions/transaction +.. _`STAC API - Collection Transaction Extension`: https://github.com/stac-api-extensions/collection-transaction diff --git a/docs/transactions.rst b/docs/transactions.rst index c9ab118d4..7ac8b03c3 100644 --- a/docs/transactions.rst +++ b/docs/transactions.rst @@ -114,5 +114,46 @@ Harvesting Harvesting is not yet supported via OGC API - Records. +Transactions using STAC API +=========================== + +pycsw's STAC API support provides transactional capabilities via the `STAC API - Transaction Extension Specification`_ and `STAC API - Collection Transaction Extension`_ specifications, +which follows RESTful patterns for insert/update/delete of resources. + +Supported Resource Types +------------------------ + +STAC Collections, Items and Item Collections are supported via OGC API - Records transactional workflow. Note that the HTTP ``Content-Type`` +header MUST be set to (i.e. ``application/json``). + +Transaction operations +---------------------- + +The below examples demonstrate transactional workflow using pycsw's OGC API - Records endpoint: + +.. code-block:: bash + + # insert STAC Item + curl -v -H "Content-Type: application/json" -XPOST http://localhost:8000/stac/collections/metadata:main/items -d @fooitem.json + + # update STAC Item + curl -v -H "Content-Type: application/json" -XPUT http://localhost:8000/stac/collections/metadata:main/items/fooitem -d @fooitem.json + + # delete STAC Item + curl -v -XDELETE http://localhost:8000/stac/collections/metadata:main/items/fooitem + + # insert STAC Item Collection + curl -v -H "Content-Type: application/json" -XPOST http://localhost:8000/stac/collections/metadata:main/items -d @fooitemcollection.json + + # insert STAC Collection + curl -v -H "Content-Type: application/json" -XPOST http://localhost:8000/stac/collections -d @foocollection.json + + # update STAC Collection + curl -v -H "Content-Type: application/json" -XPUT http://localhost:8000/stac/collections/foocollection -d @foocollection.json + + # delete STAC Collection + curl -v -XDELETE http://localhost:8000/stac/collections/foocollection .. _`OGC API - Features - Part 4: Create, Replace, Update and Delete`: https://docs.ogc.org/DRAFTS/20-002.html +.. _`STAC API - Transaction Extension Specification`: https://github.com/stac-api-extensions/transaction +.. _`STAC API - Collection Transaction Extension`: https://github.com/stac-api-extensions/collection-transaction diff --git a/pycsw/core/repository.py b/pycsw/core/repository.py index 544046bda..33813a4a1 100644 --- a/pycsw/core/repository.py +++ b/pycsw/core/repository.py @@ -147,6 +147,7 @@ def __init__(self, database, context, app_root=None, table='records', repo_filte self.query_mappings = { 'identifier': self.dataset.identifier, 'type': self.dataset.type, + 'typename': self.dataset.typename, 'parentidentifier': self.dataset.parentidentifier, 'collections': self.dataset.parentidentifier, 'updated': self.dataset.insert_date, diff --git a/pycsw/ogc/api/records.py b/pycsw/ogc/api/records.py index a100aa041..c1224c2d0 100644 --- a/pycsw/ogc/api/records.py +++ b/pycsw/ogc/api/records.py @@ -184,7 +184,8 @@ def get_response(self, status, headers, data, template=None): if headers.get('Content-Type') == 'text/html' and template is not None: content = render_j2_template(self.config, template, data) else: - content = to_json(data) + pretty_print = str2bool(self.config['server'].get('pretty_print', False)) + content = to_json(data, pretty_print) headers['Content-Length'] = len(content) diff --git a/pycsw/stac/api.py b/pycsw/stac/api.py index 566da8f59..ec4b7ecb7 100644 --- a/pycsw/stac/api.py +++ b/pycsw/stac/api.py @@ -69,7 +69,9 @@ 'https://api.stacspec.org/v1.0.0/item-search#filter', 'https://api.stacspec.org/v1.0.0/item-search#free-text', 'https://api.stacspec.org/v1.0.0-rc.1/collection-search', - 'https://api.stacspec.org/v1.0.0-rc.1/collection-search#free-text' + 'https://api.stacspec.org/v1.0.0-rc.1/collection-search#free-text', + 'https://api.stacspec.org/v1.0.0/collections/extensions/transaction', + 'https://api.stacspec.org/v1.0.0/ogcapi-features/extensions/transaction' ] @@ -321,11 +323,15 @@ def collection(self, headers_, args, collection='metadata:main'): if collection == 'metadata:main': collection_info = self.get_collection_info() else: - virtual_collection = self.repository.query_ids([collection])[0] - collection_info = self.get_collection_info( - virtual_collection.identifier, - dict(title=virtual_collection.title, + try: + virtual_collection = self.repository.query_ids([collection])[0] + collection_info = self.get_collection_info( + virtual_collection.identifier, + dict(title=virtual_collection.title, description=virtual_collection.abstract)) + except IndexError: + return self.get_exception( + 404, headers_, 'InvalidParameterValue', 'STAC collection not found') response = collection_info url_base = f"{self.config['server']['url']}/collections/{collection}" @@ -456,6 +462,11 @@ def item(self, headers_, args, collection, item): return self.get_exception(400, headers_, 'InvalidParameterValue', msg) response = json.loads(response) + + if 'id' not in response: + return self.get_exception( + 404, headers_, 'InvalidParameterValue', 'item not found') + response = links2stacassets(collection, response) return self.get_response(status, headers_, response) @@ -497,6 +508,27 @@ def get_collection_info(self, collection_name: str = 'metadata:main', }] } + def manage_collection_item(self, headers_, action='create', item=None, data=None, collection=None): + if action == 'create' and 'features' in data: + LOGGER.debug('STAC Collection detected') + + for feature in data['features']: + data2 = feature + if collection is not None: + data2['collection'] = collection + + headers, status, content = super().manage_collection_item( + headers_=headers_, action='create', data=data2) + + return self.get_response(201, headers_, {}) + + else: # default/super + if collection is not None: + data['collection'] = collection + + return super().manage_collection_item( + headers_=headers_, action=action, item=item, data=data) + def links2stacassets(collection, record): LOGGER.debug('Transforming enclosure links to STAC assets') diff --git a/pycsw/wsgi_flask.py b/pycsw/wsgi_flask.py index a468b82eb..ebef4c904 100644 --- a/pycsw/wsgi_flask.py +++ b/pycsw/wsgi_flask.py @@ -46,9 +46,6 @@ with open(os.getenv('PYCSW_CONFIG'), encoding='utf8') as fh: APP.config['PYCSW_CONFIG'] = yaml_load(fh) -pretty_print = APP.config['PYCSW_CONFIG']['server'].get('pretty_print', True) -APP.config['JSONIFY_PRETTYPRINT_REGULAR'] = pretty_print - BLUEPRINT = Blueprint('pycsw', __name__, static_folder=STATIC, static_url_path='/static') @@ -137,8 +134,8 @@ def conformance(): return get_response(api_.conformance(dict(request.headers), request.args)) -@BLUEPRINT.route('/collections') -@BLUEPRINT.route('/stac/collections') +@BLUEPRINT.route('/collections', methods=['GET', 'POST']) +@BLUEPRINT.route('/stac/collections', methods=['GET', 'POST']) def collections(): """ OGC API collections endpoint @@ -147,13 +144,19 @@ def collections(): """ if get_api_type(request.url_rule.rule) == 'stac-api': - return get_response(stacapi.collections(dict(request.headers), request.args)) # noqa + if request.method == 'POST': + data = request.get_json(silent=True) + return get_response(stacapi.manage_collection_item(dict(request.headers), + 'create', data=data)) + else: + return get_response(stacapi.collections(dict(request.headers), + request.args)) else: return get_response(api_.collections(dict(request.headers), request.args)) -@BLUEPRINT.route('/collections/') -@BLUEPRINT.route('/stac/collections/') +@BLUEPRINT.route('/collections/', methods=['GET', 'PUT', 'DELETE']) +@BLUEPRINT.route('/stac/collections/', methods=['GET', 'PUT', 'DELETE']) def collection(collection='metadata:main'): """ OGC API collection endpoint @@ -164,8 +167,18 @@ def collection(collection='metadata:main'): """ if get_api_type(request.url_rule.rule) == 'stac-api': - return get_response(stacapi.collection(dict(request.headers), - request.args, collection)) + if request.method == 'PUT': + return get_response( + stacapi.manage_collection_item( + dict(request.headers), 'update', collection, + data=request.get_json(silent=True))) + elif request.method == 'DELETE': + return get_response( + stacapi.manage_collection_item(dict(request.headers), + 'delete', collection)) + else: + return get_response(stacapi.collection(dict(request.headers), + request.args, collection)) else: return get_response(api_.collection(dict(request.headers), request.args, collection)) @@ -203,14 +216,22 @@ def items(collection='metadata:main'): :returns: HTTP response """ - if request.method == 'POST' and request.content_type not in [None, 'application/json']: # noqa + if all([get_api_type(request.url_rule.rule) == 'ogcapi-records', + request.method == 'POST', + request.content_type not in [None, 'application/json']]): + data = None if request.content_type == 'application/geo+json': # JSON grammar data = request.get_json(silent=True) elif 'xml' in request.content_type: # XML grammar data = request.data + return get_response(api_.manage_collection_item(dict(request.headers), 'create', data=data)) + elif request.method == 'POST' and get_api_type(request.url_rule.rule) == 'stac-api': + data = request.get_json(silent=True) + return get_response(stacapi.manage_collection_item(dict(request.headers), + 'create', data=data, collection=collection)) else: if get_api_type(request.url_rule.rule) == 'stac-api': return get_response(stacapi.items(dict(request.headers), diff --git a/tests/functionaltests/suites/oarec/test_oarec_functional.py b/tests/functionaltests/suites/oarec/test_oarec_functional.py index 1a211fe60..2d2343b17 100644 --- a/tests/functionaltests/suites/oarec/test_oarec_functional.py +++ b/tests/functionaltests/suites/oarec/test_oarec_functional.py @@ -107,7 +107,7 @@ def test_queryables(config): assert content['$id'] == 'http://localhost/pycsw/oarec/collections/metadata:main/queryables' # noqa assert content['$schema'] == 'http://json-schema.org/draft/2019-09/schema' - assert len(content['properties']) == 13 + assert len(content['properties']) == 14 assert 'geometry' in content['properties'] assert content['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Polygon.json' # noqa diff --git a/tests/functionaltests/suites/oarec/test_oarec_virtual_collections_functional.py b/tests/functionaltests/suites/oarec/test_oarec_virtual_collections_functional.py index 9ca71750e..af902ad3e 100644 --- a/tests/functionaltests/suites/oarec/test_oarec_virtual_collections_functional.py +++ b/tests/functionaltests/suites/oarec/test_oarec_virtual_collections_functional.py @@ -68,7 +68,7 @@ def test_queryables(config_virtual_collections): assert content['$id'] == 'http://localhost/pycsw/oarec/collections/metadata:main/queryables' # noqa assert content['$schema'] == 'http://json-schema.org/draft/2019-09/schema' - assert len(content['properties']) == 13 + assert len(content['properties']) == 14 assert 'geometry' in content['properties'] assert content['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Polygon.json' # noqa diff --git a/tests/functionaltests/suites/stac_api/conftest.py b/tests/functionaltests/suites/stac_api/conftest.py index 076c43261..dd6acb0e1 100644 --- a/tests/functionaltests/suites/stac_api/conftest.py +++ b/tests/functionaltests/suites/stac_api/conftest.py @@ -110,3 +110,210 @@ def config(): 'table': 'records', } } + + +@pytest.fixture() +def sample_collection(): + yield { + 'assets': { + '6072a0ee-0fff-4755-9cc7-660711de9b35': { + 'href': 'https://api.up42.com/v2/assets/6072a0ee-0fff-4755-9cc7-660711de9b35', + 'title': 'Original Delivery', + 'roles': [ + 'data', + 'original' + ], + 'type': 'application/zip' + } + }, + 'links': [ + { + 'rel': 'self', + 'href': 'https://api.up42.dev/catalog/hosts/oneatlas/stac/search' + } + ], + 'stac_extensions': [ + 'https://api.up42.com/stac-extensions/up42-order/v1.0.0/schema.json' + ], + 'title': 'ORT_SPOT7_20190922_094920500_000', + 'description': 'High-resolution 1.5m SPOT images acquired daily on a global basis. The datasets are available starting from 2012.', + 'keywords': [ + 'berlin', + 'optical' + ], + 'license': 'proprietary', + 'providers': [ + { + 'name': 'Airbus', + 'roles': [ + 'producer' + ], + 'url': 'https://www.airbus.com' + } + ], + 'extent': { + 'spatial': { + 'bbox': [ + [ + -86.07022916666666, + 11.900145833333333, + -86.05072916666667, + 11.942270833333334 + ] + ] + }, + 'temporal': { + 'interval': [ + [ + '2017-01-01T00:00:00Z', + '2021-12-31T00:00:00Z' + ] + ] + } + }, + 'stac_version': '1.0.0', + 'type': 'Collection', + 'id': '123e4567-e89b-12d3-a456-426614174000' + } + + +@pytest.fixture() +def sample_item(): + yield { + 'id': '20201211_223832_CS2', + 'stac_version': '1.0.0', + 'type': 'Feature', + 'geometry': None, + 'properties': { + 'datetime': '2020-12-11T22:38:32.125000Z' + }, + 'collection': 'simple-collection', + 'links': [{ + 'rel': 'collection', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }, { + 'rel': 'root', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }, { + 'rel': 'parent', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }], + 'assets': { + 'visual': { + 'href': 'https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif', + 'type': 'image/tiff; application=geotiff; profile=cloud-optimized', + 'title': '3-Band Visual', + 'roles': [ + 'visual' + ] + }, + 'thumbnail': { + 'href': 'https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg', + 'title': 'Thumbnail', + 'type': 'image/jpeg', + 'roles': [ + 'thumbnail' + ] + } + } + } + + +@pytest.fixture() +def sample_item_collection(): + yield { + 'type': 'FeatureCollection', + 'features': [{ + 'id': '20201211_223832_CS2', + 'stac_version': '1.0.0', + 'type': 'Feature', + 'geometry': None, + 'properties': { + 'datetime': '2020-12-11T22:38:32.125000Z' + }, + 'collection': 'simple-collection', + 'links': [{ + 'rel': 'collection', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }, { + 'rel': 'root', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }, { + 'rel': 'parent', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }], + 'assets': { + 'visual': { + 'href': 'https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif', + 'type': 'image/tiff; application=geotiff; profile=cloud-optimized', + 'title': '3-Band Visual', + 'roles': [ + 'visual' + ] + }, + 'thumbnail': { + 'href': 'https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg', + 'title': 'Thumbnail', + 'type': 'image/jpeg', + 'roles': [ + 'thumbnail' + ] + } + } + }, { + 'id': '20201212_223832_CS2', + 'stac_version': '1.0.0', + 'type': 'Feature', + 'geometry': None, + 'properties': { + 'datetime': '2020-12-12T22:38:32.125000Z' + }, + 'collection': 'simple-collection', + 'links': [{ + 'rel': 'collection', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }, { + 'rel': 'root', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }, { + 'rel': 'parent', + 'href': './collection.json', + 'type': 'application/json', + 'title': 'Simple Example Collection' + }], + 'assets': { + 'visual': { + 'href': 'https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.tif', + 'type': 'image/tiff; application=geotiff; profile=cloud-optimized', + 'title': '3-Band Visual', + 'roles': [ + 'visual' + ] + }, + 'thumbnail': { + 'href': 'https://storage.googleapis.com/open-cogs/stac-examples/20201211_223832_CS2.jpg', + 'title': 'Thumbnail', + 'type': 'image/jpeg', + 'roles': [ + 'thumbnail' + ] + } + } + }] + } diff --git a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py index 3d85a931e..bf26bcb91 100644 --- a/tests/functionaltests/suites/stac_api/test_stac_api_functional.py +++ b/tests/functionaltests/suites/stac_api/test_stac_api_functional.py @@ -49,7 +49,7 @@ def test_landing_page(config): assert content['stac_version'] == '1.0.0' assert content['type'] == 'Catalog' - assert len(content['conformsTo']) == 18 + assert len(content['conformsTo']) == 20 assert len(content['keywords']) == 3 @@ -70,7 +70,7 @@ def test_conformance(config): assert headers['Content-Type'] == 'application/json' assert status == 200 - assert len(content['conformsTo']) == 18 + assert len(content['conformsTo']) == 20 conformances = [ 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/simple-query', @@ -118,7 +118,7 @@ def test_queryables(config): assert content['$id'] == 'http://localhost/pycsw/oarec/stac/collections/metadata:main/queryables' # noqa assert content['$schema'] == 'http://json-schema.org/draft/2019-09/schema' - assert len(content['properties']) == 13 + assert len(content['properties']) == 14 assert 'geometry' in content['properties'] assert content['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Polygon.json' # noqa @@ -233,3 +233,151 @@ def test_item(config): headers, status, content = api.item({}, {}, 'foo', item) assert status == 400 + + +def test_json_transaction(config, sample_collection, sample_item, + sample_item_collection): + api = STACAPI(config) + request_headers = { + 'Content-Type': 'application/json' + } + + # insert item + headers, status, content = api.manage_collection_item( + request_headers, 'create', data=sample_item, collection='metadata:main') + + assert status == 201 + + # test that item is in repository + content = json.loads(api.item({}, {}, 'metadata:main', + '20201211_223832_CS2')[2]) + + assert content['id'] == '20201211_223832_CS2' + assert content['geometry'] is None + assert content['properties']['datetime'] == '2020-12-11T22:38:32.125000Z' + assert content['collection'] == 'metadata:main' + + # update item + sample_item['properties']['datetime'] = '2021-12-11T22:38:32.125000Z' + + headers, status, content = api.manage_collection_item( + request_headers, 'update', item='20201211_223832_CS2', + data=sample_item, collection='metadata:main') + + assert status == 204 + + # test that item is in repository + content = json.loads(api.item({}, {}, 'metadata:main', + '20201211_223832_CS2')[2]) + + assert content['id'] == '20201211_223832_CS2' + assert content['properties']['datetime'] == sample_item['properties']['datetime'] + assert content['collection'] == 'metadata:main' + + # delete item + headers, status, content = api.manage_collection_item( + request_headers, 'delete', item='20201211_223832_CS2') + + assert status == 200 + + # test that item is not in repository + headers, status, content = api.item({}, {}, 'metadata:main', + '20201211_223832_CS2') + + assert status == 404 + + content = api.items({}, None, {})[2] + + matched = json.loads(content)['numberMatched'] + + assert matched == 12 + + # insert item collection + headers, status, content = api.manage_collection_item( + request_headers, 'create', data=sample_item_collection, collection='metadata:main') + + assert status == 201 + + content = api.items({}, None, {})[2] + + matched = json.loads(content)['numberMatched'] + + assert matched == 14 + + # delete items from item collection + headers, status, content = api.manage_collection_item( + request_headers, 'delete', item='20201211_223832_CS2') + + assert status == 200 + + headers, status, content = api.manage_collection_item( + request_headers, 'delete', item='20201212_223832_CS2') + + assert status == 200 + + collection_id = '123e4567-e89b-12d3-a456-426614174000' + + # insert collection + headers, status, content = api.manage_collection_item( + request_headers, 'create', data=sample_collection) + + assert status == 201 + + # test that collection is in repository + headers, status, content = api.collections({}, {'f': 'json'}) + content = json.loads(content) + + collection_found = False + + for collection in content['collections']: + if collection['id'] == collection_id: + collection_found = True + + assert collection_found + + headers, status, content = api.collection({}, {'f': 'json'}, collection=collection_id) + + content = json.loads(content) + + assert content['id'] == collection_id + + assert content['title'] == 'ORT_SPOT7_20190922_094920500_000' + + # update collection + sample_collection['title'] = 'test title update' + + headers, status, content = api.manage_collection_item( + request_headers, 'update', item=collection_id, + data=sample_collection, collection='metadata:main') + + assert status == 204 + + headers, status, content = api.collection({}, {'f': 'json'}, collection=collection_id) + + content = json.loads(content) + + assert content['title'] == sample_collection['title'] + + # test that item is in repository + content = json.loads(api.item({}, {}, 'metadata:main', + '20201211_223832_CS2')[2]) + + # delete collection + headers, status, content = api.manage_collection_item( + request_headers, 'delete', item=collection_id) + + content = json.loads(content) + + assert status == 200 + + # test that collection is not in repository + headers, status, content = api.collections({}, {'f': 'json'}) + content = json.loads(content) + + collection_found = False + + for collection in content['collections']: + if collection['id'] == collection_id: + collection_found = True + + assert not collection_found