feat: support transactions with validation and authentication (#378)

### Issue

#295

### What?

- Add transactions extensions from -
#276
- Add bulk transactions extension
- Add authentication for all transactions PUT POST endpoints
- Add authentication to openapi docs
- Add request validation to all all transactions PUT POST endpoints

### Why?

- To allow stac editing (collection and item) + enable stac-admin

### Testing?

- Tested locally as well as deployed to
https://stacadmin.openveda.cloud/api/stac/docs

### TODO
- [x] Refactor the way env vars are used
- [x] Remove temporary workflow
- [x] Fix failing tests
- [x] Use `pystac` instead of `stac_pydantic`
- [x] Make transactions a feature flag
- [x] Package auth?
~~- [ ] Tests~~ (to be handled in [a separate
ticket](#413))

.example.env

@@ -25,6 +25,7 @@ VEDA_RASTER_EXPORT_ASSUME_ROLE_CREDS_AS_ENVS=False
 VEDA_RASTER_ROOT_PATH=
 
 VEDA_STAC_ROOT_PATH=
+VEDA_STAC_ENABLE_TRANSACTIONS=FALSE
 
 VEDA_USERPOOL_ID=
 VEDA_CLIENT_ID=

.github/workflows/pr.yml

@@ -77,9 +77,15 @@ jobs:
       - name: Install reqs for ingest api
         run: python -m pip install -r ingest_api/runtime/requirements_dev.txt
 
+      - name: Install veda auth for ingest api
+        run: python -m pip install common/auth
+
       - name: Ingest unit tests
         run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s
 
+      # - name: Stac-api transactions unit tests
+      #   run: python -m pytest stac_api/runtime/tests/ -vv -s
+
       - name: Stop services
         run: docker compose stop
 

common/__init__.py

@@ -0,0 +1 @@
+"""common utils shared by veda stacks"""

common/auth/setup.py

@@ -0,0 +1,17 @@
+"""Setup veda_auth
+"""
+
+from setuptools import find_packages, setup
+
+inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic<2"]
+
+setup(
+    name="veda_auth",
+    version="0.0.1",
+    description="",
+    python_requires=">=3.7",
+    packages=find_packages(),
+    zip_safe=False,
+    install_requires=inst_reqs,
+    include_package_data=True,
+)

common/auth/veda_auth/__init__.py

@@ -0,0 +1,5 @@
+"""
+    VEDA cognito auth
+"""
+
+from veda_auth.main import VedaAuth  # noqa: F401

common/auth/veda_auth/main.py

@@ -0,0 +1,128 @@
+"""Authentication handler for veda.stac and veda.ingest"""
+
+import base64
+import hashlib
+import hmac
+import logging
+from typing import Annotated, Any, Dict
+
+import boto3
+import jwt
+
+from fastapi import Depends, HTTPException, Security, security, status
+
+logger = logging.getLogger(__name__)
+
+
+class VedaAuth:
+    """Class for handling authentication"""
+
+    def __init__(self, settings) -> None:
+        """
+        Args:
+            settings: pydantic settings object containing cognito details
+        Returns:
+            None
+
+        """
+        self.oauth2_scheme = security.OAuth2AuthorizationCodeBearer(
+            authorizationUrl=settings.cognito_authorization_url,
+            tokenUrl=settings.cognito_token_url,
+            refreshUrl=settings.cognito_token_url,
+        )
+
+        self.jwks_client = jwt.PyJWKClient(settings.jwks_url)  # Caches JWKS
+
+        def validated_token(
+            token_str: Annotated[str, Security(self.oauth2_scheme)],
+            required_scopes: security.SecurityScopes,
+        ) -> Dict:
+            # Parse & validate token
+            logger.info(f"\nToken String {token_str}")
+            try:
+                token = jwt.decode(
+                    token_str,
+                    self.jwks_client.get_signing_key_from_jwt(token_str).key,
+                    algorithms=["RS256"],
+                )
+            except jwt.exceptions.InvalidTokenError as e:
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Could not validate credentials",
+                    headers={"WWW-Authenticate": "Bearer"},
+                ) from e
+
+            # Validate scopes (if required)
+            for scope in required_scopes.scopes:
+                if scope not in token["scope"]:
+                    raise HTTPException(
+                        status_code=status.HTTP_401_UNAUTHORIZED,
+                        detail="Not enough permissions",
+                        headers={
+                            "WWW-Authenticate": f'Bearer scope="{required_scopes.scope_str}"'
+                        },
+                    )
+
+            return token
+
+        self.validated_token = validated_token
+
+        def get_username(
+            token: Annotated[Dict[Any, Any], Depends(self.validated_token)]
+        ) -> str:
+            result = token["username"] if "username" in token else str(token.get("sub"))
+            return result
+
+        self.get_username = get_username
+
+    def _get_secret_hash(
+        self, username: str, client_id: str, client_secret: str
+    ) -> str:
+        # A keyed-hash message authentication code (HMAC) calculated using
+        # the secret key of a user pool client and username plus the client
+        # ID in the message.
+        message = username + client_id
+        dig = hmac.new(
+            bytearray(client_secret, "utf-8"),
+            msg=message.encode("UTF-8"),
+            digestmod=hashlib.sha256,
+        ).digest()
+        return base64.b64encode(dig).decode()
+
+    def authenticate_and_get_token(
+        self,
+        username: str,
+        password: str,
+        user_pool_id: str,
+        app_client_id: str,
+        app_client_secret: str,
+    ) -> Dict:
+        """Authenticates the credentials and returns token"""
+        client = boto3.client("cognito-idp")
+        if app_client_secret:
+            auth_params = {
+                "USERNAME": username,
+                "PASSWORD": password,
+                "SECRET_HASH": self._get_secret_hash(
+                    username, app_client_id, app_client_secret
+                ),
+            }
+        else:
+            auth_params = {
+                "USERNAME": username,
+                "PASSWORD": password,
+            }
+        try:
+            resp = client.admin_initiate_auth(
+                UserPoolId=user_pool_id,
+                ClientId=app_client_id,
+                AuthFlow="ADMIN_USER_PASSWORD_AUTH",
+                AuthParameters=auth_params,
+            )
+        except client.exceptions.NotAuthorizedException:
+            return {
+                "message": "Login failed, please make sure the credentials are correct."
+            }
+        except Exception as e:
+            return {"message": f"Login failed with exception {e}"}
+        return resp["AuthenticationResult"]

ingest_api/infrastructure/construct.py

@@ -107,12 +107,6 @@ def __init__(
             value=self.api.url,
         )
 
-        register_ssm_parameter(
-            self,
-            name="jwks_url",
-            value=self.jwks_url,
-            description="JWKS URL for Cognito user pool",
-        )
         register_ssm_parameter(
             self,
             name="dynamodb_table",

ingest_api/runtime/Dockerfile

@@ -2,6 +2,10 @@ FROM public.ecr.aws/sam/build-python3.9:latest
 
 WORKDIR /tmp
 
+COPY common/auth /tmp/common/auth
+RUN pip install /tmp/common/auth -t /asset
+RUN rm -rf /tmp/common
+
 COPY ingest_api/runtime/requirements.txt /tmp/ingestor/requirements.txt
 RUN pip install -r /tmp/ingestor/requirements.txt -t /asset --no-binary pydantic uvicorn
 RUN rm -rf /tmp/ingestor

ingest_api/runtime/requirements.txt

@@ -8,7 +8,6 @@ orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
 pydantic_ssm_settings>=0.2.0
 pydantic>=1.10.12
-pyjwt>=2.8.0
 pypgstac==0.7.4
 python-multipart==0.0.7
 requests>=2.27.1

ingest_api/runtime/src/config.py

@@ -4,6 +4,7 @@
 
 from pydantic import AnyHttpUrl, BaseSettings, Field, constr
 from pydantic_ssm_settings import AwsSsmSourceConfig
+from veda_auth import VedaAuth
 
 AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
 
@@ -63,3 +64,5 @@ def from_ssm(cls, stack: str):
         ),
     )
 )
+
+auth = VedaAuth(settings)

ingest_api/runtime/src/dependencies.py

@@ -1,9 +1,8 @@
 import logging
 
 import boto3
-import src.auth as auth
-import src.config as config
 import src.services as services
+from src.config import auth, settings
 
 from fastapi import Depends, HTTPException, security
 
@@ -14,7 +13,7 @@
 
 def get_table():
     client = boto3.resource("dynamodb")
-    return client.Table(config.settings.dynamodb_table)
+    return client.Table(settings.dynamodb_table)
 
 
 def get_db(table=Depends(get_table)) -> services.Database:

ingest_api/runtime/src/main.py

@@ -1,12 +1,11 @@
 from typing import Dict
 
-import src.auth as auth
 import src.dependencies as dependencies
 import src.schemas as schemas
 import src.services as services
 from aws_lambda_powertools.metrics import MetricUnit
 from src.collection_publisher import CollectionPublisher, ItemPublisher
-from src.config import settings
+from src.config import auth, settings
 from src.doc import DESCRIPTION
 from src.monitoring import LoggerRouteHandler, logger, metrics, tracer
 

local/Dockerfile.ingest

@@ -7,6 +7,8 @@ RUN pip install -r /tmp/ingestor/requirements.txt --no-binary pydantic uvicorn
 RUN rm -rf /tmp/ingestor
 # TODO this is temporary until we use a real packaging system like setup.py or poetry
 COPY ingest_api/runtime/src /asset/src
+COPY common/auth /tmp/common/auth
+RUN pip install /tmp/common/auth
 
 # # Reduce package size and remove useless files
 RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;

local/Dockerfile.stac

@@ -4,12 +4,14 @@ FROM ghcr.io/vincentsarago/uvicorn-gunicorn:${PYTHON_VERSION}
 
 ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
 
-RUN pip install boto3
-
-COPY stac_api/runtime /tmp/stac
 # Installing boto3, which isn't needed in the lambda container instance
 # since lambda execution environment includes boto3 by default
 RUN pip install boto3
+
+COPY stac_api/runtime /tmp/stac
+
+COPY common/auth /tmp/stac/common/auth
+RUN pip install /tmp/stac/common/auth
 RUN pip install /tmp/stac 
 RUN rm -rf /tmp/stac