diff --git a/deploy/eoepca/data-access.sh b/deploy/eoepca/data-access.sh index ce7b2d16..693b45d3 100755 --- a/deploy/eoepca/data-access.sh +++ b/deploy/eoepca/data-access.sh @@ -47,7 +47,7 @@ deployService() { serviceValues | helm ${ACTION_HELM} data-access data-access -f - \ --repo https://eoepca.github.io/helm-charts \ --namespace ${NAMESPACE} --create-namespace \ - --version 1.4.0 + --version 1.4.1 fi } @@ -61,6 +61,10 @@ global: AWS_HTTPS: "FALSE" startup_scripts: - /registrar_pycsw/registrar_pycsw/initialize-collections.sh + CREODIAS_EODATA_S3_ENDPOINT: "${CREODIAS_EODATA_S3_ENDPOINT}" + CREODIAS_EODATA_S3_ACCESS_KEY: "${CREODIAS_EODATA_S3_ACCESS_KEY}" + CREODIAS_EODATA_S3_ACCESS_SECRET: "${CREODIAS_EODATA_S3_ACCESS_SECRET}" + CREODIAS_EODATA_S3_REGION: "${CREODIAS_EODATA_S3_REGION}" # The data-access relies on the value 'ingress.tls.hosts[0]' to specify the service # hostname. So this must be supplied even if the ingress is disabled. @@ -83,9 +87,9 @@ global: data: data: type: S3 - endpoint_url: http://data.cloudferro.com - access_key_id: access - secret_access_key: access + endpoint_url: "${CREODIAS_EODATA_S3_ENDPOINT}" + access_key_id: "${CREODIAS_EODATA_S3_ACCESS_KEY}" + secret_access_key: "${CREODIAS_EODATA_S3_ACCESS_SECRET}" region_name: RegionOne validate_bucket_name: false cache: diff --git a/deploy/eoepca/eoepca-options b/deploy/eoepca/eoepca-options index aa54f381..6e9cc473 100755 --- a/deploy/eoepca/eoepca-options +++ b/deploy/eoepca/eoepca-options @@ -85,6 +85,10 @@ if [ -z "${_EOEPCA_OPTIONS_}" ]; then # Data export CREODIAS_DATA_SPECIFICATION="${CREODIAS_DATA_SPECIFICATION:-false}" + export CREODIAS_EODATA_S3_ENDPOINT="${CREODIAS_EODATA_S3_ENDPOINT:-http://data.cloudferro.com}" + export CREODIAS_EODATA_S3_ACCESS_KEY="${CREODIAS_EODATA_S3_ACCESS_KEY:-access}" + export CREODIAS_EODATA_S3_ACCESS_SECRET="${CREODIAS_EODATA_S3_ACCESS_SECRET:-access}" + export CREODIAS_EODATA_S3_REGION="${CREODIAS_EODATA_S3_REGION:-RegionOne}" # Identity Service export TEMP_FORWARDING_PORT="${TEMP_FORWARDING_PORT:-9876}" diff --git a/docs/index.md b/docs/index.md index f88b43b0..2474180b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,6 +6,7 @@ The following provides a summary of changes since the last release (v1.4)... + * **03/04/2024** - Update Data Access to chart `1.4.1` to introduce variables to remedy hard-coded harvester values for access to Creodias eodata. Ref. - `CREODIAS_EODATA_S3_ENDPOINT`, `CREODIAS_EODATA_S3_ACCESS_KEY`, `CREODIAS_EODATA_S3_ACCESS_SECRET` and `CREODIAS_EODATA_S3_REGION`. * **20/03/2024** - Correction to chart path for helm deployment of `eoepca-portal` * **20/03/2024** - Correct hardcoded OAuth client secret for Application Hub * **20/03/2024** - Clarify Gatekeeper encryption key must be 16 or 32 characters long diff --git a/docs/quickstart/creodias-deployment.md b/docs/quickstart/creodias-deployment.md index 4968d1ad..c91c42f5 100644 --- a/docs/quickstart/creodias-deployment.md +++ b/docs/quickstart/creodias-deployment.md @@ -15,6 +15,10 @@ The CREODIAS deployment applies the following configuration: With reference to the file `creodias-options`, particular attention is drawn to the following environment variables that require tailoring to your CREODIAS (Cloudferro) environment... +* Values for access to CREODIAS eodata... + * CREODIAS_EODATA_S3_ENDPOINT - if different from the default `http://data.cloudferro.com` + * Credentials required for the new clouds, including `WAW3-2` and `FRA1-2`
+ Credentials must be created at - [https://eodata-keymanager.creodias.eu/panel/s3-credentials](https://eodata-keymanager.creodias.eu/panel/s3-credentials) - and set into the variables `CREODIAS_EODATA_S3_ACCESS_KEY` and `CREODIAS_EODATA_S3_ACCESS_SECRET` * Passwords: `MINIO_ROOT_PASSWORD`, `HARBOR_ADMIN_PASSWORD` * Identity Service credentials - e.g. `IDENTITY_SERVICE_DEFAULT_SECRET`, `IDENTITY_SERVICE_ADMIN_PASSWORD`, etc. * OpenStack details: see section [Openstack Configuration](scripted-deployment.md#openstack-configuration) @@ -37,9 +41,11 @@ kubectl -n rm exec -it deployment.apps/data-access-harvester -- python3 -m harve See the [Harvester](#harvester) section below for an explanation of this harvester configuration. +See [EOData Catalogue API Manual on CREODIAS](https://creodias.docs.cloudferro.com/en/latest/eodata/EOData-Catalogue-API-Manual-on-Creodias.html) for details regarding access to the CREODIAS data offering. + ## Data Specification Walkthrough -The example scripts include optional specifcation of data-access/harvesting configuration that is tailored for the CREODIAS data offering. This is controlled via the option `CREODIAS_DATA_SPECIFICATION=true` - see [Environment Variables](scripted-deployment.md#environment-variables). +The example scripts include optional specifcation of data-access/harvesting configuration that is tailored for the CREODIAS data offering. This is controlled via the option `CREODIAS_DATA_SPECIFICATION=true` - see [Environment Variables](scripted-deployment.md#environment-variables). In addition, it may be necessary to set the variable `CREODIAS_EODATA_S3_ENDPOINT` if different from the default - for example the value `http://eodata.cloudferro.com` for the `WAW3-2` Cloudferro cloud. This section provides a walkthrough of this configuration for CREODIAS - to act as an aid to understanding by way of a worked example. diff --git a/docs/quickstart/data-access-deployment.md b/docs/quickstart/data-access-deployment.md index 46b6ac86..8a6cfccf 100644 --- a/docs/quickstart/data-access-deployment.md +++ b/docs/quickstart/data-access-deployment.md @@ -14,7 +14,8 @@ The data-access deployment applies the following configuration: * Services deployed: * Resource Catalogue for data discovery * Data Access for data visualisation and download -* Includes data specification for CREODIAS Sentinel-2, which can be exploited if running in a CREODIAS VM connected to the `eodata` network - [see description of variable `CREODIAS_DATA_SPECIFICATION`](scripted-deployment.md#environment-variables) +* Includes data specification for CREODIAS Sentinel-2, which can be exploited if running in a CREODIAS VM connected to the `eodata` network - [see description of variable `CREODIAS_DATA_SPECIFICATION`](scripted-deployment.md#environment-variables)
+ _Note that it may be necessary to set the variable `CREODIAS_EODATA_S3_ENDPOINT` if different from the default - for example the value `http://eodata.cloudferro.com` for the `WAW3-2` Cloudferro cloud._ * Open ingress are enabled for unauthenticated access to resource-catalogue and data-access services * Other eoepca services not deployed diff --git a/docs/quickstart/exploitation-deployment.md b/docs/quickstart/exploitation-deployment.md index 9080c69a..104e7617 100644 --- a/docs/quickstart/exploitation-deployment.md +++ b/docs/quickstart/exploitation-deployment.md @@ -17,7 +17,8 @@ The exploitation deployment applies the following configuration: * Data Access for data visualisation and download * Minio for S3 object storage * ADES stage-out to Minio -* Includes data specification for CREODIAS Sentinel-2, which can be exploited if running in a CREODIAS VM connected to the `eodata` network - [see description of variable `CREODIAS_DATA_SPECIFICATION`](scripted-deployment.md#environment-variables) +* Includes data specification for CREODIAS Sentinel-2, which can be exploited if running in a CREODIAS VM connected to the `eodata` network - [see description of variable `CREODIAS_DATA_SPECIFICATION`](scripted-deployment.md#environment-variables)
+ _Note that it may be necessary to set the variable `CREODIAS_EODATA_S3_ENDPOINT` if different from the default - for example the value `http://eodata.cloudferro.com` for the `WAW3-2` Cloudferro cloud._ * Open ingress are enabled for unauthenticated access to ADES, resource-catalogue and data-access services * Other eoepca services not deployed diff --git a/docs/quickstart/scripted-deployment.md b/docs/quickstart/scripted-deployment.md index 40ff8982..0f5a0e23 100644 --- a/docs/quickstart/scripted-deployment.md +++ b/docs/quickstart/scripted-deployment.md @@ -68,6 +68,10 @@ The script [`deploy/eoepca/eoepca.sh`](https://github.com/EOEPCA/deployment-guid **STAGEOUT_TARGET** | Configures the ADES with the destination to which it should push processing results:
`workspace` - via the Workspace API
`minio` - to minio S3 object storage | `workspace` **INSTALL_FLUX** | The Workspace API relies upon [Flux CI/CD](https://fluxcd.io/), and has the capability to install the required flux components to the cluster. If your deployment already has flux installed then set this value `false` to suppress the Workspace API flux install | `true` **CREODIAS_DATA_SPECIFICATION** | Apply the data specification to harvest from the CREODIAS data offering into the resource-catalogue and data-access services.
_Can only be used when running in the CREODIAS (Cloudferro) cloud, with access to the `eodata` network._ | `false` + **CREODIAS_EODATA_S3_ENDPOINT** | URL for the S3 endpoint in CREODIAS | `http://data.cloudferro.com` + **CREODIAS_EODATA_S3_ACCESS_KEY** | Access key for CREODIAS S3 endpoint | `access` + **CREODIAS_EODATA_S3_ACCESS_SECRET** | Access secret for CREODIAS S3 endpoint | `access` + **CREODIAS_EODATA_S3_REGION** | Region for the S3 endpoint in CREODIAS | `RegionOne` **TEMP_FORWARDING_PORT** | Local port used during the scripted deployment for `kubectl port-forward` operations | `9876` ### Command-line Arguments