terra.env

JUST_PROJECT_PREFIX=TERRA
JUST_VERSION="0.2.2+1dev"
source "${VSI_COMMON_DIR}/linux/python_tools.bsh"

if [ -z "${TERRA_CWD+set}" ]; then
  TERRA_CWD="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)"
fi

: ${TERRA_AUTO_ESCAPE=TERRA_.*_DIR_DOCKER}

: ${TERRA_DOCKER_REPO=terra}

: ${TERRA_HOME=/home/user}
: ${TERRA_USERNAME=$(id -u -n)}
: ${TERRA_UID=$(id -u)}
: ${TERRA_GIDS=$(id -G)}
: ${TERRA_GID=${TERRA_GIDS%% *}}
: ${TERRA_GROUP_NAMES=$(group_names)}

#**
# .. envvar:: TERRA_LOCAL
#
# By default, terra runs in a docker container. Since the docker compute type uses docker, in scenarios other than the docker socket and external docker sockets, piping the docker connection into a container can prove troublesome. For this reason, the terra master controller can also be run on the host.
#
# :envvar:`TERRA_LOCAL` defaults to 1, run terra locally. But can either be enabled in your ``local.env`` or forced to ``1`` using the ``--local`` flag
#**
: ${TERRA_LOCAL=1}

# This directory is added to the container using the docker compose file. This mechanism
# should only be used when the directory is guaranteed to exist
: ${TERRA_TERRA_DIR=${TERRA_CWD}}
: ${TERRA_TERRA_DIR_DOCKER=/terra}
: ${TERRA_TERRA_DIR_TYPE=bind}

# Register this app with Terra
TERRA_APP_PREFIXES+=(TERRA)
array_to_python_ast_list_of_strings TERRA_APP_PREFIXES_AST "${TERRA_APP_PREFIXES[@]}"
: ${TERRA_JUST_SETTINGS=${BASH_SOURCE[0]}}
set_array_default TERRA_APPS terra.tests.demo

# ARCHIVE_DIR is relative to USER_PWD (not PWD), if called relatively...
if [ -n "${ARCHIVE_DIR+set}" ] && [ "${ARCHIVE_DIR::1}" = "." ]; then
  # Make it an absolute path
  ARCHIVE_DIR="$(cd "${USER_PWD}/${ARCHIVE_DIR}"; pwd)"
fi

# Should be updated by terra app to point to it's CWD
: ${TERRA_APP_DIR=${TERRA_CWD}}
: ${TERRA_APP_DIR_DOCKER=/src}
: ${TERRA_APP_DIR_TYPE=bind}

# This represents the main directory an app runs from, so in RODEO, it's
# ARCHIVE_DIR, else it's the APP_DIR
: ${TERRA_RUN_DIR=${ARCHIVE_DIR-${TERRA_APP_DIR}}}

# The host dir should be set by the compute/service definition. Don't end in _DOCKER
: ${TERRA_SETTINGS_DOCKER_DIR=/settings}

if [ -d "/tmp/.X11-unix" ]; then
  TERRA_VOLUMES=("/tmp/.X11-unix:/tmp/.X11-unix:ro"
      ${TERRA_VOLUMES+"${TERRA_VOLUMES[@]}"})
fi

# Redis values
: ${TERRA_REDIS_PORT=6379}
: ${TERRA_REDIS_PORT_DOCKER=6379}

: ${TERRA_REDIS_DIR=terra-redis}
: ${TERRA_REDIS_DIR_DOCKER=/data}
: ${TERRA_REDIS_DIR_TYPE=volume}
# The location redis data is written to, in singularity
: ${TERRA_REDIS_DIR_HOST_SINGULAR=${TERRA_RUN_DIR}/singular/redis}

: ${TERRA_REDIS_HOSTNAME=localhost}
: ${TERRA_REDIS_HOSTNAME_DOCKER=terra-redis}
# The hostname singularity containers will use to connect to redis. Leave blank to auto determine using the ``HOSTNAME`` envvar or ``hostname`` command
: ${TERRA_REDIS_HOSTNAME_SINGULAR=}

: ${TERRA_SINGULAR_COMPOSE_FILES=${TERRA_CWD}/singular-compose.env}

#**
# .. envvar:: TERRA_REDIS_SECRET
#
# The name of the ``docker compose`` secret used.
#
# .. envvar:: TERRA_REDIS_SECRET_FILE
#
# The file name used to store the password locally. Default: ``redis_password.secret``. If the file does not exist in ``redis_secret`` mode, than it is generated with a random password. This secret file should not be added to the your git repository; ``*.secret`` files are in the ``.gitignore`` file for this reason.
#
# .. envar:: TERRA_REDIS_SECRET_FILE_DOCKER
#
# Secret file location within the docker. It is not recommended to override this variable as docker hardcodes the directory containing secrets to ``/run/secrets``.
#**
: ${TERRA_REDIS_SECRET=redis_password}
if [ "${JUST_RODEO-}" = "1" ]; then
  : ${TERRA_REDIS_SECRET_FILE=${ARCHIVE_DIR}/${TERRA_REDIS_SECRET}.secret}
else
  : ${TERRA_REDIS_SECRET_FILE=${TERRA_CWD}/${TERRA_REDIS_SECRET}.secret}
fi
: ${TERRA_REDIS_SECRET_FILE_DOCKER="/run/secrets/${TERRA_REDIS_SECRET}"}

if [ ! -f "/.dockerenv" ] && [ ! -s "${TERRA_REDIS_SECRET_FILE}" ]; then
  source "${VSI_COMMON_DIR}/linux/random.bsh"
  # Allow printable ascii characters except quotes, ';' (for an unknown redis/celery parsing reason), ':?/@' (messes with parseurl)
  # a [ or ] without a ] or [ breaks urllib.parse, so don't use those
  # # messes up urllib.parse and treats everything as a fragment, so don't use that either
  # So no: ' x22 # x23 " x27 ; / x2f x3a ? x3b : x3f @ x40 [ x5b ] x5d
  urandom_password 20 $'\x21\x24-\x26\x28-\x2e\x30\x39\x3c-\x3e\x41-\x5A\x5C\x5E\x7E' > "${TERRA_REDIS_SECRET_FILE}"
fi

#**
# .. envvar:: TERRA_CELERY_MAIN_NAME
#
# (Optional) Name of the main module if running as __main__. This is used as the prefix for auto-generated task names that are defined in the same module as ``__main__`` (Usually caused by ``python -m``). At first, python will try ``sys.modules['__main__'].__spec__.name``, before using this value, when that fails.
#
# .. envvar:: TERRA_KEEP_TEMP_DIR
#
# Optional environment variable that, when set to ``1``, will keep the temporary config files generated for containers. For debug use.
#
# .. envvar:: TERRA_DISABLE_TERRA_LOG
#
# Optional environment variable that, when set to ``1``, will disable the saving of the ``terra_log`` file in the processing dir. This is particularly useful for test script or jupyter notebooks where you do not want to litter ``terra_log`` files everywhere. For debug use.
#
# .. envvar:: TERRA_RESOLVE_HOSTNAME
#
# Optional environment variable that, when set to ``1``, will attempt to resolve the IP of the default route on the host machine.  This may correct some situations when the ``terra_log`` is missing service & task logging information due to invalid hostname resolution by the logging module.
#**

#**
# .. envvar:: TERRA_CELERY_CONF
#
# Celery reads the configuration from :envvar:`TERRA_CELERY_CONF` as the name of a module to import. Default: ``terra.celeryconfig``. You can override all the options and use your own configuration file. To easily modify one value, you can:
#
# .. code-block:: python
#
#    from terra.celeryconfig import *
#
#    result_expires=7200
#**
: ${TERRA_CELERY_CONF=terra.executor.celery.celeryconfig}
: ${TERRA_CELERY_SERVICE=celery}

#**
# .. envvar:: TERRA_CELERY_WORKERS
#
# Number of concurrent workers to start when you start a celery worker.
#**
if [ "${OS-}" = "Windows_NT" ]; then
  : ${TERRA_CELERY_WORKERS=${NUMBER_OF_PROCESSORS}}
elif [[ ${OSTYPE-} = darwin* ]]; then
  : ${TERRA_CELERY_WORKERS=$(sysctl -n hw.logicalcpu)}
elif command -v nproc &> /dev/null; then
  : ${TERRA_CELERY_WORKERS=$(nproc --all)}
elif [ -r "/proc/cpuinfo" ]; then
  : ${TERRA_CELERY_WORKERS="$(grep processor /proc/cpuinfo | wc -l)"}
else
  : ${TERRA_CELERY_WORKERS=4}
fi

#**
# .. envvar:: TERRA_CELERY_QUEUES
#
# Array of Celery queue names.  Defaults to the ``terra`` and ``celery`` queues (``celery`` is the default Celery queue name).
#**
set_array_default TERRA_CELERY_QUEUES terra celery

#**
# .. envvar:: TERRA_CELERY_INCLUDE
#
# Array of modules containing Celery tasks.
#**
set_array_default TERRA_CELERY_INCLUDE terra.task terra.tests.demo.tasks

# redis-commander configuration
: ${TERRA_REDIS_COMMANDER_PORT=4567}
: ${TERRA_REDIS_COMMANDER_PORT_DOCKER=4567}
: ${TERRA_REDIS_COMMANDER_SECRET=redis_commander_password}
if [ "${JUST_RODEO-}" = "1" ]; then
  : ${TERRA_REDIS_COMMANDER_SECRET_FILE=${ARCHIVE_DIR}/${TERRA_REDIS_COMMANDER_SECRET}.secret}
else
  : ${TERRA_REDIS_COMMANDER_SECRET_FILE=${TERRA_CWD}/${TERRA_REDIS_COMMANDER_SECRET}.secret}
fi
: ${TERRA_REDIS_COMMANDER_SECRET_FILE_DOCKER="/run/secrets/${TERRA_REDIS_COMMANDER_SECRET}"}

: ${SINGULARITY_CUSTOM_IMPORT_SCRIPT=${TERRA_TERRA_DIR}/docker/tosingular}

# Sphinx docs values
set_array_default TERRA_SPHINX_EXCLUDE_DIRS docs external
set_array_default TERRA_SPHINX_AUTODOC_DIRS terra
set_array_default TERRA_SPHINX_AUTODOC_OUTPUT_DIRS python
set_array_default TERRA_SPHINX_AUTODOC_EXCLUDE_DIRS terra/tests

###############################################################################
# Non-TERRA Settings
###############################################################################

: ${TZ=/usr/share/zoneinfo/UTC}

# Put variables that do not begin with TERRA here.

# Use this to add the user name to the docker compose project name. This is
# important when multiple users are using this docker compose project on a
# single host. This way all of the docker resources are prefixed with a unique
# name and do not collide
source "${VSI_COMMON_DIR}/linux/just_files/docker_functions.bsh"
: ${COMPOSE_PROJECT_NAME=$(docker_compose_sanitize_project_name "${TERRA_CWD}" "${TERRA_USERNAME}")}

: ${COMPOSE_FILE=${TERRA_CWD}/docker-compose-main.yml}

# `pipenv sync` fails when seeded with pip versions 20.3.0 and newer
# The reported error involves mismatched metadata
#    ERROR: Requested envcontext==2017.10.19.45 ...
#    has different version in metadata: '2017.10.19'
# This error appears to be caused by a new pip resolver introduced in pip 20.3
#    https://github.com/pypa/pip/issues/9203
# Pinning the seeded pip version to 20.2.4 resolves the issue. This is
# accomplished via a VIRTUALENV_* environment variable as per:
#   https://virtualenv.pypa.io/en/latest/cli_interface.html?highlight=environment%20variables#environment-variables
# envcontext is no longer used, but incase another library has the
# same problem uncomment the next line.
# : ${VIRTUALENV_PIP=20.2.4}