From e112088dd4e462e851739b42686e9f7185005930 Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Wed, 10 Apr 2024 03:05:37 -0700 Subject: [PATCH] fix: cleans up eventdb entrypoint and adds fix for RDS | NPG-000 (#690) This PR addresses the following issues: 1. We no longer have a dependency on graphql, so it removes all init code related to that in the eventdb entrypoint 1. The entrypoint was trying to connect to a database that didn't exist a. It was changed to connect to the "root" database on initialization (a new input was added for this) 1. For initialization to work with RDS, the root role must be assigned the role the database is being created with a. This is a weird RDS requirement where the root role isn't really "super" 1. The debug print statements in the init script were printing out the raw database password every single run a. Even though our logs are private, this is still a serious issue b. The debug statements were moved to the entrypoint instead so they could be conditionally controlled 1. The ideascale SQL for the dev environment had a syntax error in it --------- Co-authored-by: kukkok3 <93382903+kukkok3@users.noreply.github.com> --- .github/workflows/rust.yml | 2 +- containers/event-db-migrations/entry.sh | 59 ++++++++----------- src/event-db/setup/setup-db.sql | 14 ++--- .../stage_data/dev/00002_fund100_params.sql | 2 +- 4 files changed, 34 insertions(+), 43 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c216289c24..0663ba2148 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -226,7 +226,7 @@ jobs: uses: taiki-e/install-action@nextest - name: Install cargo-make - run: cargo install --force cargo-make --locked + run: cargo install --force cargo-make --version 0.37.10 --locked - name: Install refinery run: cargo install refinery_cli --version 0.8.7 --locked diff --git a/containers/event-db-migrations/entry.sh b/containers/event-db-migrations/entry.sh index f485904b84..da88fa07b5 100644 --- a/containers/event-db-migrations/entry.sh +++ b/containers/event-db-migrations/entry.sh @@ -13,6 +13,7 @@ # DB_HOST - The hostname of the database server # DB_PORT - The port of the database server # DB_NAME - The name of the database +# DB_ROOT_NAME - The name of the root database (usually postgres) # DB_SUPERUSER - The username of the database superuser # DB_SUPERUSER_PASSWORD - The password of the database superuser # DB_USER - The username of the database user @@ -20,16 +21,8 @@ # DB_SKIP_HISTORICAL_DATA - If set, historical data will not be added to the database (optional) # DB_SKIP_TEST_DATA - If set, test data will not be added to the database (optional) # DB_SKIP_STAGE_DATA - If set, stage specific data will not be added to the database (optional) -# ADMIN_ROLE_PASSWORD - The password of the cat_admin role for graphql -# ADMIN_USER_PASSWORD - The password of the admin user for graphql -# ANON_ROLE_PASSWORD - The password of the cat_anon role for graphql -# ADMIN_FIRST_NAME - The first name of the admin user for graphql (optional) -# ADMIN_LAST_NAME - The last name of the admin user for graphql (optional) -# ADMIN_ABOUT - The about of the admin user for graphql (optional) -# ADMIN_EMAIL - The email of the admin user for graphql (optional) # REINIT_EVENT_DB - If set, the database will be reinitialized (optional) (DESTRUCTIVE) # SKIP_EVENT_DB_INIT - If set, the event database will not be initialized (optional) -# SKIP_GRAPHQL_INIT - If set, graphql will not be initialized (optional) # DEBUG - If set, the script will print debug information (optional) # DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional) # STAGE - The stage being run. Currently only controls if stage specific data is applied to the DB (optional) @@ -71,34 +64,41 @@ REQUIRED_ENV=( "DB_HOST" "DB_PORT" "DB_NAME" + "DB_ROOT_NAME" "DB_SUPERUSER" "DB_SUPERUSER_PASSWORD" "DB_USER" "DB_USER_PASSWORD" - "ADMIN_ROLE_PASSWORD" - "ADMIN_USER_PASSWORD" - "ANON_ROLE_PASSWORD" ) check_env_vars "${REQUIRED_ENV[@]}" # Export environment variables export PGHOST="${DB_HOST}" export PGPORT="${DB_PORT}" -export PGUSER="${DB_SUPERUSER}" -export PGPASSWORD="${DB_SUPERUSER_PASSWORD}" -export PGDATABASE="${DB_NAME}" - -: "${ADMIN_FIRST_NAME:='Admin'}" -: "${ADMIN_LAST_NAME:='Default'}" -: "${ADMIN_ABOUT:='Default Admin User'}" -: "${ADMIN_EMAIL:='admin.default@projectcatalyst.io'}" # Sleep if DEBUG_SLEEP is set debug_sleep +if [ -n "${DEBUG:-}" ]; then + echo ">>> Environment variables:" + echo "DB_HOST: ${DB_HOST}" + echo "DB_PORT: ${DB_PORT}" + echo "DB_NAME: ${DB_NAME}" + echo "DB_ROOT_NAME: ${DB_ROOT_NAME}" + echo "DB_SUPERUSER: ${DB_SUPERUSER}" + echo "DB_SUPERUSER_PASSWORD: ${DB_SUPERUSER_PASSWORD}" + echo "DB_USER: ${DB_USER}" + echo "DB_USER_PASSWORD: ${DB_USER_PASSWORD}" +fi + # Initialize database if necessary if [[ ! -f ./tmp/initialized || -n "${REINIT_EVENT_DB:-}" ]]; then + # Connect using the superuser to create the event database + export PGUSER="${DB_SUPERUSER}" + export PGPASSWORD="${DB_SUPERUSER_PASSWORD}" + export PGDATABASE="${DB_ROOT_NAME}" + PSQL_FLAGS="" if [ -n "${DEBUG:-}" ]; then PSQL_FLAGS="-e" @@ -110,21 +110,8 @@ if [[ ! -f ./tmp/initialized || -n "${REINIT_EVENT_DB:-}" ]]; then -v dbName="${DB_NAME}" \ -v dbDescription="Catalayst Event DB" \ -v dbUser="${DB_USER}" \ - -v dbUserPw="${DB_USER_PASSWORD}" - fi - - if [[ -z "${SKIP_GRAPHQL_INIT:-}" ]]; then - echo ">>> Initializing graphql..." - psql "${PSQL_FLAGS}" -f ./setup/graphql-setup.sql \ - -v dbName="${DB_NAME}" \ - -v dbUser="${DB_USER}" \ - -v adminUserFirstName="${ADMIN_FIRST_NAME}" \ - -v adminUserLastName="${ADMIN_LAST_NAME}" \ - -v adminUserAbout="${ADMIN_ABOUT}" \ - -v adminUserEmail="${ADMIN_EMAIL}" \ - -v adminRolePw="${ADMIN_ROLE_PASSWORD}" \ - -v adminUserPw="${ADMIN_USER_PASSWORD}" \ - -v anonRolePw="${ANON_ROLE_PASSWORD}" + -v dbUserPw="${DB_USER_PASSWORD}" \ + -v dbRootUser="${DB_SUPERUSER}" fi if [[ ! -f ./tmp/initialized ]]; then @@ -135,6 +122,10 @@ else fi # Run migrations +export PGUSER="${DB_USER}" +export PGPASSWORD="${DB_USER_PASSWORD}" +export PGDATABASE="${DB_NAME}" + echo ">>> Running migrations..." export DATABASE_URL="postgres://${DB_USER}:${DB_USER_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}" ./refinery migrate -e DATABASE_URL -c ./refinery.toml -p ./migrations diff --git a/src/event-db/setup/setup-db.sql b/src/event-db/setup/setup-db.sql index 1e19b66a11..56b427004c 100644 --- a/src/event-db/setup/setup-db.sql +++ b/src/event-db/setup/setup-db.sql @@ -29,13 +29,10 @@ \set dbUserPw `echo ${DB_USER_PW:-CHANGE_ME}` \endif --- DISPLAY ALL VARIABLES -\echo VARIABLES: -\echo -> dbName ................. = :dbName -\echo -> dbDescription .......... = :dbDescription -\echo -> dbUser ................. = :dbUser -\echo -> dbUserPw / $DB_USER_PW . = :dbUserPw - +-- The root db user of the database instance (usually postgres). +\if :{?dbRootUser} \else + \set dbRootUser 'postgres' +\endif -- Cleanup if we already ran this before. DROP DATABASE IF EXISTS :"dbName"; @@ -50,6 +47,9 @@ ALTER DEFAULT privileges REVOKE EXECUTE ON functions FROM public; ALTER DEFAULT privileges IN SCHEMA public REVOKE EXECUTE ON functions FROM :"dbUser"; +-- This is necessary for RDS to work. +GRANT :"dbUser" TO :"dbRootUser"; + -- Create the database. CREATE DATABASE :"dbName" WITH OWNER :"dbUser"; diff --git a/src/event-db/stage_data/dev/00002_fund100_params.sql b/src/event-db/stage_data/dev/00002_fund100_params.sql index 3ac693b1f3..b97b283410 100644 --- a/src/event-db/stage_data/dev/00002_fund100_params.sql +++ b/src/event-db/stage_data/dev/00002_fund100_params.sql @@ -1,6 +1,6 @@ -- Define F100 IdeaScale parameters. INSERT INTO config (id, id2, id3, value) VALUES ( - 'ideascale, + 'ideascale', '100', '', '{