fix(postgresql): check the schema applies cleanly during the build, s…

…o that containers won't be created if they don't.

earthly/postgresql/Earthfile

@@ -30,6 +30,10 @@ postgres-base:
     COPY ./entry.sh .
     COPY ./setup-db.sql .
 
+    # Universal build scripts we will always need and are not target dependent.
+    COPY --dir scripts /scripts
+    COPY --dir ../../utilities/scripts+bash-scripts/include /scripts/include
+
 sqlfluff-image:
     FROM +postgres-base
 
@@ -51,7 +55,7 @@ BUILDER:
 CHECK:
     COMMAND
 
-    RUN sqlfluff lint .
+    RUN /scripts/std_checks.sh
 
 # Format sql files
 # REQUIREMENTS:
@@ -75,18 +79,17 @@ FORMAT:
 #  * prepare seed data files into the `./data` dir (optional)
 #  * prepare `refinery.toml` file
 # Arguments:
-#  * tag : The tag of the image, default value `latest`.
-#  * registry: The registry of the image.
 #  * image_name: The name of the image (required).
 BUILD:
     COMMAND
-
-    ARG tag="latest"
     ARG --required image_name
 
     USER postgres:postgres
 
+    RUN /scripts/std_build.sh
+
     ENTRYPOINT ["./entry.sh"]
 
     # Push the container...
-    SAVE IMAGE ${image_name}:$tag
+    SAVE IMAGE ${image_name}:latest
+

earthly/postgresql/entry.sh

@@ -84,7 +84,7 @@ if [[ "${DB_HOST}" == "localhost" ]]; then
     echo "POSTGRES_HOST_AUTH_METHOD is set to ${POSTGRES_HOST_AUTH_METHOD}"
 
     # Start PostgreSQL in the background
-    initdb -D /var/lib/postgresql/data || true
+    initdb -D /var/lib/postgresql/data --locale-provider=icu --icu-locale=en_US || true
     printf "\n host all all all %s \n" "${POSTGRES_HOST_AUTH_METHOD}" >> /var/lib/postgresql/data/pg_hba.conf
     pg_ctl -D /var/lib/postgresql/data start &
 fi

earthly/postgresql/scripts/README.md

@@ -0,0 +1,6 @@
+# Standard stage processing scripts
+
+These script files are used during the CI phases to simplify the Earthfiles and
+to improve maintainability.
+
+They need to be `bash` scripts, and they need to execute on an `alpine` os base.

earthly/postgresql/scripts/db_ops.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+# This script is not intended to be run by itself, and provides common functions
+# for database operations.
+
+# shellcheck disable=SC2120 
+function init_db() {
+    # Start PostgreSQL in the background
+    # $1 = Path to the DB data directory (optional)
+    local data_dir="${1:-/var/lib/postgresql/data}"
+    # $2 = Auth Method (Optional) = trust | md5 | scram-sha-256 | password | any other valid auth method
+    local trust_method="${POSTGRES_HOST_AUTH_METHOD:-${2:-trust}}"
+
+    echo "POSTGRES_HOST_AUTH_METHOD is set to ${trust_method}"
+
+    if ! initdb -D "${data_dir}" --locale-provider=icu --icu-locale=en_US; then
+        return 1
+    fi
+
+    printf "\n host all all all %s \n" "${trust_method}" >> /var/lib/postgresql/data/pg_hba.conf
+
+    return 0
+}
+
+function run_pgsql() {
+    # Function to start PostgreSQL, and wait for it to start
+    # $1 = Path to the DB data directory (optional)
+    local data_dir="${1:-/var/lib/postgresql/data}"
+    # $2 = timeout to wait in seconds.
+    local timeout="${2:-0}"
+
+    if ! pg_ctl -D /tmp/data start; then
+        return 1
+    fi
+
+    # Check if PostgreSQL is running using pg_isready
+    echo "Waiting $((timeout == 0 ? "Forever" : timeout)) for PostgreSQL to start..."
+    until pg_isready -d postgres >/dev/null 2>&1; do
+        sleep 1
+        if [[ ${timeout} -gt 0 ]]; then
+            timeout=$((timeout - 1))
+            if [[ ${timeout} -eq 0 ]]; then
+                echo "Timeout: PostgreSQL server did not start within the specified time"
+                return 1
+            fi
+        fi
+    done
+
+    echo "PostgreSQL is running"
+
+    return 0
+}
+
+function stop_pgsql() {
+    pg_ctl -D /tmp/data stop
+
+    return $?
+}
+
+# Custom function to run your desired SQL commands using psql
+function setup_db() {
+    local setup_db_sql="${1:-./setup-db.sql}"
+    local dbname="${DB_NAME:-${2:?}}"
+    local dbdesc="${DB_DESCRIPTION:-${3:?}}"
+    local dbuser="${DB_USER:-${4:?}}"
+    local dbuserpw="${DB_USER_PASSWORD:-${5:?}}"
+
+    psql -d postgres -f "${setup_db_sql}" \
+        -v dbName="${dbname}" \
+        -v dbDescription="${dbdesc}" \
+        -v dbUser="${dbuser}" \
+        -v dbUserPw="${dbuserpw}"
+
+    return $?
+}
+
+function migrate_schema() {
+    local dbname="${DB_NAME:-${1:?}}"
+    local dbhost="${DB_HOST:-${2:?}}"
+    local dbport="${DB_PORT:-${3:?}}"
+    local dbuser="${DB_USER:-${4:?}}"
+    local dbuserpw="${DB_USER_PASSWORD:-${5:?}}"
+
+    export DATABASE_URL="postgres://${dbuser}:${dbuserpw}@${dbhost}:${dbport}/${dbname}"
+    refinery migrate -e DATABASE_URL -c ./refinery.toml -p ./migrations
+}

earthly/postgresql/scripts/include

@@ -0,0 +1 @@
+../../../utilities/scripts/bash

earthly/postgresql/scripts/std_build.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# This script is run inside the `build` stage.
+# It validates that all the migrations and importable data are able to be
+# used without error.
+
+basedir=$(dirname "$0")
+
+source "${basedir}/include/colors.sh"
+source "${basedir}/db_ops.sh"
+
+# Init the db data in a tmp place
+status_and_exit "DB Initial Setup" \
+    init_db /tmp/data
+
+# Start the db server
+status_and_exit "DB Start" \
+    run_pgsql /tmp/data 10
+
+# Setup the base db namespace
+status_and_exit "DB Setup" \
+    setup_db ./setup-db.sql test "Test DB" test test
+
+# Run all migrations
+status_and_exit "DB Migrate" \
+    migrate_schema test localhost 5432 test test
+
+# Stop the database
+status_and_exit "DB Stop" \
+    stop_pgsql
+
+# We DO NOT want the tmp db in the final image, clean it up.
+rm -rf /tmp/data
+
+# These tests will immeditaley fail if the DB is not setup properly.
+exit 0

earthly/postgresql/scripts/std_checks.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# cspell: words fmtchk fmtfix rustfmt stdcfgs nextest 
+
+# This script is run inside the `check` stage for rust projects to perform all 
+# high level non-compilation checks.
+# These are the Standard checks which ALL rust targets must pass before they
+# will be scheduled to be `build`. 
+# Individual targets can add extra `check` steps, but these checks must always
+# pass. 
+
+basedir=$(dirname "$0")
+
+source "${basedir}/include/colors.sh"
+
+
+rc=0
+
+# This is set up so that ALL checks are run and it will fail if any fail.
+# This improves visibility into all issues that need to be corrected for `check`
+# to pass without needing to iterate excessively.
+
+## Check configs are as they should be.
+check_vendored_files "${rc}" .sqlfluff /root/.sqlfluff; rc=$?
+
+# Check sqlfluff linter against sql files.
+status "${rc}" "Checking SQLFluff Linter against SQL Files" sqlfluff lint -vv . ; rc=$?
+
+# Return an error if any of this fails.
+exit "${rc}"

examples/postgresql/Earthfile

@@ -40,16 +40,13 @@ format:
 
     DO ./../../earthly/postgresql+FORMAT --src=$(echo ${PWD})
 
+
 # build an event db docker image.
 #   CI target : true
-# Arguments:
-#  * tag: docker image `tag`.
 build:
     FROM +builder
 
-    ARG tag="latest"
-
-    DO ./../../earthly/postgresql+BUILD --image_name=example-db --tag=$tag
+    DO ./../../earthly/postgresql+BUILD --image_name=example-db
 
 # Internal: Test Scenario 1
 #   CI target : true

examples/postgresql/migrations/V2__example.sql

@@ -0,0 +1,7 @@
+CREATE TABLE address
+(
+  name VARCHAR PRIMARY KEY,
+  address TEXT NOT NULL,
+
+  FOREIGN KEY(name) REFERENCES users(name)
+);

examples/postgresql/seed/data2/example.sql

@@ -0,0 +1,12 @@
+INSERT INTO users (name, age) VALUES 
+('Alice', 20),
+('Bob', 30),
+('Charlie', 40),
+('Mary',22);
+
+INSERT INTO address (name, address) VALUES 
+('Alice', '123 Main St'),
+('Bob', '456 Oak St'),
+('Charlie', '789 Elm St'),
+('Mary', '321 Pine St');
+

utilities/scripts/bash/colors.sh

@@ -34,6 +34,11 @@ status() {
     return "${rc}"
 }
 
+status_and_exit() {
+    if ! status 0 "$@"; then
+        exit 1
+    fi
+}
 
 # Checks if two files that should exist DO, and are equal.
 # used to enforce consistency between local config files and the expected config locked in CI.