diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index a30381b51..a41be0ac2 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -11,6 +11,7 @@ Earthfiles errchkjson extldflags fontawesome +fontconfig ginkgolinter gitops glightbox diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..d97c5195d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +[*.sh] +# like -i=4 +indent_style = space +indent_size = 4 + +# --language-variant +shell_variant = bash +binary_next_line = true +# --case-indent +switch_case_indent = true +space_redirects = true +keep_padding = false +# --func-next-line +function_next_line = false \ No newline at end of file diff --git a/.gitignore b/.gitignore index 66f8abf70..efe357d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ node_modules /earthly/docs/local # Local Build Artefacts -target/ \ No newline at end of file +target/ + +# Python junk +**/*.pyc \ No newline at end of file diff --git a/earthly/postgresql/.sqlfluff b/.sqlfluff similarity index 96% rename from earthly/postgresql/.sqlfluff rename to .sqlfluff index bd784f286..3a43ddca9 100644 --- a/earthly/postgresql/.sqlfluff +++ b/.sqlfluff @@ -3,6 +3,7 @@ [sqlfluff] dialect = postgres large_file_skip_char_limit = 0 +max_line_length = 120 [sqlfluff:indentation] tab_space_size = 2 diff --git a/Earthfile b/Earthfile index 56a27d3bd..563135294 100644 --- a/Earthfile +++ b/Earthfile @@ -1,6 +1,5 @@ # Set the Earthly version to 0.7 -VERSION 0.7 -FROM debian:stable-slim +VERSION --global-cache 0.7 # cspell: words livedocs sitedocs @@ -18,6 +17,13 @@ markdown-check-fix: check-spelling: DO ./earthly/cspell+CHECK +# check-bash - test all bash files lint properly according to shellcheck. +check-bash: + FROM alpine:3.18 + + DO ./earthly/bash+SHELLCHECK --src=. + + ## ----------------------------------------------------------------------------- ## ## Standard CI targets. @@ -28,7 +34,9 @@ check-spelling: check: BUILD +check-spelling BUILD +check-markdown + BUILD +check-bash +# Internal: Reference to our repo root documentation used by docs builder. repo-docs: # Create artifacts of extra files we embed inside the documentation when its built. FROM scratch @@ -36,6 +44,14 @@ repo-docs: WORKDIR /repo COPY --dir *.md LICENSE-APACHE LICENSE-MIT . - #RUN ls -al /repo + + SAVE ARTIFACT /repo repo + +repo-config: + # Create artifacts of config file we need to refer to in builders. + FROM scratch + + WORKDIR /repo + COPY --dir .sqlfluff . SAVE ARTIFACT /repo repo \ No newline at end of file diff --git a/actions/configure-runner/Earthfile b/actions/configure-runner/Earthfile index d48eef04e..ed8fe45d8 100644 --- a/actions/configure-runner/Earthfile +++ b/actions/configure-runner/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM node:20-bookworm diff --git a/actions/discover/Earthfile b/actions/discover/Earthfile index d48eef04e..ed8fe45d8 100644 --- a/actions/discover/Earthfile +++ b/actions/discover/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM node:20-bookworm diff --git a/actions/install/Earthfile b/actions/install/Earthfile index d48eef04e..ed8fe45d8 100644 --- a/actions/install/Earthfile +++ b/actions/install/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM node:20-bookworm diff --git a/actions/merge/Earthfile b/actions/merge/Earthfile index d48eef04e..ed8fe45d8 100644 --- a/actions/merge/Earthfile +++ b/actions/merge/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM node:20-bookworm diff --git a/actions/push/Earthfile b/actions/push/Earthfile index d48eef04e..ed8fe45d8 100644 --- a/actions/push/Earthfile +++ b/actions/push/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM node:20-bookworm diff --git a/actions/run/Earthfile b/actions/run/Earthfile index d48eef04e..ed8fe45d8 100644 --- a/actions/run/Earthfile +++ b/actions/run/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM node:20-bookworm diff --git a/cli/Earthfile b/cli/Earthfile index c297cd46b..a623c5621 100644 --- a/cli/Earthfile +++ b/cli/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 FROM golang:1.20-alpine3.18 # cspell: words onsi ldflags extldflags diff --git a/docs/Earthfile b/docs/Earthfile index e17239dba..2960220c1 100644 --- a/docs/Earthfile +++ b/docs/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 # Copy all the source we need to build the docs src: @@ -8,6 +8,8 @@ src: # Now copy into that any artifacts we pull from the builds. COPY --dir ../+repo-docs/repo includes + # Copy docs we build in the postgres example. + COPY --dir ../examples/postgresql+build/docs src/appendix/examples/built_docs/postgresql # Build the docs here. docs: diff --git a/docs/src/appendix/earthly.md b/docs/src/appendix/earthly.md index 00add2aa4..b729cd9d6 100644 --- a/docs/src/appendix/earthly.md +++ b/docs/src/appendix/earthly.md @@ -75,7 +75,7 @@ Like a `Dockerfile`, only a single `Earthfile` can exist per directory and it *m #### Sample Structure ```Earthfile -VERSION 0.7 # This defines the "schema" that this Earthfile satisfies, much like the version of a Docker Compose file +VERSION --global-cache 0.7 # This defines the "schema version" that this Earthfile satisfies # A target, which is functionally equivalent to a `makefile` target. deps: @@ -104,7 +104,7 @@ A target can be thought of as a grouping of image layers, similar to the way mul Each target then specifies one or more commands that create the image layers associated with that target. ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM golang:1.20-alpine3.18 diff --git a/docs/src/guides/docs.md b/docs/src/guides/docs.md index d7797adc4..1c5831f9a 100644 --- a/docs/src/guides/docs.md +++ b/docs/src/guides/docs.md @@ -39,7 +39,7 @@ This folder already has an `Earthfile` in it, which contains all build process. ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 # Copy all the source we need to build the docs src: diff --git a/docs/src/guides/languages/bash.md b/docs/src/guides/languages/bash.md new file mode 100644 index 000000000..2f9948af9 --- /dev/null +++ b/docs/src/guides/languages/bash.md @@ -0,0 +1,123 @@ +--- +icon: material/bash +title: Bash Scripts +tags: + - Bash +--- + + +# :material-bash: Bash Scripts + + +## Introduction + + +!!! Tip + If you're just looking for a complete example, + [click here](https://github.com/input-output-hk/catalyst-ci/blob/master/Earthfile). + This guide will provide detailed instructions for how the example was built. + + +This guide will get you started with using the Catalyst CI to build projects that include Bash scripts. +By the end of the guide, we'll have an `Earthfile` that utilizes the Catalyst CI that can check your Bash scripts. + +Bash is not considered a stand alone target, although bash scripts are used extensively across multiple targets. +The Bash support consists solely of a single repo wide `check` target which validates: + +* Are any of the `bash` shell scripts redundant. + * This prevent maintenance issues where common scripts are copy/pasted rather than being properly organized. +* Do the bash scripts pass `shellcheck` lints. + * This forces us to follow a consistent style guide, and also checks for problematic Bash syntax. + +To begin, clone the Catalyst CI repository: + +## Adding Bash checks to your Repo that is already using Catalyst-CI + +Bash script checking is to be added to a repo that is already using Catalyst CI. + +All that needs to happen is the following be added to the `Earthfile` in the root of the repo. + +```Earthfile +# Internal: shell-check - test all bash files against our shell check rules. +shell-check: + FROM alpine:3.18 + + DO github.com/input-output-hk/catalyst-ci/earthly/bash:vx.y.z+SHELLCHECK --src=. + +# check all repo wide checks are run from here +check: + FROM alpine:3.18 + + # Lint all bash files. + BUILD +shell-check + +``` + + +!!! Note + It is expected that there may be multiple repo level `checks`. + This pattern shown above allows for this by separating the individual checks into their own targets. + The `check` target then just executed `BUILD` once for each check. + + +### Common Scripts + +It is not a good practice to copy bash scripts with common functionality. +Accordingly, the *Utility* target `./utilities/scripts+bash-scripts` exists to provide a central location for common scripts. +These are used locally to this repo and may be used by other repos using catalyst-ci. + +These scripts are intended to be used inside Earthly builds, and not locally. + +A common pattern to include these common scripts is the following: + +```Earthfile + # Copy our target specific scripts + COPY --dir scripts /scripts + + # Copy our common scripts so we can use them inside the container. + DO ../../utilities/scripts+ADD_BASH_SCRIPTS +``` + + +!!! Note + Always source scripts using `source "/scripts/include/something.sh"`. + This will ensure the scripts are properly located. + bash has no concept of the directory a script is located and so relative + source commands are unreliable. + + + +!!! Note + This is just an example, and you would adapt it to your specific requirements. + + +### Running checks + +From the root of the repo, you can check all bash scripts within the repo by running: + +```sh +earthly +check +``` + +This will also run all other repo-wide checks that are in use. + +### Build and test + +Bash scripts should not have a `build` target. +They can form part of the Build of other targets. + +### Releasing + +Bash scripts should not have a discreet `release` target. +They can form part of the `release` of other targets. + +### Publishing + +Bash scripts should not have a discreet `publish` target. +They can form part of the `publish` of other targets. + +## Conclusion + +You can see the final `Earthfile` [here](https://github.com/input-output-hk/catalyst-ci/blob/master/Earthfile). + +This `Earthfile` will check the quality of the Bash files within the Catalyst-CI repo. diff --git a/docs/src/guides/languages/go.md b/docs/src/guides/languages/go.md index 7744f01b8..595cdb279 100644 --- a/docs/src/guides/languages/go.md +++ b/docs/src/guides/languages/go.md @@ -46,7 +46,7 @@ You can choose to either delete the file and start from scratch, or read the gui ### Installing dependencies ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 deps: # This target is used to install external Go dependencies. diff --git a/docs/src/guides/languages/postgresql.md b/docs/src/guides/languages/postgresql.md index 3c31d5ad1..b6d86ac13 100644 --- a/docs/src/guides/languages/postgresql.md +++ b/docs/src/guides/languages/postgresql.md @@ -49,7 +49,7 @@ You can choose to either delete the file and start from scratch, or read the gui ### Prepare base builder ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 builder: FROM ./../../earthly/postgresql+postgres-base @@ -75,9 +75,14 @@ check: DO ./../../earthly/postgresql+CHECK +build-sqlfluff: + BUILD ./../../earthly/postgresql+sqlfluff-image + format: LOCALLY + RUN earthly +build-sqlfluff + DO ./../../earthly/postgresql+FORMAT --src=$(echo ${PWD}) ``` @@ -86,12 +91,6 @@ At this step we can begin performing static checks against `*.sql` files. These checks are intended to verify the code is healthy and well formatted to a certain standard and done with the help of the `sqlfluff` tool which is already configured during the `+postgres-base` target. - -!!! Note - Before perform formatting it is needed to build `sqlfluff-image` docker image using following command - `earthly github.com/input-output-hk/catalyst-ci/earthly/postgresql+sqlfluff-image` - - To apply and fix some formatting issues you can run `+format` target which will picks up directory where your Earthly file lies in as a source dir for formatting and run `+FORMAT` UDC target. Under the hood `+FORMAT` UDC target runs `sqlfluff-image` docker image, diff --git a/docs/src/guides/languages/python.md b/docs/src/guides/languages/python.md index 40c986516..a36d2ee24 100644 --- a/docs/src/guides/languages/python.md +++ b/docs/src/guides/languages/python.md @@ -45,7 +45,7 @@ or read the guide and follow along in the file. ### Prepare base builder ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 builder: FROM ./../../earthly/python+python-base diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md index 793db5b30..9f596ba42 100644 --- a/docs/src/guides/languages/rust.md +++ b/docs/src/guides/languages/rust.md @@ -49,7 +49,7 @@ Also we will take a look how we are setup Rust projects and what configuration i ### Prepare base builder ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 # Set up our target toolchains, and copy our files. builder: diff --git a/docs/src/style/index.md b/docs/src/style/index.md index de12dd0de..5d55acc97 100644 --- a/docs/src/style/index.md +++ b/docs/src/style/index.md @@ -19,7 +19,7 @@ Any `Earthfile` which does not adhere to this style guide will be rejected if no The following structure should be used to provide a consistent structure to `Earthfile`s: ```Earthfile -VERSION 0.7 # Should be the same across the repository +VERSION --global-cache 0.7 # Should be the same across the repository deps: FROM @@ -87,7 +87,7 @@ This target is made up of the commands that appear outside of an existing target For example: ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 FROM ubuntu:latest # Apart of the base target WORKDIR /work # Apart of the base target ``` @@ -104,7 +104,7 @@ As such, the base target should be avoided, and individual targets should be clear about their intentions: ```Earthfile -VERSION 0.7 +VERSION --global-cache 0.7 deps: FROM ubuntu:latest diff --git a/earthly/bash/Earthfile b/earthly/bash/Earthfile new file mode 100644 index 000000000..b1d09a400 --- /dev/null +++ b/earthly/bash/Earthfile @@ -0,0 +1,32 @@ +# cspell UDCs and Containers. +VERSION --global-cache 0.7 + +# Internal: builder creates a container we can use to execute shellcheck +builder: + FROM alpine:3.18 + + RUN apk add --no-cache \ + bash \ + shellcheck \ + shfmt + + WORKDIR /work + COPY check-all.sh shellcheck-dir.sh duplicated-scripts.sh . + + # Copy our common scripts so we can use them inside the container. + # Internally in the repo we use a symlink to accomplish this, but Earthly + # Will only copy the symlink and not what it references, so we need to + # manually copy what it references. This enables the script to work + # both in-repo and in-ci here. + DO ../../utilities/scripts+ADD_BASH_SCRIPTS + +# shellcheck - Check all shell files recursively in the src with shellcheck. +SHELLCHECK: + COMMAND + ARG --required src + + FROM +builder + + COPY --dir $src /src + + RUN ./check-all.sh /src diff --git a/earthly/bash/check-all.sh b/earthly/bash/check-all.sh new file mode 100755 index 000000000..482d25ead --- /dev/null +++ b/earthly/bash/check-all.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Run `shellcheck` on all files in a directory, recursively. +# $1 = The directory to check. + +# cspell: words FORCECOLOR + +# Get our includes relative to this file's location. +source "/scripts/include/colors.sh" + +rc=0 + +./duplicated-scripts.sh "$1" +rc_dup=$? + +./shellcheck-dir.sh "$1" +rc_lint=$? + +FORCECOLOR=1 shfmt -d "$1" +rc_shfmt=$? + +# Return an error if any of this fails. +status "${rc}" "Duplicated Bash Scripts" \ + [ "${rc_dup}" == 0 ] +rc=$? +status "${rc}" "Lint Errors in Bash Scripts" \ + [ "${rc_lint}" == 0 ] +rc=$? +status "${rc}" "ShellFmt Errors in Bash Scripts" \ + [ "${rc_shfmt}" == 0 ] +rc=$? + +if [[ ${rc_shfmt} -ne 0 ]]; then + echo "Shell files can be autoformatted with: 'shfmt -w .' from the root of the repo." +fi + +exit "${rc}" diff --git a/earthly/bash/duplicated-scripts.sh b/earthly/bash/duplicated-scripts.sh new file mode 100755 index 000000000..f65bab1de --- /dev/null +++ b/earthly/bash/duplicated-scripts.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Takes a single argument, the directory to check. + +# Get our includes relative to this file's location. +source "/scripts/include/colors.sh" + +rc=0 + +# Create an associative array to store the file contents and their corresponding filenames +declare -A files + +# Loop through all files with the .sh extension recursively, excluding symlinked directories +while IFS= read -r -d '' file; do + # Calculate the MD5 hash of the file's contents + hash=$(md5sum "${file}" | awk '{print $1}') || true + + # Check if the hash already exists in the array + if [[ -n "${files[${hash}]}" ]]; then + echo -e "${CYAN}Duplicated Bash Script: ${CYAN}${files[${hash}]} : ${RED}${file}${NC}" + rc=1 + else + # Add the hash and filename to the array + files[${hash}]=${file} + # Print the original file and the duplicate file + echo -e "${CYAN}New Bash Script: ${CYAN}${file}${NC}" + fi +done < <(find -P "$1" -type f -name "*.sh" -print0) || true + +exit "${rc}" diff --git a/earthly/bash/shellcheck-dir.sh b/earthly/bash/shellcheck-dir.sh new file mode 100755 index 000000000..5fd5ec49c --- /dev/null +++ b/earthly/bash/shellcheck-dir.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Run `shellcheck` on all files in a directory, recursively. +# $1 = The directory to check. + +# Import utility functions - only works inside containers. +source "/scripts/include/colors.sh" + +shopt -s globstar +rc=0 +for file in "$1"/**/*.sh; do + path=$(dirname "${file}") + filename=$(basename "${file}") + pushd "${path}" > /dev/null || return 1 + status "${rc}" "Checking Bash Lint of '${file}'" \ + shellcheck --check-sourced --external-sources --color=always --enable=all "${filename}" + popd > /dev/null || return 1 + + rc=$? +done + +exit "${rc}" diff --git a/earthly/cspell/Earthfile b/earthly/cspell/Earthfile index 38c61b3dc..5b9932233 100644 --- a/earthly/cspell/Earthfile +++ b/earthly/cspell/Earthfile @@ -1,5 +1,5 @@ # cspell UDCs and Containers. -VERSION 0.7 +VERSION --global-cache 0.7 CHECK: # Spell checking all docs and code is done with cspell diff --git a/earthly/docs/Earthfile b/earthly/docs/Earthfile index 024c4554d..e26dd20f1 100644 --- a/earthly/docs/Earthfile +++ b/earthly/docs/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 # cspell: words libgcc freetype lcms openjpeg etag @@ -14,6 +14,8 @@ deps: RUN apk add --no-cache \ bash \ graphviz \ + fontconfig \ + ttf-liberation \ curl \ zlib-dev \ jpeg-dev \ @@ -30,13 +32,18 @@ deps: tiff-dev \ tk-dev \ tcl-dev \ - git + git + + # Fix up font cache + RUN fc-cache -f # Install poetry and our python dependencies. DO ../python+BUILDER # Copy our run scripts COPY --dir scripts /scripts + # Copy our common scripts so we can use them inside the container. + DO ../../utilities/scripts+ADD_BASH_SCRIPTS # Trust directory, required for git >= 2.35.2. (mkdocs Git plugin requirement). RUN git config --global --add safe.directory /docs &&\ diff --git a/earthly/docs/pyproject.toml b/earthly/docs/pyproject.toml index bee5dc41b..4b37278b7 100644 --- a/earthly/docs/pyproject.toml +++ b/earthly/docs/pyproject.toml @@ -3,7 +3,7 @@ name = "Documentation Tooling" version = "0.1.0" description = "Common Project Catalyst Documentation Tooling." authors = ["Joshua Gilman", "Steven Johnson"] -readme = "README.md" +readme = "Readme.md" [tool.poetry.dependencies] python = "^3.12" diff --git a/earthly/docs/scripts/build.sh b/earthly/docs/scripts/build.sh index 29c07aa95..cecf14022 100755 --- a/earthly/docs/scripts/build.sh +++ b/earthly/docs/scripts/build.sh @@ -2,20 +2,19 @@ # Builds the documentation to the `/site` directory inside the container. -#!/usr/bin/env bash - -source "$(dirname "$0")/colors.sh" +source "/scripts/include/colors.sh" rc=0 ## Build the code -status $rc "Changing to Poetry Environment Workspace" \ - cd /poetry; rc=$? +status "${rc}" "Changing to Poetry Environment Workspace" \ + cd /poetry +rc=$? -## Building the documentation. -status $rc "Building Documentation" \ - poetry run mkdocs -v --color build --strict --clean -f /docs/mkdocs.yml -d /site; rc=$? +## Building the documentation. +status "${rc}" "Building Documentation" \ + poetry run mkdocs -v --color build --strict --clean -f /docs/mkdocs.yml -d /site +rc=$? # Return an error if any of this fails. -exit $rc - +exit "${rc}" diff --git a/earthly/go/Earthfile b/earthly/go/Earthfile index 6789439d5..e459ae29e 100644 --- a/earthly/go/Earthfile +++ b/earthly/go/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 DEPS: COMMAND diff --git a/earthly/mdlint/Earthfile b/earthly/mdlint/Earthfile index 2e18dd2a9..a41b5a877 100644 --- a/earthly/mdlint/Earthfile +++ b/earthly/mdlint/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 # cspell: words markdownlint MDLINT_LOCALLY: diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index 1e4bb4520..0a0e3647f 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -1,7 +1,7 @@ # Common PostgreSQL Earthly builders -VERSION 0.7 +VERSION --global-cache 0.7 -# cspell: words colordiff +# cspell: words colordiff nushell psycopg dbviz postgres-base: FROM postgres:16.0-alpine3.18 @@ -17,7 +17,15 @@ postgres-base: musl-dev \ curl \ gcc \ - py3-pip + py3-pip \ + py3-rich \ + py3-psycopg \ + graphviz \ + fontconfig \ + ttf-liberation + + # Fix up font cache + RUN fc-cache -f # Install SQLFluff RUN pip3 install sqlfluff==2.3.5 @@ -26,47 +34,119 @@ postgres-base: # Get refinery COPY ../rust+rust-base/refinery /bin - COPY ./.sqlfluff . - COPY ./entry.sh . - COPY ./setup-db.sql . + # Get dbviz + COPY ../../utilities/dbviz+build/dbviz /bin + RUN dbviz --help -sqlfluff-image: - FROM +postgres-base + # Get nushell + COPY ../../utilities/nushell+nushell-build/nu /bin - WORKDIR /sql + # Copy our set SQL files + COPY --dir sql /sql - RUN cp /root/.sqlfluff . + # Universal build scripts we will always need and are not target dependent. + COPY --dir scripts /scripts + # Copy our common scripts so we can use them inside the container. + DO ../../utilities/scripts+ADD_BASH_SCRIPTS + DO ../../utilities/scripts+ADD_PYTHON_SCRIPTS - SAVE IMAGE sqlfluff-image:latest + SAVE ARTIFACT /scripts /scripts -# Common PostgreSQL setup. -# Prepare important files needed for building image +# Common build setup steps. +# Arguments: +# sqlfluff_cfg - REQUIRED - Location of repos .sqlfluff configuration file. +# migrations - OPTIONAL - Location of Migrations directory - DEFAULT: ./migrations +# seed - OPTIONAL - Location of Seed data directory - DEFAULT: ./seed +# refinery_toml - OPTIONAL - Location of refinery,toml which configures migrations. DEFAULT: ./refinery.toml BUILDER: COMMAND - RUN cp /root/.sqlfluff . - RUN cp /root/entry.sh . - RUN chmod ugo+x ./entry.sh - RUN cp /root/setup-db.sql . + ARG migrations=./migrations + ARG seed=./seed + ARG refinery_toml=./refinery.toml + + FROM +postgres-base + + WORKDIR /build + + COPY --dir $sqlfluff_cfg . + COPY --dir $migrations . + COPY --dir $seed . + COPY --dir $refinery_toml . + +# DOCS - UDC to build the docs, needs to be run INSIDE the BUILDER like so: +# +# 1. Create a ./docs/diagrams.json which has the options needed to run to generate the docs to /docs +# 2. Define the following targets in your earthfile +# +# builder: +# DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+BUILDER --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff +# +# build: +# FROM +builder +# +# DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+BUILD --image_name= +# DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+DOCS +DOCS: + COMMAND + + ARG diagrams=./diagrams.json + ARG migrations=./migrations + ARG refinery_toml=./refinery.toml + + USER postgres:postgres + WORKDIR /docs + + COPY $diagrams ./diagrams.json + COPY --dir $migrations . + COPY --dir $refinery_toml . + + RUN /scripts/std_docs.py ./diagrams.json + + SAVE ARTIFACT docs /docs + + # Linter checks for sql files CHECK: COMMAND - RUN sqlfluff lint . + RUN /scripts/std_checks.sh # Format sql files -# REQUIREMENTS: -# * build `sqlfluff-image` image via `earthly +sqlfluff-image` target -# Arguments: -# * src : Source directory with the sql files. +# Just explains how to auto-format sql files. +# Can not do it LOCALLY as the command will change ownership of the fixed files +# if run within docker. FORMAT: COMMAND - # Where we want to run the `lint` from. - ARG --required src + # Can not run format LOCALLY as it changes permissions of files to root:root + RUN --no-cache printf "%s\n%s\n%s\n" \ + "SQL can only be formatted locally from the command line." \ + "Run locally installed 'sqlfluff' from the repo root:" \ + " $ sqlfluff fix ." + +# Internal: Integration Test container image +integration-test: + FROM earthly/dind:alpine-3.18-docker-23.0.6-r7 + + RUN apk add --no-cache \ + bash \ + postgresql15-client # We use PostgreSQL 16, but v15 clients will work OK with it. + + # Get nushell + COPY ../../utilities/nushell+nushell-build/nu /bin + + COPY +postgres-base/scripts /scripts + + WORKDIR /test + + +INTEGRATION_TEST_SETUP: + COMMAND + + FROM +integration-test - RUN docker run --rm -v $src:/sql sqlfluff-image:latest sqlfluff format . # Build PostgreSQL image. # REQUIREMENTS: @@ -74,19 +154,17 @@ FORMAT: # * prepare seed data files into the `./data` dir (optional) # * prepare `refinery.toml` file # Arguments: -# * tag : The tag of the image, default value `latest`. -# * registry: The registry of the image. # * image_name: The name of the image (required). BUILD: COMMAND - - ARG tag="latest" - ARG registry ARG --required image_name USER postgres:postgres - ENTRYPOINT ["./entry.sh"] + RUN /scripts/std_build.sh + + ENTRYPOINT ["/scripts/entry.sh"] # Push the container... - SAVE IMAGE --push ${registry}${image_name}:$tag + SAVE IMAGE ${image_name}:latest + diff --git a/earthly/postgresql/entry.sh b/earthly/postgresql/entry.sh deleted file mode 100644 index ecc1850a3..000000000 --- a/earthly/postgresql/entry.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env bash - -# cspell: words REINIT PGHOST PGPORT PGUSER PGPASSWORD psql initdb isready dotglob - -# --------------------------------------------------------------- -# Entrypoint script for database container -# --------------------------------------------------------------- -# -# This script serves as the entrypoint for the general database container. It sets up -# the environment, performing optional database initialization if configured, -# and then runs the migrations. -# -# It expects the following environment variables to be set except where noted: -# -# DB_HOST - The hostname of the database server -# DB_PORT - The port of the database server -# DB_NAME - The name of the database -# DB_DESCRIPTION - The description of the database -# DB_SUPERUSER - The username of the database superuser -# DB_SUPERUSER_PASSWORD - The password of the database superuser -# DB_USER - The username of the database user -# DB_USER_PASSWORD - The password of the database user -# DEBUG - If set, the script will print debug information (optional) -# DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional) -# STAGE - The stage being run. Currently only controls if stage specific data is applied to the DB (optional) -# --------------------------------------------------------------- -set +x -set -o errexit -set -o pipefail -set -o nounset -set -o functrace -set -o errtrace -set -o monitor -set -o posix -shopt -s dotglob - -check_env_vars() { - local env_vars=("$@") - - # Iterate over the array and check if each variable is set - for var in "${env_vars[@]}"; do - echo "Checking $var" - if [ -z "${!var:-}" ]; then - echo ">>> Error: $var is required and not set." - exit 1 - fi - done -} - -debug_sleep() { - if [ -n "${DEBUG_SLEEP:-}" ]; then - echo "DEBUG_SLEEP is set. Sleeping for ${DEBUG_SLEEP} seconds..." - sleep "${DEBUG_SLEEP}" - fi -} - -echo ">>> Starting entrypoint script..." - -# Check if all required environment variables are set -REQUIRED_ENV=( - "DB_HOST" - "DB_PORT" - "DB_NAME" - "DB_DESCRIPTION" - "DB_SUPERUSER" - "DB_SUPERUSER_PASSWORD" - "DB_USER" - "DB_USER_PASSWORD" -) -check_env_vars "${REQUIRED_ENV[@]}" - -# Export environment variables -export PGHOST="${DB_HOST}" -export PGPORT="${DB_PORT}" -export PGUSER="${DB_SUPERUSER}" -export PGPASSWORD="${DB_SUPERUSER_PASSWORD}" - -# Sleep if DEBUG_SLEEP is set -debug_sleep - -# Run postgreSQL database in this container if the host is localhost -if [ "${DB_HOST}" == "localhost" ]; then - POSTGRES_HOST_AUTH_METHOD=${POSTGRES_HOST_AUTH_METHOD:-trust} - echo "POSTGRES_HOST_AUTH_METHOD is set to ${POSTGRES_HOST_AUTH_METHOD}" - - # Start PostgreSQL in the background - initdb -D /var/lib/postgresql/data || true - printf "\n host all all all %s \n" "$POSTGRES_HOST_AUTH_METHOD" >> /var/lib/postgresql/data/pg_hba.conf - pg_ctl -D /var/lib/postgresql/data start & -fi - -# Check if PostgreSQL is running using psql -echo "Waiting for PostgreSQL to start..." -# Set the timeout value in seconds (default: 0 = wait forever) -TIMEOUT=${TIMEOUT:-0} -echo "TIMEOUT is set to ${TIMEOUT}" -until pg_isready -d postgres >/dev/null 2>&1; do - sleep 1 - if [ $TIMEOUT -gt 0 ]; then - TIMEOUT=$((TIMEOUT - 1)) - if [ $TIMEOUT -eq 0 ]; then - echo "Timeout: PostgreSQL server did not start within the specified time" - exit 1 - fi - fi -done -echo "PostgreSQL is running" - -# Initialize and drop database if necessary -if [ "${INIT_AND_DROP_DB:-}" == "true" ]; then - echo ">>> Initializing database..." - psql -d postgres -f ./setup-db.sql \ - -v dbName="${DB_NAME}" \ - -v dbDescription="${DB_DESCRIPTION}" \ - -v dbUser="${DB_USER}" \ - -v dbUserPw="${DB_USER_PASSWORD}" -fi - -# Run migrations -if [ "${WITH_MIGRATIONS:-}" == "true" ]; then - echo ">>> Running migrations..." - export DATABASE_URL="postgres://${DB_USER}:${DB_USER_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}" - refinery migrate -e DATABASE_URL -c ./refinery.toml -p ./migrations -fi - -# Apply seed data -if [ "${WITH_SEED_DATA:-}" == "true" ]; then - echo ">>> Applying seed data..." - while IFS= read -r -d '' file; do - echo "Applying seed data from $file" - psql -d $DB_NAME -f "$file" - done < <(find ./data -name '*.sql' -print0 | sort -z) -fi - -echo ">>> Finished entrypoint script" - -# Infinite loop to run until local PostgreSQL is ready -until [ "${DB_HOST}" == "localhost" ] && ! pg_isready -d postgres >/dev/null 2>&1; -do - sleep 60 -done - diff --git a/earthly/postgresql/scripts/README.md b/earthly/postgresql/scripts/README.md new file mode 100644 index 000000000..df629cafc --- /dev/null +++ b/earthly/postgresql/scripts/README.md @@ -0,0 +1,6 @@ +# Standard stage processing scripts + +These script files are used during the CI phases to simplify the Earthfiles and +to improve maintainability. + +They need to be `bash` scripts, and they need to execute on an `alpine` os base. diff --git a/earthly/postgresql/scripts/entry.sh b/earthly/postgresql/scripts/entry.sh new file mode 100755 index 000000000..cb4e244bf --- /dev/null +++ b/earthly/postgresql/scripts/entry.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# cspell: words REINIT PGHOST PGPORT PGUSER PGPASSWORD psql initdb isready dotglob +# cspell: words dbhost dbdesc dbdescription dbpath pgsql + +# --------------------------------------------------------------- +# Entrypoint script for database container +# --------------------------------------------------------------- +# +# This script serves as the entrypoint for the general database container. It sets up +# the environment, performing optional database initialization if configured, +# and then runs the migrations. +# +# All ENVVARS are optional, although the passwords should be set for security reasons. +# +# DB_HOST - The hostname of the database server +# DB_PORT - The port of the database server +# DB_NAME - The name of the database +# DB_DESCRIPTION - The description of the database +# DB_SUPERUSER - The username of the database superuser +# DB_SUPERUSER_PASSWORD - The password of the database superuser +# DB_USER - The username of the database user +# DB_USER_PASSWORD - The password of the database user +# DEBUG - If set, the script will print debug information (optional) +# DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional) +# STAGE - The stage being run. Currently only controls if stage specific data is applied to the DB (optional) +# --------------------------------------------------------------- + +source "/scripts/include/db_ops.sh" +source "/scripts/include/debug.sh" + +set +x +shopt -s dotglob + +show_db_config + +dbhost=$(get_param dbhost env_vars defaults "$@") +initdb=$(get_param init_and_drop_db env_vars defaults "$@") +migrations=$(get_param with_migrations env_vars defaults "$@") +dbdesc=$(get_param dbdescription env_vars defaults "$@") +dbpath=$(get_param dbpath env_vars defaults "$@") + +echo ">>> Starting entrypoint script for DB: ${dbdesc} @ ${dbhost}..." + +# Enforce that we must supply passwords as env vars +REQUIRED_ENV=( + "DB_SUPERUSER_PASSWORD" + "DB_USER_PASSWORD" +) +check_env_vars "${REQUIRED_ENV[@]}" + +# Sleep if DEBUG_SLEEP is set +debug_sleep + +# Run postgreSQL database in this container if the host is localhost +if [[ "${dbhost}" == "localhost" ]]; then + + if [[ "${initdb}" == "true" ]]; then + rm -rf "${dbpath}" 2> /dev/null + status 0 "Local DB Data Purge" \ + true + fi + + # Init the db data in a tmp place + status_and_exit "DB Initial Setup" \ + init_db "$@" + + # Start the db server + status_and_exit "DB Start" \ + run_pgsql "$@" +fi + +# Wait for the DB server to actually start +status_and_exit "Waiting for the DB to be Ready" \ + wait_ready_pgsql "$@" + +# Initialize and drop database if necessary +if [[ "${initdb}" == "true" ]]; then + # Setup the base db namespace + status_and_exit "Initial DB Setup - Clearing all DB data" \ + setup_db ./setup-db.sql "$@" +fi + +# Run migrations +if [[ "${migrations}" == "true" ]]; then + # Run all migrations + status_and_exit "Running Latest DB Migrations" \ + migrate_schema "$@" +fi + +# Apply seed data +seed_database + +if [[ "${dbhost}" == "localhost" ]]; then + echo ">>> Waiting until the Database terminates: ${dbdesc} @ ${dbhost}..." + # Infinite loop until the DB stops, because we are serving the DB from this container. + wait_pgsql_stopped +fi + +echo ">>> Finished DB entrypoint script for DB: ${dbdesc} @ ${dbhost}..." diff --git a/earthly/postgresql/scripts/python b/earthly/postgresql/scripts/python new file mode 120000 index 000000000..168bb203b --- /dev/null +++ b/earthly/postgresql/scripts/python @@ -0,0 +1 @@ +../../../utilities/scripts/python \ No newline at end of file diff --git a/earthly/postgresql/scripts/std_build.sh b/earthly/postgresql/scripts/std_build.sh new file mode 100755 index 000000000..149811344 --- /dev/null +++ b/earthly/postgresql/scripts/std_build.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# cspell: words dbname dbdesc dbpath pgsql dbreadytimeout + +# This script is run inside the `build` stage. +# It validates that all the migrations and importable data are able to be +# used without error. + +source "/scripts/include/colors.sh" +source "/scripts/include/db_ops.sh" + +setup_and_migrate() { + local reason="$1" + shift 1 + + # Setup the base db namespace + status_and_exit "${reason}: DB Setup" \ + setup_db ./setup-db.sql "$@" --dbname=test --dbdesc="Test DB" + + # Run all migrations + status_and_exit "${reason}: DB Migrate" \ + migrate_schema "$@" --dbname=test +} + +# Init the db data in a tmp place +status_and_exit "DB Initial Setup" \ + init_db "$@" --dbpath="/tmp/data" + +# Start the db server +status_and_exit "DB Start" \ + run_pgsql "$@" --dbpath="/tmp/data" + +# Wait for the DB server to actually start +status_and_exit "DB Ready" \ + wait_ready_pgsql "$@" --dbreadytimeout="10" + +# Setup Schema and run migrations. +setup_and_migrate "Initialization" "$@" + +# Test each seed data set can apply cleanly +rc=0 +while IFS= read -r -d '' file; do + status "${rc}" "Applying seed data from ${file}" \ + apply_seed_data "${file}" --dbname=test + rc=$? + + # Reset schema so all seed data get applied to a clean database. + setup_and_migrate "Reset" "$@" +done < <(find ./seed/* -maxdepth 1 -type d -print0 | sort -z) || true + +# Stop the database +status_and_exit "DB Stop" \ + stop_pgsql --dbpath="/tmp/data" + +# We DO NOT want the tmp db in the final image, clean it up. +rm -rf /tmp/data + +# These tests will immediately fail if the DB is not setup properly. +exit "${rc}" diff --git a/earthly/postgresql/scripts/std_checks.sh b/earthly/postgresql/scripts/std_checks.sh new file mode 100755 index 000000000..85364029a --- /dev/null +++ b/earthly/postgresql/scripts/std_checks.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# cspell: words fmtchk fmtfix rustfmt stdcfgs nextest + +# This script is run inside the `check` stage for rust projects to perform all +# high level non-compilation checks. +# These are the Standard checks which ALL rust targets must pass before they +# will be scheduled to be `build`. +# Individual targets can add extra `check` steps, but these checks must always +# pass. + +source "/scripts/include/colors.sh" + +rc=0 + +# This is set up so that ALL checks are run and it will fail if any fail. +# This improves visibility into all issues that need to be corrected for `check` +# to pass without needing to iterate excessively. + +# Check configs are as they should be. +check_vendored_files "${rc}" .sqlfluff /sql/.sqlfluff +rc=$? + +# Check sqlfluff linter against global sql files. +status "${rc}" "Checking SQLFluff Linter against Global SQL Files" sqlfluff lint -vv /sql +rc=$? + +# Check sqlfluff linter against target sql files. +status "${rc}" "Checking SQLFluff Linter against Project SQL Files" sqlfluff lint -vv . +rc=$? + +# Return an error if any of this fails. +exit "${rc}" diff --git a/earthly/postgresql/scripts/std_docs.py b/earthly/postgresql/scripts/std_docs.py new file mode 100755 index 000000000..5f088c2f7 --- /dev/null +++ b/earthly/postgresql/scripts/std_docs.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 + +# cspell: words dbmigrations dbviz dbhost dbuser dbuserpw Tsvg + +from typing import Optional +import python.cli as cli +import python.db_ops as db_ops +import argparse +import rich +from rich import print +import os +import re +import json +from dataclasses import dataclass +from textwrap import indent + + +@dataclass +class DiagramCfg: + title: str + version: int + migration_name: str + tables: Optional[list[str]] + included_tables: Optional[list[str]] + excluded_tables: Optional[list[str]] + comments: Optional[bool] + column_description_wrap: Optional[int] + table_description_wrap: Optional[int] + sql_data: str + + def include( + self, + extra_includes: Optional[list[str]] = None, + extra_excludes: Optional[list[str]] = None, + ) -> Optional[list[str]]: + # We exclude from the global include tables, any tables the migration + # itself requests to be excluded. + + include_tables = self.included_tables if self.included_tables else [] + tables = self.tables if self.tables else [] + extra_includes = extra_includes if extra_includes else [] + excluded_tables = self.excluded_tables if self.excluded_tables else [] + extra_excludes = extra_excludes if extra_excludes else [] + + for table in tables + extra_includes: + if ( + table not in excluded_tables + and table not in extra_excludes + and table not in include_tables + ): + include_tables.append(table) + + if len(include_tables) == 0: + include_tables = None + + return include_tables + + def exclude( + self, extra_excludes: Optional[list[str]] = None + ) -> Optional[list[str]]: + # We exclude from the global exclude tables, any tables the migration + # specifically includes. + exclude_tables = self.excluded_tables if self.excluded_tables else [] + extra_excludes = extra_excludes if extra_excludes else [] + for table in extra_excludes: + if table not in exclude_tables: + exclude_tables.append(table) + + if len(exclude_tables) == 0: + exclude_tables = None + + return exclude_tables + + +def process_sql_files(directory): + file_pattern = r"V(\d+)__(\w+)\.sql" + table_pattern = r"CREATE TABLE(?: IF NOT EXISTS)? (\w+)" + + diagram_option_pattern = r"^--\s*(Title|Include|Exclude|Comment|Column Description Wrap|Table Description Wrap)\s+:\s*(.*)$" + + migrations = {} + largest_version = 0 + + for filename in os.listdir(directory): + clean_sql = "" + title = None + table_names = [] + included_tables = None + excluded_tables = None + comments = None + column_description_wrap = None + table_description_wrap = None + + match = re.match(file_pattern, filename) + if match: + version = int(match.group(1)) + migration_name = match.group(2) + + if version > largest_version: + largest_version = version + + with open(os.path.join(directory, filename), "r") as file: + sql_data = file.read() + for line in sql_data.splitlines(): + match = re.match(diagram_option_pattern, line) + if match: + if match.group(1).lower() == "title" and title is None: + title = match.group(2) + elif ( + match.group(1).lower() == "include" + and len(match.group(2)) > 0 + ): + if included_tables is None: + included_tables = [] + included_tables.append(match.group(2).split()) + elif ( + match.group(1).lower() == "exclude" + and len(match.group(2)) > 0 + ): + if excluded_tables is None: + excluded_tables = [] + excluded_tables.append(match.group(2).split()) + elif match.group(1).lower() == "comment": + if match.group(2).strip().lower() == "true": + comments = True + elif match.group(1).lower() == "column description wrap": + try: + column_description_wrap = int(match.group(2)) + except: + pass + elif match.group(1).lower() == "table description wrap": + try: + table_description_wrap = int(match.group(2)) + except: + pass + else: + # We strip diagram options from the SQL. + clean_sql += line + "\n" + + match = re.match(table_pattern, line) + if match: + table_names.append(match.group(1)) + + migrations[version] = DiagramCfg( + title, + version, + migration_name, + table_names, + included_tables, + excluded_tables, + comments, + column_description_wrap, + table_description_wrap, + clean_sql, + ) + + return migrations, largest_version + + +class Migrations: + def __init__(self, args: argparse.Namespace): + """ + Initialize the class with the given arguments. + + Args: + args (argparse.Namespace): The command line arguments. + + Returns: + None + """ + self.args = args + + with open(args.diagram_config) as f: + self.config = json.load(f) + + self.migrations, self.migration_version = process_sql_files(args.dbmigrations) + + def schema_name(self) -> str: + return self.config.get("name", "Database Schema") + + def all_schema_comments(self) -> bool: + return self.config.get("all_schema", {}).get("comments", False) + + def full_schema_comments(self) -> bool: + return self.config.get("full_schema", {}).get( + "comments", self.all_schema_comments() + ) + + def all_schema_included_tables(self) -> list[str]: + return self.config.get("all_schema", {}).get("included_tables", []) + + def all_schema_excluded_tables(self) -> list[str]: + return self.config.get("all_schema", {}).get("excluded_tables", []) + + def full_schema_excluded_tables(self) -> list[str]: + return self.config.get("full_schema", {}).get( + "excluded_tables", self.all_schema_excluded_tables() + ) + + def all_schema_column_description_wrap(self) -> int: + return self.config.get("all_schema", {}).get("column_description_wrap", 50) + + def full_schema_column_description_wrap(self) -> int: + return self.config.get("full_schema", {}).get( + "column_description_wrap", self.all_schema_column_description_wrap() + ) + + def all_schema_table_description_wrap(self) -> int: + return self.config.get("all_schema", {}).get("table_description_wrap", 50) + + def full_schema_table_description_wrap(self) -> int: + return self.config.get("full_schema", {}).get( + "table_description_wrap", self.all_schema_table_description_wrap() + ) + + def dbviz( + self, + filename: str, + name: str, + title: str, + included_tables: Optional[list[str]] = None, + excluded_tables: Optional[list[str]] = None, + comments: Optional[bool] = None, + column_description_wrap: Optional[int] = None, + table_description_wrap: Optional[int] = None, + ) -> cli.Result: + if len(title) > 0: + title = f' --title "{title}"' + + includes = "" + if included_tables: + for table in included_tables: + includes += f" -i {table}" + + excludes = "" + if excluded_tables: + for table in excluded_tables: + excludes += f" -e {table}" + + if comments: + comments = " --comments" + else: + comments = "" + + if column_description_wrap and column_description_wrap > 0: + column_description_wrap = ( + f" --column-description-wrap {column_description_wrap}" + ) + else: + column_description_wrap = "" + + if table_description_wrap and table_description_wrap > 0: + table_description_wrap = ( + f" --table-description-wrap {table_description_wrap}" + ) + else: + table_description_wrap = "" + + res = cli.run( + f"dbviz -d {self.args.dbname}" + + f" -h {self.args.dbhost}" + + f" -u {self.args.dbuser}" + + f" -p {self.args.dbuserpw}" + + f"{title}" + + f"{includes}" + + f"{excludes}" + + f"{comments}" + + f"{column_description_wrap}" + + f"{table_description_wrap}" + + f" | dot -Tsvg -o {filename}", + # + f" > {filename}.dot", + name=f"Generate Schema Diagram: {name}", + verbose=True, + ) + + # if res.ok: + # cli.run( + # f"dot -Tsvg {filename}.dot -o {filename}", + # name=f"Render Schema Diagram to SVG: {name}", + # verbose=True, + # ) + + return res + + def full_schema_diagram(self) -> cli.Result: + # Create a full Schema Diagram. + return self.dbviz( + "docs/full-schema.svg", + "Full Schema", + self.schema_name(), + excluded_tables=self.full_schema_excluded_tables(), + comments=self.full_schema_comments(), + column_description_wrap=self.full_schema_column_description_wrap(), + table_description_wrap=self.full_schema_table_description_wrap(), + ) + + def migration_schema_diagram(self, ver: int) -> cli.Result: + # Create a schema diagram for an individual migration. + if ver in self.migrations: + migration = self.migrations[ver] + + include_tables = migration.include( + self.all_schema_included_tables(), self.all_schema_excluded_tables() + ) + if include_tables is None: + return cli.Result( + 0, + "", + "", + 0.0, + f"Migration {ver} has no tables to diagram.", + ) + + exclude_tables = migration.exclude(self.all_schema_excluded_tables()) + + title = f"{migration.migration_name}" + if migration.title and len(migration.title) > 0: + title = migration.title + + comments=None + if migration.comments is not None: + comments = migration.comments + else: + comments = self.all_schema_comments() + + return self.dbviz( + f"docs/migration-{ver}.svg", + f"V{ver}__{migration.migration_name}", + title, + included_tables=include_tables, + excluded_tables=exclude_tables, + comments=comments, + column_description_wrap=migration.column_description_wrap, + table_description_wrap=migration.table_description_wrap, + ) + + def create_diagrams(self, results: cli.Results) -> cli.Results: + # Create a full Schema Diagram first. + res = self.full_schema_diagram() + results.add(res) + + for ver in sorted(self.migrations.keys()): + res = self.migration_schema_diagram(ver) + results.add(res) + + # cli.run("ls -al docs", verbose=True) + + return results + + def create_markdown_file(self, file_path): + with open(file_path, "w") as markdown_file: + # Write the title with the maximum migration version + markdown_file.write( + "# Migrations (Version {}) \n\n".format(self.migration_version) + ) + + # Link the full schema diagram. + markdown_file.write('??? example "Full Schema Diagram"\n\n') + markdown_file.write( + ' ![Full Schema](./full-schema.svg "Full Schema")\n\n' + ) + + # Write the contents of each file in order + for version in sorted(self.migrations.keys()): + migration = self.migrations[version] + sql_data = migration.sql_data.strip() + + # Write the title of the file + markdown_file.write(f"## {migration.migration_name}\n\n") + + if os.path.exists(f"docs/migration-{version}.svg"): + markdown_file.write('??? example "Schema Diagram"\n\n') + markdown_file.write( + f" ![Migration {migration.migration_name}]" + + f'(./migration-{version}.svg "{migration.migration_name}")\n\n' + ) + + markdown_file.write('??? abstract "Schema Definition"\n\n') + markdown_file.write( + indent(f"```postgres\n{sql_data}\n```", " ") + "\n\n" + ) + + print("Markdown file created successfully at: {}".format(file_path)) + + +def main(): + # Force color output in CI + rich.reconfigure(color_system="256") + + parser = argparse.ArgumentParser( + description="Standard Postgresql Documentation Processing." + ) + parser.add_argument("diagram_config", help="Diagram Configuration JSON") + parser.add_argument("--verbose", action="store_true", help="Enable verbose output") + db_ops.add_args(parser) + + args = parser.parse_args() + + db = db_ops.DBOps(args) + + results = cli.Results("Generate Database Documentation") + + # Init the DB. + res = db.init_database() + results.add(res) + + if res.ok(): + db.start() + res = db.wait_ready(timeout=10) + results.add(res) + + if res.ok(): + res = db.setup() + results.add(res) + + if res.ok(): + res = db.migrate_schema() + results.add(res) + + if res.ok(): + cli.run("mkdir docs") # Where we build the docs. + + # Get all info about the migrations. + migrations = Migrations(args) + results = migrations.create_diagrams(results) + + if results.ok(): + migrations.create_markdown_file("docs/migrations.md") + # cli.run("cat /tmp/migrations.md", verbose=True) + + results.print() + + if not results.ok(): + exit(1) + + +if __name__ == "__main__": + main() diff --git a/earthly/postgresql/sql/.sqlfluff b/earthly/postgresql/sql/.sqlfluff new file mode 100644 index 000000000..3a43ddca9 --- /dev/null +++ b/earthly/postgresql/sql/.sqlfluff @@ -0,0 +1,24 @@ +# cspell: words capitalisation + +[sqlfluff] +dialect = postgres +large_file_skip_char_limit = 0 +max_line_length = 120 + +[sqlfluff:indentation] +tab_space_size = 2 + +[sqlfluff:rules:layout.long_lines] +ignore_comment_lines = True +ignore_comment_clauses = True + +[sqlfluff:rules:capitalisation.keywords] +capitalisation_policy = upper +[sqlfluff:rules:capitalisation.identifiers] +extended_capitalisation_policy = lower +[sqlfluff:rules:capitalisation.functions] +extended_capitalisation_policy = upper +[sqlfluff:rules:capitalisation.literals] +extended_capitalisation_policy = upper +[sqlfluff:rules:capitalisation.types] +extended_capitalisation_policy = upper \ No newline at end of file diff --git a/earthly/postgresql/sql/pg_hba.extra.conf b/earthly/postgresql/sql/pg_hba.extra.conf new file mode 100644 index 000000000..6f3799925 --- /dev/null +++ b/earthly/postgresql/sql/pg_hba.extra.conf @@ -0,0 +1,13 @@ +# TYPE DATABASE USER ADDRESS METHOD + +# We trust addresses that can only occur inside the local network. + +# IPv4 local connections: +host all all 127.0.0.0/8 trust +host all all 192.168.0.0/16 trust +host all all 10.0.0.0/8 trust +host all all 172.16.0.0/12 trust + +# IPv6 local connections: +host all all ::1/128 trust +host all all fc00::/7 trust \ No newline at end of file diff --git a/earthly/postgresql/setup-db.sql b/earthly/postgresql/sql/setup-db.sql similarity index 77% rename from earthly/postgresql/setup-db.sql rename to earthly/postgresql/sql/setup-db.sql index 78069ec63..a45ba3e00 100644 --- a/earthly/postgresql/setup-db.sql +++ b/earthly/postgresql/sql/setup-db.sql @@ -1,4 +1,5 @@ -- Initialize the Project Catalyst Event Database. +-- sqlfluff:dialect:postgres -- cspell: words psql @@ -15,9 +16,10 @@ \echo -> dbName ................. = :dbName \echo -> dbDescription .......... = :dbDescription \echo -> dbUser ................. = :dbUser -\echo -> dbUserPw ............... = :dbUserPw +\echo -> dbUserPw ............... = xxxx -- Cleanup if we already ran this before. +--DROP OWNED BY IF EXISTS :"dbUser"; -- noqa: PRS DROP DATABASE IF EXISTS :"dbName"; -- noqa: PRS DROP USER IF EXISTS :"dbUser"; -- noqa: PRS @@ -25,9 +27,9 @@ DROP USER IF EXISTS :"dbUser"; -- noqa: PRS CREATE USER :"dbUser" WITH PASSWORD :'dbUserPw'; -- noqa: PRS -- Privileges for this user/role. -ALTER DEFAULT privileges REVOKE EXECUTE ON functions FROM public; +ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO public; -ALTER DEFAULT privileges IN SCHEMA public REVOKE EXECUTE ON functions FROM :"dbUser"; -- noqa: PRS +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT EXECUTE ON FUNCTIONS TO :"dbUser"; -- noqa: PRS -- Create the database. CREATE DATABASE :"dbName" WITH OWNER :"dbUser"; -- noqa: PRS diff --git a/earthly/python/Earthfile b/earthly/python/Earthfile index 9f2160698..2b078886c 100644 --- a/earthly/python/Earthfile +++ b/earthly/python/Earthfile @@ -1,5 +1,5 @@ # Common Python UDCs and Builders. -VERSION 0.7 +VERSION --global-cache 0.7 # cspell: words libgcc diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index ef3264440..ae702cb41 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,5 +1,5 @@ # Common Rust UDCs and Builders. -VERSION 0.7 +VERSION --global-cache 0.7 # cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset readelf depgraph lcov # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT @@ -41,7 +41,12 @@ rust-base: ripgrep \ bash \ colordiff \ - graphviz + graphviz \ + fontconfig \ + ttf-liberation + + # Fix up font cache + RUN fc-cache -f # Make sure we have the clippy linter. RUN rustup component add clippy @@ -71,6 +76,8 @@ rust-base: # Universal build scripts we will always need and are not target dependent. COPY --dir scripts /scripts + # Copy our common scripts so we can use them inside the container. + DO ../../utilities/scripts+ADD_BASH_SCRIPTS # Standardized Rust configs. # Build will refuse to proceed if the projects rust configs do not match these. diff --git a/earthly/rust/scripts/colors.sh b/earthly/rust/scripts/colors.sh deleted file mode 100644 index 3d1cbd2be..000000000 --- a/earthly/rust/scripts/colors.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# shellcheck disable=SC2034 # This file is intended to bo sourced. - -BLACK='\033[0;30m' -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -BLUE='\033[0;34m' -MAGENTA='\033[0;35m' -CYAN='\033[0;36m' -WHITE='\033[0;37m' -NC='\033[0m' # No Color - - -status() { - local rc="$1" - local message="$2" - shift 2 - - # Check if the command returned a bad status - if "$@"; then - # Append green OK to the message - echo -e "${CYAN}${message} : ${GREEN}[OK]${NC}" - else - # Append red ERROR to the message - echo -e "${CYAN}${message} : ${RED}[ERROR]${NC}" - rc=1 - fi - - # Return the current status - return "$rc" -} diff --git a/earthly/rust/scripts/std_build.sh b/earthly/rust/scripts/std_build.sh index ffaa44abf..0829dd432 100755 --- a/earthly/rust/scripts/std_build.sh +++ b/earthly/rust/scripts/std_build.sh @@ -2,6 +2,8 @@ # cspell: words testunit testdocs RUSTDOCFLAGS Zunstable depgraph +# shellcheck disable=SC2046,SC2312 + # This script is run inside the `check` stage for rust projects to perform all # high level non-compilation checks. # These are the Standard checks which ALL rust targets must pass before they @@ -9,8 +11,7 @@ # Individual targets can add extra `check` steps, but these checks must always # pass. -source "$(dirname "$0")/colors.sh" - +source "/scripts/include/colors.sh" # This is set up so that ALL build steps are run and it will fail if any fail. # This improves visibility into all issues that need to be corrected for `build` @@ -19,56 +20,66 @@ source "$(dirname "$0")/colors.sh" rc=0 ## Build the code -status $rc "Building all code in the workspace" \ - cargo build --release --workspace --locked; rc=$? +status "${rc}" "Building all code in the workspace" \ + cargo build --release --workspace --locked +rc=$? ## Check the code passes all clippy lint checks. -status $rc "Checking all Clippy Lints in the workspace" \ - cargo lint; rc=$? +status "${rc}" "Checking all Clippy Lints in the workspace" \ + cargo lint +rc=$? ## Check we can generate all the documentation -status $rc "Checking Documentation can be generated OK" \ - cargo +nightly docs; rc=$? +status "${rc}" "Checking Documentation can be generated OK" \ + cargo docs +rc=$? ## Check if all documentation tests pass. -status $rc "Checking Documentation tests all pass" \ - cargo +nightly testdocs; rc=$? +status "${rc}" "Checking Documentation tests all pass" \ + cargo testdocs +rc=$? ## Check if any benchmarks defined run (We don;t validate the results.) -status $rc "Checking Benchmarks all run to completion" \ - cargo bench --all-targets; rc=$? +status "${rc}" "Checking Benchmarks all run to completion" \ + cargo bench --all-targets +rc=$? ## Generate dependency graphs -status $rc "Generating workspace dependency graphs" \ - $(cargo depgraph --workspace-only --dedup-transitive-deps > target/doc/workspace.dot); rc=$? -status $rc "Generating full dependency graphs" \ - $(cargo depgraph --dedup-transitive-deps > target/doc/full.dot); rc=$? -status $rc "Generating all dependency graphs" \ - $(cargo depgraph --all-deps --dedup-transitive-deps > target/doc/all.dot); rc=$? +status "${rc}" "Generating workspace dependency graphs" \ + $(cargo depgraph --workspace-only --dedup-transitive-deps > target/doc/workspace.dot) +rc=$? +status "${rc}" "Generating full dependency graphs" \ + $(cargo depgraph --dedup-transitive-deps > target/doc/full.dot) +rc=$? +status "${rc}" "Generating all dependency graphs" \ + $(cargo depgraph --all-deps --dedup-transitive-deps > target/doc/all.dot) +rc=$? export NO_COLOR=1 ## Generate Module Trees for documentation purposes. -for lib in $1; -do - status $rc "Generate Module Trees for $lib" \ +for lib in $1; do + status "${rc}" "Generate Module Trees for ${lib}" \ $(cargo modules generate tree --orphans --types --traits --tests --all-features \ - --package $lib --lib > target/doc/$lib.lib.modules.tree); rc=$? + --package "${lib}" --lib > "target/doc/${lib}.lib.modules.tree") + rc=$? - status $rc "Generate Module Graphs for $lib" \ + status "${rc}" "Generate Module Graphs for ${lib}" \ $(cargo modules generate graph --all-features --modules \ - --package $lib --lib > target/doc/$lib.lib.modules.dot); rc=$? + --package "${lib}" --lib > "target/doc/${lib}.lib.modules.dot") + rc=$? done -for bin in $2; -do - IFS="/" read -r package bin <<< "$bin" - status $rc "Generate Module Trees for $package/$bin" \ +for bin in $2; do + IFS="/" read -r package bin <<< "${bin}" + status "${rc}" "Generate Module Trees for ${package}/${bin}" \ $(cargo modules generate tree --orphans --types --traits --tests --all-features \ - --package $package --bin $bin > target/doc/$package.$bin.bin.modules.tree); rc=$? + --package "${package}" --bin "${bin}" > "target/doc/${package}.${bin}.bin.modules.tree") + rc=$? - status $rc "Generate Module Graphs for $package/$bin" \ + status "${rc}" "Generate Module Graphs for ${package}/${bin}" \ $(cargo modules generate graph --all-features --modules \ - --package $package --bin $bin > target/doc/$package.$bin.bin.modules.dot); rc=$? + --package "${package}" --bin "${bin}" > "target/doc/${package}.${bin}.bin.modules.dot") + rc=$? done # Return an error if any of this fails. -exit $rc +exit "${rc}" diff --git a/earthly/rust/scripts/std_checks.sh b/earthly/rust/scripts/std_checks.sh index 8138dde73..e3e18ad0f 100755 --- a/earthly/rust/scripts/std_checks.sh +++ b/earthly/rust/scripts/std_checks.sh @@ -1,28 +1,15 @@ #!/usr/bin/env bash -# cspell: words localfile vendorfile colordiff Naur fmtchk fmtfix rustfmt stdcfgs -# cspell: words nextest +# cspell: words fmtchk fmtfix rustfmt stdcfgs nextest -# This script is run inside the `check` stage for rust projects to perform all +# This script is run inside the `check` stage for rust projects to perform all # high level non-compilation checks. # These are the Standard checks which ALL rust targets must pass before they -# will be scheduled to be `build`. +# will be scheduled to be `build`. # Individual targets can add extra `check` steps, but these checks must always -# pass. +# pass. -source "$(dirname "$0")/colors.sh" - -# Checks if two files that should exist DO, and are equal. -# used to enforce consistency between local config files and the expected config locked in CI. -check_vendored_files() { - local rc=$1 - local localfile=$2 - local vendorfile=$3 - - status "$rc" "Checking if Local File '$localfile' == Vendored File '$vendorfile'" \ - colordiff -Naur "$localfile" "$vendorfile" - return $? -} +source "/scripts/include/colors.sh" # This is set up so that ALL checks are run and it will fail if any fail. # This improves visibility into all issues that need to be corrected for `check` @@ -30,23 +17,31 @@ check_vendored_files() { # Check if the rust src is properly formatted. # Note, we run this first so we can print help how to fix. -status 0 "Checking Rust Code Format" cargo +nightly fmtchk; rc=$? -if [ $rc -ne 0 ]; then +status 0 "Checking Rust Code Format" cargo +nightly fmtchk +rc=$? +if [[ "${rc}" -ne 0 ]]; then echo -e " ${YELLOW}You can locally fix format errors by running: \`cargo +nightly fmtfix\`${NC}" fi ## Check if .cargo.config.toml has been modified. -check_vendored_files $rc .cargo/config.toml "$CARGO_HOME"/config.toml; rc=$? -check_vendored_files $rc rustfmt.toml /stdcfgs/rustfmt.toml; rc=$? -check_vendored_files $rc .config/nextest.toml /stdcfgs/nextest.toml; rc=$? -check_vendored_files $rc clippy.toml /stdcfgs/clippy.toml; rc=$? -check_vendored_files $rc deny.toml /stdcfgs/deny.toml; rc=$? +check_vendored_files "${rc}" .cargo/config.toml "${CARGO_HOME:?}"/config.toml +rc=$? +check_vendored_files "${rc}" rustfmt.toml /stdcfgs/rustfmt.toml +rc=$? +check_vendored_files "${rc}" .config/nextest.toml /stdcfgs/nextest.toml +rc=$? +check_vendored_files "${rc}" clippy.toml /stdcfgs/clippy.toml +rc=$? +check_vendored_files "${rc}" deny.toml /stdcfgs/deny.toml +rc=$? # Check if we have unused dependencies declared in our Cargo.toml files. -status $rc "Checking for Unused Dependencies" cargo machete; rc=$? +status "${rc}" "Checking for Unused Dependencies" cargo machete +rc=$? # Check if we have any supply chain issues with dependencies. -status $rc "Checking for Supply Chain Issues" cargo deny check; rc=$? +status "${rc}" "Checking for Supply Chain Issues" cargo deny check +rc=$? # Return an error if any of this fails. -exit $rc \ No newline at end of file +exit "${rc}" diff --git a/earthly/rust/scripts/verify_toolchain.sh b/earthly/rust/scripts/verify_toolchain.sh index 5de53599f..753828602 100755 --- a/earthly/rust/scripts/verify_toolchain.sh +++ b/earthly/rust/scripts/verify_toolchain.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -source "$(dirname "$0")/colors.sh" +source "/scripts/include/colors.sh" default_rust_channel=$1 RUST_VERSION=$2 -if [ "$default_rust_channel" != "$RUST_VERSION" ]; then - echo -e "${YELLOW}Your Rust Toolchain is set to Version : ${RED}$default_rust_channel${NC}" - echo -e "${YELLOW}This Builder requires it to be : ${GREEN}$RUST_VERSION${NC}" +if [[ "${default_rust_channel}" != "${RUST_VERSION}" ]]; then + echo -e "${YELLOW}Your Rust Toolchain is set to Version : ${RED}${default_rust_channel}${NC}" + echo -e "${YELLOW}This Builder requires it to be : ${GREEN}${RUST_VERSION}${NC}" echo -e "${RED}Either use the correct Earthly Rust Builder version from CI, or correct './rust-toolchain.toml' to match.${NC}" - exit 1; + exit 1 fi diff --git a/earthly/utils/Earthfile b/earthly/utils/Earthfile deleted file mode 100644 index 42d52cf06..000000000 --- a/earthly/utils/Earthfile +++ /dev/null @@ -1,10 +0,0 @@ -# Common PostgreSQL Earthly builders -VERSION 0.7 - -shell-assert: - FROM scratch - - WORKDIR /assert-sh - - COPY assert.sh . - SAVE ARTIFACT assert.sh assert.sh \ No newline at end of file diff --git a/earthly/utils/README.md b/earthly/utils/README.md deleted file mode 100644 index 7b86c0814..000000000 --- a/earthly/utils/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Earthly Utils - -This directory contains utility Earthly targets and tools for working with Earthly. diff --git a/earthly/utils/assert.sh b/earthly/utils/assert.sh deleted file mode 100644 index ae341e1df..000000000 --- a/earthly/utils/assert.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -assert_eq() { - local expected="$1" - local actual="$2" - - if [ "$expected" == "$actual" ]; then - return 0 - else - echo "assert_eq FAILED" - echo "expected:" - echo "$expected" - echo "actual:" - echo "$actual" - echo "Diff:" - diff <(echo "$expected" ) <(echo "$actual") - return 1 - fi -} \ No newline at end of file diff --git a/examples/go/Earthfile b/examples/go/Earthfile index 0abff001a..74a9062e3 100644 --- a/examples/go/Earthfile +++ b/examples/go/Earthfile @@ -1,6 +1,6 @@ # WARNING: If you modify this file, please update the guide that it is dependent # on in docs/guides/languages/go.md. -VERSION 0.7 +VERSION --global-cache 0.7 # The structure of this Earthfile is derived from the style guide: # https://input-output-hk.github.io/catalyst-ci/style/#adhere-to-a-consistent-structure diff --git a/examples/postgresql/Earthfile b/examples/postgresql/Earthfile index da879fd55..e83b245a2 100644 --- a/examples/postgresql/Earthfile +++ b/examples/postgresql/Earthfile @@ -1,123 +1,116 @@ -VERSION 0.7 +VERSION --global-cache 0.7 # cspell: words psql +# Internal: builder is our Event db builder target. Prepares all necessary artifacts. +# CI target : dependency builder: - FROM ./../../earthly/postgresql+postgres-base - - WORKDIR /build - - COPY --dir ./migrations ./data ./refinery.toml . DO ./../../earthly/postgresql+BUILDER + # MUST Manually copy the .sqlfluff config used in the repo because the FUNCTION + # above can not be passed a reference to a local target as an argument. + COPY ./../../+repo-config/repo/.sqlfluff . + +# check if the sql files are properly formatted and pass lint quality checks. +# CI target : true check: FROM +builder DO ./../../earthly/postgresql+CHECK +# format all SQL files in the current project. Local developers tool. +# CI target : false format: LOCALLY DO ./../../earthly/postgresql+FORMAT --src=$(echo ${PWD}) + +# build an event db docker image. +# CI target : true build: FROM +builder - ARG tag="latest" - ARG registry + DO ./../../earthly/postgresql+BUILD --image_name=example-db + DO ./../../earthly/postgresql+DOCS - DO ./../../earthly/postgresql+BUILD --image_name=example-db --tag=$tag --registry=$registry +# Internal: common integration test image +all-tests: + DO ./../../earthly/postgresql+INTEGRATION_TEST_SETUP + DO ./../../utilities/scripts+ADD_BASH_SCRIPTS + + COPY --dir tests . -# Container runs PostgreSQL server, drops and initialise db, applies migrations, applies seed data. -test-1: - FROM ./../../earthly/postgresql+postgres-base - - COPY ./../../earthly/utils+shell-assert/assert.sh . +# Internal Function to run an specific integration test script. +INTEGRATION_TEST_RUN: + COMMAND - ENV INIT_AND_DROP_DB true - ENV WITH_MIGRATIONS true - ENV WITH_SEED_DATA true - COPY ./docker-compose.yml . - WITH DOCKER \ - --compose docker-compose.yml \ - --load example-db:latest=+build \ - --service example \ - --allow-privileged - RUN sleep 5;\ - res=$(psql postgresql://example-dev:example-pass@0.0.0.0:5432/ExampleDb -c "SELECT * FROM users");\ - - source assert.sh;\ - expected=$(printf " name | age \n---------+-----\n Alice | 20\n Bob | 30\n Charlie | 40\n(3 rows)");\ - assert_eq "$expected" "$res" - END + ARG seed_data + ARG test_script + ARG migrations=true + ARG compose="./tests/docker-compose-sa.yml" -# Container runs PostgreSQL server, drops and initialise db, doesn't apply migrations, doesn't apply seed data. -test-2: - FROM ./../../earthly/postgresql+postgres-base + FROM +all-tests ENV INIT_AND_DROP_DB true - ENV WITH_MIGRATIONS false - ENV WITH_SEED_DATA false - COPY ./docker-compose.yml . - WITH DOCKER \ - --compose docker-compose.yml \ - --load example-db:latest=+build \ - --service example \ - --allow-privileged - RUN sleep 5;\ - ! psql postgresql://example-dev:example-pass@0.0.0.0:5432/ExampleDb -c "SELECT * FROM users" - END + ENV WITH_MIGRATIONS $migrations + ENV WITH_SEED_DATA $seed_data + ENV DB_NAME "ExampleDb" + ENV DB_SUPERUSER postgres + ENV DB_SUPERUSER_PASSWORD postgres + ENV DB_USER example-dev + ENV DB_USER_PASSWORD example-pass -# Container runs PostgreSQL server, drops and initialise db, applies migrations, doesn't apply seed data. -test-3: - FROM ./../../earthly/postgresql+postgres-base - - COPY ./../../earthly/utils+shell-assert/assert.sh . - - ENV INIT_AND_DROP_DB true - ENV WITH_MIGRATIONS true - ENV WITH_SEED_DATA false - COPY ./docker-compose.yml . WITH DOCKER \ - --compose docker-compose.yml \ + --compose $compose \ --load example-db:latest=+build \ - --service example \ --allow-privileged - RUN sleep 5;\ - res=$(psql postgresql://example-dev:example-pass@0.0.0.0:5432/ExampleDb -c "SELECT * FROM users");\ - - source assert.sh;\ - expected=$(printf " name | age \n------+-----\n(0 rows)");\ - assert_eq "$expected" "$res" + RUN $test_script END -# PostgreSQL server runs as a separate service, drops and initialise db, applies migrations, applies seed data. -test-4: - FROM ./../../earthly/postgresql+postgres-base - - COPY ./../../earthly/utils+shell-assert/assert.sh . - ENV DB_HOST postgres - ENV INIT_AND_DROP_DB true - ENV WITH_MIGRATIONS true - ENV WITH_SEED_DATA true - COPY ./docker-compose.yml . - WITH DOCKER \ - --compose docker-compose.yml \ - --pull postgres:16 \ - --load example-db:latest=+build \ - --service example \ - --service postgres \ - --allow-privileged - RUN sleep 5;\ - res=$(psql postgresql://postgres:postgres@0.0.0.0:5433/ExampleDb -c "SELECT * FROM users");\ - - source assert.sh;\ - expected=$(printf " name | age \n---------+-----\n Alice | 20\n Bob | 30\n Charlie | 40\n(3 rows)");\ - assert_eq "$expected" "$res" - END +# Internal: Test Scenario 1 +# CI target : true +# Steps: +# * Container runs PostgreSQL server +# * drops and initialise db +# * applies migrations +# * applies seed data. +test-1: + DO +INTEGRATION_TEST_RUN --seed_data="data" --test_script=./tests/test1.sh + +# Internal: Test Scenario 2 +# CI target : dependency +# Steps: +# * Container runs PostgreSQL server +# * drops and initialise db +# * doesn't apply migrations +# * doesn't apply seed data. +test-2: + DO +INTEGRATION_TEST_RUN --migrations=false --seed_data= --test_script=./tests/test2.sh + +# Internal: Test Scenario 3 +# CI target : dependency +# Steps: +# * Container runs PostgreSQL server +# * drops and initialise db +# * applies migrations +# * doesn't apply seed data. +test-3: + DO +INTEGRATION_TEST_RUN --seed_data= --test_script=./tests/test3.sh + +# Internal: Test Scenario 4 +# CI target : dependency +# Steps: +# * PostgreSQL server runs as a separate service +# * drops and initialise db +# * applies migrations +# * applies seed data. +test-4: + DO +INTEGRATION_TEST_RUN --compose="./tests/docker-compose-svc.yml" --seed_data="data" --test_script=./tests/test1.sh -# Invoke all tests +# test the event db database schema. Invokes all tests. +# CI target : true test: BUILD +test-1 BUILD +test-2 diff --git a/examples/postgresql/data/example.sql b/examples/postgresql/data/example.sql deleted file mode 100644 index a80f80765..000000000 --- a/examples/postgresql/data/example.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO users (name, age) VALUES ('Alice', 20), -('Bob', 30), -('Charlie', 40); diff --git a/examples/postgresql/diagrams.json b/examples/postgresql/diagrams.json new file mode 100644 index 000000000..62e87058b --- /dev/null +++ b/examples/postgresql/diagrams.json @@ -0,0 +1,21 @@ +{ + "name": "Example Postgresql Database", + "all_schema": { + "comments": true, + "included_tables": [], + "excluded_tables": [ + "refinery_schema_history" + ], + "column_description_wrap": 50, + "table_description_wrap": 50 + }, + "full_schema": { + "excluded_tables": [ + "refinery_schema_history" + ], + "title": "Full Schema", + "comments": true, + "column_description_wrap": 50, + "table_description_wrap": 50 + } +} \ No newline at end of file diff --git a/examples/postgresql/migrations/V1__example.sql b/examples/postgresql/migrations/V1__users.sql similarity index 100% rename from examples/postgresql/migrations/V1__example.sql rename to examples/postgresql/migrations/V1__users.sql diff --git a/examples/postgresql/migrations/V2__addresses.sql b/examples/postgresql/migrations/V2__addresses.sql new file mode 100644 index 000000000..e2cd8be2f --- /dev/null +++ b/examples/postgresql/migrations/V2__addresses.sql @@ -0,0 +1,7 @@ +CREATE TABLE address +( + name VARCHAR PRIMARY KEY, + address TEXT NOT NULL, + + FOREIGN KEY (name) REFERENCES users (name) +); diff --git a/examples/postgresql/seed/data/example.sql b/examples/postgresql/seed/data/example.sql new file mode 100644 index 000000000..0a31a1669 --- /dev/null +++ b/examples/postgresql/seed/data/example.sql @@ -0,0 +1,4 @@ +INSERT INTO users (name, age) VALUES +('Alice', 20), +('Bob', 30), +('Charlie', 40); diff --git a/examples/postgresql/seed/data2/example.sql b/examples/postgresql/seed/data2/example.sql new file mode 100644 index 000000000..efb54d15a --- /dev/null +++ b/examples/postgresql/seed/data2/example.sql @@ -0,0 +1,11 @@ +INSERT INTO users (name, age) VALUES +('Alice', 20), +('Bob', 30), +('Charlie', 40), +('Mary', 22); + +INSERT INTO address (name, address) VALUES +('Alice', '123 Main St'), +('Bob', '456 Oak St'), +('Charlie', '789 Elm St'), +('Mary', '321 Pine St'); diff --git a/examples/postgresql/tests/docker-compose-sa.yml b/examples/postgresql/tests/docker-compose-sa.yml new file mode 100644 index 000000000..840b5ff6c --- /dev/null +++ b/examples/postgresql/tests/docker-compose-sa.yml @@ -0,0 +1,27 @@ +# Stand alone Migrations and DB Service +version: "3" + +# cspell: words healthcheck isready + +services: + example: + image: example-db:latest + container_name: example-db + environment: + # Required environment variables for migrations + - DB_HOST=localhost + - DB_PORT=5432 + - DB_NAME=ExampleDb + - DB_DESCRIPTION="Example DB" + - DB_SUPERUSER=postgres + - DB_SUPERUSER_PASSWORD=postgres + - DB_USER=example-dev + - DB_USER_PASSWORD=example-pass + + - INIT_AND_DROP_DB=${INIT_AND_DROP_DB:-true} + - WITH_MIGRATIONS=${WITH_MIGRATIONS:-true} + - WITH_SEED_DATA=${WITH_SEED_DATA:-} + ports: + - 5432:5432 + volumes: + - /var/lib/postgresql/data # Temp volume, use -V to auto delete it when container stops. diff --git a/examples/postgresql/docker-compose.yml b/examples/postgresql/tests/docker-compose-svc.yml similarity index 71% rename from examples/postgresql/docker-compose.yml rename to examples/postgresql/tests/docker-compose-svc.yml index 37dcca79e..3a647c7f8 100644 --- a/examples/postgresql/docker-compose.yml +++ b/examples/postgresql/tests/docker-compose-svc.yml @@ -1,3 +1,4 @@ +# Separate Migrations Service and DB Service version: "3" # cspell: words healthcheck isready @@ -10,22 +11,26 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres + POSTGRES_HOST_AUTH_METHOD: trust healthcheck: test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 2s timeout: 5s retries: 10 ports: - - 5433:5432 + - 5432:5432 + volumes: + - /var/lib/postgresql/data # Temp volume, use -V to auto delete it when container stops. example: image: example-db:latest + container_name: example-db environment: # Required environment variables for migrations - - DB_HOST=${DB_HOST:-localhost} + - DB_HOST=postgres - DB_PORT=5432 - DB_NAME=ExampleDb - - DB_DESCRIPTION=Example DB + - DB_DESCRIPTION="Example DB" - DB_SUPERUSER=postgres - DB_SUPERUSER_PASSWORD=postgres - DB_USER=example-dev @@ -33,6 +38,4 @@ services: - INIT_AND_DROP_DB=${INIT_AND_DROP_DB:-true} - WITH_MIGRATIONS=${WITH_MIGRATIONS:-true} - - WITH_SEED_DATA=${WITH_SEED_DATA:-true} - ports: - - 5432:5432 + - WITH_SEED_DATA=${WITH_SEED_DATA:-} diff --git a/examples/postgresql/tests/test1.sh b/examples/postgresql/tests/test1.sh new file mode 100755 index 000000000..d90561dd0 --- /dev/null +++ b/examples/postgresql/tests/test1.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# cspell: words pgsql dbreadytimeout dbconn psql + +source "/scripts/include/colors.sh" +source "/scripts/include/assert.sh" +source "/scripts/include/db_ops.sh" + +show_db_config + +# Wait for the DB Server to start up and migrate. +sleep 10 + +# Wait until the server is running +status_and_exit "DB Ready" \ + wait_ready_pgsql "$@" --dbreadytimeout="30" + +dbconn=$(pgsql_user_connection "$@") + +rc=0 +if ! res=$(psql "${dbconn}" -c "SELECT * FROM users"); then + rc=1 +fi + +status 0 "DB Query: SELECT * FROM users" \ + [ "${rc}" == 0 ] + +if [[ ${rc} -eq 0 ]]; then + expected=$(printf "%s\n%s\n%s\n%s\n%s\n%s\n" \ + " name | age " \ + "---------+-----" \ + " Alice | 20" \ + " Bob | 30" \ + " Charlie | 40" \ + "(3 rows)") + + status "${rc}" "Query Result" \ + assert_eq "${expected}" "${res}" + rc=$? +fi + +exit "${rc}" diff --git a/examples/postgresql/tests/test2.sh b/examples/postgresql/tests/test2.sh new file mode 100755 index 000000000..fe8fb1726 --- /dev/null +++ b/examples/postgresql/tests/test2.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# cspell: words pgsql dbreadytimeout dbconn psql + +source "/scripts/include/colors.sh" +source "/scripts/include/assert.sh" +source "/scripts/include/db_ops.sh" + +show_db_config + +# Wait for the DB Server to start up and migrate. +sleep 10 + +# Wait until the server is running +status_and_exit "DB Ready" \ + wait_ready_pgsql "$@" --dbreadytimeout="30" + +dbconn=$(pgsql_user_connection "$@") + +rc=1 +if ! res=$(psql "${dbconn}" -c "SELECT * FROM users" 2>&1); then + rc=0 +fi + +if [[ ${rc} -eq 0 ]]; then + expected=$(printf "%s\n%s\n%s\n" \ + "ERROR: relation \"users\" does not exist" \ + "LINE 1: SELECT * FROM users" \ + " ^") + + status "${rc}" "Query Result" \ + assert_eq "${expected}" "${res}" + rc=$? +fi + +status 0 "DB Query with No Migrations in DB: SELECT * FROM users" \ + [ "${rc}" == 0 ] + +exit "${rc}" diff --git a/examples/postgresql/tests/test3.sh b/examples/postgresql/tests/test3.sh new file mode 100755 index 000000000..c7268188d --- /dev/null +++ b/examples/postgresql/tests/test3.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# cspell: words pgsql dbreadytimeout dbconn psql + +source "/scripts/include/colors.sh" +source "/scripts/include/assert.sh" +source "/scripts/include/db_ops.sh" + +show_db_config + +# Wait for the DB Server to start up and migrate. +sleep 10 + +# Wait until the server is running +status_and_exit "DB Ready" \ + wait_ready_pgsql "$@" --dbreadytimeout="30" + +dbconn=$(pgsql_user_connection "$@") + +rc=0 +if ! res=$(psql "${dbconn}" -c "SELECT * FROM users"); then + rc=1 +fi + +if [[ ${rc} -eq 0 ]]; then + expected=$(printf "%s\n%s\n%s\n" \ + " name | age " \ + "------+-----" \ + "(0 rows)") + + status "${rc}" "Query Result" \ + assert_eq "${expected}" "${res}" + rc=$? +fi + +status 0 "DB Query with Empty DB: SELECT * FROM users" \ + [ "${rc}" == 0 ] + +exit "${rc}" diff --git a/examples/python/Earthfile b/examples/python/Earthfile index d4745ccb7..8ebb39370 100644 --- a/examples/python/Earthfile +++ b/examples/python/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 builder: FROM ./../../earthly/python+python-base diff --git a/examples/rust/Earthfile b/examples/rust/Earthfile index 303574184..975d88ade 100644 --- a/examples/rust/Earthfile +++ b/examples/rust/Earthfile @@ -1,4 +1,4 @@ -VERSION 0.7 +VERSION --global-cache 0.7 # cspell: words TARGETARCH USERARCH toolsets rustfmt nextest diff --git a/utilities/README.md b/utilities/README.md new file mode 100644 index 000000000..11130eca0 --- /dev/null +++ b/utilities/README.md @@ -0,0 +1,9 @@ +# Utilities + +Utilities are commands or scripts, that could be written in any language. +They are used to execute specific functions we need within CI. + +Examples of utilities are common bash scripts used across multiple builders, or programs to process data during builds. + +The code here should be small and relatively self contained. +If its large it should be its own stand alone project. diff --git a/utilities/dbviz/.cargo/config.toml b/utilities/dbviz/.cargo/config.toml new file mode 100644 index 000000000..0eae0eb21 --- /dev/null +++ b/utilities/dbviz/.cargo/config.toml @@ -0,0 +1,130 @@ +# Use MOLD linker where possible, but ONLY in CI applicable targets. +# cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags +# cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit +# cspell: words fmtchk fmtfix + +# Configure how Docker container targets build. + +# If you want to customize these targets for a local build, then customize them in you: +# $CARGO_HOME/config.toml +# NOT in the project itself. +# These targets are ONLY the targets used by CI and inside docker builds. + +# DO NOT remove `"-C", "target-feature=+crt-static"` from the rustflags for these targets. + +# Should be the default to have fully static rust programs in CI +[target.x86_64-unknown-linux-musl] +linker = "clang" +rustflags = [ + "-C", "link-arg=-fuse-ld=/usr/bin/mold", + "-C", "target-feature=-crt-static" +] + +# Should be the default to have fully static rust programs in CI +[target.aarch64-unknown-linux-musl] +linker = "clang" +rustflags = [ + "-C", "link-arg=-fuse-ld=/usr/bin/mold", + "-C", "target-feature=-crt-static" +] + + +[build] + +rustflags = [ + "-D", + "warnings", + "-D", + "missing_docs", + "-D", + "let_underscore_drop", + "-D", + "non_ascii_idents", + "-D", + "single_use_lifetimes", + "-D", + "trivial_casts", + "-D", + "trivial_numeric_casts", +] + +rustdocflags = [ + "--enable-index-page", + "-Z", + "unstable-options", + "-D", + "warnings", + "-D", + "missing_docs", + "-D", + "rustdoc::broken_intra_doc_links", + "-D", + "rustdoc::invalid_codeblock_attributes", + "-D", + "rustdoc::invalid_html_tags", + "-D", + "rustdoc::invalid_rust_codeblocks", + "-D", + "rustdoc::bare_urls", + "-D", + "rustdoc::unescaped_backticks", +] + +[profile.dev] +opt-level = 1 +debug = true +debug-assertions = true +overflow-checks = true +lto = false +panic = 'unwind' +incremental = true +codegen-units = 256 + +[profile.release] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = "thin" +panic = 'unwind' +incremental = false +codegen-units = 16 + +[profile.test] +opt-level = 3 +debug = true +lto = false +debug-assertions = true +incremental = true +codegen-units = 256 + +[profile.bench] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = "thin" +incremental = false +codegen-units = 16 + +[alias] +lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" + +docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +# nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" +testfast = "nextest --release --workspace --locked" +testdocs = "test --doc --release --workspace --locked" + +# Rust formatting, MUST be run with +nightly +fmtchk = "fmt -- --check -v --color=always" +fmtfix = "fmt -- -v" + +[term] +quiet = false # whether cargo output is quiet +verbose = true # whether cargo provides verbose output +color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'auto' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/utilities/dbviz/.config/nextest.toml b/utilities/dbviz/.config/nextest.toml new file mode 100644 index 000000000..ebc7d7390 --- /dev/null +++ b/utilities/dbviz/.config/nextest.toml @@ -0,0 +1,49 @@ +# cspell: words nextest scrollability testcase +[store] +# The directory under the workspace root at which nextest-related files are +# written. Profile-specific storage is currently written to dir/. +# dir = "target/nextest" + +[profile.default] +# Print out output for failing tests as soon as they fail, and also at the end +# of the run (for easy scrollability). +failure-output = "immediate-final" + +# Do not cancel the test run on the first failure. +fail-fast = true + +status-level = "all" +final-status-level = "all" + +[profile.ci] +# Print out output for failing tests as soon as they fail, and also at the end +# of the run (for easy scrollability). +failure-output = "immediate-final" +# Do not cancel the test run on the first failure. +fail-fast = false + +status-level = "all" +final-status-level = "all" + + +[profile.ci.junit] +# Output a JUnit report into the given file inside 'store.dir/'. +# If unspecified, JUnit is not written out. + +path = "junit.xml" + +# The name of the top-level "report" element in JUnit report. If aggregating +# reports across different test runs, it may be useful to provide separate names +# for each report. +report-name = "cat-gateway" + +# Whether standard output and standard error for passing tests should be stored in the JUnit report. +# Output is stored in the and elements of the element. +store-success-output = true + +# Whether standard output and standard error for failing tests should be stored in the JUnit report. +# Output is stored in the and elements of the element. +# +# Note that if a description can be extracted from the output, it is always stored in the +# element. +store-failure-output = true diff --git a/utilities/dbviz/Cargo.lock b/utilities/dbviz/Cargo.lock new file mode 100644 index 000000000..f2d5ca834 --- /dev/null +++ b/utilities/dbviz/Cargo.lock @@ -0,0 +1,1103 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "anstream" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "dbviz" +version = "1.4.0" +dependencies = [ + "anyhow", + "clap", + "itertools", + "minijinja", + "postgres", + "serde", + "textwrap", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + +[[package]] +name = "js-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "minijinja" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb" +dependencies = [ + "serde", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "postgres" +version = "0.19.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7915b33ed60abc46040cbcaa25ffa1c7ec240668e0477c4f3070786f5916d451" +dependencies = [ + "bytes", + "fallible-iterator", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "2.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" + +[[package]] +name = "web-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" diff --git a/utilities/dbviz/Cargo.toml b/utilities/dbviz/Cargo.toml new file mode 100644 index 000000000..548fa0779 --- /dev/null +++ b/utilities/dbviz/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dbviz" +version = "1.4.0" +authors = ["Steven Johnson"] +edition = "2018" +license = "Apache-2.0/MIT" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0" +itertools = "0.12.0" +postgres = "0.19.4" +minijinja = "1.0.10" +textwrap = "0.16.0" +serde = { version = "1.0", features = ["derive"] } +clap = { version = "4.4.8", features = ["derive"] } diff --git a/utilities/dbviz/Earthfile b/utilities/dbviz/Earthfile new file mode 100644 index 000000000..8b0ab7c4e --- /dev/null +++ b/utilities/dbviz/Earthfile @@ -0,0 +1,91 @@ +# Common Rust UDCs and Builders. +VERSION --global-cache 0.7 + +# cspell: words rustup rustc automake autotools xutils miri nextest kani +# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT +# cspell: words USERPLATFORM USEROS USERARCH USERVARIANT +# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets readelf + +# Internal: Set up our target toolchains, and copy our files. +builder: + FROM ../../earthly/rust+rust-base + + DO ../../earthly/rust+SETUP --toolchain=rust-toolchain.toml + + COPY --dir .cargo .config Cargo.* clippy.toml deny.toml rustfmt.toml src . + +# Internal: Run Project Checks - Use best architecture host tools. +check-hosted: + FROM +builder + + DO ../../earthly/rust+CHECK + +# check-all-hosts - A developers test which runs check with all supported host tooling. +# Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets by developers. +check-all-hosts: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check-hosted + +build-hosted: + ARG TARGETPLATFORM + + # Build the service + FROM +builder + + RUN /scripts/std_build.sh + + DO github.com/input-output-hk/catalyst-ci/earthly/rust:v2.0.3+SMOKE_TEST --bin=dbviz + + SAVE ARTIFACT target/$TARGETARCH/doc doc + SAVE ARTIFACT target/$TARGETARCH/release/dbviz dbviz + +# build-all-hosts - A developers test which runs build with all supported host tooling. +# Needs qemu or rosetta to run. +# Only used to validate build tooling is working across host toolsets. +build-all-hosts: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted + + +## ----------------------------------------------------------------------------- +## +## Standard CI targets. +## +## These targets are discovered and executed automatically by CI. + +# check - Run check using the most efficient host tooling +# CI Automated Entry point. +check: + FROM busybox + # This is necessary to pick the correct architecture build to suit the native machine. + # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. + # All target implementation of this should follow this pattern. + ARG USERARCH + + IF [ "$USERARCH" == "arm64" ] + BUILD --platform=linux/arm64 +check-hosted + ELSE + BUILD --platform=linux/amd64 +check-hosted + END + +# build - Run build using the most efficient host tooling +# CI Automated Entry point. +build: + FROM busybox + # This is necessary to pick the correct architecture build to suit the native machine. + # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. + # All target implementation of this should follow this pattern. + ARG USERARCH + + IF [ "$USERARCH" == "arm64" ] + BUILD --platform=linux/arm64 +build-hosted + COPY +build-hosted/doc /doc + ELSE + BUILD --platform=linux/amd64 +build-hosted + END + + COPY --dir +build-hosted/doc . + COPY +build-hosted/dbviz . + + SAVE ARTIFACT doc doc + SAVE ARTIFACT dbviz dbviz + diff --git a/utilities/dbviz/README.md b/utilities/dbviz/README.md new file mode 100644 index 000000000..07f40b1ad --- /dev/null +++ b/utilities/dbviz/README.md @@ -0,0 +1,11 @@ +# dbviz + +Simple tool to create database diagrams from postgres schemas. +The diagrams themselves are just text files, and are controlled by `.jinja` templates. +The tool builds in a default template which produces `.dot` files. + +## Usage + +```sh +dbviz -d database_name | dot -Tpng > schema.png +``` diff --git a/utilities/dbviz/clippy.toml b/utilities/dbviz/clippy.toml new file mode 100644 index 000000000..6933b8164 --- /dev/null +++ b/utilities/dbviz/clippy.toml @@ -0,0 +1 @@ +allow-expect-in-tests = true diff --git a/utilities/dbviz/cspell.json b/utilities/dbviz/cspell.json new file mode 100644 index 000000000..a45613263 --- /dev/null +++ b/utilities/dbviz/cspell.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", + "version": "0.2", + "import": "../../cspell.json", + "words": [ + "dbviz", + "Tpng", + "dbname", + "regclass", + "nspname", + "schemaname", + "relname", + "attname", + "attrf", + "conrelid", + "attnum", + "conkey", + "relnamespace", + "confrelid", + "attrelid", + "confkey", + "indrelid", + "indisprimary", + "indisunique", + "indexdef", + "indexrelid", + "indkey", + "relam", + "minijinja", + "pkey", + "endmacro", + "varchar", + "startingwith", + "nextval", + "endmacro", + "labelloc", + "rankdir", + "cellborder", + "endfor", + "bytea", + "Autoincrement" + ] +} \ No newline at end of file diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml new file mode 100644 index 000000000..d28c1100d --- /dev/null +++ b/utilities/dbviz/deny.toml @@ -0,0 +1,275 @@ +# This template contains all of the possible sections and their default values + +# cspell: words rustc RUSTSEC dotgraphs reqwest rustls pemfile webpki + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #{ triple = "x86_64-unknown-linux-musl" }, + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory database is cloned/fetched into +db-path = "~/.cargo/advisory-db" +# The url(s) of the advisory databases to use +db-urls = ["https://github.com/rustsec/advisory-db"] +# The lint level for security vulnerabilities +vulnerability = "deny" +# The lint level for unmaintained crates +unmaintained = "warn" +# The lint level for crates that have been yanked from their source registry +yanked = "warn" +# The lint level for crates with security notices. +notice = "warn" +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", +] +# Threshold for security vulnerabilities, any vulnerability with a CVSS score +# lower than the range specified will be ignored. Note that ignored advisories +# will still output a note when they are encountered. +# * None - CVSS Score 0.0 +# * Low - CVSS Score 0.1 - 3.9 +# * Medium - CVSS Score 4.0 - 6.9 +# * High - CVSS Score 7.0 - 8.9 +# * Critical - CVSS Score 9.0 - 10.0 +#severity-threshold = + +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# The lint level for crates which do not have a detectable license +unlicensed = "deny" +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "MIT", + "Apache-2.0", + "Unicode-DFS-2016", + "BSD-3-Clause", + "BlueOak-1.0.0" +] +# List of explicitly disallowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +deny = [ + #"Nokia", +] +# Lint level for licenses considered copyleft +copyleft = "deny" + +# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses +# * both - The license will be approved if it is both OSI-approved *AND* FSF +# * either - The license will be approved if it is either OSI-approved *OR* FSF +# * osi - The license will be approved if it is OSI approved +# * fsf - The license will be approved if it is FSF Free +# * osi-only - The license will be approved if it is OSI-approved *AND NOT* FSF +# * fsf-only - The license will be approved if it is FSF *AND NOT* OSI-approved +# * neither - This predicate is ignored and the default lint level is used +allow-osi-fsf-free = "neither" +# Lint level used when no other predicates are matched +# 1. License isn't in the allow or deny lists +# 2. License isn't copyleft +# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" +default = "deny" +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], name = "adler32", version = "*" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The name of the crate the clarification applies to +#name = "ring" +# The optional version constraint for the crate +#version = "*" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ + # Each entry is a crate relative path, and the (opaque) hash of its contents + #{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "deny" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #{ name = "ansi_term", version = "=0.11.0" }, +] +# List of crates to deny +deny = [ + # Each entry the name of a crate and a version range. If version is + # not specified, all versions will be matched. + #{ name = "ansi_term", version = "=0.11.0" }, + # + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ name = "ansi_term", version = "=0.11.0", wrappers = [] }, + { name = "openssl" }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#name = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + #{ name = "ansi_term", version = "=0.11.0" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "deny" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "deny" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] + +[sources.allow-org] +# 1 or more github.com organizations to allow git sources for +#github = [""] +# 1 or more gitlab.com organizations to allow git sources for +#gitlab = [""] +# 1 or more bitbucket.org organizations to allow git sources for +#bitbucket = [""] diff --git a/utilities/dbviz/rust-toolchain.toml b/utilities/dbviz/rust-toolchain.toml new file mode 100644 index 000000000..dca770e1b --- /dev/null +++ b/utilities/dbviz/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +channel = "1.73.0" +profile = "default" +components = [ ] +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", "aarch64-apple-darwin"] \ No newline at end of file diff --git a/utilities/dbviz/rustfmt.toml b/utilities/dbviz/rustfmt.toml new file mode 100644 index 000000000..b0f20832c --- /dev/null +++ b/utilities/dbviz/rustfmt.toml @@ -0,0 +1,68 @@ +# Enable unstable features: +# * imports_indent +# * imports_layout +# * imports_granularity +# * group_imports +# * reorder_impl_items +# * trailing_comma +# * where_single_line +# * wrap_comments +# * comment_width +# * blank_lines_upper_bound +# * condense_wildcard_suffixes +# * force_multiline_blocks +# * format_code_in_doc_comments +# * format_generated_files +# * hex_literal_case +# * inline_attribute_width +# * normalize_comments +# * normalize_doc_attributes +# * overflow_delimited_expr +unstable_features = true + +# Compatibility: +edition = "2021" + +# Tabs & spaces - Defaults, listed for clarity +tab_spaces = 4 +hard_tabs = false + +# Commas. +trailing_comma = "Vertical" +match_block_trailing_comma = true + +# General width constraints. +max_width = 100 + +# Comments: +normalize_comments = true +normalize_doc_attributes = true +wrap_comments = true +comment_width = 90 # small excess is okay but prefer 80 +format_code_in_doc_comments = true +format_generated_files = false + +# Imports. +imports_indent = "Block" +imports_layout = "Mixed" +group_imports = "StdExternalCrate" +reorder_imports = true +imports_granularity = "Crate" + +# Arguments: +use_small_heuristics = "Default" +fn_params_layout = "Compressed" +overflow_delimited_expr = true +where_single_line = true + +# Misc: +inline_attribute_width = 0 +blank_lines_upper_bound = 1 +reorder_impl_items = true +use_field_init_shorthand = true +force_multiline_blocks = true +condense_wildcard_suffixes = true +hex_literal_case = "Upper" + +# Ignored files: +ignore = [] diff --git a/utilities/dbviz/src/default_template.jinja b/utilities/dbviz/src/default_template.jinja new file mode 100644 index 000000000..64488162b --- /dev/null +++ b/utilities/dbviz/src/default_template.jinja @@ -0,0 +1,222 @@ +{%- macro theme(x) -%} +{{ { + "title_loc" : "t", + "title_size" : 30, + "title_color" : "blue", + "graph_direction" : "LR", + "default_fontsize" : 16, + "pkey_bgcolor" : "seagreen1", + "field_desc" : "color='grey50' face='Monospace' point-size='14'", + "column_heading" : "color='black' face='Courier bold' point-size='18'", + "table_heading_bgcolor" : "#009879", + "table_heading" : "color='white' face='Courier bold italic' point-size='20'", + "table_desc_bgcolor" : "grey20", + "table_description" : "color='white' face='Monospace' point-size='14'" +}[x] }} +{%- endmacro -%} + +{%- macro data_type(data_type, default, max_chars, nullable) -%} + + {%- if nullable == "YES" -%} + + {%- endif -%} + + {%- if data_type == "character varying" -%} + varchar + {%- elif data_type == "timestamp without time zone" -%} + timestamp + {%- else -%} + {{- data_type -}} + {%- endif -%} + + {%- if max_chars is not none -%} + ({{max_chars}}) + {%- endif -%} + + {%- if default is startingwith("nextval") -%} + + + {%- endif -%} + + {%- if nullable == "YES" -%} + + {%- endif -%} + +{%- endmacro -%} + +{%- macro pkey_bgcolor(pkey) -%} + {%- if pkey -%} + bgcolor="{{ theme('pkey_bgcolor') }}" + {%- endif -%} +{%- endmacro -%} + +{%- macro column_name_x(name, pkey) -%} + {{name}}
+{%- endmacro -%} + + +{%- macro column_name(field) -%} + {{- column_name_x(field.column, field.primary_key) -}} +{%- endmacro -%} + +{%- macro final(field, final=true) -%} + {%- if final -%} + port="{{field.column}}_out" + {%- endif -%} +{%- endmacro -%} + +{%- macro column_type(field, last=false) -%} + {{ data_type(field.data_type, field.default, field.max_chars, field.nullable ) }} +{%- endmacro -%} + +{%- macro column_description(field) -%} + {{- field.description|trim|escape|replace("\n", "
") -}}

+{%- endmacro -%} + +digraph erd { + + {% if opts.title is not none %} + label = "{{ opts.title }}" + labelloc = {{ theme("title_loc") }} + fontsize = {{ theme("title_size") }} + fontcolor = {{ theme("title_color") }} + {% endif %} + + graph [ + rankdir = "{{theme('graph_direction')}}" + ]; + + node [ + fontsize = "{{theme('default_fontsize')}}" + shape = "plaintext" + ]; + + edge [ + ]; + + {% for table in schema.tables %} + {% if opts.comments %} + + "{{table.name}}" [shape=plain label=< + + + + + + + + + + + {% for field in table.fields %} + + {{ column_name(field) }} + {{ column_type(field, false) }} + {{ column_description(field) }} + + {% endfor %} + + {% if table.description is not none %} + + + + {% endif %} + +
{{table.name}}
ColumnTypeDescription
{{- table.description|trim|escape|replace("\n", "
") -}}

+ >]; + + {% else %} + + "{{table.name}}" [label=< + + + + + + + + + + {% for field in table.fields %} + + {{ column_name(field) }} + {{ column_type(field, true) }} + + {% endfor %} + +
{{table.name}}
ColumnType
+ >]; + + {% endif %} + {% endfor %} + + {% for partial in schema.partial_tables %} + + "{{partial}}" [label=< + + + + + + + + + {% for field in schema.partial_tables[partial] %} + + {{ column_name_x(field,false) }} + + {% endfor %} + + + + +
{{partial}}
Column
ABRIDGED
+ >]; + + {% endfor %} + + + "LEGEND" [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LEGEND
TypeExample
Primary Key
{{ data_type("integer", "nextval", none, "NO") }}
Standard Field
{{ data_type("bytea", none, none, "NO") }}
Nullable Field
{{ data_type("text", none, none, "YES") }}
Sized Field
{{ data_type("varchar", none, 32, "NO") }}
Autoincrement Field
{{ data_type("integer", "nextval", none, "NO") }}
+ >]; + + {% for relation in schema.relations %} + "{{relation.on_table}}":"{{relation.on_field}}_out" -> "{{relation.to_table}}":"{{relation.to_field}}" + {% endfor %} + + +} diff --git a/utilities/dbviz/src/lib.rs b/utilities/dbviz/src/lib.rs new file mode 100644 index 000000000..2a0f48add --- /dev/null +++ b/utilities/dbviz/src/lib.rs @@ -0,0 +1,2 @@ +//! Intentionally empty +//! This file exists, so that doc tests can be used inside binary crates. diff --git a/utilities/dbviz/src/main.rs b/utilities/dbviz/src/main.rs new file mode 100644 index 000000000..a22d08ac4 --- /dev/null +++ b/utilities/dbviz/src/main.rs @@ -0,0 +1,39 @@ +//! `DBViz` - Database Diagram Generator +//! +//! `DBViz` is a tool for generating database diagrams. + +mod opts; +mod postgresql; +mod schema; + +use std::fs; + +use anyhow::Result; +use minijinja::{context, Environment}; + +fn main() -> Result<()> { + let opts = opts::load(); + + let loader = postgresql::Conn::new(&opts)?; + let schema = loader.load()?; + + let template_file = match &opts.template { + Some(fname) => fs::read_to_string(fname)?, + None => include_str!("default_template.jinja").to_string(), + }; + + let mut env = Environment::new(); + env.add_template("diagram", &template_file)?; + let tmpl = env.get_template("diagram")?; + + let ctx = context!( + opts => opts, + schema => schema + ); + + let rendered = tmpl.render(ctx)?; + + println!("{rendered}"); + + Ok(()) +} diff --git a/utilities/dbviz/src/opts.rs b/utilities/dbviz/src/opts.rs new file mode 100644 index 000000000..43d38d13d --- /dev/null +++ b/utilities/dbviz/src/opts.rs @@ -0,0 +1,74 @@ +//! CLI Option parsing + +use std::path::PathBuf; + +use clap::{Args, Parser}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Parser, Clone, Serialize, Deserialize)] +#[command(author, version, about, long_about = None)] +/// `DBViz` a tool for generating database diagrams. +pub(crate) struct Cli { + #[command(flatten)] + /// Postgres connection options + pub(crate) pg_opts: Pg, + + #[arg(short, long)] + /// Tables to include in the current diagram. + pub(crate) include_tables: Option>, + + #[arg(short, long)] + /// Tables to completely exclude in the current diagram. + pub(crate) exclude_tables: Option>, + + /// Title to give the Diagram + #[arg(short, long)] + pub(crate) title: Option, + + /// How wide is the Column Description before we wrap it? + #[arg(long)] + pub(crate) column_description_wrap: Option, + + /// How wide is the Table Description before we wrap it? + #[arg(long)] + pub(crate) table_description_wrap: Option, + + /// Do we include comments in the diagram? + #[arg(long)] + pub(crate) comments: bool, + + /// Input file + pub(crate) template: Option, + + /// Output file + pub(crate) output: Option, +} + +#[derive(Debug, Args, Clone, Serialize, Deserialize)] +/// Postgres connection options +pub(crate) struct Pg { + #[arg(short, long, default_value = "localhost")] + /// Hostname to connect to + pub(crate) hostname: String, + + #[arg(short, long, default_value = "postgres")] + /// Username to use when connecting + pub(crate) username: String, + + #[arg(short, long, default_value = "postgres")] + /// Password to use when connecting + pub(crate) password: String, + + #[arg(short, long, default_value = "postgres")] + /// Database name to connect to + pub(crate) database: String, + + #[arg(short, long, default_value = "public")] + /// Schema name to use + pub(crate) schema: String, +} + +/// Load CLI Options. +pub(crate) fn load() -> Cli { + Cli::parse() +} diff --git a/utilities/dbviz/src/postgresql.rs b/utilities/dbviz/src/postgresql.rs new file mode 100644 index 000000000..297eea796 --- /dev/null +++ b/utilities/dbviz/src/postgresql.rs @@ -0,0 +1,312 @@ +//! Loader for postgresql. + +// This is not production code, so while we should get rid of these lints, its not +// worth the time it would take, for no return value. +#![allow(clippy::unwrap_used)] +#![allow(clippy::indexing_slicing)] + +use std::{ + cell::RefCell, + collections::HashMap, + convert::{TryFrom, TryInto}, +}; + +use anyhow::Result; +use itertools::Itertools; +use postgres::{tls::NoTls, Client, Row}; + +use crate::{ + opts, + schema::{Index, Relation, Schema, Table, TableColumn}, +}; + +/// Struct that manages the loading and implements `Loader` trait. +pub struct Conn { + /// Postgres client + pg_client: RefCell, + /// Schema name + schema: String, + /// Options + opts: opts::Cli, +} + +/// Check if a column is a primary key +fn is_primary_key(table: &str, column: &str, indexes: &[Index]) -> bool { + indexes + .iter() + .any(|idx| idx.table == table && idx.fields.contains(&column.to_string()) && idx.primary) +} + +impl Conn { + /// Make a new postgres connection + pub(crate) fn new(opts: &opts::Cli) -> Result { + let pg_client = postgres::Config::new() + .user(&opts.pg_opts.username) + .password(&opts.pg_opts.password) + .dbname(&opts.pg_opts.database) + .host(&opts.pg_opts.hostname) + .connect(NoTls)?; + + let pg_client = RefCell::new(pg_client); + let schema = opts.pg_opts.schema.clone(); + Ok(Conn { + pg_client, + schema, + opts: opts.clone(), + }) + } + + /// Do we include this table name? + fn include_table(&self, name: &String) -> bool { + match &self.opts.include_tables { + Some(inc) => inc.contains(name), + None => true, + } + } + + /// Do we exclude this table name? + fn exclude_table(&self, name: &String) -> bool { + match &self.opts.exclude_tables { + Some(inc) => inc.contains(name), + None => false, + } + } + + /// Load the schema + pub(crate) fn load(&self) -> Result { + let mut client = self.pg_client.borrow_mut(); + let tables_rows = client.query(tables_query(), &[&self.schema])?; + let relations_rows = client.query(relations_query(), &[&self.schema])?; + let index_rows = client.query(index_query(), &[])?; + + let mut partial_tables: HashMap> = HashMap::new(); + + let indexes: Vec<_> = index_rows + .into_iter() + .filter(|row| { + let row_name: String = row.get(0); + self.include_table(&row_name) && !self.exclude_table(&row_name) + }) + .map(|row| { + let idx: Index = row.try_into().unwrap(); + idx + }) + .collect(); + + let tables: Vec<_> = tables_rows + .into_iter() + .group_by(|row| row.get(0)) + .into_iter() + .filter(|(name, _rows)| self.include_table(name) && !self.exclude_table(name)) + .map(|(name, rows)| { + let fields: Vec<_> = rows + .into_iter() + .map(|row| { + let mut field: TableColumn = row.try_into().unwrap(); + field.primary_key = is_primary_key(&name, &field.column, &indexes); + + let desc = match field.description { + Some(desc) => { + match self.opts.column_description_wrap { + Some(wrap) => Some(textwrap::fill(&desc, wrap)), + None => Some(desc), + } + }, + None => None, + }; + field.description = desc; + + field + }) + .collect(); + + let desc = match &fields[0].table_description { + Some(desc) => { + match self.opts.table_description_wrap { + Some(wrap) => Some(textwrap::fill(desc, wrap)), + None => Some(desc).cloned(), + } + }, + None => None, + }; + + Table { + name, + description: desc, + fields, + } + }) + .collect(); + + let relations: Vec<_> = relations_rows + .into_iter() + .map(|row| { + let relation: Relation = row.try_into().unwrap(); + relation + }) + .filter(|relation| { + if self.include_table(&relation.on_table) + && !self.exclude_table(&relation.on_table) + && !self.exclude_table(&relation.to_table) + { + if !self.include_table(&relation.to_table) { + match partial_tables.get_mut(&relation.to_table) { + Some(value) => { + if !value.contains(&relation.to_field) { + value.push(relation.to_field.clone()); + } + }, + None => { + partial_tables.insert(relation.to_table.clone(), vec![relation + .to_field + .clone()]); + }, + } + } + true + } else { + false + } + }) + .collect(); + + Ok(Schema { + tables, + relations, + partial_tables, + }) + } +} + +impl TryFrom for Index { + type Error = String; + + fn try_from(row: Row) -> std::result::Result { + let all_fields: String = row.get(4); + let braces: &[_] = &['{', '}']; + + let fields: Vec<_> = all_fields + .trim_matches(braces) + .split(',') + .map(std::string::ToString::to_string) + .collect(); + + Ok(Self { + table: row.get(0), + // name: row.get(1), + primary: row.get(2), + // unique: row.get(3), + fields, + }) + } +} + +impl TryFrom for Relation { + type Error = String; + + fn try_from(row: Row) -> std::result::Result { + let fields: HashMap = row + .columns() + .iter() + .enumerate() + .map(|(i, c)| (c.name().to_string(), row.get(i))) + .collect(); + + Ok(Self { + on_table: fetch_field(&fields, "on_table")?, + on_field: fetch_field(&fields, "on_field")?, + to_table: fetch_field(&fields, "to_table")?, + to_field: fetch_field(&fields, "to_field")?, + }) + } +} + +impl TryFrom for TableColumn { + type Error = String; + + fn try_from(row: Row) -> std::result::Result { + Ok(Self { + column: row.get(1), + data_type: row.get(2), + index: row.get(3), + default: row.get(4), + nullable: row.get(5), + max_chars: row.get(6), + description: row.get(7), + table_description: row.get(8), + primary_key: false, + }) + } +} + +/// Fetch a field from a hashmap +fn fetch_field(map: &HashMap, key: &str) -> std::result::Result { + map.get(key) + .cloned() + .ok_or(format!("could not find field {key}")) +} + +/// Query all tables and columns +fn tables_query() -> &'static str { + " + select table_name, column_name, data_type, ordinal_position, column_default, is_nullable, character_maximum_length, col_description(table_name::regclass, ordinal_position), obj_description(table_name::regclass) + from information_schema.columns + where table_schema = $1 + order by table_name, ordinal_position + " +} + +/// Query all relationships +fn relations_query() -> &'static str { + " + select * + from ( + select ns.nspname AS schemaname, + cl.relname AS on_table, + attr.attname AS on_field, + clf.relname AS to_table, + attrf.attname AS to_field + from pg_constraint con + join pg_class cl + on con.conrelid = cl.oid + join pg_namespace ns + on cl.relnamespace = ns.oid + join pg_class clf + on con.confrelid = clf.oid + join pg_attribute attr + on attr.attnum = ANY(con.conkey) and + attr.attrelid = con.conrelid + join pg_attribute attrf + on attrf.attnum = ANY(con.confkey) and + attrf.attrelid = con.confrelid + ) as fk + where fk.schemaname = $1 + " +} + +/// Query all indexes +fn index_query() -> &'static str { + " +SELECT + CAST(idx.indrelid::regclass as varchar) as table_name, + i.relname as index_name, + idx.indisprimary as primary_key, + idx.indisunique as unique, + CAST( + ARRAY( + SELECT pg_get_indexdef(idx.indexrelid, k + 1, true) + FROM generate_subscripts(idx.indkey, 1) as k + ORDER BY k + ) as varchar + ) as columns +FROM pg_index as idx +JOIN pg_class as i +ON i.oid = idx.indexrelid +JOIN pg_am as am +ON i.relam = am.oid +JOIN pg_namespace as ns +ON ns.oid = i.relnamespace +AND ns.nspname = ANY(current_schemas(false)) +ORDER BY idx.indrelid +" +} diff --git a/utilities/dbviz/src/schema.rs b/utilities/dbviz/src/schema.rs new file mode 100644 index 000000000..da07eb9b5 --- /dev/null +++ b/utilities/dbviz/src/schema.rs @@ -0,0 +1,84 @@ +//! Core entities. +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// All the schema information. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct Schema { + /// List of tables in the database. + pub(crate) tables: Vec, + /// List of relations in the database. + pub(crate) relations: Vec, + /// Partial Tables + pub(crate) partial_tables: HashMap>, +} + +/// Table information. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct TableColumn { + /// Column name. + pub(crate) column: String, + /// Column data type. + pub(crate) data_type: String, + /// Column index. + pub(crate) index: i32, + /// Column default. + pub(crate) default: Option, + /// Column nullable. + pub(crate) nullable: String, + /// Column max chars. + pub(crate) max_chars: Option, + /// Column description. + pub(crate) description: Option, + /// Table description. + pub(crate) table_description: Option, // Redundant but easiest way to get it. + /// Column primary key. + pub(crate) primary_key: bool, +} + +/// Table information. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct Table { + /// Table name. + pub(crate) name: String, + /// Table Description + pub(crate) description: Option, + /// List of fields. + pub(crate) fields: Vec, +} + +/// Row description. +//#[derive(Debug)] +// pub(crate)struct Field(pub(crate)FieldName, pub(crate)FieldType); + +/// Relation node. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct Relation { + /// Table that the constraint references. + pub(crate) on_table: TableName, + /// Field that the constraint references. + pub(crate) on_field: FieldName, + /// Table which the fk references. + pub(crate) to_table: TableName, + /// Field which the fk references. + pub(crate) to_field: FieldName, +} + +/// Table name +pub(crate) type TableName = String; +/// Field name +pub(crate) type FieldName = String; +// pub(crate)type FieldType = String; + +/// Index Definition +pub(crate) struct Index { + /// Table name + pub(crate) table: TableName, + // pub(crate)name: String, + /// Primary Key + pub(crate) primary: bool, + // pub(crate)unique: bool, + /// Fields + pub(crate) fields: Vec, +} diff --git a/utilities/nushell/Earthfile b/utilities/nushell/Earthfile new file mode 100644 index 000000000..e047c3367 --- /dev/null +++ b/utilities/nushell/Earthfile @@ -0,0 +1,65 @@ +# Common Rust UDCs and Builders. +VERSION --global-cache 0.7 + +# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT +# cspell: words nushell dataframe nufmt rustup toolset + +# Base Rustup build container. +nushell-build: + ARG TARGETPLATFORM + ARG TARGETOS + ARG TARGETARCH + ARG TARGETVARIANT + ARG USERPLATFORM + ARG USEROS + ARG USERARCH + ARG USERVARIANT + + # This is our base Host toolset, and rustup. + # The ACTUAL version of rust that will be used, and available targets + # is controlled by a `rust-toolchain.toml` file when the `SETUP` UDC is run. + # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version. + FROM rust:1.73-alpine3.18 + + RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \ + echo "TARGETOS = $TARGETOS"; \ + echo "TARGETARCH = $TARGETARCH"; \ + echo "TARGETVARIANT = $TARGETVARIANT"; \ + echo "USERPLATFORM = $USERPLATFORM"; \ + echo "USEROS = $USEROS"; \ + echo "USERARCH = $USERARCH"; \ + echo "USERVARIANT = $USERVARIANT"; + + WORKDIR /root + + # Install necessary packages + # Expand this list as needed, rather than adding more tools in later containers. + RUN apk add --no-cache \ + musl-dev \ + mold \ + clang \ + alpine-sdk \ + openssl-dev \ + perl \ + openssl-libs-static + + RUN OPENSSL_NO_VENDOR=Y cargo install nu --version=0.87.1 --features static-link-openssl,extra,dataframe + + # Reduce the size of `nu` and make sure it can execute. + RUN strip $CARGO_HOME/bin/nu + RUN $CARGO_HOME/bin/nu --help + + # Build nufmt from latest git. + RUN git clone https://github.com/nushell/nufmt + + WORKDIR /root/nufmt + + RUN cargo build --release + RUN cargo install --path . + + # Reduce the size of `nufmt` and make sure it can execute. + RUN strip $CARGO_HOME/bin/nufmt + RUN $CARGO_HOME/bin/nufmt --help + + SAVE ARTIFACT $CARGO_HOME/bin/nu nu + SAVE ARTIFACT $CARGO_HOME/bin/nufmt nufmt \ No newline at end of file diff --git a/utilities/scripts/Earthfile b/utilities/scripts/Earthfile new file mode 100644 index 000000000..b9b32b5db --- /dev/null +++ b/utilities/scripts/Earthfile @@ -0,0 +1,30 @@ +# Earthfile containing common scripts for easy reference +VERSION --global-cache 0.7 + +# Internal: bash-scripts : Common bash scripts. +bash-scripts: + FROM scratch + + COPY --dir bash / + + SAVE ARTIFACT /bash /include + +# Internal: bash-scripts : Common bash scripts. +python-scripts: + FROM scratch + + COPY --dir python / + + SAVE ARTIFACT /python /python + +# UDC to add our common bash scripts to a container image. +ADD_BASH_SCRIPTS: + COMMAND + + COPY --dir +bash-scripts/include /scripts/include + +# UDC to add our common bash scripts to a container image. +ADD_PYTHON_SCRIPTS: + COMMAND + + COPY --dir +python-scripts/python /scripts/python \ No newline at end of file diff --git a/utilities/scripts/bash/assert.sh b/utilities/scripts/bash/assert.sh new file mode 100644 index 000000000..c402dedeb --- /dev/null +++ b/utilities/scripts/bash/assert.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +assert_eq() { + local expected="$1" + local actual="$2" + + if [[ "${expected}" == "${actual}" ]]; then + return 0 + else + echo "assert_eq FAILED" + echo "expected:" + echo "${expected}" + echo "actual:" + echo "${actual}" + echo "Diff:" + diff <(echo "${expected}") <(echo "${actual}") + return 1 + fi +} diff --git a/earthly/docs/scripts/colors.sh b/utilities/scripts/bash/colors.sh old mode 100755 new mode 100644 similarity index 53% rename from earthly/docs/scripts/colors.sh rename to utilities/scripts/bash/colors.sh index 3d1cbd2be..3518d861a --- a/earthly/docs/scripts/colors.sh +++ b/utilities/scripts/bash/colors.sh @@ -1,5 +1,7 @@ #!/bin/bash +# cspell: words localfile vendorfile colordiff Naur + # shellcheck disable=SC2034 # This file is intended to bo sourced. BLACK='\033[0;30m' @@ -12,7 +14,6 @@ CYAN='\033[0;36m' WHITE='\033[0;37m' NC='\033[0m' # No Color - status() { local rc="$1" local message="$2" @@ -29,5 +30,23 @@ status() { fi # Return the current status - return "$rc" + return "${rc}" +} + +status_and_exit() { + if ! status 0 "$@"; then + exit 1 + fi +} + +# Checks if two files that should exist DO, and are equal. +# used to enforce consistency between local config files and the expected config locked in CI. +check_vendored_files() { + local rc=$1 + local localfile=$2 + local vendorfile=$3 + + status "${rc}" "Checking if Local File '${localfile}' == Vendored File '${vendorfile}'" \ + colordiff -Naur "${localfile}" "${vendorfile}" + return $? } diff --git a/utilities/scripts/bash/db_ops.sh b/utilities/scripts/bash/db_ops.sh new file mode 100755 index 000000000..2147e8d9c --- /dev/null +++ b/utilities/scripts/bash/db_ops.sh @@ -0,0 +1,363 @@ +#!/usr/bin/env bash + +# cspell: words dbhost dbport dbuser dbuserpw dbname dbsuperuser dbsuperuserpw dbnamesuperuser dbdescription dbpath +# cspell: words dbauthmethod dbcollation dbreadytimeout setupdbsql dbrefinerytoml dbmigrations dbseeddatasrc +# cspell: words pgsql initdb pwfile dbconn isready dbdesc psql + +# This script is not intended to be run by itself, and provides common functions +# for database operations. + +source "/scripts/include/colors.sh" +source "/scripts/include/params.sh" + +# define all the defaults we could need +declare -A defaults +# shellcheck disable=SC2034 # It really is used +defaults=( + ["dbhost"]="localhost" + ["dbport"]="5432" + ["dbuser"]="postgres" + ["dbuserpw"]="CHANGE_ME" + ["dbname"]="postgres" + ["dbsuperuser"]="admin" + ["dbsuperuserpw"]="CHANGE_ME" + ["dbnamesuperuser"]="postgres" + ["dbdescription"]="PostgreSQL Database" + ["dbpath"]="/var/lib/postgresql/data" + ["dbauthmethod"]="trust" + ["dbcollation"]="en_US.utf8" + ["dbreadytimeout"]="-1" + ["setupdbsql"]="/sql/setup-db.sql" + ["dbrefinerytoml"]="./refinery.toml" + ["dbmigrations"]="./migrations" + ["dbseeddatasrc"]="./seed" + ["init_and_drop_db"]=true + ["with_migrations"]=true + ["with_seed_data"]="" +) + +# Define how parameters map to env vars +declare -A env_vars +# shellcheck disable=SC2034 # It really is used +env_vars=( + ["dbhost"]="DB_HOST" + ["dbport"]="DB_PORT" + ["dbname"]="DB_NAME" + ["dbdescription"]="DB_DESCRIPTION" + ["dbuser"]="DB_USER" + ["dbuserpw"]="DB_USER_PASSWORD" + ["dbsuperuser"]="DB_SUPERUSER" + ["dbsuperuserpw"]="DB_SUPERUSER_PASSWORD" + ["dbnamesuperuser"]="DB_NAME_SUPERUSER" + ["dbpath"]="DB_PATH" + ["dbauthmethod"]="DB_AUTH_METHOD" + ["dbcollation"]="DB_COLLATION" + ["dbreadytimeout"]="DB_READY_TIMEOUT" + ["setupdbsql"]="SETUP_DB_SQL" + ["dbrefinerytoml"]="DB_REFINERY_TOML" + ["dbmigrations"]="DB_MIGRATIONS" + ["dbseeddatasrc"]="DB_SEED_DATA_SRC" + ["init_and_drop_db"]="INIT_AND_DROP_DB" + ["with_migrations"]="WITH_MIGRATIONS" + ["with_seed_data"]="WITH_SEED_DATA" +) + +function pgsql_user_connection() { + # shellcheck disable=SC2155 # Can not fail + local dbname=$(get_param dbname env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbhost=$(get_param dbhost env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbport=$(get_param dbport env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbuser=$(get_param dbuser env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbuserpw=$(get_param dbuserpw env_vars defaults "$@") + + echo "postgres://${dbuser}:${dbuserpw}@${dbhost}:${dbport}/${dbname}" +} + +function pgsql_superuser_connection() { + # shellcheck disable=SC2155 # Can not fail + local dbname=$(get_param dbnamesuperuser env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbhost=$(get_param dbhost env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbport=$(get_param dbport env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbsuperuser=$(get_param dbsuperuser env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local dbsuperuserpw=$(get_param dbsuperuserpw env_vars defaults "$@") + + echo "postgres://${dbsuperuser}:${dbsuperuserpw}@${dbhost}:${dbport}/${dbname}" +} + +# Initialize the database +# --dbpath = +# --dbauthmethod = +function init_db() { + # Start PostgreSQL in the background + + # shellcheck disable=SC2155 # Can not fail + local data_dir=$(get_param dbpath env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local auth_method=$(get_param dbauthmethod env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local super_user=$(get_param dbsuperuser env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local super_user_pw=$(get_param dbsuperuserpw env_vars defaults "$@") + # shellcheck disable=SC2155 # Can not fail + local collation=$(get_param dbcollation env_vars defaults "$@") + + echo "data dir: ${data_dir}" + echo "auth method: ${auth_method}" + + echo "POSTGRES_HOST_AUTH_METHOD is set to ${auth_method}" + + if ! initdb -D "${data_dir}" \ + --locale-provider=icu --icu-locale="${collation}" --locale="${collation}" \ + -A "${auth_method}" \ + -U "${super_user}" --pwfile=<(echo "${super_user_pw}"); then + return 1 + fi + + echo "include_if_exists ${data_dir}/pg_hba.extra.conf" >> "${data_dir}/pg_hba.conf" + echo "include_if_exists /sql/pg_hba.extra.conf" >> "${data_dir}/pg_hba.conf" + + return 0 +} + +# Start PostgreSQL Local database server in the background +# --dbpath = +function run_pgsql() { + # Function to start the local PostgreSQL server + + # shellcheck disable=SC2155 # Can not fail + local data_dir=$(get_param dbpath env_vars defaults "$@") + + if ! pg_ctl -D "${data_dir}" start; then + return 1 + fi + + echo "PostgreSQL is starting" + + return 0 +} + +# Wait until PostgreSQL is ready to serve requests +# --dbreadytimeout =