From a8bdcafd74fe16844e4d9880187cce09c57eb3b4 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Fri, 11 Oct 2024 09:11:51 +0000
Subject: [PATCH 01/10] chore(tooling): swap out dbviz custom database diagram
 tool for SchemaSpy, remove dependencies in catalyst-ci postgresql base

---
 earthly/postgresql/Earthfile           |  50 +++-
 earthly/postgresql/scripts/std_docs.py | 361 +++----------------------
 earthly/postgresql/templates/schema.md |  64 +++++
 3 files changed, 138 insertions(+), 337 deletions(-)
 create mode 100644 earthly/postgresql/templates/schema.md

diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
index 37f75b076..4d45cfab0 100644
--- a/earthly/postgresql/Earthfile
+++ b/earthly/postgresql/Earthfile
@@ -2,10 +2,9 @@
 VERSION 0.8
 
 IMPORT ../rust/tools AS rust-tools
-IMPORT ../../utilities/dbviz AS dbviz
 IMPORT ../../utilities/scripts AS scripts
 
-# cspell: words psycopg dbviz
+# cspell: words psycopg
 
 postgres-base:
     FROM postgres:16.4-bookworm
@@ -43,10 +42,6 @@ postgres-base:
     # Get refinery
     COPY rust-tools+tool-refinery/refinery /bin
 
-    # Get dbviz
-    COPY dbviz+build/dbviz /bin
-    RUN dbviz --help
-
     # Copy our set SQL files
     COPY --dir sql /sql
 
@@ -56,7 +51,11 @@ postgres-base:
     DO scripts+ADD_BASH_SCRIPTS
     DO scripts+ADD_PYTHON_SCRIPTS
 
+    # Copy templates to the working directory
+    COPY --dir templates /templates
+
     SAVE ARTIFACT /scripts /scripts
+    SAVE ARTIFACT /templates /templates
 
 # Common build setup steps.
 # Arguments:
@@ -82,8 +81,13 @@ BUILDER:
 
 # DOCS - FUNCTION to build the docs, needs to be run INSIDE the BUILDER like so:
 # 
-# 1. Create a ./docs/diagrams.json which has the options needed to run to generate the docs to /docs
-# 2. Define the following targets in your earthfile
+# This function uses SchemaSpy to generate database documentation.
+# SchemaSpy creates detailed, browsable ER diagrams and schema documentation.
+#
+# To use this function:
+# 1. Ensure your migrations are in the ./migrations directory
+# 2. Have a refinery.toml file to configure the migrations
+# 3. Define the following targets in your earthfile:
 #
 # builder:
 #    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:<ver>+BUILDER --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff
@@ -93,25 +97,46 @@ BUILDER:
 #
 #    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:<ver>+BUILD --image_name=<name>
 #    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:<ver>+DOCS
+#
+# The generated documentation will be saved in the ./docs artifact.
 DOCS:
     FUNCTION
 
-    ARG diagrams=./diagrams.json
     ARG migrations=./migrations
     ARG refinery_toml=./refinery.toml
 
+    FROM +docs-base
+      
     USER postgres:postgres
     WORKDIR /docs
 
-    COPY $diagrams ./diagrams.json
     COPY --dir $migrations .
     COPY --dir $refinery_toml .
 
-    RUN /scripts/std_docs.py ./diagrams.json
+    RUN /scripts/std_docs.py
+    # Pull templates artifact from postgres-base
+    COPY +postgres-base/templates/schema.md ./docs/schema.md
 
-    SAVE ARTIFACT docs /docs
+    SAVE ARTIFACT docs ./docs
 
+docs-base:
+    FROM +postgres-base
 
+    # Define ARGs for versions specific to docs
+    ARG SCHEMASPY_VERSION=6.2.4
+    ARG POSTGRESQL_JDBC_VERSION=42.7.4
+    ARG OPENJDK_VERSION=21
+
+    WORKDIR /
+        
+    # Setup SchemaSpy requirements, commands combined to reduce layers
+    RUN echo "deb http://ftp.de.debian.org/debian sid main" > /etc/apt/sources.list && \
+        apt-get update && \
+        apt-get install -y \
+            openjdk-${OPENJDK_VERSION}-jre-headless && \
+        wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar && \
+        wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar && \
+        apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Linter checks for sql files
 CHECK:
@@ -171,3 +196,4 @@ BUILD:
     # Push the container...
     SAVE IMAGE ${image_name}:latest
 
+
diff --git a/earthly/postgresql/scripts/std_docs.py b/earthly/postgresql/scripts/std_docs.py
index 09789a49c..b5dbc8ea7 100755
--- a/earthly/postgresql/scripts/std_docs.py
+++ b/earthly/postgresql/scripts/std_docs.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# cspell: words dbmigrations dbviz dbhost dbuser dbuserpw Tsvg
+# cspell: words dbmigrations dbhost dbuser dbuserpw Tsvg
 
 from typing import Optional
 import python.exec_manager as exec_manager
@@ -10,87 +10,14 @@
 from rich import print
 import os
 import re
-import json
-from dataclasses import dataclass
 from textwrap import indent
 
-
-@dataclass
-class DiagramCfg:
-    title: str
-    version: int
-    migration_name: str
-    tables: Optional[list[str]]
-    included_tables: Optional[list[str]]
-    excluded_tables: Optional[list[str]]
-    comments: Optional[bool]
-    column_description_wrap: Optional[int]
-    table_description_wrap: Optional[int]
-    sql_data: str
-
-    def include(
-        self,
-        extra_includes: Optional[list[str]] = None,
-        extra_excludes: Optional[list[str]] = None,
-    ) -> Optional[list[str]]:
-        # We exclude from the global include tables, any tables the migration
-        # itself requests to be excluded.
-
-        include_tables = self.included_tables if self.included_tables else []
-        tables = self.tables if self.tables else []
-        extra_includes = extra_includes if extra_includes else []
-        excluded_tables = self.excluded_tables if self.excluded_tables else []
-        extra_excludes = extra_excludes if extra_excludes else []
-
-        for table in tables + extra_includes:
-            if (
-                table not in excluded_tables
-                and table not in extra_excludes
-                and table not in include_tables
-            ):
-                include_tables.append(table)
-
-        if len(include_tables) == 0:
-            include_tables = None
-
-        return include_tables
-
-    def exclude(
-        self, extra_excludes: Optional[list[str]] = None
-    ) -> Optional[list[str]]:
-        # We exclude from the global exclude tables, any tables the migration
-        # specifically includes.
-        exclude_tables = self.excluded_tables if self.excluded_tables else []
-        extra_excludes = extra_excludes if extra_excludes else []
-        for table in extra_excludes:
-            if table not in exclude_tables:
-                exclude_tables.append(table)
-
-        if len(exclude_tables) == 0:
-            exclude_tables = None
-
-        return exclude_tables
-
-
 def process_sql_files(directory):
     file_pattern = r"V(\d+)__(\w+)\.sql"
-    table_pattern = r"CREATE TABLE(?: IF NOT EXISTS)? (\w+)"
-
-    diagram_option_pattern = r"^--\s*(Title|Include|Exclude|Comment|Column Description Wrap|Table Description Wrap)\s+:\s*(.*)$"
-
     migrations = {}
     largest_version = 0
 
     for filename in os.listdir(directory):
-        clean_sql = ""
-        title = None
-        table_names = []
-        included_tables = None
-        excluded_tables = None
-        comments = None
-        column_description_wrap = None
-        table_description_wrap = None
-
         match = re.match(file_pattern, filename)
         if match:
             version = int(match.group(1))
@@ -101,61 +28,14 @@ def process_sql_files(directory):
 
             with open(os.path.join(directory, filename), "r") as file:
                 sql_data = file.read()
-                for line in sql_data.splitlines():
-                    match = re.match(diagram_option_pattern, line)
-                    if match:
-                        if match.group(1).lower() == "title" and title is None:
-                            title = match.group(2)
-                        elif (
-                            match.group(1).lower() == "include"
-                            and len(match.group(2)) > 0
-                        ):
-                            if included_tables is None:
-                                included_tables = []
-                            included_tables.append(match.group(2).split())
-                        elif (
-                            match.group(1).lower() == "exclude"
-                            and len(match.group(2)) > 0
-                        ):
-                            if excluded_tables is None:
-                                excluded_tables = []
-                            excluded_tables.append(match.group(2).split())
-                        elif match.group(1).lower() == "comment":
-                            if match.group(2).strip().lower() == "true":
-                                comments = True
-                        elif match.group(1).lower() == "column description wrap":
-                            try:
-                                column_description_wrap = int(match.group(2))
-                            except:
-                                pass
-                        elif match.group(1).lower() == "table description wrap":
-                            try:
-                                table_description_wrap = int(match.group(2))
-                            except:
-                                pass
-                    else:
-                        # We strip diagram options from the SQL.
-                        clean_sql += line + "\n"
-
-                        match = re.match(table_pattern, line)
-                        if match:
-                            table_names.append(match.group(1))
-
-            migrations[version] = DiagramCfg(
-                title,
-                version,
-                migration_name,
-                table_names,
-                included_tables,
-                excluded_tables,
-                comments,
-                column_description_wrap,
-                table_description_wrap,
-                clean_sql,
-            )
 
-    return migrations, largest_version
+            migrations[version] = {
+                "version": version,
+                "migration_name": migration_name,
+                "sql_data": sql_data
+            }
 
+    return migrations, largest_version
 
 class Migrations:
     def __init__(self, args: argparse.Namespace):
@@ -169,184 +49,8 @@ def __init__(self, args: argparse.Namespace):
             None
         """
         self.args = args
-
-        with open(args.diagram_config) as f:
-            self.config = json.load(f)
-
         self.migrations, self.migration_version = process_sql_files(args.dbmigrations)
 
-    def schema_name(self) -> str:
-        return self.config.get("name", "Database Schema")
-
-    def all_schema_comments(self) -> bool:
-        return self.config.get("all_schema", {}).get("comments", False)
-
-    def full_schema_comments(self) -> bool:
-        return self.config.get("full_schema", {}).get(
-            "comments", self.all_schema_comments()
-        )
-
-    def all_schema_included_tables(self) -> list[str]:
-        return self.config.get("all_schema", {}).get("included_tables", [])
-
-    def all_schema_excluded_tables(self) -> list[str]:
-        return self.config.get("all_schema", {}).get("excluded_tables", [])
-
-    def full_schema_excluded_tables(self) -> list[str]:
-        return self.config.get("full_schema", {}).get(
-            "excluded_tables", self.all_schema_excluded_tables()
-        )
-
-    def all_schema_column_description_wrap(self) -> int:
-        return self.config.get("all_schema", {}).get("column_description_wrap", 50)
-
-    def full_schema_column_description_wrap(self) -> int:
-        return self.config.get("full_schema", {}).get(
-            "column_description_wrap", self.all_schema_column_description_wrap()
-        )
-
-    def all_schema_table_description_wrap(self) -> int:
-        return self.config.get("all_schema", {}).get("table_description_wrap", 50)
-
-    def full_schema_table_description_wrap(self) -> int:
-        return self.config.get("full_schema", {}).get(
-            "table_description_wrap", self.all_schema_table_description_wrap()
-        )
-
-    def dbviz(
-        self,
-        filename: str,
-        name: str,
-        title: str,
-        included_tables: Optional[list[str]] = None,
-        excluded_tables: Optional[list[str]] = None,
-        comments: Optional[bool] = None,
-        column_description_wrap: Optional[int] = None,
-        table_description_wrap: Optional[int] = None,
-    ) -> exec_manager.Result:
-        if len(title) > 0:
-            title = f' --title "{title}"'
-
-        includes = ""
-        if included_tables:
-            for table in included_tables:
-                includes += f" -i {table}"
-
-        excludes = ""
-        if excluded_tables:
-            for table in excluded_tables:
-                excludes += f" -e {table}"
-
-        if comments:
-            comments = " --comments"
-        else:
-            comments = ""
-
-        if column_description_wrap and column_description_wrap > 0:
-            column_description_wrap = (
-                f" --column-description-wrap {column_description_wrap}"
-            )
-        else:
-            column_description_wrap = ""
-
-        if table_description_wrap and table_description_wrap > 0:
-            table_description_wrap = (
-                f" --table-description-wrap {table_description_wrap}"
-            )
-        else:
-            table_description_wrap = ""
-
-        res = exec_manager.cli_run(
-            f"dbviz -d {self.args.dbname}"
-            + f" -h {self.args.dbhost}"
-            + f" -u {self.args.dbuser}"
-            + f" -p {self.args.dbuserpw}"
-            + f"{title}"
-            + f"{includes}"
-            + f"{excludes}"
-            + f"{comments}"
-            + f"{column_description_wrap}"
-            + f"{table_description_wrap}"
-            + f" | dot -Tsvg -o {filename}",
-            # + f" > {filename}.dot",
-            name=f"Generate Schema Diagram: {name}",
-            verbose=True,
-        )
-
-        # if res.ok:
-        #    exec_manager.cli_run(
-        #        f"dot -Tsvg {filename}.dot -o {filename}",
-        #        name=f"Render Schema Diagram to SVG: {name}",
-        #        verbose=True,
-        #    )
-
-        return res
-
-    def full_schema_diagram(self) -> exec_manager.Result:
-        # Create a full Schema Diagram.
-        return self.dbviz(
-            "docs/full-schema.svg",
-            "Full Schema",
-            self.schema_name(),
-            excluded_tables=self.full_schema_excluded_tables(),
-            comments=self.full_schema_comments(),
-            column_description_wrap=self.full_schema_column_description_wrap(),
-            table_description_wrap=self.full_schema_table_description_wrap(),
-        )
-
-    def migration_schema_diagram(self, ver: int) -> exec_manager.Result:
-        # Create a schema diagram for an individual migration.
-        if ver in self.migrations:
-            migration = self.migrations[ver]
-
-            include_tables = migration.include(
-                self.all_schema_included_tables(), self.all_schema_excluded_tables()
-            )
-            if include_tables is None:
-                return exec_manager.Result(
-                    0,
-                    "",
-                    "",
-                    0.0,
-                    f"Migration {ver} has no tables to diagram.",
-                )
-
-            exclude_tables = migration.exclude(self.all_schema_excluded_tables())
-
-            title = f"{migration.migration_name}"
-            if migration.title and len(migration.title) > 0:
-                title = migration.title
-
-            comments = None
-            if migration.comments is not None:
-                comments = migration.comments
-            else:
-                comments = self.all_schema_comments()
-
-            return self.dbviz(
-                f"docs/migration-{ver}.svg",
-                f"V{ver}__{migration.migration_name}",
-                title,
-                included_tables=include_tables,
-                excluded_tables=exclude_tables,
-                comments=comments,
-                column_description_wrap=migration.column_description_wrap,
-                table_description_wrap=migration.table_description_wrap,
-            )
-
-    def create_diagrams(self, results: exec_manager.Results) -> exec_manager.Results:
-        # Create a full Schema Diagram first.
-        res = self.full_schema_diagram()
-        results.add(res)
-
-        for ver in sorted(self.migrations.keys()):
-            res = self.migration_schema_diagram(ver)
-            results.add(res)
-
-        # exec_manager.cli_run("ls -al docs", verbose=True)
-
-        return results
-
     def create_markdown_file(self, file_path):
         with open(file_path, "w") as markdown_file:
             # Write the title with the maximum migration version
@@ -354,26 +58,13 @@ def create_markdown_file(self, file_path):
                 "# Migrations (Version {}) \n\n".format(self.migration_version)
             )
 
-            # Link the full schema diagram.
-            markdown_file.write('??? example "Full Schema Diagram"\n\n')
-            markdown_file.write(
-                '    ![Full Schema](./full-schema.svg "Full Schema")\n\n'
-            )
-
             # Write the contents of each file in order
             for version in sorted(self.migrations.keys()):
                 migration = self.migrations[version]
-                sql_data = migration.sql_data.strip()
+                sql_data = migration["sql_data"].strip()
 
                 # Write the title of the file
-                markdown_file.write(f"## {migration.migration_name}\n\n")
-
-                if os.path.exists(f"docs/migration-{version}.svg"):
-                    markdown_file.write('??? example "Schema Diagram"\n\n')
-                    markdown_file.write(
-                        f"    ![Migration {migration.migration_name}]"
-                        + f'(./migration-{version}.svg "{migration.migration_name}")\n\n'
-                    )
+                markdown_file.write(f"## {migration['migration_name']}\n\n")
 
                 markdown_file.write('??? abstract "Schema Definition"\n\n')
                 markdown_file.write(
@@ -382,7 +73,6 @@ def create_markdown_file(self, file_path):
 
         print("Markdown file created successfully at: {}".format(file_path))
 
-
 def main():
     # Force color output in CI
     rich.reconfigure(color_system="256")
@@ -390,7 +80,6 @@ def main():
     parser = argparse.ArgumentParser(
         description="Standard Postgresql Documentation Processing."
     )
-    parser.add_argument("diagram_config", help="Diagram Configuration JSON")
     parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
     db_ops.add_args(parser)
 
@@ -418,21 +107,43 @@ def main():
         results.add(res)
 
     if res.ok():
-        exec_manager.cli_run("mkdir docs")  # Where we build the docs.
+        # Create the docs directory
+        exec_manager.cli_run("mkdir -p docs")  # Where we build the docs.
 
         # Get all info about the migrations.
         migrations = Migrations(args)
-        results = migrations.create_diagrams(results)
 
     if results.ok():
+        schemaspy_cmd = (
+            f"java -jar /bin/schemaspy.jar -t pgsql11 "
+            f"-dp /bin/postgresql.jar "
+            f"-db {args.dbname} "
+            f"-host {args.dbhost} "
+            f"-u {args.dbuser} "
+            f"-p {args.dbuserpw} "
+            f"-o docs/database_schema/ "
+        )
+        res = exec_manager.cli_run(
+            schemaspy_cmd,
+            name="Generate SchemaSpy Documentation",
+            verbose=True
+        )
+        results.add(res)
+
+        # If SchemaSpy command completes without error, create .pages file to hide the schema folder
+        if res.ok():
+            exec_manager.cli_run(
+                'echo "hide: true" > docs/database_schema/.pages',
+                name="Create .pages file",
+                verbose=True
+            )
+
         migrations.create_markdown_file("docs/migrations.md")
-        # exec_manager.cli_run("cat /tmp/migrations.md", verbose=True)
 
     results.print()
 
     if not results.ok():
         exit(1)
 
-
 if __name__ == "__main__":
-    main()
+    main()
\ No newline at end of file
diff --git a/earthly/postgresql/templates/schema.md b/earthly/postgresql/templates/schema.md
new file mode 100644
index 000000000..bb442edc6
--- /dev/null
+++ b/earthly/postgresql/templates/schema.md
@@ -0,0 +1,64 @@
+---
+icon: material/database
+hide:
+  - navigation
+  - toc
+---
+
+<!--
+  This HTML block configures the page to display a full-width iframe containing the database schema,
+  positioned below the header and above the footer removing the second scrollbar.
+  
+  Key features:
+  1. Custom styling to ensure the iframe takes up the full width and available height between header and footer.
+  2. JavaScript to dynamically adjust the iframe height and handle internal navigation.
+  3. Prevention of the default link behavior inside the iframe to avoid page reloads and flashing.
+
+  The hide metadata removes the navigation and table of contents from the markdown page.
+-->
+
+<!-- markdownlint-disable no-inline-html -->
+<style>
+  iframe {
+    position: fixed;
+    left: 0;
+    width: 100%;
+    border: none;
+    z-index: 1;
+  }
+</style>
+
+<iframe src="../database_schema/index.html" id="schemaFrame"></iframe>
+
+<script>
+  document.addEventListener('DOMContentLoaded', function() {
+    var iframe = document.getElementById('schemaFrame');
+    var header = document.querySelector('header');
+    var footer = document.querySelector('footer');
+    
+    function adjustIframePosition() {
+      var headerHeight = header.offsetHeight;
+      var footerHeight = footer.offsetHeight;
+      iframe.style.top = (headerHeight) + 'px';
+      iframe.style.height = 'calc(100% - ' + (headerHeight + footerHeight) + 'px)';
+    }
+
+    iframe.onload = function() {
+      adjustIframePosition();
+      
+      // Prevent default link behavior inside iframe to avoid page reloads
+      iframe.contentWindow.addEventListener('click', function(e) {
+        if (e.target.tagName === 'A') {
+          e.preventDefault();
+          iframe.src = e.target.href;
+        }
+      }, true);
+    };
+
+    // Adjust iframe position and height when window is resized
+    window.addEventListener('resize', adjustIframePosition);
+
+    // Initial adjustment
+    adjustIframePosition();
+  });
+</script>
\ No newline at end of file

From a4f234673cee4251873e0ccb525a0c5dd5de6e0a Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Fri, 11 Oct 2024 09:12:25 +0000
Subject: [PATCH 02/10] chore(tooling): swap out dbviz custom database diagram
 tool for SchemaSpy, remove dependencies in catalyst-ci postgresql base

---
 earthly/postgresql/Earthfile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
index 4d45cfab0..4c21c3eb9 100644
--- a/earthly/postgresql/Earthfile
+++ b/earthly/postgresql/Earthfile
@@ -128,15 +128,14 @@ docs-base:
     ARG OPENJDK_VERSION=21
 
     WORKDIR /
-        
-    # Setup SchemaSpy requirements, commands combined to reduce layers
+
+    # Setup SchemaSpy requirements
     RUN echo "deb http://ftp.de.debian.org/debian sid main" > /etc/apt/sources.list && \
         apt-get update && \
-        apt-get install -y \
-            openjdk-${OPENJDK_VERSION}-jre-headless && \
+        apt-get install -y openjdk-${OPENJDK_VERSION}-jre-headless && \
+        apt-get clean && rm -rf /var/lib/apt/lists/* && \
         wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar && \
-        wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar && \
-        apt-get clean && rm -rf /var/lib/apt/lists/*
+        wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
 
 # Linter checks for sql files
 CHECK:

From 89da9ffa6d27ee7adb007c6d52bab4b3efe0bef7 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Fri, 11 Oct 2024 13:23:19 +0000
Subject: [PATCH 03/10] chore(tooling): refactored how the dependencies are
 downloaded and cached to avoid apt install executing every run (reduced
 network calls)

---
 earthly/postgresql/Earthfile | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
index 4c21c3eb9..e68af34ea 100644
--- a/earthly/postgresql/Earthfile
+++ b/earthly/postgresql/Earthfile
@@ -3,6 +3,7 @@ VERSION 0.8
 
 IMPORT ../rust/tools AS rust-tools
 IMPORT ../../utilities/scripts AS scripts
+IMPORT ../java AS java
 
 # cspell: words psycopg
 
@@ -21,7 +22,7 @@ postgres-base:
         colordiff \
         findutils \
         fontconfig \
-        fonts-liberation2        \
+        fonts-liberation2 \
         graphviz \
         libssl-dev \
         mold \
@@ -122,20 +123,28 @@ DOCS:
 docs-base:
     FROM +postgres-base
 
-    # Define ARGs for versions specific to docs
-    ARG SCHEMASPY_VERSION=6.2.4
-    ARG POSTGRESQL_JDBC_VERSION=42.7.4
-    ARG OPENJDK_VERSION=21
-
     WORKDIR /
 
-    # Setup SchemaSpy requirements
-    RUN echo "deb http://ftp.de.debian.org/debian sid main" > /etc/apt/sources.list && \
-        apt-get update && \
-        apt-get install -y openjdk-${OPENJDK_VERSION}-jre-headless && \
-        apt-get clean && rm -rf /var/lib/apt/lists/* && \
-        wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar && \
-        wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
+    # Copy the JAR files from the SCHEMASPY_ARTIFACTS target
+    COPY +schemaspy-artifacts/schemaspy.jar /bin/schemaspy.jar
+    COPY +schemaspy-artifacts/postgresql.jar /bin/postgresql.jar
+
+    # Use the cached java installation from the java Earthfile
+    DO java+COPY_DEPS
+
+    RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# seperate target to download and cache schemaspy and postgresql jdbc jars, avoids nework calls on every run
+schemaspy-artifacts:
+    FROM alpine:3.20.3
+    ARG SCHEMASPY_VERSION=6.2.4
+    ARG POSTGRESQL_JDBC_VERSION=42.7.4
+    WORKDIR /build
+    RUN apk add --no-cache wget && \
+        wget -O schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar && \
+        wget -O postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
+    SAVE ARTIFACT schemaspy.jar /schemaspy.jar
+    SAVE ARTIFACT postgresql.jar /postgresql.jar
 
 # Linter checks for sql files
 CHECK:

From 03ba2932648dc1506cef91bb2c4dbdf980dfc98e Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Fri, 11 Oct 2024 13:23:38 +0000
Subject: [PATCH 04/10] chore(tooling): refactored how the dependencies are
 downloaded and cached to avoid apt install executing every run (reduced
 network calls)

---
 earthly/java/Earthfile | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 earthly/java/Earthfile

diff --git a/earthly/java/Earthfile b/earthly/java/Earthfile
new file mode 100644
index 000000000..4c0268fae
--- /dev/null
+++ b/earthly/java/Earthfile
@@ -0,0 +1,17 @@
+VERSION 0.8
+
+# Base image for Java used in other targets to avoid using apt update / get-installs
+# Network operations are not cached: Earthly doesn’t cache network 
+# operations (like apt-get update or wget) because network resources can change over time. 
+java-base:
+    FROM openjdk:21-jdk-slim
+
+    SAVE ARTIFACT /usr/local/openjdk-21 /java
+
+COPY_DEPS:
+    FUNCTION
+    COPY +java-base/java /usr/local/openjdk-21
+
+    # Set environment variables for Java
+    ENV JAVA_HOME=/usr/local/openjdk-21
+    ENV PATH=$JAVA_HOME/bin:$PATH
\ No newline at end of file

From e0e2d6db81c6d1bc713bb6f116a6078e6fc76ad6 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Mon, 14 Oct 2024 14:52:26 +0000
Subject: [PATCH 05/10] chore(tooling): Fix markdown warnings breaking the CI
 build. No warnings required on md file as it is just a container for
 SchemaSpy

---
 earthly/postgresql/templates/schema.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/earthly/postgresql/templates/schema.md b/earthly/postgresql/templates/schema.md
index bb442edc6..db40d5efd 100644
--- a/earthly/postgresql/templates/schema.md
+++ b/earthly/postgresql/templates/schema.md
@@ -4,7 +4,7 @@ hide:
   - navigation
   - toc
 ---
-
+<!-- markdownlint-disable -->
 <!--
   This HTML block configures the page to display a full-width iframe containing the database schema,
   positioned below the header and above the footer removing the second scrollbar.
@@ -17,7 +17,6 @@ hide:
   The hide metadata removes the navigation and table of contents from the markdown page.
 -->
 
-<!-- markdownlint-disable no-inline-html -->
 <style>
   iframe {
     position: fixed;
@@ -35,7 +34,6 @@ hide:
     var iframe = document.getElementById('schemaFrame');
     var header = document.querySelector('header');
     var footer = document.querySelector('footer');
-    
     function adjustIframePosition() {
       var headerHeight = header.offsetHeight;
       var footerHeight = footer.offsetHeight;
@@ -45,7 +43,6 @@ hide:
 
     iframe.onload = function() {
       adjustIframePosition();
-      
       // Prevent default link behavior inside iframe to avoid page reloads
       iframe.contentWindow.addEventListener('click', function(e) {
         if (e.target.tagName === 'A') {
@@ -61,4 +58,4 @@ hide:
     // Initial adjustment
     adjustIframePosition();
   });
-</script>
\ No newline at end of file
+</script>

From 1ec01c2bf079cdb0940741810843979bc29fa568 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Mon, 14 Oct 2024 15:11:47 +0000
Subject: [PATCH 06/10] chore(tooling): fix breaking CI issues

---
 .config/dictionaries/project.dic       |  1 +
 earthly/postgresql/scripts/std_docs.py |  2 +-
 examples/postgresql/diagrams.json      | 21 ---------------------
 3 files changed, 2 insertions(+), 22 deletions(-)
 delete mode 100644 examples/postgresql/diagrams.json

diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic
index be1184498..b4c8ffb3f 100644
--- a/.config/dictionaries/project.dic
+++ b/.config/dictionaries/project.dic
@@ -87,6 +87,7 @@ rustflags
 rustfmt
 rustup
 sanchonet
+schemaspy
 sqlfluff
 stdcfgs
 subproject
diff --git a/earthly/postgresql/scripts/std_docs.py b/earthly/postgresql/scripts/std_docs.py
index b5dbc8ea7..e64c7e3ad 100755
--- a/earthly/postgresql/scripts/std_docs.py
+++ b/earthly/postgresql/scripts/std_docs.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# cspell: words dbmigrations dbhost dbuser dbuserpw Tsvg
+# cspell: words dbmigrations dbhost dbuser dbuserpw Tsvg pgsql11
 
 from typing import Optional
 import python.exec_manager as exec_manager
diff --git a/examples/postgresql/diagrams.json b/examples/postgresql/diagrams.json
deleted file mode 100644
index 62e87058b..000000000
--- a/examples/postgresql/diagrams.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-    "name": "Example Postgresql Database",
-    "all_schema": {
-        "comments": true,
-        "included_tables": [],
-        "excluded_tables": [
-            "refinery_schema_history"
-        ],
-        "column_description_wrap": 50,
-        "table_description_wrap": 50
-    },
-    "full_schema": {
-        "excluded_tables": [
-            "refinery_schema_history"
-        ],
-        "title": "Full Schema",
-        "comments": true,
-        "column_description_wrap": 50,
-        "table_description_wrap": 50
-    }
-}
\ No newline at end of file

From e25974325a6840f6b741bbcca04085179e599b94 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Mon, 14 Oct 2024 15:21:58 +0000
Subject: [PATCH 07/10] chore(tooling): fix breaking CI issues

---
 .config/dictionaries/project.dic | 1 +
 earthly/postgresql/Earthfile     | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic
index b4c8ffb3f..d725801fc 100644
--- a/.config/dictionaries/project.dic
+++ b/.config/dictionaries/project.dic
@@ -113,3 +113,4 @@ WORKDIR
 xerrors
 zstd
 zstdcat
+JDBC
diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
index e68af34ea..81c9cee91 100644
--- a/earthly/postgresql/Earthfile
+++ b/earthly/postgresql/Earthfile
@@ -83,7 +83,7 @@ BUILDER:
 # DOCS - FUNCTION to build the docs, needs to be run INSIDE the BUILDER like so:
 # 
 # This function uses SchemaSpy to generate database documentation.
-# SchemaSpy creates detailed, browsable ER diagrams and schema documentation.
+# SchemaSpy creates detailed, Discoverable ER diagrams and schema documentation.
 #
 # To use this function:
 # 1. Ensure your migrations are in the ./migrations directory
@@ -134,7 +134,7 @@ docs-base:
 
     RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# seperate target to download and cache schemaspy and postgresql jdbc jars, avoids nework calls on every run
+# separate target to download and cache schemaspy and postgresql JDBC jars, avoids network calls on every run
 schemaspy-artifacts:
     FROM alpine:3.20.3
     ARG SCHEMASPY_VERSION=6.2.4

From 6a235bede9004e8684af6ae09df349f2d8569e63 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Tue, 15 Oct 2024 13:35:38 +0000
Subject: [PATCH 08/10] refactor(tooling): modified approach to align with repo
 goals based on PR feedback. Removed seperate layers for package installs

---
 earthly/java/Earthfile       |  4 +---
 earthly/postgresql/Earthfile | 40 ++++++++++--------------------------
 2 files changed, 12 insertions(+), 32 deletions(-)

diff --git a/earthly/java/Earthfile b/earthly/java/Earthfile
index 4c0268fae..98c8570aa 100644
--- a/earthly/java/Earthfile
+++ b/earthly/java/Earthfile
@@ -1,8 +1,6 @@
 VERSION 0.8
 
-# Base image for Java used in other targets to avoid using apt update / get-installs
-# Network operations are not cached: Earthly doesn’t cache network 
-# operations (like apt-get update or wget) because network resources can change over time. 
+# Base image for Java used in other targets to avoid improve caching
 java-base:
     FROM openjdk:21-jdk-slim
 
diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
index 81c9cee91..bae08f45f 100644
--- a/earthly/postgresql/Earthfile
+++ b/earthly/postgresql/Earthfile
@@ -10,6 +10,9 @@ IMPORT ../java AS java
 postgres-base:
     FROM postgres:16.4-bookworm
 
+    ARG SCHEMASPY_VERSION=6.2.4
+    ARG POSTGRESQL_JDBC_VERSION=42.7.4
+
     WORKDIR /root
 
     # Install necessary packages 
@@ -34,8 +37,10 @@ postgres-base:
         ripgrep \
         wget \
         pipx \
-        && rm -rf /var/lib/apt/lists/*
-
+        && rm -rf /var/lib/apt/lists/* \
+        && wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar \
+        && wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
+   
     # Install SQLFluff - Check
     ENV PATH="/root/.local/bin:${PATH}"
     RUN pipx install sqlfluff==3.1.1 && sqlfluff version
@@ -55,6 +60,9 @@ postgres-base:
     # Copy templates to the working directory
     COPY --dir templates /templates
 
+    # Use the cached java installation from the java Earthfile
+    DO java+COPY_DEPS
+
     SAVE ARTIFACT /scripts /scripts
     SAVE ARTIFACT /templates /templates
 
@@ -106,7 +114,7 @@ DOCS:
     ARG migrations=./migrations
     ARG refinery_toml=./refinery.toml
 
-    FROM +docs-base
+    FROM +postgres-base
       
     USER postgres:postgres
     WORKDIR /docs
@@ -120,32 +128,6 @@ DOCS:
 
     SAVE ARTIFACT docs ./docs
 
-docs-base:
-    FROM +postgres-base
-
-    WORKDIR /
-
-    # Copy the JAR files from the SCHEMASPY_ARTIFACTS target
-    COPY +schemaspy-artifacts/schemaspy.jar /bin/schemaspy.jar
-    COPY +schemaspy-artifacts/postgresql.jar /bin/postgresql.jar
-
-    # Use the cached java installation from the java Earthfile
-    DO java+COPY_DEPS
-
-    RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# separate target to download and cache schemaspy and postgresql JDBC jars, avoids network calls on every run
-schemaspy-artifacts:
-    FROM alpine:3.20.3
-    ARG SCHEMASPY_VERSION=6.2.4
-    ARG POSTGRESQL_JDBC_VERSION=42.7.4
-    WORKDIR /build
-    RUN apk add --no-cache wget && \
-        wget -O schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar && \
-        wget -O postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
-    SAVE ARTIFACT schemaspy.jar /schemaspy.jar
-    SAVE ARTIFACT postgresql.jar /postgresql.jar
-
 # Linter checks for sql files
 CHECK:
     FUNCTION

From 58e8a6bf3362a9c9e5a227e9c3802d9b3f811a90 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Wed, 16 Oct 2024 11:07:17 +0000
Subject: [PATCH 09/10] refactor(tooling): re-ordered package install and
 download steps for improved caching

---
 earthly/postgresql/Earthfile | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
index bae08f45f..611e09d62 100644
--- a/earthly/postgresql/Earthfile
+++ b/earthly/postgresql/Earthfile
@@ -37,14 +37,19 @@ postgres-base:
         ripgrep \
         wget \
         pipx \
-        && rm -rf /var/lib/apt/lists/* \
-        && wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar \
-        && wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
-   
+        && rm -rf /var/lib/apt/lists/*
+
+    # Use the cached java installation from the java Earthfile
+    DO java+COPY_DEPS
+
     # Install SQLFluff - Check
     ENV PATH="/root/.local/bin:${PATH}"
     RUN pipx install sqlfluff==3.1.1 && sqlfluff version
 
+    # Install SchemaSpy and required Postgresql JDBC driver
+    RUN wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
+    RUN wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar
+
     # Get refinery
     COPY rust-tools+tool-refinery/refinery /bin
 
@@ -60,9 +65,6 @@ postgres-base:
     # Copy templates to the working directory
     COPY --dir templates /templates
 
-    # Use the cached java installation from the java Earthfile
-    DO java+COPY_DEPS
-
     SAVE ARTIFACT /scripts /scripts
     SAVE ARTIFACT /templates /templates
 

From a18179f25a939cc8ca6e0db62644b92cd312c879 Mon Sep 17 00:00:00 2001
From: neil <neil.mcauliffe@iohk.io>
Date: Wed, 16 Oct 2024 11:14:12 +0000
Subject: [PATCH 10/10] refactor(tooling): removed dbviz utility from
 repository, dbviz is no longer required and has been replaced with SchemaSpy
 for database schema visualizations diagram generation

---
 utilities/dbviz/.cargo/config.toml         |  93 ------
 utilities/dbviz/.config/nextest.toml       |  49 ----
 utilities/dbviz/Cargo.toml                 |  53 ----
 utilities/dbviz/Earthfile                  |  49 ----
 utilities/dbviz/README.md                  |  11 -
 utilities/dbviz/clippy.toml                |   2 -
 utilities/dbviz/cspell.json                |  43 ---
 utilities/dbviz/deny.toml                  | 123 --------
 utilities/dbviz/rust-toolchain.toml        |   3 -
 utilities/dbviz/rustfmt.toml               |  68 -----
 utilities/dbviz/src/default_template.jinja | 222 ---------------
 utilities/dbviz/src/lib.rs                 |   2 -
 utilities/dbviz/src/main.rs                |  39 ---
 utilities/dbviz/src/opts.rs                |  74 -----
 utilities/dbviz/src/postgresql.rs          | 312 ---------------------
 utilities/dbviz/src/schema.rs              |  84 ------
 16 files changed, 1227 deletions(-)
 delete mode 100644 utilities/dbviz/.cargo/config.toml
 delete mode 100644 utilities/dbviz/.config/nextest.toml
 delete mode 100644 utilities/dbviz/Cargo.toml
 delete mode 100644 utilities/dbviz/Earthfile
 delete mode 100644 utilities/dbviz/README.md
 delete mode 100644 utilities/dbviz/clippy.toml
 delete mode 100644 utilities/dbviz/cspell.json
 delete mode 100644 utilities/dbviz/deny.toml
 delete mode 100644 utilities/dbviz/rust-toolchain.toml
 delete mode 100644 utilities/dbviz/rustfmt.toml
 delete mode 100644 utilities/dbviz/src/default_template.jinja
 delete mode 100644 utilities/dbviz/src/lib.rs
 delete mode 100644 utilities/dbviz/src/main.rs
 delete mode 100644 utilities/dbviz/src/opts.rs
 delete mode 100644 utilities/dbviz/src/postgresql.rs
 delete mode 100644 utilities/dbviz/src/schema.rs

diff --git a/utilities/dbviz/.cargo/config.toml b/utilities/dbviz/.cargo/config.toml
deleted file mode 100644
index 02c231407..000000000
--- a/utilities/dbviz/.cargo/config.toml
+++ /dev/null
@@ -1,93 +0,0 @@
-# Use MOLD linker where possible, but ONLY in CI applicable targets.
-
-# Configure how Docker container targets build.
-
-# If you want to customize these targets for a local build, then customize them in your:
-#  $CARGO_HOME/config.toml
-# NOT in the project itself.
-# These targets are ONLY the targets used by CI and inside docker builds.
-
-# DO NOT remove `"-C", "target-feature=+crt-static"` from the rustflags for these targets.
-
-# Should be the default to have fully static rust programs in CI
-[target.x86_64-unknown-linux-musl]
-linker = "clang"
-rustflags = [
-    "-C", "link-arg=-fuse-ld=/usr/bin/mold",
-    "-C", "target-feature=-crt-static"
-]
-
-# Should be the default to have fully static rust programs in CI
-[target.aarch64-unknown-linux-musl]
-linker = "clang"
-rustflags = [
-    "-C", "link-arg=-fuse-ld=/usr/bin/mold",
-    "-C", "target-feature=-crt-static"
-]
-
-[build]
-rustflags = []
-rustdocflags = [
-    "--enable-index-page",
-    "-Z",
-    "unstable-options",
-]
-
-[profile.dev]
-opt-level = 1
-debug = true
-debug-assertions = true
-overflow-checks = true
-lto = false
-panic = "unwind"
-incremental = true
-codegen-units = 256
-
-[profile.release]
-opt-level = 3
-debug = false
-debug-assertions = false
-overflow-checks = false
-lto = "thin"
-panic = "unwind"
-incremental = false
-codegen-units = 16
-
-[profile.test]
-opt-level = 3
-debug = true
-lto = false
-debug-assertions = true
-incremental = true
-codegen-units = 256
-
-[profile.bench]
-opt-level = 3
-debug = false
-debug-assertions = false
-overflow-checks = false
-lto = "thin"
-incremental = false
-codegen-units = 16
-
-[alias]
-lint = "clippy --all-targets"
-lintfix = "clippy --all-targets --fix --allow-dirty"
-lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets"
-
-docs = "doc --release --no-deps --document-private-items --bins --lib --examples"
-# nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options"
-testunit = "nextest run --release --bins --lib --tests --benches --no-fail-fast -P ci"
-testcov = "llvm-cov nextest --release --bins --lib --tests --benches --no-fail-fast -P ci"
-testdocs = "test --doc --release"
-
-# Rust formatting, MUST be run with +nightly
-fmtchk = "fmt -- --check -v --color=always"
-fmtfix = "fmt -- -v"
-
-[term]
-quiet = false           # whether cargo output is quiet
-verbose = false         # whether cargo provides verbose output
-color = "auto"          # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable.
-progress.when = "never" # whether cargo shows progress bar
-progress.width = 80     # width of progress bar
diff --git a/utilities/dbviz/.config/nextest.toml b/utilities/dbviz/.config/nextest.toml
deleted file mode 100644
index be3673830..000000000
--- a/utilities/dbviz/.config/nextest.toml
+++ /dev/null
@@ -1,49 +0,0 @@
-# cspell: words scrollability testcase
-[store]
-# The directory under the workspace root at which nextest-related files are
-# written. Profile-specific storage is currently written to dir/<profile-name>.
-# dir = "target/nextest"
-
-[profile.default]
-# Print out output for failing tests as soon as they fail, and also at the end
-# of the run (for easy scrollability).
-failure-output = "immediate-final"
-
-# Do not cancel the test run on the first failure.
-fail-fast = true
-
-status-level = "all"
-final-status-level = "all"
-
-[profile.ci]
-# Print out output for failing tests as soon as they fail, and also at the end
-# of the run (for easy scrollability).
-failure-output = "immediate-final"
-# Do not cancel the test run on the first failure.
-fail-fast = false
-
-status-level = "all"
-final-status-level = "all"
-
-
-[profile.ci.junit]
-# Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
-# If unspecified, JUnit is not written out.
-
-path = "junit.xml"
-
-# The name of the top-level "report" element in JUnit report. If aggregating
-# reports across different test runs, it may be useful to provide separate names
-# for each report.
-report-name = "nextest"
-
-# Whether standard output and standard error for passing tests should be stored in the JUnit report.
-# Output is stored in the <system-out> and <system-err> elements of the <testcase> element.
-store-success-output = true
-
-# Whether standard output and standard error for failing tests should be stored in the JUnit report.
-# Output is stored in the <system-out> and <system-err> elements of the <testcase> element.
-#
-# Note that if a description can be extracted from the output, it is always stored in the
-# <description> element.
-store-failure-output = true
diff --git a/utilities/dbviz/Cargo.toml b/utilities/dbviz/Cargo.toml
deleted file mode 100644
index c1c58329b..000000000
--- a/utilities/dbviz/Cargo.toml
+++ /dev/null
@@ -1,53 +0,0 @@
-[package]
-name = "dbviz"
-version = "1.4.0"
-authors = ["Steven Johnson"]
-edition = "2021"
-license = "Apache-2.0/MIT"
-
-[lints.rust]
-warnings = "deny"
-missing_docs = "deny"
-let_underscore_drop = "deny"
-non_ascii_idents = "deny"
-single_use_lifetimes = "deny"
-trivial_casts = "deny"
-trivial_numeric_casts = "deny"
-
-[lints.rustdoc]
-broken_intra_doc_links = "deny"
-invalid_codeblock_attributes = "deny"
-invalid_html_tags = "deny"
-invalid_rust_codeblocks = "deny"
-bare_urls = "deny"
-unescaped_backticks = "deny"
-
-[lints.clippy]
-pedantic = { level = "deny", priority = -1 }
-unwrap_used = "deny"
-expect_used = "deny"
-todo = "deny"
-unimplemented = "deny"
-exit = "deny"
-get_unwrap = "deny"
-index_refutable_slice = "deny"
-indexing_slicing = "deny"
-match_on_vec_items = "deny"
-match_wild_err_arm = "deny"
-missing_panics_doc = "deny"
-panic = "deny"
-string_slice = "deny"
-unchecked_duration_subtraction = "deny"
-unreachable = "deny"
-missing_docs_in_private_items = "deny"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-anyhow = "1.0"
-itertools = "0.12.0"
-postgres = "0.19.4"
-minijinja = "1.0.10"
-textwrap = "0.16.0"
-serde = { version = "1.0", features = ["derive"] }
-clap = { version = "4.4.8", features = ["derive"] }
diff --git a/utilities/dbviz/Earthfile b/utilities/dbviz/Earthfile
deleted file mode 100644
index efdd1d336..000000000
--- a/utilities/dbviz/Earthfile
+++ /dev/null
@@ -1,49 +0,0 @@
-VERSION 0.8
-
-IMPORT ./../../earthly/rust AS rust-ci
-
-# cspell: words stdcfgs toolset toolsets
-
-# Internal: Set up our target toolchains, and copy our files.
-builder:
-    DO rust-ci+SETUP
-
-    COPY --keep-ts --dir \
-        .cargo .config Cargo.toml clippy.toml deny.toml rustfmt.toml \
-        src \
-        .
-
-## -----------------------------------------------------------------------------
-##
-## Standard CI targets.
-##
-## These targets are discovered and executed automatically by CI.
-
-# check - Run check using the most efficient host tooling
-# CI Automated Entry point.
-check:
-    FROM +builder
-
-    DO rust-ci+EXECUTE --cmd="/scripts/std_checks.py"
-
-# Test which runs check with all supported host tooling.  Needs qemu or rosetta to run.
-# Only used to validate tooling is working across host toolsets.
-all-hosts-check:
-    BUILD --platform=linux/amd64 --platform=linux/arm64 +check
-
-# build - Run build using the most efficient host tooling
-# CI Automated Entry point.
-build:
-    FROM +builder
-
-    DO rust-ci+EXECUTE \
-        --cmd="/scripts/std_build.py --bins=dbviz/dbviz" \
-        --output="release/[^\./]+" \
-        --docs="true"
-
-    SAVE ARTIFACT target/release/dbviz dbviz
-
-# Test which runs check with all supported host tooling.  Needs qemu or rosetta to run.
-# Only used to validate tooling is working across host toolsets.
-all-hosts-build:
-    BUILD --platform=linux/amd64 --platform=linux/arm64 +build
diff --git a/utilities/dbviz/README.md b/utilities/dbviz/README.md
deleted file mode 100644
index 07f40b1ad..000000000
--- a/utilities/dbviz/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# dbviz
-
-Simple tool to create database diagrams from postgres schemas.
-The diagrams themselves are just text files, and are controlled by `.jinja` templates.
-The tool builds in a default template which produces `.dot` files.
-
-## Usage
-
-```sh
-dbviz -d database_name | dot -Tpng > schema.png
-```
diff --git a/utilities/dbviz/clippy.toml b/utilities/dbviz/clippy.toml
deleted file mode 100644
index 0358cdb50..000000000
--- a/utilities/dbviz/clippy.toml
+++ /dev/null
@@ -1,2 +0,0 @@
-allow-unwrap-in-tests = true
-allow-expect-in-tests = true
diff --git a/utilities/dbviz/cspell.json b/utilities/dbviz/cspell.json
deleted file mode 100644
index a45613263..000000000
--- a/utilities/dbviz/cspell.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
-    "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json",
-    "version": "0.2",
-    "import": "../../cspell.json",
-    "words": [
-        "dbviz",
-        "Tpng",
-        "dbname",
-        "regclass",
-        "nspname",
-        "schemaname",
-        "relname",
-        "attname",
-        "attrf",
-        "conrelid",
-        "attnum",
-        "conkey",
-        "relnamespace",
-        "confrelid",
-        "attrelid",
-        "confkey",
-        "indrelid",
-        "indisprimary",
-        "indisunique",
-        "indexdef",
-        "indexrelid",
-        "indkey",
-        "relam",
-        "minijinja",
-        "pkey",
-        "endmacro",
-        "varchar",
-        "startingwith",
-        "nextval",
-        "endmacro",
-        "labelloc",
-        "rankdir",
-        "cellborder",
-        "endfor",
-        "bytea",
-        "Autoincrement"
-    ]
-}
\ No newline at end of file
diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml
deleted file mode 100644
index 26ec8794b..000000000
--- a/utilities/dbviz/deny.toml
+++ /dev/null
@@ -1,123 +0,0 @@
-# cspell: words msvc, wasip, RUSTSEC, rustls, libssh, reqwest, tinyvec, Leay, webpki
-
-[graph]
-# cargo-deny is really only ever intended to run on the "normal" tier-1 targets
-targets = [
-    "x86_64-unknown-linux-gnu",
-    "aarch64-unknown-linux-gnu",
-    "x86_64-unknown-linux-musl",
-    "aarch64-apple-darwin",
-    "x86_64-apple-darwin",
-    "x86_64-pc-windows-msvc",
-    "wasm32-unknown-unknown",
-    "wasm32-wasip1",
-    "wasm32-wasip2",
-]
-
-[advisories]
-version = 2
-ignore = [
-    { id = "RUSTSEC-2020-0168", reason = "`mach` is used by wasmtime and we have no control over that." },
-    { id = "RUSTSEC-2021-0145", reason = "we don't target windows, and don't use a custom global allocator." },
-    { id = "RUSTSEC-2024-0370", reason = "`proc-macro-error` is used by crates we rely on, we can't control what they use."},
-]
-
-[bans]
-multiple-versions = "warn"
-wildcards = 'deny'
-deny = [
-    # Scylla DB Drivers currently require OpenSSL.  Its unavoidable.
-    # However, there is movement to enable support for Rustls. 
-    # So, for now, allow open-ssl but it needs to be disabled as soon as Scylla DB enables Rustls.
-    #{ crate = "openssl", use-instead = "rustls" },
-    #{ crate = "openssl-sys", use-instead = "rustls" },
-    "libssh2-sys",
-    # { crate = "git2", use-instead = "gix" },
-    # { crate = "cmake", use-instead = "cc" },
-    # { crate = "windows", reason = "bloated and unnecessary", use-instead = "ideally inline bindings, practically, windows-sys" },
-]
-skip = [
-    # { crate = "bitflags@1.3.2", reason = "https://github.com/seanmonstar/reqwest/pull/2130 should be in the next version" },
-    # { crate = "winnow@0.5.40", reason = "gix 0.59 was yanked, see https://github.com/Byron/gitoxide/issues/1309" },
-    # { crate = "heck@0.4.1", reason = "strum_macros uses this old version" },
-    # { crate = "base64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" },
-    # { crate = "byte-array-literalsase64@0.21.7", reason = "gix-transport pulls in this old version, as well as a newer version via reqwest" },
-]
-skip-tree = [
-    { crate = "windows-sys@0.48.0", reason = "a foundational crate for many that bumps far too frequently to ever have a shared version" },
-]
-
-[sources]
-unknown-registry = "deny"
-unknown-git = "deny"
-
-# List of URLs for allowed Git repositories
-allow-git = [
-    "https://github.com/input-output-hk/catalyst-libs.git",
-    "https://github.com/input-output-hk/catalyst-pallas.git",
-    "https://github.com/input-output-hk/catalyst-mithril.git",
-    "https://github.com/bytecodealliance/wasmtime",
-    "https://github.com/aldanor/hdf5-rust",
-]
-
-[licenses]
-version = 2
-# Don't warn if a listed license isn't found
-unused-allowed-license="allow"
-# We want really high confidence when inferring licenses from text
-confidence-threshold = 0.93
-allow = [
-    "MIT",
-    "Apache-2.0",
-    "Unicode-DFS-2016",
-    "BSD-3-Clause",
-    "BSD-2-Clause",
-    "BlueOak-1.0.0",
-    "Apache-2.0 WITH LLVM-exception",
-    "CC0-1.0",
-    "ISC",
-    "Unicode-3.0",
-    "MPL-2.0",
-    "Zlib",
-]
-exceptions = [
-    #{ allow = ["Zlib"], crate = "tinyvec" },
-    #{ allow = ["Unicode-DFS-2016"], crate = "unicode-ident" },
-    #{ allow = ["OpenSSL"], crate = "ring" },
-]
-
-[[licenses.clarify]]
-crate = "byte-array-literals"
-expression = "Apache-2.0 WITH LLVM-exception"
-license-files = [{ path = "../../../LICENSE", hash = 0x001c7e6c }]
-
-[[licenses.clarify]]
-crate = "hdf5-src"
-expression = "MIT"
-license-files = [{ path = "../LICENSE-MIT", hash = 0x001c7e6c }]
-
-[[licenses.clarify]]
-crate = "ring"
-expression = "MIT"
-license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
-
-# SPDX considers OpenSSL to encompass both the OpenSSL and SSLeay licenses
-# https://spdx.org/licenses/OpenSSL.html
-# ISC - Both BoringSSL and ring use this for their new files
-# MIT - "Files in third_party/ have their own licenses, as described therein. The MIT
-# license, for third_party/fiat, which, unlike other third_party directories, is
-# compiled into non-test libraries, is included below."
-# OpenSSL - Obviously
-#expression = "ISC AND MIT AND OpenSSL"
-#license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
-
-#[[licenses.clarify]]
-#crate = "webpki"
-#expression = "ISC"
-#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]
-
-# Actually "ISC-style"
-#[[licenses.clarify]]
-#crate = "rustls-webpki"
-#expression = "ISC"
-#license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]
diff --git a/utilities/dbviz/rust-toolchain.toml b/utilities/dbviz/rust-toolchain.toml
deleted file mode 100644
index f01d02df3..000000000
--- a/utilities/dbviz/rust-toolchain.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-[toolchain]
-channel = "1.81"
-profile = "default"
\ No newline at end of file
diff --git a/utilities/dbviz/rustfmt.toml b/utilities/dbviz/rustfmt.toml
deleted file mode 100644
index b0f20832c..000000000
--- a/utilities/dbviz/rustfmt.toml
+++ /dev/null
@@ -1,68 +0,0 @@
-# Enable unstable features:
-# * imports_indent
-# * imports_layout
-# * imports_granularity
-# * group_imports
-# * reorder_impl_items
-# * trailing_comma
-# * where_single_line
-# * wrap_comments
-# * comment_width
-# * blank_lines_upper_bound
-# * condense_wildcard_suffixes
-# * force_multiline_blocks
-# * format_code_in_doc_comments
-# * format_generated_files
-# * hex_literal_case
-# * inline_attribute_width
-# * normalize_comments
-# * normalize_doc_attributes
-# * overflow_delimited_expr
-unstable_features = true
-
-# Compatibility:
-edition = "2021"
-
-# Tabs & spaces - Defaults, listed for clarity
-tab_spaces = 4
-hard_tabs = false
-
-# Commas.
-trailing_comma = "Vertical"
-match_block_trailing_comma = true
-
-# General width constraints.
-max_width = 100
-
-# Comments:
-normalize_comments = true
-normalize_doc_attributes = true
-wrap_comments = true
-comment_width = 90      # small excess is okay but prefer 80
-format_code_in_doc_comments = true
-format_generated_files = false
-
-# Imports.
-imports_indent = "Block"
-imports_layout = "Mixed"
-group_imports = "StdExternalCrate"
-reorder_imports = true
-imports_granularity = "Crate"
-
-# Arguments:
-use_small_heuristics = "Default"
-fn_params_layout = "Compressed"
-overflow_delimited_expr = true
-where_single_line = true
-
-# Misc:
-inline_attribute_width = 0
-blank_lines_upper_bound = 1
-reorder_impl_items = true
-use_field_init_shorthand = true
-force_multiline_blocks = true
-condense_wildcard_suffixes = true
-hex_literal_case = "Upper"
-
-# Ignored files:
-ignore = []
diff --git a/utilities/dbviz/src/default_template.jinja b/utilities/dbviz/src/default_template.jinja
deleted file mode 100644
index 64488162b..000000000
--- a/utilities/dbviz/src/default_template.jinja
+++ /dev/null
@@ -1,222 +0,0 @@
-{%- macro theme(x) -%}
-{{ {
-    "title_loc"             : "t",
-    "title_size"            : 30,
-    "title_color"           : "blue",
-    "graph_direction"       : "LR",
-    "default_fontsize"      : 16,
-    "pkey_bgcolor"          : "seagreen1",
-    "field_desc"            : "color='grey50' face='Monospace' point-size='14'",
-    "column_heading"        : "color='black' face='Courier bold' point-size='18'",
-    "table_heading_bgcolor" : "#009879",
-    "table_heading"         : "color='white' face='Courier bold italic' point-size='20'",
-    "table_desc_bgcolor"    : "grey20",
-    "table_description"     : "color='white' face='Monospace' point-size='14'"
-}[x] }}
-{%- endmacro -%}
-
-{%- macro data_type(data_type, default, max_chars, nullable) -%}
-
-    {%- if nullable == "YES" -%}
-    <i>
-    {%- endif -%}
-
-    {%- if data_type == "character varying" -%}
-    varchar
-    {%- elif data_type == "timestamp without time zone" -%}
-    timestamp
-    {%- else -%}
-    {{- data_type -}}
-    {%- endif -%}
-
-    {%- if max_chars is not none -%}
-    ({{max_chars}})
-    {%- endif -%}
-
-    {%- if default is startingwith("nextval") -%}
-    +
-    {%- endif -%}
-
-    {%- if nullable == "YES" -%}
-    </i>
-    {%- endif -%}
-
-{%- endmacro -%}
-
-{%- macro pkey_bgcolor(pkey) -%}
-    {%- if pkey -%}
-    bgcolor="{{ theme('pkey_bgcolor') }}"
-    {%- endif -%}
-{%- endmacro -%}
-
-{%- macro column_name_x(name, pkey) -%}
-    <td port="{{name}}" {{ pkey_bgcolor(pkey) }} align='text'><font>{{name}}</font><br align='left'/></td>
-{%- endmacro -%}
-
-
-{%- macro column_name(field) -%}
-    {{- column_name_x(field.column, field.primary_key) -}}
-{%- endmacro -%}
-
-{%- macro final(field, final=true) -%}
-    {%- if final -%}
-    port="{{field.column}}_out"
-    {%- endif -%}
-{%- endmacro -%}
-
-{%- macro column_type(field, last=false) -%}
-    <td {{ final(field,last) }} {{ pkey_bgcolor(field.primary_key) }} ><font>{{ data_type(field.data_type, field.default, field.max_chars, field.nullable ) }}</font></td>
-{%- endmacro -%}
-
-{%- macro column_description(field) -%}
-    <td {{ final(field,true) }} {{ pkey_bgcolor(field.primary_key) }} align='text'><font {{ theme("field_desc") }} ><i>{{- field.description|trim|escape|replace("\n", "<br align='left'/>") -}}</i><br align="left"/></font></td>
-{%- endmacro -%}
-
-digraph erd {
-
-    {% if opts.title is not none %}
-    label = "{{ opts.title }}"
-    labelloc = {{ theme("title_loc") }}
-    fontsize = {{ theme("title_size") }}
-    fontcolor = {{ theme("title_color") }}
-    {% endif %}
-
-    graph [
-        rankdir = "{{theme('graph_direction')}}"
-    ];
-
-    node [
-        fontsize = "{{theme('default_fontsize')}}"
-        shape = "plaintext"
-    ];
-
-    edge [
-    ];
-
-    {% for table in schema.tables %}
-        {% if opts.comments %}
-
-    "{{table.name}}" [shape=plain label=<
-        <table border='0' cellborder='1' cellspacing='0' >
-            <tr>
-                <td colspan='3' bgcolor='{{theme("table_heading_bgcolor")}}' port='__title'><font {{theme("table_heading")}}><b>{{table.name}}</b></font></td>
-            </tr>
-            <tr>
-                <td><font {{ theme("column_heading") }} ><b>Column</b></font></td>
-                <td><font {{ theme("column_heading") }} ><b>Type</b></font></td>
-                <td><font {{ theme("column_heading") }} ><b>Description</b></font></td>
-            </tr>
-
-            {% for field in table.fields %}
-            <tr>
-                {{ column_name(field) }}
-                {{ column_type(field, false) }}
-                {{ column_description(field) }}
-            </tr>
-            {% endfor %}
-
-            {% if table.description is not none %}
-            <tr>
-                <td colspan='3' bgcolor="{{theme('table_desc_bgcolor')}}" align='text'><font {{theme("table_description")}}><i>{{- table.description|trim|escape|replace("\n", "<br align='left'/>") -}}</i><br align="left"/></font></td>
-            </tr>
-            {% endif %}
-
-        </table>
-    >];
-
-        {% else %}
-
-    "{{table.name}}" [label=<
-        <table border='0' cellborder='1' cellspacing='0'>
-            <tr>
-                <td colspan='2' bgcolor='{{theme("table_heading_bgcolor")}}' port='__title'><font {{theme("table_heading")}}><b>{{table.name}}</b></font></td>
-            </tr>
-            <tr>
-                <td><font {{ theme("column_heading") }} ><b>Column</b></font></td>
-                <td><font {{ theme("column_heading") }} ><b>Type</b></font></td>
-            </tr>
-
-            {% for field in table.fields %}
-            <tr>
-                {{ column_name(field) }}
-                {{ column_type(field, true) }}
-            </tr>
-            {% endfor %}
-
-        </table>
-    >];
-
-        {% endif %}
-    {% endfor %}
-
-    {% for partial in schema.partial_tables %}
-
-    "{{partial}}" [label=<
-        <table border='0' cellborder='1' cellspacing='0'>
-            <tr>
-                <td bgcolor='{{theme("table_heading_bgcolor")}}' port='__title'><font {{theme("table_heading")}}><b>{{partial}}</b></font></td>
-            </tr>
-            <tr>
-                <td><font {{ theme("column_heading") }} ><b>Column</b></font></td>
-            </tr>
-
-            {% for field in schema.partial_tables[partial] %}
-            <tr>
-                {{ column_name_x(field,false) }}
-            </tr>
-            {% endfor %}
-
-            <tr>
-                <td colspan='3' bgcolor="{{theme('table_desc_bgcolor')}}" align='text'><font {{theme("table_description")}}><i>ABRIDGED</i><br align="left"/></font></td>
-            </tr>
-        </table>
-    >];
-
-    {% endfor %}
-
-
-    "LEGEND" [label=<
-        <table border='0' cellborder='1' cellspacing='0'>
-            <tr>
-                <td colspan='2' bgcolor='Pink' port='__title'><font color='white' face='Courier bold italic' point-size='20'><b>LEGEND</b></font></td>
-            </tr>
-            <tr>
-                <td><font {{theme("column_heading")}}><b>Type</b></font></td>
-                <td><font {{theme("column_heading")}}><b>Example</b></font></td>
-            </tr>
-
-            <tr>
-                <td align='text' bgcolor="seagreen1" ><font>Primary Key</font><br align='left'/></td>
-                <td  bgcolor="{{theme('pkey_bgcolor')}}" ><font>{{ data_type("integer", "nextval", none, "NO") }}</font></td>
-            </tr>
-
-
-            <tr>
-                <td align='text'><font>Standard Field</font><br align='left'/></td>
-                <td><font>{{ data_type("bytea", none, none, "NO") }}</font></td>
-            </tr>
-
-            <tr>
-                <td align='text'><font>Nullable Field</font><br align='left'/></td>
-                <td><font>{{ data_type("text", none, none, "YES") }}</font></td>
-            </tr>
-
-            <tr>
-                <td align='text'><font>Sized Field</font><br align='left'/></td>
-                <td><font>{{ data_type("varchar", none, 32, "NO") }}</font></td>
-            </tr>
-
-            <tr>
-                <td align='text'><font>Autoincrement Field</font><br align='left'/></td>
-                <td><font>{{ data_type("integer", "nextval", none, "NO") }}</font></td>
-            </tr>
-
-        </table>
-    >];
-
-    {% for relation in schema.relations %}
-    "{{relation.on_table}}":"{{relation.on_field}}_out" -> "{{relation.to_table}}":"{{relation.to_field}}"
-    {% endfor %}
-
-
-}
diff --git a/utilities/dbviz/src/lib.rs b/utilities/dbviz/src/lib.rs
deleted file mode 100644
index 2a0f48add..000000000
--- a/utilities/dbviz/src/lib.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-//! Intentionally empty
-//! This file exists, so that doc tests can be used inside binary crates.
diff --git a/utilities/dbviz/src/main.rs b/utilities/dbviz/src/main.rs
deleted file mode 100644
index a22d08ac4..000000000
--- a/utilities/dbviz/src/main.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-//! `DBViz` - Database Diagram Generator
-//!
-//! `DBViz` is a tool for generating database diagrams.
-
-mod opts;
-mod postgresql;
-mod schema;
-
-use std::fs;
-
-use anyhow::Result;
-use minijinja::{context, Environment};
-
-fn main() -> Result<()> {
-    let opts = opts::load();
-
-    let loader = postgresql::Conn::new(&opts)?;
-    let schema = loader.load()?;
-
-    let template_file = match &opts.template {
-        Some(fname) => fs::read_to_string(fname)?,
-        None => include_str!("default_template.jinja").to_string(),
-    };
-
-    let mut env = Environment::new();
-    env.add_template("diagram", &template_file)?;
-    let tmpl = env.get_template("diagram")?;
-
-    let ctx = context!(
-        opts => opts,
-        schema => schema
-    );
-
-    let rendered = tmpl.render(ctx)?;
-
-    println!("{rendered}");
-
-    Ok(())
-}
diff --git a/utilities/dbviz/src/opts.rs b/utilities/dbviz/src/opts.rs
deleted file mode 100644
index 43d38d13d..000000000
--- a/utilities/dbviz/src/opts.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-//! CLI Option parsing
-
-use std::path::PathBuf;
-
-use clap::{Args, Parser};
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Parser, Clone, Serialize, Deserialize)]
-#[command(author, version, about, long_about = None)]
-/// `DBViz` a tool for generating database diagrams.
-pub(crate) struct Cli {
-    #[command(flatten)]
-    /// Postgres connection options
-    pub(crate) pg_opts: Pg,
-
-    #[arg(short, long)]
-    /// Tables to include in the current diagram.
-    pub(crate) include_tables: Option<Vec<String>>,
-
-    #[arg(short, long)]
-    /// Tables to completely exclude in the current diagram.
-    pub(crate) exclude_tables: Option<Vec<String>>,
-
-    /// Title to give the Diagram
-    #[arg(short, long)]
-    pub(crate) title: Option<String>,
-
-    /// How wide is the Column Description before we wrap it?
-    #[arg(long)]
-    pub(crate) column_description_wrap: Option<usize>,
-
-    /// How wide is the Table Description before we wrap it?
-    #[arg(long)]
-    pub(crate) table_description_wrap: Option<usize>,
-
-    /// Do we include comments in the diagram?
-    #[arg(long)]
-    pub(crate) comments: bool,
-
-    /// Input file
-    pub(crate) template: Option<PathBuf>,
-
-    /// Output file
-    pub(crate) output: Option<PathBuf>,
-}
-
-#[derive(Debug, Args, Clone, Serialize, Deserialize)]
-/// Postgres connection options
-pub(crate) struct Pg {
-    #[arg(short, long, default_value = "localhost")]
-    /// Hostname to connect to
-    pub(crate) hostname: String,
-
-    #[arg(short, long, default_value = "postgres")]
-    /// Username to use when connecting
-    pub(crate) username: String,
-
-    #[arg(short, long, default_value = "postgres")]
-    /// Password to use when connecting
-    pub(crate) password: String,
-
-    #[arg(short, long, default_value = "postgres")]
-    /// Database name to connect to
-    pub(crate) database: String,
-
-    #[arg(short, long, default_value = "public")]
-    /// Schema name to use
-    pub(crate) schema: String,
-}
-
-/// Load CLI Options.
-pub(crate) fn load() -> Cli {
-    Cli::parse()
-}
diff --git a/utilities/dbviz/src/postgresql.rs b/utilities/dbviz/src/postgresql.rs
deleted file mode 100644
index 297eea796..000000000
--- a/utilities/dbviz/src/postgresql.rs
+++ /dev/null
@@ -1,312 +0,0 @@
-//! Loader for postgresql.
-
-// This is not production code, so while we should get rid of these lints, its not
-// worth the time it would take, for no return value.
-#![allow(clippy::unwrap_used)]
-#![allow(clippy::indexing_slicing)]
-
-use std::{
-    cell::RefCell,
-    collections::HashMap,
-    convert::{TryFrom, TryInto},
-};
-
-use anyhow::Result;
-use itertools::Itertools;
-use postgres::{tls::NoTls, Client, Row};
-
-use crate::{
-    opts,
-    schema::{Index, Relation, Schema, Table, TableColumn},
-};
-
-/// Struct that manages the loading and implements `Loader` trait.
-pub struct Conn {
-    /// Postgres client
-    pg_client: RefCell<Client>,
-    /// Schema name
-    schema: String,
-    /// Options
-    opts: opts::Cli,
-}
-
-/// Check if a column is a primary key
-fn is_primary_key(table: &str, column: &str, indexes: &[Index]) -> bool {
-    indexes
-        .iter()
-        .any(|idx| idx.table == table && idx.fields.contains(&column.to_string()) && idx.primary)
-}
-
-impl Conn {
-    /// Make a new postgres connection
-    pub(crate) fn new(opts: &opts::Cli) -> Result<Conn> {
-        let pg_client = postgres::Config::new()
-            .user(&opts.pg_opts.username)
-            .password(&opts.pg_opts.password)
-            .dbname(&opts.pg_opts.database)
-            .host(&opts.pg_opts.hostname)
-            .connect(NoTls)?;
-
-        let pg_client = RefCell::new(pg_client);
-        let schema = opts.pg_opts.schema.clone();
-        Ok(Conn {
-            pg_client,
-            schema,
-            opts: opts.clone(),
-        })
-    }
-
-    /// Do we include this table name?
-    fn include_table(&self, name: &String) -> bool {
-        match &self.opts.include_tables {
-            Some(inc) => inc.contains(name),
-            None => true,
-        }
-    }
-
-    /// Do we exclude this table name?
-    fn exclude_table(&self, name: &String) -> bool {
-        match &self.opts.exclude_tables {
-            Some(inc) => inc.contains(name),
-            None => false,
-        }
-    }
-
-    /// Load the schema
-    pub(crate) fn load(&self) -> Result<Schema> {
-        let mut client = self.pg_client.borrow_mut();
-        let tables_rows = client.query(tables_query(), &[&self.schema])?;
-        let relations_rows = client.query(relations_query(), &[&self.schema])?;
-        let index_rows = client.query(index_query(), &[])?;
-
-        let mut partial_tables: HashMap<String, Vec<String>> = HashMap::new();
-
-        let indexes: Vec<_> = index_rows
-            .into_iter()
-            .filter(|row| {
-                let row_name: String = row.get(0);
-                self.include_table(&row_name) && !self.exclude_table(&row_name)
-            })
-            .map(|row| {
-                let idx: Index = row.try_into().unwrap();
-                idx
-            })
-            .collect();
-
-        let tables: Vec<_> = tables_rows
-            .into_iter()
-            .group_by(|row| row.get(0))
-            .into_iter()
-            .filter(|(name, _rows)| self.include_table(name) && !self.exclude_table(name))
-            .map(|(name, rows)| {
-                let fields: Vec<_> = rows
-                    .into_iter()
-                    .map(|row| {
-                        let mut field: TableColumn = row.try_into().unwrap();
-                        field.primary_key = is_primary_key(&name, &field.column, &indexes);
-
-                        let desc = match field.description {
-                            Some(desc) => {
-                                match self.opts.column_description_wrap {
-                                    Some(wrap) => Some(textwrap::fill(&desc, wrap)),
-                                    None => Some(desc),
-                                }
-                            },
-                            None => None,
-                        };
-                        field.description = desc;
-
-                        field
-                    })
-                    .collect();
-
-                let desc = match &fields[0].table_description {
-                    Some(desc) => {
-                        match self.opts.table_description_wrap {
-                            Some(wrap) => Some(textwrap::fill(desc, wrap)),
-                            None => Some(desc).cloned(),
-                        }
-                    },
-                    None => None,
-                };
-
-                Table {
-                    name,
-                    description: desc,
-                    fields,
-                }
-            })
-            .collect();
-
-        let relations: Vec<_> = relations_rows
-            .into_iter()
-            .map(|row| {
-                let relation: Relation = row.try_into().unwrap();
-                relation
-            })
-            .filter(|relation| {
-                if self.include_table(&relation.on_table)
-                    && !self.exclude_table(&relation.on_table)
-                    && !self.exclude_table(&relation.to_table)
-                {
-                    if !self.include_table(&relation.to_table) {
-                        match partial_tables.get_mut(&relation.to_table) {
-                            Some(value) => {
-                                if !value.contains(&relation.to_field) {
-                                    value.push(relation.to_field.clone());
-                                }
-                            },
-                            None => {
-                                partial_tables.insert(relation.to_table.clone(), vec![relation
-                                    .to_field
-                                    .clone()]);
-                            },
-                        }
-                    }
-                    true
-                } else {
-                    false
-                }
-            })
-            .collect();
-
-        Ok(Schema {
-            tables,
-            relations,
-            partial_tables,
-        })
-    }
-}
-
-impl TryFrom<Row> for Index {
-    type Error = String;
-
-    fn try_from(row: Row) -> std::result::Result<Self, String> {
-        let all_fields: String = row.get(4);
-        let braces: &[_] = &['{', '}'];
-
-        let fields: Vec<_> = all_fields
-            .trim_matches(braces)
-            .split(',')
-            .map(std::string::ToString::to_string)
-            .collect();
-
-        Ok(Self {
-            table: row.get(0),
-            // name: row.get(1),
-            primary: row.get(2),
-            // unique: row.get(3),
-            fields,
-        })
-    }
-}
-
-impl TryFrom<Row> for Relation {
-    type Error = String;
-
-    fn try_from(row: Row) -> std::result::Result<Self, String> {
-        let fields: HashMap<String, String> = row
-            .columns()
-            .iter()
-            .enumerate()
-            .map(|(i, c)| (c.name().to_string(), row.get(i)))
-            .collect();
-
-        Ok(Self {
-            on_table: fetch_field(&fields, "on_table")?,
-            on_field: fetch_field(&fields, "on_field")?,
-            to_table: fetch_field(&fields, "to_table")?,
-            to_field: fetch_field(&fields, "to_field")?,
-        })
-    }
-}
-
-impl TryFrom<Row> for TableColumn {
-    type Error = String;
-
-    fn try_from(row: Row) -> std::result::Result<Self, String> {
-        Ok(Self {
-            column: row.get(1),
-            data_type: row.get(2),
-            index: row.get(3),
-            default: row.get(4),
-            nullable: row.get(5),
-            max_chars: row.get(6),
-            description: row.get(7),
-            table_description: row.get(8),
-            primary_key: false,
-        })
-    }
-}
-
-/// Fetch a field from a hashmap
-fn fetch_field(map: &HashMap<String, String>, key: &str) -> std::result::Result<String, String> {
-    map.get(key)
-        .cloned()
-        .ok_or(format!("could not find field {key}"))
-}
-
-/// Query all tables and columns
-fn tables_query() -> &'static str {
-    "
-    select table_name, column_name, data_type, ordinal_position, column_default, is_nullable, character_maximum_length, col_description(table_name::regclass, ordinal_position), obj_description(table_name::regclass)
-      from information_schema.columns
-     where table_schema = $1
-     order by table_name, ordinal_position
-    "
-}
-
-/// Query all relationships
-fn relations_query() -> &'static str {
-    "
-    select *
-      from (
-        select ns.nspname AS schemaname,
-               cl.relname AS on_table,
-               attr.attname AS on_field,
-               clf.relname AS to_table,
-               attrf.attname AS to_field
-          from pg_constraint con
-                 join pg_class cl
-                     on con.conrelid = cl.oid
-                 join pg_namespace ns
-                     on cl.relnamespace = ns.oid
-                 join pg_class clf
-                     on con.confrelid = clf.oid
-                 join pg_attribute attr
-                     on attr.attnum = ANY(con.conkey) and
-                 attr.attrelid = con.conrelid
-                 join pg_attribute attrf
-                     on attrf.attnum = ANY(con.confkey) and
-                 attrf.attrelid = con.confrelid
-      ) as fk
-     where fk.schemaname = $1
-    "
-}
-
-/// Query all indexes
-fn index_query() -> &'static str {
-    "
-SELECT
-    CAST(idx.indrelid::regclass as varchar) as table_name,
-    i.relname as index_name,
-    idx.indisprimary as primary_key,
-    idx.indisunique as unique,
-    CAST(
-        ARRAY(
-            SELECT pg_get_indexdef(idx.indexrelid, k + 1, true)
-            FROM generate_subscripts(idx.indkey, 1) as k
-            ORDER BY k
-        ) as varchar
-    ) as columns
-FROM   pg_index as idx
-JOIN   pg_class as i
-ON     i.oid = idx.indexrelid
-JOIN   pg_am as am
-ON     i.relam = am.oid
-JOIN   pg_namespace as ns
-ON     ns.oid = i.relnamespace
-AND    ns.nspname = ANY(current_schemas(false))
-ORDER BY idx.indrelid
-"
-}
diff --git a/utilities/dbviz/src/schema.rs b/utilities/dbviz/src/schema.rs
deleted file mode 100644
index da07eb9b5..000000000
--- a/utilities/dbviz/src/schema.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-//! Core entities.
-use std::collections::HashMap;
-
-use serde::{Deserialize, Serialize};
-
-/// All the schema information.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub(crate) struct Schema {
-    /// List of tables in the database.
-    pub(crate) tables: Vec<Table>,
-    /// List of relations in the database.
-    pub(crate) relations: Vec<Relation>,
-    /// Partial Tables
-    pub(crate) partial_tables: HashMap<String, Vec<String>>,
-}
-
-/// Table information.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub(crate) struct TableColumn {
-    /// Column name.
-    pub(crate) column: String,
-    /// Column data type.
-    pub(crate) data_type: String,
-    /// Column index.
-    pub(crate) index: i32,
-    /// Column default.
-    pub(crate) default: Option<String>,
-    /// Column nullable.
-    pub(crate) nullable: String,
-    /// Column max chars.
-    pub(crate) max_chars: Option<i32>,
-    /// Column description.
-    pub(crate) description: Option<String>,
-    /// Table description.
-    pub(crate) table_description: Option<String>, // Redundant but easiest way to get it.
-    /// Column primary key.
-    pub(crate) primary_key: bool,
-}
-
-/// Table information.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub(crate) struct Table {
-    /// Table name.
-    pub(crate) name: String,
-    /// Table Description
-    pub(crate) description: Option<String>,
-    /// List of fields.
-    pub(crate) fields: Vec<TableColumn>,
-}
-
-/// Row description.
-//#[derive(Debug)]
-// pub(crate)struct Field(pub(crate)FieldName, pub(crate)FieldType);
-
-/// Relation node.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub(crate) struct Relation {
-    /// Table that the constraint references.
-    pub(crate) on_table: TableName,
-    /// Field that the constraint references.
-    pub(crate) on_field: FieldName,
-    /// Table which the fk references.
-    pub(crate) to_table: TableName,
-    /// Field which the fk references.
-    pub(crate) to_field: FieldName,
-}
-
-/// Table name
-pub(crate) type TableName = String;
-/// Field name
-pub(crate) type FieldName = String;
-// pub(crate)type FieldType = String;
-
-/// Index Definition
-pub(crate) struct Index {
-    /// Table name
-    pub(crate) table: TableName,
-    // pub(crate)name: String,
-    /// Primary Key
-    pub(crate) primary: bool,
-    // pub(crate)unique: bool,
-    /// Fields
-    pub(crate) fields: Vec<String>,
-}