refactor: Mount unified catalog dir

.pre-commit-config.yaml

@@ -1,5 +1,8 @@
 # pre-commit autoupdate
 ---
+default_language_version:
+  python: python3
+minimum_pre_commit_version: '3.4.0'
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -12,7 +15,7 @@ repos:
       - id: check-toml
       - id: check-xml
       - id: check-yaml
-        exclude: ^ci/meta.yaml$
+        exclude: ^.pre-commit-config.yaml$
       - id: debug-statements
       - id: end-of-file-fixer
         exclude: \.txt$
@@ -60,7 +63,6 @@ repos:
           - Flake8-pyproject
           - pydocstyle>=5.0.0
 
-
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
@@ -96,6 +98,13 @@ repos:
         verbose: true
         stages: [ commit-msg ]
 
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+        name: 'git conflict enforcement'
+        verbose: true
+
   # - repo: https://github.com/pre-commit/pre-commit-hooks
   #   rev: v4.4.0
   #   hooks:
@@ -111,3 +120,16 @@ repos:
       - id: codespell
         types_or: [ python, rst, markdown, cython, c ]
         additional_dependencies: [ tomli ]
+
+  - repo: https://github.com/dbt-checkpoint/dbt-checkpoint
+    rev: v1.1.1
+    hooks:
+      # - id: dbt-deps
+      # name: 'dbt deps'
+      # verbose: true
+      - id: dbt-compile
+        name: 'dbt compile'
+        verbose: true
+      - id: dbt-test
+        name: 'dbt test'
+        verbose: true

docker-compose.yml

@@ -52,7 +52,7 @@ services:
       - ./docker/volume/postgres:/var/lib/postgresql/data
       - ./docker/postgres/init-database.sh:/docker-entrypoint-initdb.d/init-database.sh
     healthcheck:
-      test: ["CMD", "pg_isready", "-U", "postgres"]
+      test: [ "CMD", "pg_isready", "-U", "postgres" ]
       interval: 10s
       retries: 5
       start_period: 5s
@@ -94,6 +94,8 @@ services:
 
   minio-job:
     image: 'minio/mc'
+    container_name: minio-job
+    hostname: minio-job
     entrypoint: |
       /bin/bash -c "
       sleep 5;
@@ -122,28 +124,26 @@ services:
     container_name: trino
     hostname: trino
     image: "trinodb/trino:425"
+    restart: always
     ports:
       - "8889:8889"
     volumes:
       - ./docker/trino/etc-coordinator:/etc/trino
-    healthcheck:
-      test: ["CMD-SHELL", "curl -sS http://localhost:8889/ || exit 1"]
-      interval: 5s
-      retries: 5
-      start_period: 15s
+      - ./docker/trino/catalog:/etc/trino/catalog
     depends_on:
       - hive-metastore
 
   trino-worker:
-    profiles: [ "trino" ]
+    profiles: [ "trino-worker" ]
     container_name: trino-worker
     hostname: trino-worker
     image: "trinodb/trino:425"
+    restart: always
     volumes:
       - ./docker/trino/etc-worker:/etc/trino
+      - ./docker/trino/catalog:/etc/trino/catalog
     depends_on:
-      trino:
-        condition: service_healthy
+      - trino
 
   ####################################################################################################
   #
@@ -232,7 +232,7 @@ services:
   ####################################################################################################
 
   flink-jobmanager:
-    profiles: ["flink"]
+    profiles: [ "flink" ]
     build:
       dockerfile: ./docker/flink/Dockerfile-flink1.16
     image: 1ambda/lakehouse:flink-1.16
@@ -311,11 +311,11 @@ services:
     depends_on:
       - flink-jobmanager
 
-####################################################################################################
-#
-# Airflow
-#
-####################################################################################################
+  ####################################################################################################
+  #
+  # Airflow
+  #
+  ####################################################################################################
   airflow-webserver:
     <<: *airflow-common
     profiles: [ "airflow" ]
@@ -324,7 +324,7 @@ services:
     ports:
       - "8080:8080"
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      test: [ "CMD", "curl", "--fail", "http://localhost:8080/health" ]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -341,7 +341,7 @@ services:
     container_name: airflow-scheduler
     command: scheduler
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
+      test: [ "CMD", "curl", "--fail", "http://localhost:8974/health" ]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -380,7 +380,7 @@ services:
     container_name: airflow-triggerer
     command: triggerer
     healthcheck:
-      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
+      test: [ "CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"' ]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -414,7 +414,7 @@ services:
       - -c
       - |
         mkdir -p /sources/logs /sources/dags /sources/plugins
-        chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
+        chown -R "50000:0" /sources/{logs,dags,plugins}
         exec /entrypoint airflow version
     # yamllint enable rule:line-length
     environment:
@@ -434,7 +434,7 @@ services:
     expose:
       - 6379
     healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
+      test: [ "CMD", "redis-cli", "ping" ]
       interval: 10s
       timeout: 30s
       retries: 50
@@ -444,4 +444,4 @@ services:
 # Configure Network
 networks:
   default:
-    name: lakehouse
+    name: lakehouse

docker/trino/etc-coordinator/catalog/hive.properties → docker/trino/catalog/hive.properties

@@ -8,4 +8,4 @@ hive.s3.aws-secret-key=minio123
 
 hive.metastore-cache-ttl=0s
 hive.metastore-refresh-interval=5s
-hive.metastore-timeout=10s
+hive.metastore-timeout=10s

docker/trino/etc-coordinator/catalog/iceberg.properties → docker/trino/catalog/iceberg.properties

@@ -15,4 +15,8 @@ iceberg.file-format=parquet
 iceberg.compression-codec=ZSTD
 iceberg.target-max-file-size=256MB
 
+iceberg.expire_snapshots.min-retention=1m
+iceberg.remove_orphan_files.min-retention=1m
+
+iceberg.unique-table-location=false
 iceberg.materialized-views.storage-schema=common

docker/trino/etc-worker/catalog/postgresql.properties → docker/trino/catalog/postgresql.properties

@@ -1,4 +1,4 @@
 connector.name=postgresql
 connection-url=jdbc:postgresql://metastore_db:5432/hive
 connection-user=hive
-connection-password=hive
+connection-password=hive

docker/trino/etc-coordinator/config.properties

@@ -1,5 +1,5 @@
 coordinator=true
-node-scheduler.include-coordinator=false
+node-scheduler.include-coordinator=true
 http-server.http.port=8889
 discovery.uri=http://trino:8889
 
@@ -8,4 +8,4 @@ internal-communication.shared-secret=acN0eEVueRJVGkQ1djBFh9A6OU0+Utk+UMYzSPOO0W+
 
 #http-server.authentication.type=PASSWORD
 web-ui.authentication.type=FIXED
-web-ui.user=admin
+web-ui.user=admin

docker/trino/etc-coordinator/log.properties

@@ -1,4 +1,4 @@
-io.trino=DEBUG
+io.trino=INFO
 
 # show classpath for plugins
 io.trino.server.PluginManager=DEBUG

docker/trino/etc-worker/log.properties

@@ -1,4 +1,4 @@
-io.trino=DEBUG
+io.trino=INFO
 
 # show classpath for plugins
 io.trino.server.PluginManager=DEBUG