feat: Add dbt aggregation models

.pre-commit-config.yaml

@@ -133,3 +133,17 @@ repos:
   #     - id: dbt-test
   #       name: 'dbt test'
   #       verbose: true
+
+
+  # SQL
+  - repo: https://github.com/sqlfluff/sqlfluff
+    rev: 2.3.1
+    hooks:
+      - id: sqlfluff-fix
+        name: 'sqlfluff fix'
+        verbose: true
+        files: ^(dags|dbts)/
+      - id: sqlfluff-lint
+        name: 'sqlfluff lint'
+        verbose: true
+        files: ^(dags|dbts)/

dbts/.gitignore

@@ -1,3 +1,5 @@
+__sqlfluff_tmp_*.sql
+
 # Created by https://www.toptal.com/developers/gitignore/api/dbt,python
 # Edit at https://www.toptal.com/developers/gitignore?templates=dbt,python
 

dbts/dbt-model/mart/_aggr.yml

@@ -0,0 +1,12 @@
+version: 2
+
+models:
+  - name: aggr_location
+    tags:
+      - iceberg
+      - aggregation
+    columns:
+      - name: loc
+        tests:
+          - not_null
+      - name: count

dbts/dbt-model/mart/aggr_location.sql

@@ -0,0 +1,10 @@
+WITH location AS (
+    SELECT *
+    FROM {{ ref('int_location') }}
+)
+
+SELECT
+    concat_ws(' - ', location.region, location.nation) AS loc,
+    count(*) AS count
+FROM location
+GROUP BY 1

dbts/dbt-model/staging/_int.yml

@@ -0,0 +1,15 @@
+version: 2
+
+models:
+  - name: location
+    tags:
+      - iceberg
+      - intermediate
+    columns:
+      - name: nation_key
+        tests:
+          - unique
+          - not_null
+      - name: region_key
+        tests:
+          - not_null

dbts/dbt-model/staging/_staging.yml → dbts/dbt-model/staging/_stg.yml

@@ -1,21 +1,21 @@
 version: 2
 
 models:
-  - name: staging_nations
+  - name: stg_nations
     description: staging layer for nations data
     tags:
-      - "iceberg"
-      - "staging"
+      - iceberg
+      - staging
     columns:
       - name: nation_key
         tests:
           - not_null
 
-  - name: staging_regions
+  - name: stg_regions
     description: staging layer for regions data
     tags:
-      - "iceberg"
-      - "staging"
+      - iceberg
+      - staging
     columns:
       - name: region_key
         tests:

dbts/dbt-model/staging/int_location.sql

@@ -0,0 +1,23 @@
+WITH nation AS (
+    SELECT *
+    FROM {{ ref('stg_nations') }}
+),
+
+region AS (
+    SELECT *
+    FROM {{ ref('stg_regions') }}
+),
+
+final AS (
+    SELECT
+        nation.nation_key,
+        nation.name AS nation,
+        nation.region_key,
+        region.name AS region
+    FROM nation
+    INNER JOIN region
+        ON nation.region_key = region.region_key
+)
+
+SELECT *
+FROM final

dbts/dbt-model/staging/stg_nations.sql

@@ -0,0 +1,30 @@
+WITH source AS (
+    SELECT *
+    FROM {{ source('tpch', 'nation') }}
+),
+
+renamed AS (
+    SELECT
+        nationkey AS nation_key,
+        name AS name,
+        regionkey AS region_key,
+        comment AS comment
+
+    FROM source
+
+    UNION ALL
+    (
+
+    -- make sure to test for dupes!!!
+        SELECT
+            nationkey AS nation_key,
+            NULL AS name,
+            NULL AS region_key,
+            NULL AS comment
+
+        FROM source LIMIT 1
+    )
+)
+
+SELECT *
+FROM renamed

dbts/dbt-model/staging/stg_regions.sql

@@ -0,0 +1,16 @@
+WITH source AS (
+    SELECT *
+    FROM {{ source('tpch', 'region') }}
+),
+
+renamed AS (
+    SELECT
+        regionkey AS region_key,
+        name AS name,
+        comment AS comment
+
+    FROM source
+)
+
+SELECT *
+FROM renamed

dbts/dbt_project.yml

@@ -15,11 +15,10 @@ seed-paths: [ "dbt-seeds" ]
 macro-paths: [ "dbt-macro" ]
 snapshot-paths: [ "dbt-snapshot" ]
 target-path: "target"
-clean-targets: # directories to be removed by `dbt clean`
+clean-targets:
   - "target"
   - "logs"
   - "dbt_modules"
-  # - "dbt_packages"
 
 # Configure variables
 vars:
@@ -28,11 +27,13 @@ vars:
 # Configure models
 models:
   lakehouse:
+    +on_table_exists: drop
     staging:
       +schema: staging
+      +materialized: view
+    mart:
+      +schema: mart
       +materialized: table
-      +on_table_exists: drop
-
 
 # Configure dispatches
 dispatch: