From 90673ec65299e6046b84c8832d5f2bfe889168d6 Mon Sep 17 00:00:00 2001
From: Matthew Parker <matthew.parker@octoenergy.com>
Date: Mon, 30 Sep 2024 10:30:39 +0100
Subject: [PATCH] Dbt gatekeeper submission

---
 .vscode/settings.json                         |  4 ++
 .../models/final/finance/_exposures.yml       | 14 +++++
 jaffle_shop/models/final/finance/_models.yml  | 17 ++++++
 .../finance/fnl_finance_returned_orders.sql   | 14 +++++
 jaffle_shop/models/final/sales/_exposures.yml | 14 +++++
 jaffle_shop/models/final/sales/_models.yml    | 16 +++++
 .../final/sales/fnl_sales_first_orders.sql    | 14 +++++
 jaffle_shop/models/staging/schema.yml         | 31 ----------
 .../{docs.md => staging/src_seeds/_docs.yml}  |  0
 .../models/staging/src_seeds/_models.yml      | 61 +++++++++++++++++++
 .../staging/src_seeds/stg_customers.sql       |  3 +
 .../stg_customers_pii.sql}                    |  0
 .../staging/{ => src_seeds}/stg_orders.sql    |  0
 .../staging/{ => src_seeds}/stg_payments.sql  |  0
 .../{schema.yml => warehouse/_models.yml}     |  0
 .../wh_customers.sql}                         |  0
 .../{orders.sql => warehouse/wh_orders.sql}   |  0
 jaffle_shop/seeds/_seeds.yml                  | 30 +++++++++
 .../dbt_project_evaluator_exceptions.csv      |  2 +
 19 files changed, 189 insertions(+), 31 deletions(-)
 create mode 100644 .vscode/settings.json
 create mode 100644 jaffle_shop/models/final/finance/_exposures.yml
 create mode 100644 jaffle_shop/models/final/finance/_models.yml
 create mode 100644 jaffle_shop/models/final/finance/fnl_finance_returned_orders.sql
 create mode 100644 jaffle_shop/models/final/sales/_exposures.yml
 create mode 100644 jaffle_shop/models/final/sales/_models.yml
 create mode 100644 jaffle_shop/models/final/sales/fnl_sales_first_orders.sql
 delete mode 100644 jaffle_shop/models/staging/schema.yml
 rename jaffle_shop/models/{docs.md => staging/src_seeds/_docs.yml} (100%)
 create mode 100644 jaffle_shop/models/staging/src_seeds/_models.yml
 create mode 100644 jaffle_shop/models/staging/src_seeds/stg_customers.sql
 rename jaffle_shop/models/staging/{stg_customers.sql => src_seeds/stg_customers_pii.sql} (100%)
 rename jaffle_shop/models/staging/{ => src_seeds}/stg_orders.sql (100%)
 rename jaffle_shop/models/staging/{ => src_seeds}/stg_payments.sql (100%)
 rename jaffle_shop/models/{schema.yml => warehouse/_models.yml} (100%)
 rename jaffle_shop/models/{customers.sql => warehouse/wh_customers.sql} (100%)
 rename jaffle_shop/models/{orders.sql => warehouse/wh_orders.sql} (100%)
 create mode 100644 jaffle_shop/seeds/_seeds.yml
 create mode 100644 jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 000000000..5592767af
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,4 @@
+{
+    "editor.tabSize": 2,
+    "editor.detectIndentation": false
+}
\ No newline at end of file
diff --git a/jaffle_shop/models/final/finance/_exposures.yml b/jaffle_shop/models/final/finance/_exposures.yml
new file mode 100644
index 000000000..9fa93a9bd
--- /dev/null
+++ b/jaffle_shop/models/final/finance/_exposures.yml
@@ -0,0 +1,14 @@
+version: 2
+
+################################################################################
+# EXPOSURES      https://docs.getdbt.com/reference/exposure-properties
+################################################################################
+exposures:
+  - name: fnl_finance_returned_orders
+    description: Inksacio Dashboard
+    type: dashboard
+    url: url: https://inksacio.uk.octopus.engineering/my_certification_finance_dashboard/
+    owner:
+      email: matthew.parker@octoenergy.com
+    depends_on:
+      - ref('fnl_finance_returned_orders')
diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml
new file mode 100644
index 000000000..05b700d08
--- /dev/null
+++ b/jaffle_shop/models/final/finance/_models.yml
@@ -0,0 +1,17 @@
+version: 2
+
+models: 
+  - name: fnl_finance_returned_orders
+    config:
+      tags: ['access:public']
+    meta:
+      owner: 'matthew.parker@octoenergy.com'
+    description: |
+      Report on the value of returned orders. 1 row per customer with total returned value for 
+      that customer
+    columns:
+      - name: customer_id
+        description: Primary key
+        tests:
+          - unique
+          - not_null
\ No newline at end of file
diff --git a/jaffle_shop/models/final/finance/fnl_finance_returned_orders.sql b/jaffle_shop/models/final/finance/fnl_finance_returned_orders.sql
new file mode 100644
index 000000000..088377389
--- /dev/null
+++ b/jaffle_shop/models/final/finance/fnl_finance_returned_orders.sql
@@ -0,0 +1,14 @@
+WITH returned_orders AS (
+    SELECT
+      customer_id
+      , amount AS value
+    FROM {{ ref('wh_orders') }}
+    WHERE status = 'returned'
+    GROUP BY customer_id
+)
+
+SELECT
+  customer_id
+  , SUM(value) AS total_value
+FROM returned_orders
+GROUP BY customer_id
diff --git a/jaffle_shop/models/final/sales/_exposures.yml b/jaffle_shop/models/final/sales/_exposures.yml
new file mode 100644
index 000000000..9ee479c63
--- /dev/null
+++ b/jaffle_shop/models/final/sales/_exposures.yml
@@ -0,0 +1,14 @@
+version: 2
+
+################################################################################
+# EXPOSURES      https://docs.getdbt.com/reference/exposure-properties
+################################################################################
+exposures:
+  - name: fnl_sales_first_orders
+    description: Inksacio Dashboard
+    type: dashboard
+    url: url: https://inksacio.uk.octopus.engineering/my_certification_sales_dashboard/
+    owner:
+      email: matthew.parker@octoenergy.com
+    depends_on:
+      - ref('fnl_sales_first_orders')
diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml
new file mode 100644
index 000000000..88f937dfd
--- /dev/null
+++ b/jaffle_shop/models/final/sales/_models.yml
@@ -0,0 +1,16 @@
+version: 2
+
+models: 
+  - name: fnl_sales_first_orders
+    config:
+      tags: ['access:public']
+    meta:
+      owner: 'matthew.parker@octoenergy.com'
+    description: |
+      Report on the numbers of customers making their first order per month.
+    columns:
+      - name: first_order_month
+        description: Primary key
+        tests:
+          - unique
+          - not_null
\ No newline at end of file
diff --git a/jaffle_shop/models/final/sales/fnl_sales_first_orders.sql b/jaffle_shop/models/final/sales/fnl_sales_first_orders.sql
new file mode 100644
index 000000000..c5a0bb6ca
--- /dev/null
+++ b/jaffle_shop/models/final/sales/fnl_sales_first_orders.sql
@@ -0,0 +1,14 @@
+WITH customer_first_orders AS (
+  SELECT
+    customer_id
+    -- Get just the month from the first order date
+    , DATE_FORMAT(first_order, 'MMMM') As first_order_month
+  FROM {{ ref('wh_customers') }}
+)
+
+SELECT
+  first_order_month
+  , COUNT(*) AS number_of_customers
+FROM customer_first_orders
+GROUP BY first_order_month
+
diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/schema.yml
deleted file mode 100644
index c207e4cf5..000000000
--- a/jaffle_shop/models/staging/schema.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-version: 2
-
-models:
-  - name: stg_customers
-    columns:
-      - name: customer_id
-        tests:
-          - unique
-          - not_null
-
-  - name: stg_orders
-    columns:
-      - name: order_id
-        tests:
-          - unique
-          - not_null
-      - name: status
-        tests:
-          - accepted_values:
-              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
-
-  - name: stg_payments
-    columns:
-      - name: payment_id
-        tests:
-          - unique
-          - not_null
-      - name: payment_method
-        tests:
-          - accepted_values:
-              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']
diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/staging/src_seeds/_docs.yml
similarity index 100%
rename from jaffle_shop/models/docs.md
rename to jaffle_shop/models/staging/src_seeds/_docs.yml
diff --git a/jaffle_shop/models/staging/src_seeds/_models.yml b/jaffle_shop/models/staging/src_seeds/_models.yml
new file mode 100644
index 000000000..0fd8f0ea1
--- /dev/null
+++ b/jaffle_shop/models/staging/src_seeds/_models.yml
@@ -0,0 +1,61 @@
+version: 2
+
+models:
+  - name: stg_customers_pii
+    config:
+      tags: ['access:private']
+    meta: 
+      owner: 'matthew.parker@octoenergy.com'
+    description: |
+      Staging table for csv containing customer information.
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+      - name: first_name
+        meta:
+          sensitive: true
+      - name: last_name
+        meta:
+          sensitive: true
+  - name: stg_customers
+    config:
+      tags: ['access:private']
+    meta: 
+      owner: 'matthew.parker@octoenergy.com'
+    description: |
+      Staging table for csv containing customer information with PII columns hashed.
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+      - name: first_name_hash
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: last_name_hash
+        tests:
+          - dbt_expectations.expect_column_to_exist
+
+  - name: stg_orders
+    columns:
+      - name: order_id
+        tests:
+          - unique
+          - not_null
+      - name: status
+        tests:
+          - accepted_values:
+              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+
+  - name: stg_payments
+    columns:
+      - name: payment_id
+        tests:
+          - unique
+          - not_null
+      - name: payment_method
+        tests:
+          - accepted_values:
+              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']
diff --git a/jaffle_shop/models/staging/src_seeds/stg_customers.sql b/jaffle_shop/models/staging/src_seeds/stg_customers.sql
new file mode 100644
index 000000000..27b915a99
--- /dev/null
+++ b/jaffle_shop/models/staging/src_seeds/stg_customers.sql
@@ -0,0 +1,3 @@
+SELECT
+  {{ hash_sensitive_columns('stg_customers_pii') }}
+FROM {{ ref('stg_customers_pii') }}
\ No newline at end of file
diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seeds/stg_customers_pii.sql
similarity index 100%
rename from jaffle_shop/models/staging/stg_customers.sql
rename to jaffle_shop/models/staging/src_seeds/stg_customers_pii.sql
diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seeds/stg_orders.sql
similarity index 100%
rename from jaffle_shop/models/staging/stg_orders.sql
rename to jaffle_shop/models/staging/src_seeds/stg_orders.sql
diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seeds/stg_payments.sql
similarity index 100%
rename from jaffle_shop/models/staging/stg_payments.sql
rename to jaffle_shop/models/staging/src_seeds/stg_payments.sql
diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml
similarity index 100%
rename from jaffle_shop/models/schema.yml
rename to jaffle_shop/models/warehouse/_models.yml
diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql
similarity index 100%
rename from jaffle_shop/models/customers.sql
rename to jaffle_shop/models/warehouse/wh_customers.sql
diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql
similarity index 100%
rename from jaffle_shop/models/orders.sql
rename to jaffle_shop/models/warehouse/wh_orders.sql
diff --git a/jaffle_shop/seeds/_seeds.yml b/jaffle_shop/seeds/_seeds.yml
new file mode 100644
index 000000000..9ff9c3087
--- /dev/null
+++ b/jaffle_shop/seeds/_seeds.yml
@@ -0,0 +1,30 @@
+version: 2
+
+seeds:
+  - name: raw_customers
+    description: |
+      Table containing customer_id, and their first and last name.
+    config:
+      column_types:
+        id: int
+        first_name: string
+        last_name: string
+  - name: raw_orders
+    description: |
+      Table containing information about customers orders
+    config:
+      column_types:
+        id: int
+        user_id: int
+        order_date: date
+        status: string
+  - name: raw_payments
+    description: |
+      Table containing information about customers payments
+    config:
+      column_types:
+        id: int
+        order_id: int
+        payment_method: string
+        amount: int
+
diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv
new file mode 100644
index 000000000..13dd3066f
--- /dev/null
+++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv
@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.