Sm jaffle test

.circleci/config.yml

@@ -79,4 +79,7 @@ workflows:
   test:
     jobs:
       - run-python-tests
-      - run-dbt-project-evaluator
+      - run-dbt-project-evaluator
+
+
+

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-dbt-core = "==1.3.0"
+dbt-core = "==1.7.17"
 dbt-databricks = {extras = ["odbc"]}
 pytest-mock = "*"
 glob2 = "*"

dbt_project.yml

@@ -21,9 +21,16 @@ require-dbt-version: [">=1.0.0", "<2.0.0"]
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['dbt_project_evaluator', 'spark_utils', 'dbt_utils']
+  - macro_namespace: dbt
+    search_order: ['dbt_project_evaluator', 'dbt']
 
 models:
   jaffle_shop:
       materialized: table
       staging:
         materialized: view
+
+seeds:
+  dbt_project_evaluator:
+    dbt_project_evaluator_exceptions:
+      +enabled: false

jaffle_shop/macros/sensitive/hash_of_column.sql

@@ -3,7 +3,7 @@
     SHA2(
         TRIM(
             LOWER(
-                CAST({{ column|lower }} AS {{ dbt_utils.type_string() }})
+                CAST({{ column|lower }} AS {{ dbt.type_string() }})
                 || '{{ jaffle_shop.get_salt(column|lower) }}'
             )
         ),

jaffle_shop/models/final/finance/_models.yml

@@ -0,0 +1,55 @@
+version: 2
+
+exposures:
+  - name: value_of_returned_orders_dashboard
+    description: |
+      Value of returned orders by customer
+    type: dashboard
+    url: https://inksacio.eks.octopus.engineering/my_certification_dashboard2/
+    owner:
+      name: Sam Milligan 
+      email: [email protected]
+    depends_on:
+     - ref('wh_orders')
+     - ref('wh_customers')
+
+models:
+  - name: fnl_finance_returned_orders
+    meta:
+      owner: '[email protected]'
+      description: |
+        Number of new customer orders placed by month
+    columns:
+      - name: customer_id
+        description: Primary Key
+        tests:
+          - unique
+          - not_null
+      - name: first_name_hash
+        description: Hash of customer's first name
+        tests:
+          - not_null
+      - name: last_name_hash
+        description: Hash of customer's last name
+        tests:
+          - not_null
+      - name: first_order
+        description: Date of a customer's first order
+        tests:
+          - not_null
+      - name: most_recent_order
+        description: Date of a customer's most recent order
+        tests:
+          - not_null
+      - name: number_of_orders
+        description: Count of the number of orders a customer has placed
+        tests:
+          - not_null
+      - name: customer_lifetime_value
+        description: Total value of a customer's orders
+        tests:
+          - not_null
+      - name: total_returned_amount
+        description: Total value of orders returned by customer
+        tests:
+          - not_null

jaffle_shop/models/final/finance/fnl_finance_returned_orders.sql

@@ -0,0 +1,23 @@
+WITH returned as (
+    SELECT 
+        customer_id,
+        COUNT(status) as number_of_returned_orders,
+        SUM(amount) as total_returned_amount
+    FROM {{ ref('wh_orders') }}
+    WHERE status='returned'
+    GROUP BY customer_id
+    )
+
+SELECT
+    r.customer_id,
+    r.number_of_returned_orders,
+    r.total_returned_amount,
+    c.first_name_hash,
+    c.last_name_hash,
+    c.first_order,
+    c.most_recent_order,
+    c.number_of_orders,
+    c.customer_lifetime_value
+FROM returned r
+LEFT JOIN {{ ref('wh_customers') }} c
+ON r.customer_id = c.customer_id

jaffle_shop/models/final/sales/_models.yml

@@ -0,0 +1,32 @@
+version: 2
+
+exposures:
+  - name: new_customer_sales_dashboard
+    description: |
+      Number of new customer orders placed by month
+    type: dashboard
+    url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/
+    owner:
+      name: Sam Milligan 
+      email: [email protected]
+    depends_on:
+     - ref('wh_customers')
+
+models:
+  - name: fnl_sales_new_customers
+    meta:
+      owner: '[email protected]'
+      description: Total value of orders returned by customer
+    columns:
+      - name: month
+        description: Primary Key
+        tests:
+          - unique
+          - not_null
+      - name: month_formatted
+        description: Formatted month
+        tests:
+          - not_null
+      - name: count_new_customer_orders
+        tests:
+          - not_null

jaffle_shop/models/final/sales/fnl_sales_new_customers.sql

@@ -0,0 +1,8 @@
+SELECT 
+    DATE_TRUNC('MONTH', first_order) AS month,
+    DATE_FORMAT(month, "MMM y") AS month_formatted,
+    COUNT(*) as count_new_customer_orders
+FROM {{ ref('wh_customers')}}
+GROUP BY month
+HAVING isnotnull(month)
+ORDER BY month

jaffle_shop/models/staging/src_seed/_models.yml

@@ -0,0 +1,74 @@
+version: 2
+
+
+models:
+  - name: stg_customers_pii
+    meta:
+      owner: '[email protected]'
+      sensitive: true
+      description: |
+        Customers staging table containing PII
+    columns:
+      - name: customer_id
+        description: This is a unique identifier for a customer
+        tests:
+          - unique
+          - not_null
+
+      - name: first_name
+        meta:
+          sensitive: true
+        description: Customer's first name. PII.
+
+      - name: last_name
+        meta:
+          sensitive: true
+        description: Customer's last name. PII.
+
+  - name: stg_customers
+    meta:
+      owner: '[email protected]'
+    description: |
+      Customers staging table with PII removed
+    columns:
+      - name: customer_id
+        description: This is a unique identifier for a customer
+        tests:
+          - unique
+          - not_null
+
+      - name: first_name
+        description: Customer's first name.
+
+      - name: last_name
+        description: Customer's last name.
+
+  - name: stg_orders
+    meta:
+      owner: '[email protected]'
+      description: |
+        Orders staging table
+      columns:
+        - name: order_id
+          description: Primary Key
+          tests:
+            - unique
+            - not_null
+        - name: status
+          tests:
+            - accepted_values:
+                values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+  - name: stg_payments
+    meta:
+      owner: '[email protected]'
+      description: |
+        Payments staging table
+    columns:
+      - name: payment_id
+        tests:
+          - unique
+          - not_null
+      - name: payment_method
+        tests:
+          - accepted_values:
+              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']

jaffle_shop/models/staging/src_seed/stg_customers.sql

@@ -0,0 +1,3 @@
+SELECT 
+    {{hash_sensitive_columns('stg_customers_pii')}}
+FROM {{ ref('stg_customers_pii') }}

jaffle_shop/models/schema.yml → jaffle_shop/models/warehouse/_models.yml

@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: customers
+  - name: wh_customers
     description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
 
     columns:
@@ -29,7 +29,7 @@ models:
       - name: total_order_amount
         description: Total value (AUD) of a customer's orders
 
-  - name: orders
+  - name: wh_orders
     description: This table has basic information about orders, as well as some derived facts based on payments
 
     columns:
@@ -44,7 +44,7 @@ models:
         tests:
           - not_null
           - relationships:
-              to: ref('customers')
+              to: ref('stg_customers')
               field: customer_id
 
       - name: order_date

jaffle_shop/models/customers.sql → jaffle_shop/models/warehouse/wh_customers.sql

@@ -49,8 +49,8 @@ final as (
 
     select
         customers.customer_id,
-        customers.first_name,
-        customers.last_name,
+        customers.first_name_hash,
+        customers.last_name_hash,
         customer_orders.first_order,
         customer_orders.most_recent_order,
         customer_orders.number_of_orders,

jaffle_shop/seeds/_seeds.yml

@@ -0,0 +1,6 @@
+version: 2
+
+seeds:
+  - name: raw_customers
+  - name: raw_orders
+  - name: raw_payments

jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.

package-lock.yml

@@ -0,0 +1,10 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.3.0
+- package: dbt-labs/spark_utils
+  version: 0.3.0
+- package: dbt-labs/codegen
+  version: 0.13.1
+- package: dbt-labs/dbt_project_evaluator
+  version: 0.14.3
+sha1_hash: 59adf0a667e307048209cbea5ef70e871d6ec49a

packages.yml

@@ -1,9 +1,9 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: 0.8.6
+    version: '>=0.8.6'
   - package: dbt-labs/spark_utils
-    version: 0.3.0
+    version: '>=0.3.0'
   - package: dbt-labs/codegen
-    version: 0.7.0
+    version: '>=0.7.0'
   - package: dbt-labs/dbt_project_evaluator
-    version: 0.1.3
+    version: '>=0.1.3'