diff --git a/jaffle_shop/models/final/finance/_exposures.yml b/jaffle_shop/models/final/finance/_exposures.yml deleted file mode 100644 index b38da1da7..000000000 --- a/jaffle_shop/models/final/finance/_exposures.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -exposures: - - name: customer_return_total - description: Inksacio App to show the total sum of customer returns, per customer (for both completed and upcoming) - type: dashboard - url: https://inksacio.eks.octojaffle.engineering/customer_return_totals/ - owner: - name: 'Elliot Taylor' - email: elliot.taylor@octojaffle.com - depends_on: - - ref('fnl_finance_customerreturnstotal') \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml index ced7510c5..668ec266b 100644 --- a/jaffle_shop/models/final/finance/_models.yml +++ b/jaffle_shop/models/final/finance/_models.yml @@ -1,7 +1,21 @@ version: 2 +exposures: + - name: customer_return_total + description: Inksacio App to show the total sum of customer returns, per customer (for both completed and upcoming) + type: dashboard + url: https://inksacio.eks.octojaffle.engineering/customer_return_totals/ + owner: + name: 'Elliot Taylor' + email: elliot.taylor@octojaffle.com + depends_on: + - ref('fnl_finance_customerreturnstotal') + models: - name: fnl_finance_customerreturnstotal + meta: + owner: 'matthew.elsham@octoenergy.com' + team_owner: '!subteam^S7K9CR63D' #@datateam description: > This table gives one row per customer, with the total of their completed returns, pending_returns and the sum of the 2 diff --git a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql index a832f0dae..e107b0f85 100644 --- a/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql +++ b/jaffle_shop/models/final/finance/fnl_finance_customerreturnstotal.sql @@ -1,6 +1,6 @@ {% set return_states = ['returned', 'return_pending'] %} -select +SELECT orders.customer_id {% for return_state in return_states -%} @@ -9,6 +9,6 @@ select , SUM(orders.amount_dollars) AS sum_return_amount_dollars -from {{ ref('wh_orders') }} AS orders -where orders.status IN ('returned', 'return_pending') -GROUP BY orders.customer_id \ No newline at end of file +FROM {{ ref('wh_orders') }} AS orders +WHERE orders.status IN ('returned', 'return_pending') +GROUP BY orders.customer_id diff --git a/jaffle_shop/models/final/sales/_exposures.yml b/jaffle_shop/models/final/sales/_exposures.yml deleted file mode 100644 index c12bb1be5..000000000 --- a/jaffle_shop/models/final/sales/_exposures.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -exposures: - - name: new_customer_history - description: Inksacio App to show monthly new customer history - type: dashboard - url: https://inksacio.eks.octojaffle.engineering/new_customer_history/ - owner: - name: 'Elliot Taylor' - email: elliot.taylor@octojaffle.com - depends_on: - - ref('fnl_sales_newcustomerhistory') \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml index 8186417cf..167454016 100644 --- a/jaffle_shop/models/final/sales/_models.yml +++ b/jaffle_shop/models/final/sales/_models.yml @@ -1,7 +1,21 @@ version: 2 +exposures: + - name: new_customer_history + description: Inksacio App to show monthly new customer history + type: dashboard + url: https://inksacio.eks.octojaffle.engineering/new_customer_history/ + owner: + name: 'Elliot Taylor' + email: elliot.taylor@octojaffle.com + depends_on: + - ref('fnl_sales_newcustomerhistory') + models: - name: fnl_sales_newcustomerhistory + meta: + owner: 'matthew.elsham@octoenergy.com' + team_owner: '!subteam^S7K9CR63D' #@datateam description: > This table gives one row per truncated month with the count of customers that have created their first order within this month @@ -10,4 +24,4 @@ models: description: primary key tests: - unique - - not_null \ No newline at end of file + - not_null diff --git a/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql index 8258f25f4..45220de8b 100644 --- a/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql +++ b/jaffle_shop/models/final/sales/fnl_sales_newcustomerhistory.sql @@ -1,4 +1,4 @@ -select - date_trunc('month', first_order) AS first_order_month - , count(*) AS number_customers -from {{ ref('wh_customers') }} \ No newline at end of file +SELECT + DATE_TRUNC('month', cust.first_order) AS first_order_month + , COUNT(DISTINCT cust.customer_id) AS number_customers +FROM {{ ref('wh_customers') }} AS cust diff --git a/jaffle_shop/models/staging/src_seed/_models.yml b/jaffle_shop/models/staging/src_seed/_models.yml index 981a7e3b9..cd5fe32b2 100644 --- a/jaffle_shop/models/staging/src_seed/_models.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -1,6 +1,37 @@ version: 2 models: + - name: stg_customers_pii + description: > + Table that includes all info about all customers, with PII hashed + columns: + - name: customer_id + description: Primary Key + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true + + - name: stg_customers + description: staging layer for all customers, PII hashed + columns: + - name: customer_id + description: Primary Key + tests: + - unique + - not_null + - name: first_name_hash + tests: + - dbt_expectations.expect_column_to_exist + - name: last_name_hash + tests: + - dbt_expectations.expect_column_to_exist + - name: stg_orders description: staging layer for all orders columns: @@ -26,12 +57,3 @@ models: tests: - accepted_values: values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] - - - name: stg_customers - description: staging layer for all customers, PII hashed - columns: - - name: customer_id - description: Primary Key - tests: - - unique - - not_null \ No newline at end of file diff --git a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml b/jaffle_shop/models/staging/src_seed/sensitive/_models.yml deleted file mode 100644 index e75f52c6c..000000000 --- a/jaffle_shop/models/staging/src_seed/sensitive/_models.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 - -models: - - name: stg_customers_pii - description: > - Table that includes all info about all customers, with PII hashed - columns: - - name: customer_id - description: Primary Key - tests: - - unique - - not_null - - name: first_name - meta: - sensitive: true - - name: last_name - meta: - sensitive: true \ No newline at end of file diff --git a/jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql b/jaffle_shop/models/staging/src_seed/stg_customers_pii.sql similarity index 100% rename from jaffle_shop/models/staging/src_seed/sensitive/stg_customers_pii.sql rename to jaffle_shop/models/staging/src_seed/stg_customers_pii.sql diff --git a/jaffle_shop/models/staging/src_seed/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql index a654dcb94..1caa10276 100644 --- a/jaffle_shop/models/staging/src_seed/stg_orders.sql +++ b/jaffle_shop/models/staging/src_seed/stg_orders.sql @@ -1,23 +1,24 @@ -with source as ( +WITH source AS ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_orders') }} + SELECT * FROM {{ ref('raw_orders') }} ), -renamed as ( +renamed AS ( - select - id as order_id, - user_id as customer_id, - order_date, - status + SELECT + id AS order_id + , user_id AS customer_id + , order_date + , status - from source + FROM source ) -select * from renamed +SELECT * +FROM renamed diff --git a/jaffle_shop/models/staging/src_seed/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql index 502a3ecdc..b2618dfd5 100644 --- a/jaffle_shop/models/staging/src_seed/stg_payments.sql +++ b/jaffle_shop/models/staging/src_seed/stg_payments.sql @@ -1,25 +1,25 @@ -with source as ( +WITH source AS ( {#- Normally we would select from the table here, but we are using seeds to load our data in this project #} - select * from {{ ref('raw_payments') }} + SELECT * FROM {{ ref('raw_payments') }} ), -renamed as ( - - select - id as payment_id, - order_id, - payment_method, +renamed AS ( + SELECT + id AS payment_id + , order_id + , payment_method -- `amount` is currently stored in cents, so we convert it to dollars - amount / 100 as amount_dollars + , amount / 100 AS amount_dollars - from source + FROM source ) -select * from renamed +SELECT * +FROM renamed diff --git a/jaffle_shop/models/warehouse/wh_customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql index 3f96d6e00..99d994802 100644 --- a/jaffle_shop/models/warehouse/wh_customers.sql +++ b/jaffle_shop/models/warehouse/wh_customers.sql @@ -1,70 +1,15 @@ -with customers as ( - - select * from {{ ref('stg_customers') }} - -), - -orders as ( - - select * from {{ ref('stg_orders') }} - -), - -payments as ( - - select * from {{ ref('stg_payments') }} - -), - -customer_orders as ( - - select - customer_id, - - min(order_date) as first_order, - max(order_date) as most_recent_order, - count(order_id) as number_of_orders - from orders - - group by customer_id - -), - -customer_payments as ( - - select - orders.customer_id, - sum(amount) as total_amount - - from payments - - left join orders on - payments.order_id = orders.order_id - - group by orders.customer_id - -), - -final as ( - - select - customers.customer_id, - customers.first_name, - customers.last_name, - - customer_orders.first_order, - customer_orders.most_recent_order, - customer_orders.number_of_orders, - customer_payments.total_amount as customer_lifetime_value - - from customers - - left join customer_orders - on customers.customer_id = customer_orders.customer_id - - left join customer_payments - on customers.customer_id = customer_payments.customer_id - -) - -select * from final +SELECT + orders.customer_id + , customers.first_name_hash + , customers.last_name_hash + , MIN(orders.order_date) AS first_order + , MAX(orders.order_date) AS most_recent_order + , COUNT(orders.order_id) AS number_of_orders + , SUM(payments.amount) AS customer_lifetime_value +FROM {{ ref('stg_orders') }} orders +LEFT JOIN {{ ref('stg_customers') }} customers + ON orders.customer_id = customers.customer_id +LEFT JOIN {{ ref('stg_payments') }} payments + ON orders.order_id = payments.order_id + +GROUP BY orders.customer_id, customers.first_name_hash, customers.last_name_hash diff --git a/jaffle_shop/models/warehouse/wh_orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql index b693d3775..7bc263c12 100644 --- a/jaffle_shop/models/warehouse/wh_orders.sql +++ b/jaffle_shop/models/warehouse/wh_orders.sql @@ -1,56 +1,37 @@ {% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} -with orders as ( +WITH order_payments AS ( - select * from {{ ref('stg_orders') }} - -), - -payments as ( - - select * from {{ ref('stg_payments') }} - -), - -order_payments as ( - - select - order_id, + SELECT + payments.order_id {% for payment_method in payment_methods -%} - sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, + , sum(CASE WHEN payment_method = '{{ payment_method }}' THEN payments.amount ELSE 0 END) AS {{ payment_method }}_amount {% endfor -%} - sum(amount) as total_amount - - from payments + , sum(payments.amount) AS total_amount - group by order_id + FROM {{ ref('stg_payments') }} payments -), + GROUP BY payments.order_id -final as ( - - select - orders.order_id, - orders.customer_id, - orders.order_date, - orders.status, - - {% for payment_method in payment_methods -%} - - order_payments.{{ payment_method }}_amount, +) - {% endfor -%} +SELECT + orders.order_id + , orders.customer_id + , orders.order_date + , orders.status - order_payments.total_amount as amount_dollars + {% for payment_method in payment_methods -%} - from orders + , order_payments.{{ payment_method }}_amount + {% endfor -%} - left join order_payments - on orders.order_id = order_payments.order_id + , order_payments.total_amount AS amount_dollars -) +FROM {{ ref('stg_orders') }} orders -select * from final +LEFT JOIN order_payments + ON orders.order_id = order_payments.order_id diff --git a/jaffle_shop/seeds/_seeds.yml b/jaffle_shop/seeds/_seeds.yml new file mode 100644 index 000000000..e2a88cdcf --- /dev/null +++ b/jaffle_shop/seeds/_seeds.yml @@ -0,0 +1,38 @@ +version: 2 + +seeds: + - name: dbt_project_evaluator_exceptions + description: > + Stop check on staging models refer to other staging models from failing + config: + column_types: + fct_name: string + column_name: string + id_to_exclude: string + comment: string + - name: raw_customers + description: > + Csv of all customers names and id + config: + column_types: + id: bigint + first_name: string + last_name: string + - name: raw_payments + description: > + Csv of all orders, payment methods and amounts + config: + column_types: + id: bigint + order_id: bigint + payment_method: string + amount: bigint + - name: raw_orders + description: > + Csv of all orders, dates and status + config: + column_types: + id: bigint + user_id: bigint + order_date: date + status: string