From bde8752643371cde4bb52f5b440249d311d3ac8a Mon Sep 17 00:00:00 2001 From: Joshua Dias Date: Fri, 12 Apr 2024 16:12:15 +0100 Subject: [PATCH 1/4] Rename models in warehouse directory and add owner metadata --- jaffle_shop/models/{docs.md => warehouse/_docs.md} | 0 jaffle_shop/models/{schema.yml => warehouse/_models.yml} | 9 ++++++--- .../models/{customers.sql => warehouse/wh_customers.sql} | 0 .../models/{orders.sql => warehouse/wh_orders.sql} | 0 4 files changed, 6 insertions(+), 3 deletions(-) rename jaffle_shop/models/{docs.md => warehouse/_docs.md} (100%) rename jaffle_shop/models/{schema.yml => warehouse/_models.yml} (93%) rename jaffle_shop/models/{customers.sql => warehouse/wh_customers.sql} (100%) rename jaffle_shop/models/{orders.sql => warehouse/wh_orders.sql} (100%) diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/_docs.md similarity index 100% rename from jaffle_shop/models/docs.md rename to jaffle_shop/models/warehouse/_docs.md diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml similarity index 93% rename from jaffle_shop/models/schema.yml rename to jaffle_shop/models/warehouse/_models.yml index 381349cfd..f818b1188 100644 --- a/jaffle_shop/models/schema.yml +++ b/jaffle_shop/models/warehouse/_models.yml @@ -1,8 +1,10 @@ version: 2 models: - - name: customers + - name: wh_customers description: This table has basic information about a customer, as well as some derived facts based on a customer's orders + meta: + owner: "joshua.oconnor@kraken.tech" columns: - name: customer_id @@ -29,9 +31,10 @@ models: - name: total_order_amount description: Total value (AUD) of a customer's orders - - name: orders + - name: wh_orders description: This table has basic information about orders, as well as some derived facts based on payments - + meta: + owner: "joshua.oconnor@kraken.tech" columns: - name: order_id tests: diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql similarity index 100% rename from jaffle_shop/models/customers.sql rename to jaffle_shop/models/warehouse/wh_customers.sql diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql similarity index 100% rename from jaffle_shop/models/orders.sql rename to jaffle_shop/models/warehouse/wh_orders.sql From 72b62a87df6004b735923b13c38a80bbd63104ca Mon Sep 17 00:00:00 2001 From: Joshua Dias Date: Fri, 12 Apr 2024 16:12:52 +0100 Subject: [PATCH 2/4] Add seed models for raw_customers, raw_orders, and raw_payments --- jaffle_shop/seeds/_models.yml | 6 ++++++ jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 jaffle_shop/seeds/_models.yml create mode 100644 jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv diff --git a/jaffle_shop/seeds/_models.yml b/jaffle_shop/seeds/_models.yml new file mode 100644 index 000000000..807eb865c --- /dev/null +++ b/jaffle_shop/seeds/_models.yml @@ -0,0 +1,6 @@ +version: 2 + +seed: + - name: raw_customers + - name: raw_orders + - name: raw_payments \ No newline at end of file diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv new file mode 100644 index 000000000..b9f94e579 --- /dev/null +++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv @@ -0,0 +1,2 @@ +fct_name,column_name,id_to_exclude,comment +fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer. \ No newline at end of file From 728f10be88da1e2b34f7f7f62e27549485c8fabf Mon Sep 17 00:00:00 2001 From: Joshua Dias Date: Fri, 12 Apr 2024 16:14:12 +0100 Subject: [PATCH 3/4] Update models for staging layer --- .../{schema.yml => src_seed/_models.yml} | 29 +++++++++++++++++++ .../models/staging/src_seed/stg_customers.sql | 4 +++ .../stg_customers_pii.sql} | 0 .../staging/{ => src_seed}/stg_orders.sql | 0 .../staging/{ => src_seed}/stg_payments.sql | 0 5 files changed, 33 insertions(+) rename jaffle_shop/models/staging/{schema.yml => src_seed/_models.yml} (50%) create mode 100644 jaffle_shop/models/staging/src_seed/stg_customers.sql rename jaffle_shop/models/staging/{stg_customers.sql => src_seed/stg_customers_pii.sql} (100%) rename jaffle_shop/models/staging/{ => src_seed}/stg_orders.sql (100%) rename jaffle_shop/models/staging/{ => src_seed}/stg_payments.sql (100%) diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/src_seed/_models.yml similarity index 50% rename from jaffle_shop/models/staging/schema.yml rename to jaffle_shop/models/staging/src_seed/_models.yml index c207e4cf5..54c6bbe0d 100644 --- a/jaffle_shop/models/staging/schema.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -1,14 +1,41 @@ version: 2 models: + - name: stg_customers + meta: + owner: "joshua.oconnor@kraken.tech" + columns: + - name: customer_id + tests: + - unique + - not_null + - name: first_name + tests: + - dbt_expectations.expect_column_to_exist + - name: last_name + tests: + - dbt_expectations.expect_column_to_exist + + - name: stg_customers_pii + meta: + owner: "joshua.oconnor@kraken.tech" + sensitive: true columns: - name: customer_id tests: - unique - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true - name: stg_orders + meta: + owner: "joshua.oconnor@kraken.tech" columns: - name: order_id tests: @@ -20,6 +47,8 @@ models: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - name: stg_payments + meta: + owner: "joshua.oconnor@kraken.tech" columns: - name: payment_id tests: diff --git a/jaffle_shop/models/staging/src_seed/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers.sql new file mode 100644 index 000000000..7f112c71d --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/stg_customers.sql @@ -0,0 +1,4 @@ + +SELECT + {{hash_sensitive_columns('stg_customers_pii')}} +FROM {{ref('stg_customers_pii')}} diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers_pii.sql similarity index 100% rename from jaffle_shop/models/staging/stg_customers.sql rename to jaffle_shop/models/staging/src_seed/stg_customers_pii.sql diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql similarity index 100% rename from jaffle_shop/models/staging/stg_orders.sql rename to jaffle_shop/models/staging/src_seed/stg_orders.sql diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql similarity index 100% rename from jaffle_shop/models/staging/stg_payments.sql rename to jaffle_shop/models/staging/src_seed/stg_payments.sql From 2036dacbc596200aa7b9f87ed14cd8fb727e6c78 Mon Sep 17 00:00:00 2001 From: Joshua Dias Date: Fri, 12 Apr 2024 16:14:31 +0100 Subject: [PATCH 4/4] Add finance and sales models for final layer --- conftest.py | 1 + jaffle_shop/models/final/finance/_models.yml | 28 +++++++++++++++++++ .../finance/fnl_returned_order_value.sql | 13 +++++++++ jaffle_shop/models/final/sales/_models.yml | 26 +++++++++++++++++ .../sales/fnl_monthly_customer_count.sql | 12 ++++++++ 5 files changed, 80 insertions(+) create mode 100644 conftest.py create mode 100644 jaffle_shop/models/final/finance/_models.yml create mode 100644 jaffle_shop/models/final/finance/fnl_returned_order_value.sql create mode 100644 jaffle_shop/models/final/sales/_models.yml create mode 100644 jaffle_shop/models/final/sales/fnl_monthly_customer_count.sql diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..b5b7f3629 --- /dev/null +++ b/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ["dbt.tests.fixtures.project"] \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml new file mode 100644 index 000000000..d96149cd7 --- /dev/null +++ b/jaffle_shop/models/final/finance/_models.yml @@ -0,0 +1,28 @@ +version: 2 + + +exposures: + - name: fnl_returned_order_value + label: Inkacio + description: fnl_returned_order_value + type: dashboard + url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/ + owner: + email: joshua.oconnor@kraken.tech + depends_on: + - ref('wh_orders') + + +models: +- name: fnl_returned_order_value + meta: + owner: joshua.oconnor@kraken.tech + description: | + One row per customer for total value of returns + columns: + - name: customer_id + description: Primary key + tests: + - unique + - not_null + \ No newline at end of file diff --git a/jaffle_shop/models/final/finance/fnl_returned_order_value.sql b/jaffle_shop/models/final/finance/fnl_returned_order_value.sql new file mode 100644 index 000000000..52f71e22f --- /dev/null +++ b/jaffle_shop/models/final/finance/fnl_returned_order_value.sql @@ -0,0 +1,13 @@ +WITH returned_orders AS ( + SELECT + customer_id AS customer_id + ,amount as total_amount + FROM {{ ref('wh_orders') }} + WHERE status = 'returned' +) + +SELECT + customer_id + ,SUM(COALESCE(total_amount, 0)) AS total_value +FROM returned_orders +GROUP BY customer_id \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml new file mode 100644 index 000000000..58ad7105a --- /dev/null +++ b/jaffle_shop/models/final/sales/_models.yml @@ -0,0 +1,26 @@ +version: 2 + + +exposures: + - name: fnl_monthly_customer_count + label: Inkacio + description: fnl_monthly_customer_count + type: dashboard + url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/ + owner: + email: joshua.oconnor@kraken.tech + depends_on: + - ref('wh_customers') + +models: +- name: fnl_monthly_customer_count + meta: + owner: joshua.oconnor@kraken.tech + description: | + One row per customer count for each month based on their first orders. + columns: + - name: first_order_month + description: Primary key + tests: + - unique + - not_null \ No newline at end of file diff --git a/jaffle_shop/models/final/sales/fnl_monthly_customer_count.sql b/jaffle_shop/models/final/sales/fnl_monthly_customer_count.sql new file mode 100644 index 000000000..8ba784926 --- /dev/null +++ b/jaffle_shop/models/final/sales/fnl_monthly_customer_count.sql @@ -0,0 +1,12 @@ +WITH customer_orders AS ( + SELECT + customer_id + , DATE_FORMAT(first_order, 'MMMM') As order_month + FROM {{ ref('wh_customers') }} +) + +SELECT + first_order AS first_order_month + , COUNT(*) AS customer_count +FROM customer_orders +GROUP BY first_order \ No newline at end of file