Releases/v0.12.latest (#82)

* Update shopify__daily_shop.sql Update code to include new 'delayed' status from Shopify * Bump package version * Update CHANGELOG.md * Bump project version * Update link to PR in changelog * testing * validation tests * docs * joe feedback * change package ref * avinash feedback --------- Co-authored-by: Brent Shreve <[email protected]>
fivetran · Jun 11, 2024 · 9dc9de3 · 9dc9de3
1 parent fc20889
commit 9dc9de3
Show file tree

Hide file tree

Showing 15 changed files with 269 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,14 @@
+# dbt_shopify v0.12.1
+
+## 🪲 Bug Fixes 🪛
+- Added support for a new `delayed` fulfillment event status from Shopify. This produces a new `count_fulfillment_delayed` field in the `shopify__daily_shop` model ([PR #81](https://github.com/fivetran/dbt_shopify/pull/81)).
+
+## 🚘 Under the Hood 🚘
+- Added validation tests to be used by package maintainers to evaluate the consistency and integrity of subsequent model updates ([PR #82](https://github.com/fivetran/dbt_shopify/pull/82)).
+
+## Contributors
+- [@shreveasaurus](https://github.com/shreveasaurus) ([PR #81](https://github.com/fivetran/dbt_shopify/pull/81))
+
 # dbt_shopify v0.12.0
 
 [PR #76](https://github.com/fivetran/dbt_shopify/pull/76) includes the following updates: 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.12.0'
+version: '0.12.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -47,4 +47,4 @@ vars:
     shopify_abandoned_checkout_discount_code: "{{ ref('stg_shopify__abandoned_checkout_discount_code') }}"
     shopify_order_discount_code: "{{ ref('stg_shopify__order_discount_code') }}"
     shopify_abandoned_checkout_shipping_line: "{{ ref('stg_shopify__abandoned_checkout_shipping_line') }}"
-    shopify_fulfillment_event: "{{ ref('stg_shopify__fulfillment_event') }}"
+    shopify_fulfillment_event: "{{ ref('stg_shopify__fulfillment_event') }}"
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.12.0'
+version: '0.12.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -55,6 +55,7 @@ dispatch:
 models:
   shopify:
     +schema: "{{ 'shopify_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'shopify' }}"
+  # +schema: "shopify_{{ var('directed_schema','dev') }}"
 
 seeds:
   shopify_integration_tests:
@@ -191,4 +192,4 @@ seeds:
     shopify_tax_line_data:
       +column_types:
         price: float
-        rate: float
+        rate: float
diff --git a/integration_tests/seeds/shopify_fulfillment_event_data.csv b/integration_tests/seeds/shopify_fulfillment_event_data.csv
@@ -1,6 +1,6 @@
 id,_fivetran_synced,address_1,city,country,created_at,estimated_delivery_at,fulfillment_id,happened_at,latitude,longitude,message,order_id,province,shop_id,status,updated_at,zip,_fivetran_deleted
 451435,2022-11-18 04:39:07.945000,,,,2022-08-29 20:52:39.000000,,40495,2022-08-29 20:52:39.000000,,,,4502987,,89440612,delivered,2022-08-29 20:52:39.000000,,false
 48779,2022-11-18 05:48:01.773000,,LONDON,GB,2022-09-13 08:07:57.000000,,4064737,2022-08-15 12:41:00.000000,101.349998474121094,-14.0333000011742115,Delay,4588203,,320612,out_for_delivery,2022-09-13 08:07:57.000000,CR0,false
-1481515,2022-11-18 05:41:00.745000,,ECHO PARK,AU,2022-09-14 14:16:52.000000,2022-09-14 08:00:00.000000,4019339,2022-09-14 01:26:00.000000,-3.797698974609375,190.783958203125,Delay,451915,,89320612,out_for_delivery,2022-09-14 14:16:52.000000,2759,false
+1481515,2022-11-18 05:41:00.745000,,ECHO PARK,AU,2022-09-14 14:16:52.000000,2022-09-14 08:00:00.000000,4019339,2022-09-14 01:26:00.000000,-3.797698974609375,190.783958203125,Delay,451915,,89320612,delayed,2022-09-14 14:16:52.000000,2759,false
 558955,2022-11-18 10:51:24.286000,,LAZYTOWN,US,2022-08-13 12:40:26.000000,,402947,2022-03-01 10:36:39.000000,22.337699890136719,-71.731002807617188,Delay,429188587,MA,89420612,in_transit,2022-08-13 12:40:26.000000,01505,false
 6904235,2022-11-18 08:58:00.458000,,LA,US,2022-08-24 06:29:21.000000,2022-08-24 23:59:59.000000,4060491,2022-08-24 05:30:57.000000,12.287498474121094,-21.3573989868164,Delay,4242667,MA,89420612,in_transit,2022-08-24 06:29:21.000000,01760,false
diff --git a/integration_tests/tests/consistency/consistency_daily_shop.sql b/integration_tests/tests/consistency/consistency_daily_shop.sql
@@ -0,0 +1,91 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select
+        date_day,
+        shop_id, 
+        source_relation,
+        sum(count_orders) as count_orders,
+        count(count_customers) as count_customers,
+        sum(order_adjusted_total) as order_adjusted_total,
+        sum(count_abandoned_checkouts) as count_abandoned_checkouts
+
+        {% if var('shopify_using_fulfillment_event', false) %}
+        , sum(count_fulfillment_attempted_delivery) as count_fulfillment_attempted_delivery
+        , sum(count_fulfillment_confirmed) as count_fulfillment_confirmed
+        , count(count_fulfillment_in_transit) as count_fulfillment_in_transit
+        {% endif %}
+
+    from {{ target.schema }}_shopify_prod.shopify__daily_shop
+    group by 1,2,3
+),
+
+dev as (
+    select
+        date_day,
+        shop_id, 
+        source_relation,
+        sum(count_orders) as count_orders,
+        count(count_customers) as count_customers,
+        sum(order_adjusted_total) as order_adjusted_total,
+        sum(count_abandoned_checkouts) as count_abandoned_checkouts
+
+        {% if var('shopify_using_fulfillment_event', false) %}
+        , sum(count_fulfillment_attempted_delivery) as count_fulfillment_attempted_delivery
+        , sum(count_fulfillment_confirmed) as count_fulfillment_confirmed
+        , count(count_fulfillment_in_transit) as count_fulfillment_in_transit
+        {% endif %}
+
+    from {{ target.schema }}_shopify_dev.shopify__daily_shop
+    group by 1,2,3
+),
+
+final as (
+    select 
+        prod.date_day as prod_date_day,
+        dev.date_day as dev_date_day,
+        prod.shop_id as prod_shop_id,
+        dev.shop_id as dev_shop_id,
+        prod.source_relation as prod_source_relation,
+        dev.source_relation as dev_source_relation,
+        prod.count_orders as prod_count_orders,
+        dev.count_orders as dev_count_orders,
+        prod.count_customers as prod_count_customers,
+        dev.count_customers as dev_count_customers,
+        prod.order_adjusted_total as prod_order_adjusted_total,
+        dev.order_adjusted_total as dev_order_adjusted_total,
+        prod.count_abandoned_checkouts as prod_count_abandoned_checkouts,
+        dev.count_abandoned_checkouts as dev_count_abandoned_checkouts
+
+        {% if var('shopify_using_fulfillment_event', false) %}
+        , prod.count_fulfillment_attempted_delivery as prod_count_fulfillment_attempted_delivery
+        , dev.count_fulfillment_attempted_delivery as dev_count_fulfillment_attempted_delivery
+        , prod.count_fulfillment_confirmed as prod_count_fulfillment_confirmed
+        , dev.count_fulfillment_confirmed as dev_count_fulfillment_confirmed
+        {% endif %}
+
+    from prod
+    full outer join dev 
+        on dev.date_day = prod.date_day
+        and dev.shop_id = prod.shop_id
+        and dev.source_relation = prod.source_relation
+)
+
+select *
+from final
+where 
+    prod_date_day != dev_date_day or
+    prod_shop_id != dev_shop_id or
+    prod_source_relation != dev_source_relation or
+    abs(prod_count_orders - dev_count_orders) > .001 or
+    abs(prod_count_customers - dev_count_customers) > .001 or
+    abs(prod_order_adjusted_total - dev_order_adjusted_total) > .001 or
+    abs(prod_count_abandoned_checkouts - dev_count_abandoned_checkouts) > .001
+
+    {% if var('shopify_using_fulfillment_event', false) %}
+    or abs(prod_count_fulfillment_attempted_delivery - dev_count_fulfillment_attempted_delivery) > .001
+    or abs(prod_count_fulfillment_confirmed - dev_count_fulfillment_confirmed) > .001
+    {% endif %}
diff --git a/integration_tests/tests/integrity/horizontal_integrity_daily_shop_orders.sql b/integration_tests/tests/integrity/horizontal_integrity_daily_shop_orders.sql
@@ -0,0 +1,63 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with orders as (
+
+    select *
+    from {{ ref('shopify__orders') }}
+),
+
+order_metrics as (
+
+    select 
+        source_relation,
+        cast({{ dbt.date_trunc('day','created_timestamp') }} as date) as date_day,
+        count(distinct order_id) as count_orders,
+        sum(order_adjusted_total) as order_adjusted_total
+
+    from orders
+    where created_timestamp > '2020-01-01' and created_timestamp < '2024-06-10'
+    group by 1,2
+),
+
+daily_shop as (
+
+    select *
+    from {{ ref('shopify__daily_shop') }}
+),
+
+daily_shop_metrics as (
+
+    select 
+        source_relation,
+        date_day,
+        sum(count_orders) as count_orders,
+        sum(order_adjusted_total) as order_adjusted_total
+
+    from daily_shop
+    where date_day >= '2020-01-01' and date_day < '2024-06-10'
+    group by 1,2
+),
+
+final as (
+
+    select
+        daily_shop_metrics.source_relation,
+        coalesce(daily_shop_metrics.count_orders, 0) as daily_shop_count_orders,
+        coalesce(order_metrics.count_orders, 0) as order_count_orders,
+        coalesce(daily_shop_metrics.order_adjusted_total, 0) as daily_shop_order_adjusted_total,
+        coalesce(order_metrics.order_adjusted_total, 0) as order_order_adjusted_total
+
+    from daily_shop_metrics
+    full outer join order_metrics
+        on daily_shop_metrics.source_relation = order_metrics.source_relation
+        and daily_shop_metrics.date_day = order_metrics.date_day
+)
+
+select *
+from final
+where 
+    abs(daily_shop_count_orders - order_count_orders) > 0 or
+    abs(daily_shop_order_adjusted_total - order_order_adjusted_total) > .1
diff --git a/integration_tests/tests/integrity/vertical_integrity_daily_shop_fulfillment.sql b/integration_tests/tests/integrity/vertical_integrity_daily_shop_fulfillment.sql
@@ -0,0 +1,81 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+-- test will only run if shopify_using_fulfillment_event is set to True
+
+with source as (
+
+    select *
+    from {{ ref('stg_shopify__fulfillment_event') }}
+),
+
+source_metrics as (
+
+    select 
+        source_relation,
+        cast({{ dbt.date_trunc('day','happened_at') }} as date) as date_day,
+        shop_id,
+        count(distinct case when status = 'delayed' then fulfillment_id end) as count_fulfillment_delayed,
+        count(distinct case when status = 'in_transit' then fulfillment_id end) as count_fulfillment_in_transit,
+        count(distinct case when status = 'confirmed' then fulfillment_id end) as count_fulfillment_confirmed,
+        count(distinct case when status = 'delivered' then fulfillment_id end) as count_fulfillment_delivered
+
+    from source
+    where happened_at > '2020-01-01' and happened_at < '2024-06-10'
+    group by 1,2,3
+),
+
+model as (
+
+    select *
+    from {{ ref('shopify__daily_shop') }}
+),
+
+model_metrics as (
+
+    select 
+        source_relation,
+        date_day,
+        shop_id,
+        sum(count_fulfillment_delayed) as count_fulfillment_delayed,
+        sum(count_fulfillment_in_transit) as count_fulfillment_in_transit,
+        sum(count_fulfillment_confirmed) as count_fulfillment_confirmed,
+        sum(count_fulfillment_delivered) as count_fulfillment_delivered
+
+    from model
+    where date_day >= '2020-01-01' and date_day < '2024-06-10'
+    group by 1,2,3
+),
+
+final as (
+
+    select
+        model_metrics.source_relation,
+        model_metrics.shop_id as model_shop_id,
+        source_metrics.shop_id as source_shop_id,
+        model_metrics.date_day as model_date_day,
+        source_metrics.date_day as source_date_day,
+        coalesce(model_metrics.count_fulfillment_delayed, 0) as model_count_fulfillment_delayed,
+        coalesce(source_metrics.count_fulfillment_delayed, 0) as source_count_fulfillment_delayed,
+        coalesce(model_metrics.count_fulfillment_in_transit, 0) as model_count_fulfillment_in_transit,
+        coalesce(source_metrics.count_fulfillment_in_transit, 0) as source_count_fulfillment_in_transit,
+        coalesce(model_metrics.count_fulfillment_confirmed, 0) as model_count_fulfillment_confirmed,
+        coalesce(source_metrics.count_fulfillment_confirmed, 0) as source_count_fulfillment_confirmed,
+        coalesce(model_metrics.count_fulfillment_delivered, 0) as model_count_fulfillment_delivered,
+        coalesce(source_metrics.count_fulfillment_delivered, 0) as source_count_fulfillment_delivered
+
+    from model_metrics
+    full outer join source_metrics
+        on model_metrics.source_relation = source_metrics.source_relation
+        and model_metrics.shop_id = source_metrics.shop_id
+        and model_metrics.date_day = source_metrics.date_day
+)
+
+select *
+from final
+where 
+model_count_fulfillment_delayed != source_count_fulfillment_delayed or
+model_count_fulfillment_in_transit != source_count_fulfillment_in_transit or
+model_count_fulfillment_confirmed != source_count_fulfillment_confirmed or
+model_count_fulfillment_delivered != source_count_fulfillment_delivered
diff --git a/models/intermediate/int_shopify__daily_fulfillment.sql b/models/intermediate/int_shopify__daily_fulfillment.sql
@@ -12,7 +12,7 @@ fulfillment_aggregates as (
         source_relation,
         cast({{ dbt.date_trunc('day','happened_at') }} as date) as date_day
 
-        {% for status in ['attempted_delivery', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'picked_up', 'label_printed', 'label_purchased', 'confirmed']%}
+        {% for status in ['attempted_delivery', 'delayed', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'picked_up', 'label_printed', 'label_purchased', 'confirmed']%}
         , count(distinct case when lower(status) = '{{ status }}' then fulfillment_id end) as count_fulfillment_{{ status }}
         {% endfor %}
 

diff --git a/models/shopify.yml b/models/shopify.yml
@@ -1139,6 +1139,8 @@ models:
         description: Count of distinct fulfillments for which a shipping label has been purchased (but not printed yet) on this day.
       - name: count_fulfillment_confirmed
         description: Count of confirmed distinct fulfillments. This is the default status when no other information is available about a fulfillment.
+      - name: count_fulfillment_delayed
+        description: Count of delayed distinct fulfillments.
       - name: avg_line_item_count
         description: Average line item count for orders placed on this day.
       - name: avg_discount

diff --git a/models/shopify__daily_shop.sql b/models/shopify__daily_shop.sql
@@ -97,7 +97,7 @@ final as (
         coalesce(daily_abandoned_checkouts.count_customer_emails_abandoned_checkout, 0) as count_customer_emails_abandoned_checkout
 
         {% if var('shopify_using_fulfillment_event', false) %}
-            {% for status in ['attempted_delivery', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'picked_up', 'label_printed', 'label_purchased', 'confirmed']%}
+            {% for status in ['attempted_delivery', 'delayed', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'picked_up', 'label_printed', 'label_purchased', 'confirmed']%}
         , coalesce(count_fulfillment_{{ status }}, 0) as count_fulfillment_{{ status }}
             {% endfor %}
         {% endif %}
@@ -119,4 +119,4 @@ final as (
 
 
 select *
-from final
+from final
diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,3 @@
 packages:
 - package: fivetran/shopify_source
-  version: [">=0.12.0", "<0.13.0"]
+  version: [">=0.12.0", "<0.13.0"]