Bug Fix: Coalesce incremental fields (#87)

* Coalesce Incremental Load Fields in `shopify__customer_cohorts` (#86) * coalesce incremental load values * coalesce windows columns * validation tests + docs * CHANGELOG --------- Co-authored-by: advolut-team <[email protected]>
fivetran · Aug 27, 2024 · 470b8da · 470b8da
1 parent 3888e08
commit 470b8da
Show file tree

Hide file tree

Showing 11 changed files with 146 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,15 @@
+# dbt_shopify v0.13.1
+[PR #87](https://github.com/fivetran/dbt_shopify/pull/87) includes the following changes:
+
+## Bug Fixes
+- Coalesces the `backfill_lifetime_sums` fields from incremental loads, as well as `cohort_month_number` in the rare cases there are no orders from an incremental period. This fixes the issue of NULL values in the lifetime columns in `shopify__customer_cohorts` table. ([PR #86](https://github.com/fivetran/dbt_shopify/pull/86)).
+
+## Under the Hood:
+- Added consistency and integrity tests within `integration_tests` for the `shopify__customer_cohorts` model. ([PR #87](https://github.com/fivetran/dbt_shopify/pull/87)).
+
+## Contributors
+- [@advolut-team](https://github.com/advolut-team) ([PR #86](https://github.com/fivetran/dbt_shopify/pull/86))
+
 # dbt_shopify v0.13.0
 [PR #83](https://github.com/fivetran/dbt_shopify/pull/83) includes the following changes:
 
@@ -6,7 +18,7 @@
   - This model is currently disabled by default. You may enable it by setting the `shopify__standardized_billing_model_enabled` as `true` in your `dbt_project.yml`.
 
 ## Under the Hood:
-- Added consistency test within integration_tests for the `stripe__line_item_enhanced` model.
+- Added consistency test within integration_tests for the `shopify__line_item_enhanced` model.
 
 # dbt_shopify v0.12.2
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.13.0'
+version: '0.13.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_integration_tests_10
+      schema: shopify_integration_tests_11
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_integration_tests_10
+      schema: shopify_integration_tests_11
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_integration_tests_10
+      schema: shopify_integration_tests_11
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_integration_tests_10
+      schema: shopify_integration_tests_11
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_integration_tests_10
+      schema: shopify_integration_tests_11
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.13.0'
+version: '0.13.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -9,9 +9,9 @@ clean-targets:         # directories to be removed by `dbt clean`
   - "dbt_modules"
 
 vars:
-  # shopify_using_fulfillment_event: true # set to true when regenerating docs
-  # shopify_using_all_metafields: true # set to true when regenerating docs
-  shopify_schema: shopify_integration_tests_10
+  shopify_using_fulfillment_event: true # set to true when regenerating docs
+  shopify_using_all_metafields: true # set to true when regenerating docs
+  shopify_schema: shopify_integration_tests_11
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"

diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
@@ -1,2 +1,2 @@
 packages:
-  - local: ../
+  - local: ../
diff --git a/integration_tests/tests/consistency/consistency_customer_cohorts.sql b/integration_tests/tests/consistency/consistency_customer_cohorts.sql
@@ -0,0 +1,54 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+
+    select
+        customer_cohort_id, 
+        source_relation,
+        total_price_lifetime,
+        order_count_lifetime,
+        line_item_count_lifetime
+    from {{ target.schema }}_shopify_prod.shopify__customer_cohorts
+),
+
+dev as (
+
+    select
+        customer_cohort_id, 
+        source_relation,
+        total_price_lifetime,
+        order_count_lifetime,
+        line_item_count_lifetime
+    from {{ target.schema }}_shopify_dev.shopify__customer_cohorts 
+),
+
+final as (
+
+    select 
+        prod.customer_cohort_id as prod_customer_cohort_id,
+        dev.customer_cohort_id as dev_customer_cohort_id,
+        prod.source_relation as prod_source_relation,
+        dev.source_relation as dev_source_relation,
+        prod.total_price_lifetime as prod_total_price_lifetime,
+        dev.total_price_lifetime as dev_total_price_lifetime,
+        prod.order_count_lifetime as prod_order_count_lifetime,
+        dev.order_count_lifetime as dev_order_count_lifetime,
+        prod.line_item_count_lifetime as prod_line_item_count_lifetime,
+        dev.line_item_count_lifetime as dev_line_item_count_lifetime
+    from prod
+    full outer join dev 
+        on dev.customer_cohort_id = prod.customer_cohort_id
+        and dev.source_relation = prod.source_relation
+)
+
+select *
+from final
+where 
+    prod_customer_cohort_id != dev_customer_cohort_id or 
+    prod_source_relation != dev_source_relation or
+    abs(prod_total_price_lifetime - dev_total_price_lifetime) > .001 or
+    abs(prod_order_count_lifetime - dev_order_count_lifetime) > .001 or
+    abs(prod_line_item_count_lifetime - dev_line_item_count_lifetime) > .001
diff --git a/integration_tests/tests/integrity/vertical_integrity_customer_cohorts.sql b/integration_tests/tests/integrity/vertical_integrity_customer_cohorts.sql
@@ -0,0 +1,61 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with calendar as (
+
+    select *
+    from {{ ref('shopify__calendar') }}
+    where cast({{ dbt.date_trunc('month','date_day') }} as date) = date_day
+
+), 
+
+customers as (
+
+    select *
+    from {{ ref('shopify__customers') }}
+
+), 
+
+orders as (
+
+    select *
+    from {{ ref('shopify__orders') }}
+
+), 
+
+customer_cohort_source as (
+
+    select 
+        customers.source_relation,
+        count(*) as source_rows 
+    from calendar
+    inner join customers
+        on cast({{ dbt.date_trunc('month', 'first_order_timestamp') }} as date) <= calendar.date_day
+    group by 1
+),
+
+customer_cohort_end as (
+
+    select 
+        source_relation,
+        count(*) as end_rows
+    from {{ ref('shopify__customer_cohorts') }}
+    group by 1
+),
+
+final as (
+    select
+        customer_cohort_source.source_relation,
+        source_rows,
+        end_rows
+    from customer_cohort_source
+    join customer_cohort_end
+    on customer_cohort_source.source_relation = customer_cohort_end.source_relation
+    where customer_cohort_source.source_rows != customer_cohort_end.end_rows
+)
+
+select *
+from final
+
diff --git a/models/shopify__customer_cohorts.sql b/models/shopify__customer_cohorts.sql
@@ -103,10 +103,10 @@ with calendar as (
         windows.order_count_in_month,
         windows.total_price_in_month,
         windows.line_item_count_in_month,
-        backfill_lifetime_sums.previous_cohort_month_number + windows.cohort_month_number as cohort_month_number,
-        backfill_lifetime_sums.previous_total_price_lifetime + windows.total_price_lifetime as total_price_lifetime,
-        backfill_lifetime_sums.previous_order_count_lifetime + windows.order_count_lifetime as order_count_lifetime,
-        backfill_lifetime_sums.previous_line_item_count_lifetime + windows.line_item_count_lifetime as line_item_count_lifetime,
+        coalesce(backfill_lifetime_sums.previous_cohort_month_number, 0) + coalesce(windows.cohort_month_number, 0) as cohort_month_number,
+        coalesce(backfill_lifetime_sums.previous_total_price_lifetime, 0) + coalesce(windows.total_price_lifetime, 0) as total_price_lifetime,
+        coalesce(backfill_lifetime_sums.previous_order_count_lifetime, 0) + coalesce(windows.order_count_lifetime, 0) as order_count_lifetime,
+        coalesce(backfill_lifetime_sums.previous_line_item_count_lifetime, 0) + coalesce(windows.line_item_count_lifetime, 0) as line_item_count_lifetime,
         {{ dbt_utils.generate_surrogate_key(['windows.date_month','windows.customer_id','windows.source_relation']) }} as customer_cohort_id
     from windows
     left join backfill_lifetime_sums