bugfix/klaviyo-too-many-partitions (#26)

* bugfix/klaviyo-too-many-partitions * docs regen and changelog update * schema change * schema name shortening * changelog update and partition removal * Update CHANGELOG.md Co-authored-by: fivetran-catfritz <[email protected]> * Apply suggestions from code review Co-authored-by: Avinash Kunnath <[email protected]> * review updates and docs regen * Update CHANGELOG.md Co-authored-by: Avinash Kunnath <[email protected]> * docs fix and dependency update --------- Co-authored-by: fivetran-catfritz <[email protected]> Co-authored-by: Avinash Kunnath <[email protected]>
fivetran · Sep 16, 2024 · 33ab35a · 33ab35a
1 parent 849a697
commit 33ab35a
Show file tree

Hide file tree

Showing 16 changed files with 182 additions and 100 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,4 +4,5 @@ dbt_modules/
 logs/
 .DS_Store
 dbt_packages/
-package-lock.yml
+package-lock.yml
+env/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,20 @@
+# dbt_shopify_holistic_reporting v0.7.0
+
+[PR #26](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/26) includes the following changes:
+
+## Upstream Klaviyo and Shopify Holistic Reporting Breaking Changes (Full refresh required after upgrading)
+- Upstream and immediate incremental models within the dbt_klaviyo and dbt_shopify_holistic_reporting packages running on BigQuery have had the `partition_by` logic removed. This change affects only BigQuery warehouses and resolves the `too many partitions` error that some users encountered. The partitioning was also deemed unnecessary for the aforementioned models and their downstream references, offering no performance benefit. By removing it, we eliminate both the error risk and an unneeded configuration. Refer to the [v0.8.0 dbt_klaviyo release notes](https://github.com/fivetran/dbt_klaviyo/releases/tag/v0.8.0) for more details regarding the upstream changes. This change applies to the following models:
+  - `int_klaviyo__event_attribution`
+  - `klaviyo__events`
+  - `shopify_holistic_reporting__orders_attribution`
+
+## Under the Hood
+- Added consistency validation tests for the following models:
+  - `shopify_holistic_reporting__customer_enhanced`
+  - `shopify_holistic_reporting__daily_customer_metrics`
+  - `shopify_holistic_reporting__orders_attribution`
+- Cleaned up unnecessary variable configuration within the `integration_tests/dbt_project.yml` file.
+
 # dbt_shopify_holistic_reporting v0.6.0
 [PR #21](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/21) includes the following changes:
 

diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ Include the following shopify_holistic_reporting package version in your `packag
 ```yml
 packages:
   - package: fivetran/shopify_holistic_reporting
-    version: [">=0.6.0", "<0.7.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.7.0", "<0.8.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do **NOT** include the `shopify`, `shopify_source`, `klaviyo`, or `klaviyo_source` packages in this file. The combo package itself has a dependency on these and will install the transformation and source packages as well.
@@ -136,7 +136,7 @@ packages:
       version: [">=0.11.0", "<0.13.0"]
 
     - package: fivetran/klaviyo
-      version: [">=0.7.0", "<0.8.0"]
+      version: [">=0.8.0", "<0.9.0"]
 
     - package: fivetran/klaviyo_source
       version: [">=0.7.0", "<0.8.0"]

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_holistic_reporting'
-version: '0.6.0'
+version: '0.7.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_holistic_reporting_integration_tests_3
+      schema: combo_integration_tests_12
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_holistic_reporting_integration_tests_3
+      schema: combo_integration_tests_12
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_holistic_reporting_integration_tests_3
+      schema: combo_integration_tests_12
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_holistic_reporting_integration_tests_3
+      schema: combo_integration_tests_12
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_holistic_reporting_integration_tests_3
+      schema: combo_integration_tests_12
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_holistic_reporting_integration_tests'
-version: '0.6.0'
+version: '0.7.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -11,58 +11,19 @@ clean-targets:         # directories to be removed by `dbt clean`
 vars:
   shopify_source:
     shopify_using_fulfillment_event: true
-    shopify_schema: 'shopify_holistic_reporting_integration_tests_3'
-    shopify_order_identifier:             'order_table' # reserved keyword in snowflake
+    shopify_schema: 'combo_integration_tests_12'
+    shopify_order_identifier: 'order_table' # reserved keyword in snowflake
 
   klaviyo_source:
-    klaviyo_schema: 'shopify_holistic_reporting_integration_tests_3'
-    klaviyo__event_pass_through_columns: []
-    klaviyo__person_pass_through_columns: []
+    klaviyo_schema: 'combo_integration_tests_12'
     klaviyo_campaign_identifier: "klaviyo_campaign"
     klaviyo_event_identifier: "klaviyo_event"
     klaviyo_flow_identifier: "klaviyo_flow"
     klaviyo_person_identifier: "klaviyo_person"
     klaviyo_metric_identifier: "klaviyo_metric"
 
-  klaviyo:
-    klaviyo__eligible_attribution_events: # default in klaviyo -> https://help.klaviyo.com/hc/en-us/articles/115005248128 
-      - 'opened email'
-      - 'clicked email'
-      - 'clicked sms'
-
-    klaviyo__count_metrics: 
-      - 'Active on Site' # default API metrics
-      - 'Viewed Product'
-      - 'Ordered Product' # default Shopify metrics
-      - 'Placed Order'
-      - 'Refunded Order' # note: klaviyo only tracks FULL refunds (not partial)
-      - 'Cancelled Order'
-      - 'Fulfilled Order'
-      - 'Received Email' # default email and sms conversion metrics in Klaviyo
-      - 'Clicked Email'
-      - 'Opened Email'
-      - 'Bounced Email'
-      - 'Marked Email as Spam'
-      - 'Dropped Email'
-      - 'Subscribed to List'
-      - 'Unsubscribed to List'
-      - 'Unsubscribed'
-      - 'Updated Email Preferences'
-      - 'Subscribed to Back in Stock'
-      - 'Merged Profile'
-      - 'Received SMS'
-      - 'Clicked SMS'
-      - 'Consented to Receive SMS'
-      - 'Sent SMS'
-      - 'Unsubscribed from SMS'
-      - 'Failed to Deliver SMS'
-
-    klaviyo__sum_revenue_metrics:
-      - 'Refunded Order'
-      - 'Placed Order'
-      - 'Ordered Product'
-      - 'checkout started'
-      - 'cancelled order'
+models:
+  +schema: "combo_reporting_{{ var('directed_schema','dev') }}"
 
 dispatch:
   - macro_namespace: dbt_utils

diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
@@ -1,2 +1,2 @@
 packages:
-  - local: ../
+  - local: ../
diff --git a/integration_tests/tests/consistency/consistency_customer_enhanced_row_comparison.sql b/integration_tests/tests/consistency/consistency_customer_enhanced_row_comparison.sql
@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_combo_reporting_prod.shopify_holistic_reporting__customer_enhanced
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_combo_reporting_dev.shopify_holistic_reporting__customer_enhanced
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final
diff --git a/integration_tests/tests/consistency/consistency_daily_customer_metrics_row_comparison.sql b/integration_tests/tests/consistency/consistency_daily_customer_metrics_row_comparison.sql
@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_combo_reporting_prod.shopify_holistic_reporting__daily_customer_metrics
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_combo_reporting_dev.shopify_holistic_reporting__daily_customer_metrics
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final
diff --git a/integration_tests/tests/consistency/consistency_order_attribution_row_comparison.sql b/integration_tests/tests/consistency/consistency_order_attribution_row_comparison.sql
@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_combo_reporting_prod.shopify_holistic_reporting__orders_attribution
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_combo_reporting_dev.shopify_holistic_reporting__orders_attribution
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final
diff --git a/models/shopify_holistic_reporting__orders_attribution.sql b/models/shopify_holistic_reporting__orders_attribution.sql
@@ -2,10 +2,6 @@
     config(
         materialized='incremental',
         unique_key='unique_order_key',
-        partition_by={
-            "field": "created_timestamp",
-            "data_type": "timestamp"
-        } if target.type == 'bigquery' else none,
         incremental_strategy = 'merge' if target.type not in ('postgres', 'redshift') else 'delete+insert',
         file_format = 'delta'
     )

diff --git a/packages.yml b/packages.yml
@@ -3,4 +3,4 @@ packages:
   version: [">=0.11.0", "<0.14.0"]
 
 - package: fivetran/klaviyo
-  version: [">=0.7.0", "<0.8.0"]
+  version: [">=0.8.0", "<0.9.0"]