Merge pull request #58 from fivetran/bug/aggregate-only-distinct-cust…

…omer-ids aggregate distinct customer ids in email rollup
fivetran · May 17, 2023 · 2049f8d · 2049f8d
2 parents e298118 + 50c2198
commit 2049f8d
Show file tree

Hide file tree

Showing 12 changed files with 50 additions and 25 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,23 @@
-# dbt_shopify v0.UPDATE.UPDATE
+# dbt_shopify v0.8.1
 
- ## Under the Hood:
+[PR #58](https://github.com/fivetran/dbt_shopify/pull/58) applies the following changes:
 
-- Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
-- Updated the pull request [templates](/.github).
+## Bug Fixes
+- Adjusts the `int_shopify__customer_email_rollup` model to aggregate _distinct_ `customer_ids`.
+- Ensures that each order is tagged with the orderer's `email` before aggregating order metrics in `int_shopify__emails__order_aggregates`.
+
+## Under the Hood
+- Ensures transaction `kinds` are being read correctly by applying a `lower()` function. 
+- Removes unused and potentially problematic fields from `int_shopify__customer_email_rollup`. The removed fields include `orders_count` and `total_spent`, which are actually calculated in `int_shopify__emails__order_aggregates` before being passed to `shopify__customer_emails` (which is unaffected by this change).
+- Removes `updated_timestamp` and `created_timestamp` from `shopify__customer_emails`. Refer to the following fields instead:
+  - `first_account_created_at`
+  - `last_account_created_at`
+  - `last_updated_at`
+- Incorporates the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job ([PR #57](https://github.com/fivetran/dbt_shopify/pull/57)).
+- Updates the pull request [templates](/.github) ([PR #57](https://github.com/fivetran/dbt_shopify/pull/57)).
+
+## Related-Package Releases:
+- https://github.com/fivetran/dbt_shopify_holistic_reporting/releases/tag/v0.4.0
 
 # dbt_shopify v0.8.0
 

diff --git a/README.md b/README.md
@@ -48,8 +48,8 @@ To use this dbt package, you must have the following:
 - At least one Fivetran Shopify connector syncing data into your destination.
 - A **BigQuery**, **Snowflake**, **Redshift**, **Databricks**, or **PostgreSQL** destination.
 
-## Step 2: Install the package
-Include the following shopify package version in your `packages.yml` file:
+## Step 2: Install the package (skip if also using the `shopify_holistic_reporting` package)
+If you are **not** using the [Shopify Holistic reporting package](https://github.com/fivetran/dbt_shopify_holistic_reporting), include the following shopify package version in your `packages.yml` file:
 > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 ```yml
 packages:

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.8.0'
+version: '0.8.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,8 +1,12 @@
 name: 'shopify_integration_tests'
-version: '0.8.0'
+version: '0.8.1'
 profile: 'integration_tests'
 config-version: 2
 
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+  - "dbt_modules"
 
 vars:
   shopify_schema: shopify_integration_tests_6

diff --git a/models/intermediate/int_shopify__customer_email_rollup.sql b/models/intermediate/int_shopify__customer_email_rollup.sql
@@ -21,7 +21,7 @@ with customers as (
         customers.source_relation,
 
         -- fields to string agg together
-        {{ fivetran_utils.string_agg("cast(customers.customer_id as " ~ dbt.type_string() ~ ")", "', '") }} as customer_ids,
+        {{ fivetran_utils.string_agg("distinct cast(customers.customer_id as " ~ dbt.type_string() ~ ")", "', '") }} as customer_ids,
         {{ fivetran_utils.string_agg("distinct cast(customers.phone as " ~ dbt.type_string() ~ ")", "', '") }} as phone_numbers,
         {{ fivetran_utils.string_agg("distinct cast(customer_tags.value as " ~ dbt.type_string() ~ ")", "', '") }} as customer_tags,
 
@@ -31,8 +31,6 @@ with customers as (
         max(customers.updated_timestamp) as last_updated_at,
         max(customers.marketing_consent_updated_at) as marketing_consent_updated_at,
         max(customers._fivetran_synced) as last_fivetran_synced,
-        sum(customers.orders_count) as orders_count,
-        sum(customers.total_spent) as total_spent,
 
         -- take true if ever given for boolean fields
         {{ fivetran_utils.max_bool("case when customers.customer_index = 1 then customers.is_tax_exempt else null end") }} as is_tax_exempt, -- since this changes every year
@@ -41,7 +39,7 @@ with customers as (
         -- for all other fields, just take the latest value
         {% set cols = adapter.get_columns_in_relation(ref('stg_shopify__customer')) %}
         {% set except_cols = ['_fivetran_synced', 'email', 'source_relation', 'customer_id', 'phone', 'created_at', 
-                                'updated_at', 'marketing_consent_updated_at', 'orders_count', 'total_spent',
+                                'marketing_consent_updated_at', 'orders_count', 'total_spent', 'created_timestamp', 'updated_timestamp',
                                 'is_tax_exempt', 'is_verified_email'] %}
         {% for col in cols %}
             {% if col.column|lower not in except_cols %}

diff --git a/models/intermediate/int_shopify__emails__order_aggregates.sql b/models/intermediate/int_shopify__emails__order_aggregates.sql
@@ -2,7 +2,6 @@ with orders as (
 
     select *
     from {{ var('shopify_order') }}
-    where email is not null
 
 ), order_aggregates as (
 
@@ -23,16 +22,27 @@ with orders as (
     select 
         order_id,
         source_relation,
-        kind,
+        lower(kind) as kind,
         sum(currency_exchange_calculated_amount) as currency_exchange_calculated_amount
 
     from transactions
     {{ dbt_utils.group_by(n=3) }}
 
+), customer_emails as (
+-- in case any orders records don't have the customer email attached yet
+    select 
+        customer_id, 
+        source_relation,
+        email
+
+    from {{ var('shopify_customer') }}
+    where email is not null
+    {{ dbt_utils.group_by(n=3) }}
+
 ), aggregated as (
 
     select
-        orders.email,
+        lower(customer_emails.email) as email,
         orders.source_relation,
         min(orders.created_timestamp) as first_order_timestamp,
         max(orders.created_timestamp) as most_recent_order_timestamp,
@@ -52,6 +62,9 @@ with orders as (
         sum(order_aggregates.order_total_shipping_tax) as lifetime_total_shipping_tax,
         avg(order_aggregates.order_total_shipping_tax) as avg_shipping_tax_per_order
     from orders
+    join customer_emails
+        on orders.customer_id = customer_emails.customer_id
+        and orders.source_relation = customer_emails.source_relation
     left join transaction_aggregates 
         on orders.order_id = transaction_aggregates.order_id
         and orders.source_relation = transaction_aggregates.source_relation

diff --git a/models/intermediate/shopify__customers__order_aggregates.sql b/models/intermediate/shopify__customers__order_aggregates.sql
@@ -23,7 +23,7 @@ with orders as (
     select 
         order_id,
         source_relation,
-        kind,
+        lower(kind) as kind,
         sum(currency_exchange_calculated_amount) as currency_exchange_calculated_amount
 
     from transactions
@@ -35,7 +35,7 @@ with orders as (
     from {{ var('shopify_customer_tag' )}}
 
 ), customer_tags_aggregated as (
-    
+
     select 
         customer_id,
         source_relation,

diff --git a/models/shopify.yml b/models/shopify.yml
@@ -392,8 +392,6 @@ models:
     columns:
       - name: last_fivetran_synced
         description: "{{ doc('_fivetran_synced') }}"
-      - name: created_timestamp
-        description: The date and time when the customer was created.
       - name: default_address_id
         description: The default address for the customer. 
       - name: email
@@ -470,8 +468,6 @@ models:
         description: The current email marketing state for the customer. New version of `accepts_marketing` field.
       - name: marketing_opt_in_level
         description: Deprecated. The package will coalesce with `email_marketing_consent_opt_in_level`.
-      - name: updated_timestamp
-        description: The date and time when the customer information was last updated.
       - name: note
         description: A note about the customer.
       - name: marketing_consent_updated_at