Merge pull request #18 from fivetran/test/update-shopify-versions

Test/update shopify versions
fivetran · Nov 16, 2023 · ab5b8b6 · ab5b8b6
2 parents 14f105e + 2f00dd6
commit ab5b8b6
Show file tree

Hide file tree

Showing 19 changed files with 97 additions and 55 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,28 @@
+# dbt_shopify_holistic_reporting v0.5.0
+[PR #18](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/18) includes the following changes:
+
+## 🚨 Breaking Changes 🚨:
+
+- This package now points to the following ranges of the upstream packages. We recommend a `dbt run --full-refresh` to capture all the latest data within incremental models.
+
+```
+- package: fivetran/shopify
+  version: [">=0.10.0", "<0.11.0"]
+- package: fivetran/klaviyo
+  version: [">=0.7.0", "<0.8.0"]
+```
+
+For more information on the changes in the underlying upstream pacakges, refer to the changelogs for [Shopify](https://github.com/fivetran/dbt_shopify/compare/v0.8.1...v0.10.0) and [Klaviyo](https://github.com/fivetran/dbt_klaviyo/compare/v0.5.0...v0.7.1). 
+
+## Additions
+- Adds field `last_touch_integration_id` to `shopify_holistic_reporting__orders_attribution`
+
+## Under the Hood:
+- Replace seed files with the new ones from the respective upstream packages.
+- Renamed all Klaviyo seed files so that they start with prefix `klaviyo` in order to differentiate which seed files come from Shopify versus Klaviyo.
+- Removed the flow_snowflake seed file now that `trigger` has been removed from the Klaviyo flow object and as such will not cause run errors in Snowflake.
+- Populate the order_line_refund seed file as it was empty previously.
+
 # dbt_shopify_holistic_reporting v0.4.0
 
 [PR #16](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/16) includes the following changes:

diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ Include the following shopify_holistic_reporting package version in your `packag
 ```yml
 packages:
   - package: fivetran/shopify_holistic_reporting
-    version: [">=0.4.0", "<0.5.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do **NOT** include the `shopify`, `shopify_source`, `klaviyo`, or `klaviyo_source` packages in this file. The combo package itself has a dependency on these and will install the transformation and source packages as well. 
@@ -130,16 +130,16 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/shopify
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.10.0", "<0.11.0"]
 
     - package: fivetran/shopify_source
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.10.0", "<0.11.0"]
 
     - package: fivetran/klaviyo
-      version: [">=0.5.0", "<0.6.0"]
+      version: [">=0.7.0", "<0.8.0"]
 
     - package: fivetran/klaviyo_source
-      version: [">=0.5.0", "<0.6.0"]
+      version: [">=0.7.0", "<0.8.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_holistic_reporting'
-version: '0.4.0'
+version: '0.5.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_holistic_reporting_integration_tests_1
+      schema: shopify_holistic_reporting_integration_tests_3
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_holistic_reporting_integration_tests_1
+      schema: shopify_holistic_reporting_integration_tests_3
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_holistic_reporting_integration_tests_1
+      schema: shopify_holistic_reporting_integration_tests_3
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_holistic_reporting_integration_tests_1
+      schema: shopify_holistic_reporting_integration_tests_3
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_holistic_reporting_integration_tests_1
-      threads: 2
+      schema: shopify_holistic_reporting_integration_tests_3
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_holistic_reporting_integration_tests'
-version: '0.4.0'
+version: '0.5.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -10,13 +10,19 @@ clean-targets:         # directories to be removed by `dbt clean`
 
 vars:
   shopify_source:
-    shopify_schema: 'shopify_holistic_reporting_integration_tests_1'
+    shopify_using_fulfillment_event: true
+    shopify_schema: 'shopify_holistic_reporting_integration_tests_3'
     shopify_order_identifier:             'order_table' # reserved keyword in snowflake
-    
+
   klaviyo_source:
-    klaviyo_schema: 'shopify_holistic_reporting_integration_tests_1'
+    klaviyo_schema: 'shopify_holistic_reporting_integration_tests_3'
     klaviyo__event_pass_through_columns: []
     klaviyo__person_pass_through_columns: []
+    klaviyo_campaign_identifier: "klaviyo_campaign"
+    klaviyo_event_identifier: "klaviyo_event"
+    klaviyo_flow_identifier: "klaviyo_flow"
+    klaviyo_person_identifier: "klaviyo_person"
+    klaviyo_metric_identifier: "klaviyo_metric"
 
   klaviyo:
     klaviyo__eligible_attribution_events: # default in klaviyo -> https://help.klaviyo.com/hc/en-us/articles/115005248128 
@@ -194,29 +200,22 @@ seeds:
         price: float
         rate: float
     # klaviyo
-    person:
+    klaviyo_person:
       +tags: klaviyo
       +column_types:
         phone_number: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
-    flow:
+    klaviyo_flow:
       +tags: klaviyo
       +enabled: "{{ true if target.type != 'snowflake' else false }}"
       +quote_columns: "{{ true if target.type == 'redshift' else false }}"
-      +column_types:
-        trigger: "{{ 'string' if target.type in  ('bigquery', 'spark', 'databricks') else 'varchar' }}"
-    flow_snowflake:
-      +tags: klaviyo
-      +enabled: "{{ true if target.type == 'snowflake' else false }}"
-    event:
+    klaviyo_event:
       +tags: klaviyo
       +column_types:
         flow_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
         campaign_id: "{{ 'string' if target.type in  ('bigquery', 'spark', 'databricks') else 'varchar' }}"
         datetime: timestamp
         timestamp: timestamp
-    integration:
-      +tags: klaviyo
-    metric:
+    klaviyo_metric:
       +tags: klaviyo
-    campaign:
+    klaviyo_campaign:
       +tags: klaviyo
diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -1,7 +1,7 @@
 dbt-snowflake>=1.3.0,<2.0.0
 dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<1.5.0
+dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.6.0,<2.0.0
diff --git a/integration_tests/seeds/campaign.csv b/integration_tests/seeds/campaign.csv
diff --git a/integration_tests/seeds/flow.csv b/integration_tests/seeds/flow.csv
diff --git a/integration_tests/seeds/flow_snowflake.csv b/integration_tests/seeds/flow_snowflake.csv
diff --git a/integration_tests/seeds/integration.csv b/integration_tests/seeds/integration.csv
diff --git a/integration_tests/seeds/klaviyo_campaign.csv b/integration_tests/seeds/klaviyo_campaign.csv
@@ -0,0 +1,3 @@
+id,from_name,status_id,campaign_type,status_label,from_email,subject,name,status,created,updated,send_time,sent_at,_fivetran_deleted,email_template_id,_fivetran_synced,archived,scheduled
+WMddDe,friend,2,Regular,Draft,[email protected],Ready to do stuff?,Jan-2 Email,draft,2021-04-12 23:42:35.000,2021-04-12 23:42:44.000,,,FALSE,YzD79H,2021-04-14 07:45:40.981,false,2021-04-22 23:42:44.000
+WViD3c,friend,1,Regular,Sent,[email protected],say hello,camp,sent,2021-03-16 19:16:37.000,2021-03-17 15:38:59.000,2021-03-17 15:38:28.000,2021-03-17 15:38:59.000,FALSE,Y8hHH4,2021-04-14 07:45:41.005,false,2021-03-22 15:38:59.000
diff --git a/integration_tests/seeds/event.csv → integration_tests/seeds/klaviyo_event.csv b/integration_tests/seeds/event.csv → integration_tests/seeds/klaviyo_event.csv
@@ -1,3 +1,3 @@
 id,person_id,campaign_id,flow_id,flow_message_id,uuid,datetime,timestamp,_fivetran_deleted,metric_id,type,_variation,property_value,property_source_name,property_extra,property_shipping_rate,property_items,property_tags,property_item_count,property_collections,_fivetran_synced,property_currency_code,property_product_id,property_quantity,property_name,property_variant_option_title,property_sku,property_variant_name,property_vendor,property_event_id,property_total_discounts,property_attribution,property_discount_codes,property_fulfillment_hours,property_fulfillment_status,property_subject,property_title,property_message_interaction,property_url,property_shipment_type,property_courier_name,property_current_status,property_tracking_ship_date,property_tracking_postal_code,property_shipment_package_count,property_campaign_name,property_cohort_message_send_cohort,property_email_domain,property_bounce_type,property_client_type,property_client_os,property_client_name,property_client_os_family,property_client_canonical,property_page,property_is_session_activity,property_os,property_session_end,property_browser,property_list,property_compare_at_price,property_price,property_image_url,property_brand,property_categories,property_variant_option_size,property_attribute_workout,property_attribute_age,property_attribute_fitness_goal,property_attribute_breakfast,property_variant_option_type,property_cohort_variation_send_cohort,property_method,property_to_number,property_message_type,property_from_number,property_message_format,property_message_name,property_carrier_delivery_status,property_failure_type,property_failure_source,property_message_body,property_email,property_attribute_kit,property_vendor_error_code,property_ordr_details,property_date_to_send_gift,property_recipient_name,property_attribute_facebook_order_retailer_id,property_sender_s_name,property_attribute_kitid,property_conversation_id,property_conversation_link,property_conversation_channel,property_rating,property_score,property_from_phone_region,property_shipment_carrier,property_shipment_status,property_to_phone_region
-3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,"brand new chopstcks",,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,"brand new shorts",,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,brand new chopstcks,,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,brand new shorts,,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,,
diff --git a/integration_tests/seeds/klaviyo_flow.csv b/integration_tests/seeds/klaviyo_flow.csv
@@ -0,0 +1,3 @@
+id,name,status,created,updated,customer_filter,_fivetran_deleted,_fivetran_synced,archived,trigger_type
+X839zt,Welcome SMS -  Redesign,live,2020-10-12 22:56:59.000,2021-02-10 18:34:39.000,"{""stanzas"": [ {""criteria"": [{""timeframe"": ""alltime"",""type"": ""customer-in-flow""}]}]}",FALSE,2021-04-14 07:46:13.280,false,Metric
+S6uajk,Abandoned Cart - NEW VERSION - Katelyn & DW Working,draft,2020-07-08 20:55:56.000,2020-07-09 16:27:58.000,"{""stanzas"": [{""criteria"": [{""operator"": ""nin"",""region_id"": ""EUROPEAN_UNION"",""type"": ""customer-location""}]}]}",FALSE,2021-04-14 07:46:13.285,false,Added to List
diff --git a/integration_tests/seeds/metric.csv → integration_tests/seeds/klaviyo_metric.csv b/integration_tests/seeds/metric.csv → integration_tests/seeds/klaviyo_metric.csv
@@ -1,3 +1,3 @@
-id,name,integration_id,created,updated,_fivetran_deleted,_fivetran_synced
-Vx2d6i,Received SMS,0rG4eQ,2020-04-15 15:15:18.000,2020-04-15 15:15:18.000,FALSE,2021-04-14 07:46:12.974
-MNdFpi,Cancelled Order,0eMvjm,2019-06-19 19:10:26.000,2019-07-01 15:16:51.000,FALSE,2021-04-14 07:46:12.973
+id,name,integration_id,created,updated,_fivetran_deleted,_fivetran_synced,integration_category,integration_name
+MNdFpi,Cancelled Order,0eMvjm,2019-06-19 19:10:26.000,2019-07-01 15:16:51.000,FALSE,2021-04-14 07:46:12.973,Email,Mailchimp
+Vx2d6i,Received SMS,0rG4eQ,2020-04-15 15:15:18.000,2020-04-15 15:15:18.000,FALSE,2021-04-14 07:46:12.974,ecommerce,Shopify
diff --git a/integration_tests/seeds/person.csv → integration_tests/seeds/klaviyo_person.csv b/integration_tests/seeds/person.csv → integration_tests/seeds/klaviyo_person.csv
@@ -1,3 +1,3 @@
-id,updated,created,_fivetran_deleted,first_name,last_name,address_1,address_2,title,timezone,organization,region,longitude,latitude,phone_number,country,zip,city,email,custom_object,custom_email,custom_accepts_marketing,custom_shopify_tags,_fivetran_synced,custom_consent_form_id,custom_source,custom_consent_method,custom_consent,custom_gift_giver_name,custom_state,custom_flow,custom_gift_recipient_name,custom_last_sign_in,custom_consent_form_version,custom_consent_timestamp,custom_mail_chimp_rating,custom_gift_recipient_email,custom_gift_options,custom_address,custom_subscription_expiration,custom_city,custom_address_line_2,custom_zipcode,custom_expected_date_of_next_order,custom_sms_attentive_signup,custom_phone,custom_afterpay_order,custom_phone_number_region,custom_name,custom_referrer_name,custom_referrer_email,custom_birthday,custom_first_purchase_date_,custom_unengaged,custom_landing_page_tag,custom_fitness_goal,custom_age,custom_workout,custom_quiz,custom_breakfast,custom_quiz_data,custom_raf_subscribe,custom_suppress,custom_sms_consent,custom_gift_option_dreambelt,custom_gift_option_starter,custom_gift_option_machine,custom_landingpage_tag
-01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,+15165555555,United States,,Brooklyn,[email protected],person,[email protected],,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,,
-01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,+15165555555,United States,64224,Jefferson City,[email protected],person,[email protected],false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,,
+id,updated,created,_fivetran_deleted,first_name,last_name,address_1,address_2,title,timezone,organization,region,longitude,latitude,phone_number,country,zip,city,email,custom_object,custom_email,custom_accepts_marketing,custom_shopify_tags,_fivetran_synced,custom_consent_form_id,custom_source,custom_consent_method,custom_consent,custom_gift_giver_name,custom_state,custom_flow,custom_gift_recipient_name,custom_last_sign_in,custom_consent_form_version,custom_consent_timestamp,custom_mail_chimp_rating,custom_gift_recipient_email,custom_gift_options,custom_address,custom_subscription_expiration,custom_city,custom_address_line_2,custom_zipcode,custom_expected_date_of_next_order,custom_sms_attentive_signup,custom_phone,custom_afterpay_order,custom_phone_number_region,custom_name,custom_referrer_name,custom_referrer_email,custom_birthday,custom_first_purchase_date_,custom_unengaged,custom_landing_page_tag,custom_fitness_goal,custom_age,custom_workout,custom_quiz,custom_breakfast,custom_quiz_data,custom_raf_subscribe,custom_suppress,custom_sms_consent,custom_gift_option_dreambelt,custom_gift_option_starter,custom_gift_option_machine,custom_landingpage_tag,last_event_date
+01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,5165555555,United States,,Brooklyn,[email protected],person,[email protected],,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,,,2023-07-18 12:43:52 UTC
+01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,5165555555,United States,64224,Jefferson City,[email protected],person,[email protected],false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,,,2023-09-08 09:19:46 UTC
diff --git a/integration_tests/seeds/order_line_refund.csv b/integration_tests/seeds/order_line_refund.csv
@@ -1 +1,6 @@
-id,location_id,refund_id,restock_type,quantity,order_line_id,_fivetran_synced
+id,location_id,refund_id,restock_type,quantity,order_line_id,_fivetran_synced,subtotal,total_tax_set,subtotal_set,total_tax
+189012115527,32131711046,679976206407,return,1,6113984839751,2020-11-14 07:52:56.522,415,,,19.74
+289901510727,32131711046,800919683143,return,1,9698959196231,2020-11-14 07:52:56.522,415,,,56.33
+196428005447,32131711046,686409187399,return,1,6423996530759,2020-11-14 07:52:56.522,415,,,16.18
+286567268423,,798222680135,no_restock,1,6367161483335,2020-11-14 07:52:56.522,415,,,26.17
+185936773191,,677359190087,no_restock,1,6009460064327,2020-11-14 07:52:56.522,415,,,13.75