From e0c1f15b078be0769ddd44b5f7f9be70c0b93069 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:46:09 -0500 Subject: [PATCH 01/21] test/update-shopify-versions --- integration_tests/ci/sample.profiles.yml | 2 +- integration_tests/requirements.txt | 4 +++- packages.yml | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index d3239b2..b65e771 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -49,6 +49,6 @@ integration_tests: host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: shopify_holistic_reporting_integration_tests_1 - threads: 2 + threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 56a7f6d..42b50b8 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,6 @@ dbt-redshift>=1.3.0,<1.5.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 + +oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file diff --git a/packages.yml b/packages.yml index 96b11dc..0d5d9df 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ packages: - package: fivetran/shopify - version: [">=0.8.0", "<0.9.0"] + version: [">=0.10.0", "<0.11.0"] - package: fivetran/klaviyo - version: [">=0.5.0", "<0.6.0"] + version: [">=0.6.0", "<0.7.0"] From f787aa8d9ab522c10085ed8ca0f0f2293612ee2a Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 24 Oct 2023 19:33:21 -0500 Subject: [PATCH 02/21] update seed --- integration_tests/seeds/order_line_refund.csv | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/integration_tests/seeds/order_line_refund.csv b/integration_tests/seeds/order_line_refund.csv index 1572667..8637809 100644 --- a/integration_tests/seeds/order_line_refund.csv +++ b/integration_tests/seeds/order_line_refund.csv @@ -1 +1,6 @@ -id,location_id,refund_id,restock_type,quantity,order_line_id,_fivetran_synced +id,location_id,refund_id,restock_type,quantity,order_line_id,_fivetran_synced,subtotal,total_tax_set,subtotal_set,total_tax +189012115527,32131711046,679976206407,return,1,6113984839751,2020-11-14 07:52:56.522,415,,,19.74 +289901510727,32131711046,800919683143,return,1,9698959196231,2020-11-14 07:52:56.522,415,,,56.33 +196428005447,32131711046,686409187399,return,1,6423996530759,2020-11-14 07:52:56.522,415,,,16.18 +286567268423,,798222680135,no_restock,1,6367161483335,2020-11-14 07:52:56.522,415,,,26.17 +185936773191,,677359190087,no_restock,1,6009460064327,2020-11-14 07:52:56.522,415,,,13.75 From 2108d94f4f5475fab3eeab27701709a4d403d48d Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 7 Nov 2023 17:26:09 -0500 Subject: [PATCH 03/21] update seeds and rename for clarity --- integration_tests/dbt_project.yml | 17 +++++------------ integration_tests/seeds/campaign.csv | 3 --- integration_tests/seeds/event.csv | 3 --- integration_tests/seeds/flow.csv | 3 --- integration_tests/seeds/flow_snowflake.csv | 3 --- integration_tests/seeds/integration.csv | 3 --- integration_tests/seeds/klaviyo_campaign.csv | 3 +++ integration_tests/seeds/klaviyo_event.csv | 3 +++ integration_tests/seeds/klaviyo_flow.csv | 3 +++ integration_tests/seeds/klaviyo_metric.csv | 3 +++ integration_tests/seeds/klaviyo_person.csv | 3 +++ integration_tests/seeds/metric.csv | 3 --- integration_tests/seeds/person.csv | 3 --- packages.yml | 2 +- 14 files changed, 21 insertions(+), 34 deletions(-) delete mode 100644 integration_tests/seeds/campaign.csv delete mode 100644 integration_tests/seeds/event.csv delete mode 100644 integration_tests/seeds/flow.csv delete mode 100644 integration_tests/seeds/flow_snowflake.csv delete mode 100644 integration_tests/seeds/integration.csv create mode 100644 integration_tests/seeds/klaviyo_campaign.csv create mode 100644 integration_tests/seeds/klaviyo_event.csv create mode 100644 integration_tests/seeds/klaviyo_flow.csv create mode 100644 integration_tests/seeds/klaviyo_metric.csv create mode 100644 integration_tests/seeds/klaviyo_person.csv delete mode 100644 integration_tests/seeds/metric.csv delete mode 100644 integration_tests/seeds/person.csv diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6758aef..06db246 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -194,29 +194,22 @@ seeds: price: float rate: float # klaviyo - person: + klaviyo_person: +tags: klaviyo +column_types: phone_number: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - flow: + klaviyo_flow: +tags: klaviyo +enabled: "{{ true if target.type != 'snowflake' else false }}" +quote_columns: "{{ true if target.type == 'redshift' else false }}" - +column_types: - trigger: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - flow_snowflake: - +tags: klaviyo - +enabled: "{{ true if target.type == 'snowflake' else false }}" - event: + klaviyo_event: +tags: klaviyo +column_types: flow_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" campaign_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" datetime: timestamp timestamp: timestamp - integration: - +tags: klaviyo - metric: + klaviyo_metric: +tags: klaviyo - campaign: + klaviyo_campaign: +tags: klaviyo \ No newline at end of file diff --git a/integration_tests/seeds/campaign.csv b/integration_tests/seeds/campaign.csv deleted file mode 100644 index 3d33134..0000000 --- a/integration_tests/seeds/campaign.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,from_name,status_id,is_segmented,campaign_type,status_label,from_email,subject,name,status,created,updated,send_time,sent_at,_fivetran_deleted,email_template_id,_fivetran_synced -WMddDe,friend,2,FALSE,Regular,Draft,hello@gmail.com,Ready to do stuff?,Jan-2 Email,draft,2021-04-12 23:42:35.000,2021-04-12 23:42:44.000,,,FALSE,YzD79H,2021-04-14 07:45:40.981 -WViD3c,friend,1,FALSE,Regular,Sent,hello@gmail.com,say hello,camp,sent,2021-03-16 19:16:37.000,2021-03-17 15:38:59.000,2021-03-17 15:38:28.000,2021-03-17 15:38:59.000,FALSE,Y8hHH4,2021-04-14 07:45:41.005 \ No newline at end of file diff --git a/integration_tests/seeds/event.csv b/integration_tests/seeds/event.csv deleted file mode 100644 index 3c0c952..0000000 --- a/integration_tests/seeds/event.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,person_id,campaign_id,flow_id,flow_message_id,uuid,datetime,timestamp,_fivetran_deleted,metric_id,type,_variation,property_value,property_source_name,property_extra,property_shipping_rate,property_items,property_tags,property_item_count,property_collections,_fivetran_synced,property_currency_code,property_product_id,property_quantity,property_name,property_variant_option_title,property_sku,property_variant_name,property_vendor,property_event_id,property_total_discounts,property_attribution,property_discount_codes,property_fulfillment_hours,property_fulfillment_status,property_subject,property_title,property_message_interaction,property_url,property_shipment_type,property_courier_name,property_current_status,property_tracking_ship_date,property_tracking_postal_code,property_shipment_package_count,property_campaign_name,property_cohort_message_send_cohort,property_email_domain,property_bounce_type,property_client_type,property_client_os,property_client_name,property_client_os_family,property_client_canonical,property_page,property_is_session_activity,property_os,property_session_end,property_browser,property_list,property_compare_at_price,property_price,property_image_url,property_brand,property_categories,property_variant_option_size,property_attribute_workout,property_attribute_age,property_attribute_fitness_goal,property_attribute_breakfast,property_variant_option_type,property_cohort_variation_send_cohort,property_method,property_to_number,property_message_type,property_from_number,property_message_format,property_message_name,property_carrier_delivery_status,property_failure_type,property_failure_source,property_message_body,property_email,property_attribute_kit,property_vendor_error_code,property_ordr_details,property_date_to_send_gift,property_recipient_name,property_attribute_facebook_order_retailer_id,property_sender_s_name,property_attribute_kitid,property_conversation_id,property_conversation_link,property_conversation_channel,property_rating,property_score,property_from_phone_region,property_shipment_carrier,property_shipment_status,property_to_phone_region -3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,"brand new chopstcks",,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, -3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,"brand new shorts",,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/integration_tests/seeds/flow.csv b/integration_tests/seeds/flow.csv deleted file mode 100644 index 0ac6b68..0000000 --- a/integration_tests/seeds/flow.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name,status,created,updated,customer_filter,trigger,_fivetran_deleted,_fivetran_synced -X839zt,Welcome SMS - Redesign,live,2020-10-12 22:56:59.000,2021-02-10 18:34:39.000,"{""stanzas"": [ {""criteria"": [{""timeframe"": ""alltime"",""type"": ""customer-in-flow""}]}]}","{""filter"": null,""metric"": {""id"": ""UE7prv"",""name"": ""Consented to Receive SMS""},""type"": ""Metric""}",FALSE,2021-04-14 07:46:13.280 -S6uajk,Abandoned Cart - NEW VERSION - Katelyn & DW Working,draft,2020-07-08 20:55:56.000,2020-07-09 16:27:58.000,"{""stanzas"": [{""criteria"": [{""operator"": ""nin"",""region_id"": ""EUROPEAN_UNION"",""type"": ""customer-location""}]}]}","{""filter"": null,""metric"": {""id"": ""NXCwha"",""name"": ""Checkout Started""},""type"": ""Metric""}",FALSE,2021-04-14 07:46:13.285 \ No newline at end of file diff --git a/integration_tests/seeds/flow_snowflake.csv b/integration_tests/seeds/flow_snowflake.csv deleted file mode 100644 index 17ffeb8..0000000 --- a/integration_tests/seeds/flow_snowflake.csv +++ /dev/null @@ -1,3 +0,0 @@ -ID,NAME,STATUS,CREATED,UPDATED,CUSTOMER_FILTER,"""TRIGGER""",_FIVETRAN_DELETED,_fivetran_synced -X839zt,Welcome SMS - Redesign,live,2020-10-12 22:56:59.000,2021-02-10 18:34:39.000,"{""stanzas"": [ {""criteria"": [{""timeframe"": ""alltime"",""type"": ""customer-in-flow""}]}]}","{""filter"": null,""metric"": {""id"": ""UE7prv"",""name"": ""Consented to Receive SMS""},""type"": ""Metric""}",FALSE,2021-04-14 07:46:13.280 -S6uajk,Abandoned Cart - NEW VERSION - Katelyn & DW Working,draft,2020-07-08 20:55:56.000,2020-07-09 16:27:58.000,"{""stanzas"": [{""criteria"": [{""operator"": ""nin"",""region_id"": ""EUROPEAN_UNION"",""type"": ""customer-location""}]}]}","{""filter"": null,""metric"": {""id"": ""NXCwha"",""name"": ""Checkout Started""},""type"": ""Metric""}",FALSE,2021-04-14 07:46:13.285 \ No newline at end of file diff --git a/integration_tests/seeds/integration.csv b/integration_tests/seeds/integration.csv deleted file mode 100644 index c070d67..0000000 --- a/integration_tests/seeds/integration.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,category,name,_fivetran_deleted,_fivetran_synced -4Eerea,Email,Mailchimp,FALSE,2021-04-14 07:46:12.974 -ghSWe4,ecommerce,Shopify,FALSE,2021-04-14 07:46:12.974 \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_campaign.csv b/integration_tests/seeds/klaviyo_campaign.csv new file mode 100644 index 0000000..97c128f --- /dev/null +++ b/integration_tests/seeds/klaviyo_campaign.csv @@ -0,0 +1,3 @@ +ID,FROM_NAME,STATUS_ID,CAMPAIGN_TYPE,STATUS_LABEL,FROM_EMAIL,SUBJECT,NAME,STATUS,CREATED,UPDATED,SEND_TIME,SENT_AT,_FIVETRAN_DELETED,EMAIL_TEMPLATE_ID,_FIVETRAN_SYNCED,ARCHIVED,SCHEDULED +WMddDe,friend,2,Regular,Draft,hello@gmail.com,Ready to do stuff?,Jan-2 Email,draft,2021-04-12 23:42:35.000,2021-04-12 23:42:44.000,,,FALSE,YzD79H,2021-04-14 07:45:40.981,false,2021-04-22 23:42:44.000 +WViD3c,friend,1,Regular,Sent,hello@gmail.com,say hello,camp,sent,2021-03-16 19:16:37.000,2021-03-17 15:38:59.000,2021-03-17 15:38:28.000,2021-03-17 15:38:59.000,FALSE,Y8hHH4,2021-04-14 07:45:41.005,false,2021-03-22 15:38:59.000 \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_event.csv b/integration_tests/seeds/klaviyo_event.csv new file mode 100644 index 0000000..b4f66c5 --- /dev/null +++ b/integration_tests/seeds/klaviyo_event.csv @@ -0,0 +1,3 @@ +ID,PERSON_ID,CAMPAIGN_ID,FLOW_ID,FLOW_MESSAGE_ID,UUID,DATETIME,TIMESTAMP,_FIVETRAN_DELETED,METRIC_ID,TYPE,_VARIATION,PROPERTY_VALUE,PROPERTY_SOURCE_NAME,PROPERTY_EXTRA,PROPERTY_SHIPPING_RATE,PROPERTY_ITEMS,PROPERTY_TAGS,PROPERTY_ITEM_COUNT,PROPERTY_COLLECTIONS,_FIVETRAN_SYNCED,PROPERTY_CURRENCY_CODE,PROPERTY_PRODUCT_ID,PROPERTY_QUANTITY,PROPERTY_NAME,PROPERTY_VARIANT_OPTION_TITLE,PROPERTY_SKU,PROPERTY_VARIANT_NAME,PROPERTY_VENDOR,PROPERTY_EVENT_ID,PROPERTY_TOTAL_DISCOUNTS,PROPERTY_ATTRIBUTION,PROPERTY_DISCOUNT_CODES,PROPERTY_FULFILLMENT_HOURS,PROPERTY_FULFILLMENT_STATUS,PROPERTY_SUBJECT,PROPERTY_TITLE,PROPERTY_MESSAGE_INTERACTION,PROPERTY_URL,PROPERTY_SHIPMENT_TYPE,PROPERTY_COURIER_NAME,PROPERTY_CURRENT_STATUS,PROPERTY_TRACKING_SHIP_DATE,PROPERTY_TRACKING_POSTAL_CODE,PROPERTY_SHIPMENT_PACKAGE_COUNT,PROPERTY_CAMPAIGN_NAME,PROPERTY_COHORT_MESSAGE_SEND_COHORT,PROPERTY_EMAIL_DOMAIN,PROPERTY_BOUNCE_TYPE,PROPERTY_CLIENT_TYPE,PROPERTY_CLIENT_OS,PROPERTY_CLIENT_NAME,PROPERTY_CLIENT_OS_FAMILY,PROPERTY_CLIENT_CANONICAL,PROPERTY_PAGE,PROPERTY_IS_SESSION_ACTIVITY,PROPERTY_OS,PROPERTY_SESSION_END,PROPERTY_BROWSER,PROPERTY_LIST,PROPERTY_COMPARE_AT_PRICE,PROPERTY_PRICE,PROPERTY_IMAGE_URL,PROPERTY_BRAND,PROPERTY_CATEGORIES,PROPERTY_VARIANT_OPTION_SIZE,PROPERTY_ATTRIBUTE_WORKOUT,PROPERTY_ATTRIBUTE_AGE,PROPERTY_ATTRIBUTE_FITNESS_GOAL,PROPERTY_ATTRIBUTE_BREAKFAST,PROPERTY_VARIANT_OPTION_TYPE,PROPERTY_COHORT_VARIATION_SEND_COHORT,PROPERTY_METHOD,PROPERTY_TO_NUMBER,PROPERTY_MESSAGE_TYPE,PROPERTY_FROM_NUMBER,PROPERTY_MESSAGE_FORMAT,PROPERTY_MESSAGE_NAME,PROPERTY_CARRIER_DELIVERY_STATUS,PROPERTY_FAILURE_TYPE,PROPERTY_FAILURE_SOURCE,PROPERTY_MESSAGE_BODY,PROPERTY_EMAIL,PROPERTY_ATTRIBUTE_KIT,PROPERTY_VENDOR_ERROR_CODE,PROPERTY_ORDR_DETAILS,PROPERTY_DATE_TO_SEND_GIFT,PROPERTY_RECIPIENT_NAME,PROPERTY_ATTRIBUTE_FACEBOOK_ORDER_RETAILER_ID,PROPERTY_SENDER_S_NAME,PROPERTY_ATTRIBUTE_KITID,PROPERTY_CONVERSATION_ID,PROPERTY_CONVERSATION_LINK,PROPERTY_CONVERSATION_CHANNEL,PROPERTY_RATING,PROPERTY_SCORE,PROPERTY_FROM_PHONE_REGION,PROPERTY_SHIPMENT_CARRIER,PROPERTY_SHIPMENT_STATUS,PROPERTY_TO_PHONE_REGION +3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,"brand new chopstcks",,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, +3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,"brand new shorts",,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_flow.csv b/integration_tests/seeds/klaviyo_flow.csv new file mode 100644 index 0000000..3d20a68 --- /dev/null +++ b/integration_tests/seeds/klaviyo_flow.csv @@ -0,0 +1,3 @@ +ID,NAME,STATUS,CREATED,UPDATED,CUSTOMER_FILTER,_FIVETRAN_DELETED,_FIVETRAN_SYNCED,ARCHIVED,TRIGGER_TYPE +X839zt,Welcome SMS - Redesign,live,2020-10-12 22:56:59.000,2021-02-10 18:34:39.000,"{""stanzas"": [ {""criteria"": [{""timeframe"": ""alltime"",""type"": ""customer-in-flow""}]}]}",FALSE,2021-04-14 07:46:13.280,false,Metric +S6uajk,Abandoned Cart - NEW VERSION - Katelyn & DW Working,draft,2020-07-08 20:55:56.000,2020-07-09 16:27:58.000,"{""stanzas"": [{""criteria"": [{""operator"": ""nin"",""region_id"": ""EUROPEAN_UNION"",""type"": ""customer-location""}]}]}",FALSE,2021-04-14 07:46:13.285,false,Added to List \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_metric.csv b/integration_tests/seeds/klaviyo_metric.csv new file mode 100644 index 0000000..a13bd79 --- /dev/null +++ b/integration_tests/seeds/klaviyo_metric.csv @@ -0,0 +1,3 @@ +ID,NAME,INTEGRATION_ID,CREATED,UPDATED,_FIVETRAN_DELETED,_FIVETRAN_SYNCED,INTEGRATION_CATEGORY,INTEGRATION_NAME +MNdFpi,Cancelled Order,0eMvjm,2019-06-19 19:10:26.000,2019-07-01 15:16:51.000,FALSE,2021-04-14 07:46:12.973,Email,Mailchimp +Vx2d6i,Received SMS,0rG4eQ,2020-04-15 15:15:18.000,2020-04-15 15:15:18.000,FALSE,2021-04-14 07:46:12.974,ecommerce,Shopify \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_person.csv b/integration_tests/seeds/klaviyo_person.csv new file mode 100644 index 0000000..ffb591b --- /dev/null +++ b/integration_tests/seeds/klaviyo_person.csv @@ -0,0 +1,3 @@ +ID,UPDATED,CREATED,_FIVETRAN_DELETED,FIRST_NAME,LAST_NAME,ADDRESS_1,ADDRESS_2,TITLE,TIMEZONE,ORGANIZATION,REGION,LONGITUDE,LATITUDE,PHONE_NUMBER,COUNTRY,ZIP,CITY,EMAIL,CUSTOM_OBJECT,CUSTOM_EMAIL,CUSTOM_ACCEPTS_MARKETING,CUSTOM_SHOPIFY_TAGS,_FIVETRAN_SYNCED,CUSTOM_CONSENT_FORM_ID,CUSTOM_SOURCE,CUSTOM_CONSENT_METHOD,CUSTOM_CONSENT,CUSTOM_GIFT_GIVER_NAME,CUSTOM_STATE,CUSTOM_FLOW,CUSTOM_GIFT_RECIPIENT_NAME,CUSTOM_LAST_SIGN_IN,CUSTOM_CONSENT_FORM_VERSION,CUSTOM_CONSENT_TIMESTAMP,CUSTOM_MAIL_CHIMP_RATING,CUSTOM_GIFT_RECIPIENT_EMAIL,CUSTOM_GIFT_OPTIONS,CUSTOM_ADDRESS,CUSTOM_SUBSCRIPTION_EXPIRATION,CUSTOM_CITY,CUSTOM_ADDRESS_LINE_2,CUSTOM_ZIPCODE,CUSTOM_EXPECTED_DATE_OF_NEXT_ORDER,CUSTOM_SMS_ATTENTIVE_SIGNUP,CUSTOM_PHONE,CUSTOM_AFTERPAY_ORDER,CUSTOM_PHONE_NUMBER_REGION,CUSTOM_NAME,CUSTOM_REFERRER_NAME,CUSTOM_REFERRER_EMAIL,CUSTOM_BIRTHDAY,CUSTOM_FIRST_PURCHASE_DATE_,CUSTOM_UNENGAGED,CUSTOM_LANDING_PAGE_TAG,CUSTOM_FITNESS_GOAL,CUSTOM_AGE,CUSTOM_WORKOUT,CUSTOM_QUIZ,CUSTOM_BREAKFAST,CUSTOM_QUIZ_DATA,CUSTOM_RAF_SUBSCRIBE,CUSTOM_SUPPRESS,CUSTOM_SMS_CONSENT,CUSTOM_GIFT_OPTION_DREAMBELT,CUSTOM_GIFT_OPTION_STARTER,CUSTOM_GIFT_OPTION_MACHINE,CUSTOM_LANDINGPAGE_TAG,LAST_EVENT_DATE +01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,+15165555555,United States,,Brooklyn,nobody@hotmail.com,person,nobody@hotmail.com,,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,,,2023-07-18 12:43:52 UTC +01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,+15165555555,United States,64224,Jefferson City,mrgoogle@gmail.com,person,mrgoogle@gmail.com,false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,,,2023-09-08 09:19:46 UTC \ No newline at end of file diff --git a/integration_tests/seeds/metric.csv b/integration_tests/seeds/metric.csv deleted file mode 100644 index b00c484..0000000 --- a/integration_tests/seeds/metric.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name,integration_id,created,updated,_fivetran_deleted,_fivetran_synced -Vx2d6i,Received SMS,0rG4eQ,2020-04-15 15:15:18.000,2020-04-15 15:15:18.000,FALSE,2021-04-14 07:46:12.974 -MNdFpi,Cancelled Order,0eMvjm,2019-06-19 19:10:26.000,2019-07-01 15:16:51.000,FALSE,2021-04-14 07:46:12.973 \ No newline at end of file diff --git a/integration_tests/seeds/person.csv b/integration_tests/seeds/person.csv deleted file mode 100644 index ac6dbfc..0000000 --- a/integration_tests/seeds/person.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,updated,created,_fivetran_deleted,first_name,last_name,address_1,address_2,title,timezone,organization,region,longitude,latitude,phone_number,country,zip,city,email,custom_object,custom_email,custom_accepts_marketing,custom_shopify_tags,_fivetran_synced,custom_consent_form_id,custom_source,custom_consent_method,custom_consent,custom_gift_giver_name,custom_state,custom_flow,custom_gift_recipient_name,custom_last_sign_in,custom_consent_form_version,custom_consent_timestamp,custom_mail_chimp_rating,custom_gift_recipient_email,custom_gift_options,custom_address,custom_subscription_expiration,custom_city,custom_address_line_2,custom_zipcode,custom_expected_date_of_next_order,custom_sms_attentive_signup,custom_phone,custom_afterpay_order,custom_phone_number_region,custom_name,custom_referrer_name,custom_referrer_email,custom_birthday,custom_first_purchase_date_,custom_unengaged,custom_landing_page_tag,custom_fitness_goal,custom_age,custom_workout,custom_quiz,custom_breakfast,custom_quiz_data,custom_raf_subscribe,custom_suppress,custom_sms_consent,custom_gift_option_dreambelt,custom_gift_option_starter,custom_gift_option_machine,custom_landingpage_tag -01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,+15165555555,United States,,Brooklyn,nobody@hotmail.com,person,nobody@hotmail.com,,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,, -01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,+15165555555,United States,64224,Jefferson City,mrgoogle@gmail.com,person,mrgoogle@gmail.com,false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/packages.yml b/packages.yml index 0d5d9df..0458d36 100644 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,4 @@ packages: - package: fivetran/shopify version: [">=0.10.0", "<0.11.0"] - package: fivetran/klaviyo - version: [">=0.6.0", "<0.7.0"] + version: [">=0.7.0", "<0.8.0"] From 2897c4d7b0cfe2bcec1cc857149ec9cc09384f28 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 7 Nov 2023 17:42:30 -0500 Subject: [PATCH 04/21] versions --- CHANGELOG.md | 2 ++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 420ccbb..5841498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +# dbt_shopify_holistic_reporting v0.5.0 + # dbt_shopify_holistic_reporting v0.4.0 [PR #16](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/16) includes the following changes: diff --git a/dbt_project.yml b/dbt_project.yml index d357726..129fd82 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_holistic_reporting' -version: '0.4.0' +version: '0.5.0' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 06db246..889444d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_holistic_reporting_integration_tests' -version: '0.4.0' +version: '0.5.0' profile: 'integration_tests' config-version: 2 From fc01f5ddaaf20a1535bf20f341a20a726bf74b54 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 15:13:41 -0500 Subject: [PATCH 05/21] more updates --- CHANGELOG.md | 18 ++++++++++++++++++ integration_tests/dbt_project.yml | 3 ++- .../seeds/{klaviyo_person.csv => person.csv} | 0 .../int__daily_shopify_customer_orders.sql | 9 ++++++--- ..._holistic_reporting__orders_attribution.sql | 1 + 5 files changed, 27 insertions(+), 4 deletions(-) rename integration_tests/seeds/{klaviyo_person.csv => person.csv} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5841498..68efb75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,22 @@ # dbt_shopify_holistic_reporting v0.5.0 +[PR #18](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/18) includes the following changes: + +## 🚨 Breaking Changes 🚨: + +- This package now points to the following ranges of the upstream packages: + +``` +- package: fivetran/shopify + version: [">=0.10.0", "<0.11.0"] +- package: fivetran/klaviyo + version: [">=0.7.0", "<0.8.0"] +``` + +For more information on the changes in the underlying upstream pacakges, refer to the changelogs for [Shopify](https://github.com/fivetran/dbt_shopify/compare/v0.8.1...v0.10.0) and [Klaviyo](https://github.com/fivetran/dbt_klaviyo/compare/v0.5.0...v0.7.1). + +## + +## Under the Hood: # dbt_shopify_holistic_reporting v0.4.0 diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 889444d..7fabd4d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -10,9 +10,10 @@ clean-targets: # directories to be removed by `dbt clean` vars: shopify_source: + shopify_using_fulfillment_event: true shopify_schema: 'shopify_holistic_reporting_integration_tests_1' shopify_order_identifier: 'order_table' # reserved keyword in snowflake - + klaviyo_source: klaviyo_schema: 'shopify_holistic_reporting_integration_tests_1' klaviyo__event_pass_through_columns: [] diff --git a/integration_tests/seeds/klaviyo_person.csv b/integration_tests/seeds/person.csv similarity index 100% rename from integration_tests/seeds/klaviyo_person.csv rename to integration_tests/seeds/person.csv diff --git a/models/intermediate/int__daily_shopify_customer_orders.sql b/models/intermediate/int__daily_shopify_customer_orders.sql index 9a79e2c..220b817 100644 --- a/models/intermediate/int__daily_shopify_customer_orders.sql +++ b/models/intermediate/int__daily_shopify_customer_orders.sql @@ -15,7 +15,8 @@ with orders as ( source_relation, count(distinct product_id) as count_products, count(distinct product_id || '-' || variant_id) as count_product_variants, - sum(quantity) as sum_quantity + sum(quantity) as sum_quantity, + sum(subtotal) as sum_subtotal from order_lines group by 1,2 @@ -26,7 +27,8 @@ with orders as ( orders.*, order_line_metrics.count_products, order_line_metrics.count_product_variants, - order_line_metrics.sum_quantity + order_line_metrics.sum_quantity, + order_line_metrics.sum_subtotal from orders left join order_line_metrics @@ -66,7 +68,8 @@ with orders as ( sum(case when cancelled_timestamp is not null then 1 else 0 end) as count_cancelled_orders, sum(count_products) as count_products, sum(count_product_variants) as count_product_variants, - sum(sum_quantity) as sum_quantity + sum(sum_quantity) as sum_quantity, + sum(sum_subtotal) as sum_subtotal, {% if var('shopify__using_order_adjustment', true) %} , sum(order_adjustment_amount) as total_order_adjustment_amount diff --git a/models/shopify_holistic_reporting__orders_attribution.sql b/models/shopify_holistic_reporting__orders_attribution.sql index 8d50c7a..7a19829 100644 --- a/models/shopify_holistic_reporting__orders_attribution.sql +++ b/models/shopify_holistic_reporting__orders_attribution.sql @@ -52,6 +52,7 @@ with orders as ( events.event_id as last_touch_event_id, events.occurred_at as last_touch_event_occurred_at, events.type as last_touch_event_type, + events.integration_id as last_touch_integration_id, events.integration_name as last_touch_integration_name, events.integration_category as last_touch_integration_category, events.source_relation as klaviyo_source_relation From 2d4af9690a38f4fdfc9299bd952b2b86903c5b12 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 15:16:21 -0500 Subject: [PATCH 06/21] rename person seed to specify klaviyo --- integration_tests/seeds/{person.csv => klaviyo_person.csv} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integration_tests/seeds/{person.csv => klaviyo_person.csv} (100%) diff --git a/integration_tests/seeds/person.csv b/integration_tests/seeds/klaviyo_person.csv similarity index 100% rename from integration_tests/seeds/person.csv rename to integration_tests/seeds/klaviyo_person.csv From 1fa77b2b612c4c96c2586d7b8fd6ab154cb6887c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 15:25:02 -0500 Subject: [PATCH 07/21] changelog --- CHANGELOG.md | 9 +++++++-- integration_tests/requirements.txt | 4 +--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68efb75..82cad42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,15 @@ For more information on the changes in the underlying upstream pacakges, refer to the changelogs for [Shopify](https://github.com/fivetran/dbt_shopify/compare/v0.8.1...v0.10.0) and [Klaviyo](https://github.com/fivetran/dbt_klaviyo/compare/v0.5.0...v0.7.1). -## +## Additions +- Adds field `sum_subtotal` to `int__daily_shopify_customer_orders` +- Adds field `last_touch_integration_id` to `shopify_holistic_reporting__orders_attribution` ## Under the Hood: - +- Renamed all Klaviyo seed files so that they start with prefix `klaviyo` in order to differentiate which seed files come from Shopify versus Klaviyo. +- Removed the flow_snowflake seed file now that `trigger` has been removed from the Klaviyo flow object and as such will not cause run errors in Snowflake. +- Populate the order_line_refund seed file as it was empty previously. + # dbt_shopify_holistic_reporting v0.4.0 [PR #16](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/16) includes the following changes: diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 42b50b8..56a7f6d 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,6 +4,4 @@ dbt-redshift>=1.3.0,<1.5.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 - -oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file From a057f5b91d1e3af61be4a8885a7c254115ef2616 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 15:27:19 -0500 Subject: [PATCH 08/21] extra comma --- models/intermediate/int__daily_shopify_customer_orders.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/intermediate/int__daily_shopify_customer_orders.sql b/models/intermediate/int__daily_shopify_customer_orders.sql index 220b817..502f48b 100644 --- a/models/intermediate/int__daily_shopify_customer_orders.sql +++ b/models/intermediate/int__daily_shopify_customer_orders.sql @@ -69,7 +69,7 @@ with orders as ( sum(count_products) as count_products, sum(count_product_variants) as count_product_variants, sum(sum_quantity) as sum_quantity, - sum(sum_subtotal) as sum_subtotal, + sum(sum_subtotal) as sum_subtotal {% if var('shopify__using_order_adjustment', true) %} , sum(order_adjustment_amount) as total_order_adjustment_amount From 6ded3f3006fc3c6dee455b666ec579a9b9f7bde3 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 15:59:00 -0500 Subject: [PATCH 09/21] got the wrong field --- .../int__daily_shopify_customer_orders.sql | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/models/intermediate/int__daily_shopify_customer_orders.sql b/models/intermediate/int__daily_shopify_customer_orders.sql index 502f48b..2144efa 100644 --- a/models/intermediate/int__daily_shopify_customer_orders.sql +++ b/models/intermediate/int__daily_shopify_customer_orders.sql @@ -16,7 +16,9 @@ with orders as ( count(distinct product_id) as count_products, count(distinct product_id || '-' || variant_id) as count_product_variants, sum(quantity) as sum_quantity, - sum(subtotal) as sum_subtotal + sum(price) as sum_price, + sum(pre_tax_price) as sum_pre_tax_price, + sum(total_discount) as sum_total_discount from order_lines group by 1,2 @@ -28,7 +30,9 @@ with orders as ( order_line_metrics.count_products, order_line_metrics.count_product_variants, order_line_metrics.sum_quantity, - order_line_metrics.sum_subtotal + order_line_metrics.sum_price, + order_line_metrics.sum_pre_tax_price, + order_line_metrics.sum_total_discount from orders left join order_line_metrics @@ -69,7 +73,9 @@ with orders as ( sum(count_products) as count_products, sum(count_product_variants) as count_product_variants, sum(sum_quantity) as sum_quantity, - sum(sum_subtotal) as sum_subtotal + sum(sum_price) as sum_price, + sum(sum_pre_tax_price) as sum_pre_tax_price, + sum(sum_total_discount) as sum_total_discount {% if var('shopify__using_order_adjustment', true) %} , sum(order_adjustment_amount) as total_order_adjustment_amount From 49ab4b344b9e11b333cccd70d5f9748b77e86fe6 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 16:10:15 -0500 Subject: [PATCH 10/21] try new schema since it seems stuck on phone number being numeric still --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index b65e771..4f3b815 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_holistic_reporting_integration_tests_1 + schema: shopify_holistic_reporting_integration_tests_2 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_holistic_reporting_integration_tests_1 + schema: shopify_holistic_reporting_integration_tests_2 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_holistic_reporting_integration_tests_1 + schema: shopify_holistic_reporting_integration_tests_2 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_holistic_reporting_integration_tests_1 + schema: shopify_holistic_reporting_integration_tests_2 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_holistic_reporting_integration_tests_1 + schema: shopify_holistic_reporting_integration_tests_2 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 7fabd4d..2722f5e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -11,11 +11,11 @@ clean-targets: # directories to be removed by `dbt clean` vars: shopify_source: shopify_using_fulfillment_event: true - shopify_schema: 'shopify_holistic_reporting_integration_tests_1' + shopify_schema: 'shopify_holistic_reporting_integration_tests_2' shopify_order_identifier: 'order_table' # reserved keyword in snowflake klaviyo_source: - klaviyo_schema: 'shopify_holistic_reporting_integration_tests_1' + klaviyo_schema: 'shopify_holistic_reporting_integration_tests_2' klaviyo__event_pass_through_columns: [] klaviyo__person_pass_through_columns: [] From 5871e1066d80e1bfb17b304b8832a1752402ec42 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 8 Nov 2023 16:26:15 -0500 Subject: [PATCH 11/21] try2 --- integration_tests/seeds/klaviyo_person.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/seeds/klaviyo_person.csv b/integration_tests/seeds/klaviyo_person.csv index ffb591b..b78a8da 100644 --- a/integration_tests/seeds/klaviyo_person.csv +++ b/integration_tests/seeds/klaviyo_person.csv @@ -1,3 +1,3 @@ ID,UPDATED,CREATED,_FIVETRAN_DELETED,FIRST_NAME,LAST_NAME,ADDRESS_1,ADDRESS_2,TITLE,TIMEZONE,ORGANIZATION,REGION,LONGITUDE,LATITUDE,PHONE_NUMBER,COUNTRY,ZIP,CITY,EMAIL,CUSTOM_OBJECT,CUSTOM_EMAIL,CUSTOM_ACCEPTS_MARKETING,CUSTOM_SHOPIFY_TAGS,_FIVETRAN_SYNCED,CUSTOM_CONSENT_FORM_ID,CUSTOM_SOURCE,CUSTOM_CONSENT_METHOD,CUSTOM_CONSENT,CUSTOM_GIFT_GIVER_NAME,CUSTOM_STATE,CUSTOM_FLOW,CUSTOM_GIFT_RECIPIENT_NAME,CUSTOM_LAST_SIGN_IN,CUSTOM_CONSENT_FORM_VERSION,CUSTOM_CONSENT_TIMESTAMP,CUSTOM_MAIL_CHIMP_RATING,CUSTOM_GIFT_RECIPIENT_EMAIL,CUSTOM_GIFT_OPTIONS,CUSTOM_ADDRESS,CUSTOM_SUBSCRIPTION_EXPIRATION,CUSTOM_CITY,CUSTOM_ADDRESS_LINE_2,CUSTOM_ZIPCODE,CUSTOM_EXPECTED_DATE_OF_NEXT_ORDER,CUSTOM_SMS_ATTENTIVE_SIGNUP,CUSTOM_PHONE,CUSTOM_AFTERPAY_ORDER,CUSTOM_PHONE_NUMBER_REGION,CUSTOM_NAME,CUSTOM_REFERRER_NAME,CUSTOM_REFERRER_EMAIL,CUSTOM_BIRTHDAY,CUSTOM_FIRST_PURCHASE_DATE_,CUSTOM_UNENGAGED,CUSTOM_LANDING_PAGE_TAG,CUSTOM_FITNESS_GOAL,CUSTOM_AGE,CUSTOM_WORKOUT,CUSTOM_QUIZ,CUSTOM_BREAKFAST,CUSTOM_QUIZ_DATA,CUSTOM_RAF_SUBSCRIBE,CUSTOM_SUPPRESS,CUSTOM_SMS_CONSENT,CUSTOM_GIFT_OPTION_DREAMBELT,CUSTOM_GIFT_OPTION_STARTER,CUSTOM_GIFT_OPTION_MACHINE,CUSTOM_LANDINGPAGE_TAG,LAST_EVENT_DATE -01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,+15165555555,United States,,Brooklyn,nobody@hotmail.com,person,nobody@hotmail.com,,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,,,2023-07-18 12:43:52 UTC -01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,+15165555555,United States,64224,Jefferson City,mrgoogle@gmail.com,person,mrgoogle@gmail.com,false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,,,2023-09-08 09:19:46 UTC \ No newline at end of file +01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,5165555555,United States,,Brooklyn,nobody@hotmail.com,person,nobody@hotmail.com,,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,,,2023-07-18 12:43:52 UTC +01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,5165555555,United States,64224,Jefferson City,mrgoogle@gmail.com,person,mrgoogle@gmail.com,false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,,,2023-09-08 09:19:46 UTC \ No newline at end of file From 6555a316ab38a10b7668502616652cc5e8945178 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 9 Nov 2023 13:27:24 -0500 Subject: [PATCH 12/21] try3 --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2722f5e..9c3999c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -198,7 +198,7 @@ seeds: klaviyo_person: +tags: klaviyo +column_types: - phone_number: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + phone_number: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar(100)' }}" klaviyo_flow: +tags: klaviyo +enabled: "{{ true if target.type != 'snowflake' else false }}" From 4cb73d046a1b905148383a291af2ed538fdac413 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 9 Nov 2023 14:21:58 -0500 Subject: [PATCH 13/21] seed fix --- integration_tests/dbt_project.yml | 10 +++++----- .../int__daily_shopify_customer_orders.sql | 15 +++------------ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 9c3999c..e124ad4 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -198,7 +198,7 @@ seeds: klaviyo_person: +tags: klaviyo +column_types: - phone_number: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar(100)' }}" + PHONE_NUMBER: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar(100)' }}" klaviyo_flow: +tags: klaviyo +enabled: "{{ true if target.type != 'snowflake' else false }}" @@ -206,10 +206,10 @@ seeds: klaviyo_event: +tags: klaviyo +column_types: - flow_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - campaign_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - datetime: timestamp - timestamp: timestamp + FLOW_ID: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + CAMPAIGN_ID: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + DATETIME: timestamp + TIMESTAMP: timestamp klaviyo_metric: +tags: klaviyo klaviyo_campaign: diff --git a/models/intermediate/int__daily_shopify_customer_orders.sql b/models/intermediate/int__daily_shopify_customer_orders.sql index 2144efa..9a79e2c 100644 --- a/models/intermediate/int__daily_shopify_customer_orders.sql +++ b/models/intermediate/int__daily_shopify_customer_orders.sql @@ -15,10 +15,7 @@ with orders as ( source_relation, count(distinct product_id) as count_products, count(distinct product_id || '-' || variant_id) as count_product_variants, - sum(quantity) as sum_quantity, - sum(price) as sum_price, - sum(pre_tax_price) as sum_pre_tax_price, - sum(total_discount) as sum_total_discount + sum(quantity) as sum_quantity from order_lines group by 1,2 @@ -29,10 +26,7 @@ with orders as ( orders.*, order_line_metrics.count_products, order_line_metrics.count_product_variants, - order_line_metrics.sum_quantity, - order_line_metrics.sum_price, - order_line_metrics.sum_pre_tax_price, - order_line_metrics.sum_total_discount + order_line_metrics.sum_quantity from orders left join order_line_metrics @@ -72,10 +66,7 @@ with orders as ( sum(case when cancelled_timestamp is not null then 1 else 0 end) as count_cancelled_orders, sum(count_products) as count_products, sum(count_product_variants) as count_product_variants, - sum(sum_quantity) as sum_quantity, - sum(sum_price) as sum_price, - sum(sum_pre_tax_price) as sum_pre_tax_price, - sum(sum_total_discount) as sum_total_discount + sum(sum_quantity) as sum_quantity {% if var('shopify__using_order_adjustment', true) %} , sum(order_adjustment_amount) as total_order_adjustment_amount From aabcf4625d0a327d76e424e56f758016ad9da977 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 9 Nov 2023 17:00:54 -0500 Subject: [PATCH 14/21] seed update --- integration_tests/dbt_project.yml | 2 +- integration_tests/seeds/klaviyo_event.csv | 4 ++-- models/shopify_holistic_reporting.yml | 22 ++++++++++++++++++++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e124ad4..9c61e15 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -198,7 +198,7 @@ seeds: klaviyo_person: +tags: klaviyo +column_types: - PHONE_NUMBER: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar(100)' }}" + PHONE_NUMBER: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" klaviyo_flow: +tags: klaviyo +enabled: "{{ true if target.type != 'snowflake' else false }}" diff --git a/integration_tests/seeds/klaviyo_event.csv b/integration_tests/seeds/klaviyo_event.csv index b4f66c5..b0018f5 100644 --- a/integration_tests/seeds/klaviyo_event.csv +++ b/integration_tests/seeds/klaviyo_event.csv @@ -1,3 +1,3 @@ ID,PERSON_ID,CAMPAIGN_ID,FLOW_ID,FLOW_MESSAGE_ID,UUID,DATETIME,TIMESTAMP,_FIVETRAN_DELETED,METRIC_ID,TYPE,_VARIATION,PROPERTY_VALUE,PROPERTY_SOURCE_NAME,PROPERTY_EXTRA,PROPERTY_SHIPPING_RATE,PROPERTY_ITEMS,PROPERTY_TAGS,PROPERTY_ITEM_COUNT,PROPERTY_COLLECTIONS,_FIVETRAN_SYNCED,PROPERTY_CURRENCY_CODE,PROPERTY_PRODUCT_ID,PROPERTY_QUANTITY,PROPERTY_NAME,PROPERTY_VARIANT_OPTION_TITLE,PROPERTY_SKU,PROPERTY_VARIANT_NAME,PROPERTY_VENDOR,PROPERTY_EVENT_ID,PROPERTY_TOTAL_DISCOUNTS,PROPERTY_ATTRIBUTION,PROPERTY_DISCOUNT_CODES,PROPERTY_FULFILLMENT_HOURS,PROPERTY_FULFILLMENT_STATUS,PROPERTY_SUBJECT,PROPERTY_TITLE,PROPERTY_MESSAGE_INTERACTION,PROPERTY_URL,PROPERTY_SHIPMENT_TYPE,PROPERTY_COURIER_NAME,PROPERTY_CURRENT_STATUS,PROPERTY_TRACKING_SHIP_DATE,PROPERTY_TRACKING_POSTAL_CODE,PROPERTY_SHIPMENT_PACKAGE_COUNT,PROPERTY_CAMPAIGN_NAME,PROPERTY_COHORT_MESSAGE_SEND_COHORT,PROPERTY_EMAIL_DOMAIN,PROPERTY_BOUNCE_TYPE,PROPERTY_CLIENT_TYPE,PROPERTY_CLIENT_OS,PROPERTY_CLIENT_NAME,PROPERTY_CLIENT_OS_FAMILY,PROPERTY_CLIENT_CANONICAL,PROPERTY_PAGE,PROPERTY_IS_SESSION_ACTIVITY,PROPERTY_OS,PROPERTY_SESSION_END,PROPERTY_BROWSER,PROPERTY_LIST,PROPERTY_COMPARE_AT_PRICE,PROPERTY_PRICE,PROPERTY_IMAGE_URL,PROPERTY_BRAND,PROPERTY_CATEGORIES,PROPERTY_VARIANT_OPTION_SIZE,PROPERTY_ATTRIBUTE_WORKOUT,PROPERTY_ATTRIBUTE_AGE,PROPERTY_ATTRIBUTE_FITNESS_GOAL,PROPERTY_ATTRIBUTE_BREAKFAST,PROPERTY_VARIANT_OPTION_TYPE,PROPERTY_COHORT_VARIATION_SEND_COHORT,PROPERTY_METHOD,PROPERTY_TO_NUMBER,PROPERTY_MESSAGE_TYPE,PROPERTY_FROM_NUMBER,PROPERTY_MESSAGE_FORMAT,PROPERTY_MESSAGE_NAME,PROPERTY_CARRIER_DELIVERY_STATUS,PROPERTY_FAILURE_TYPE,PROPERTY_FAILURE_SOURCE,PROPERTY_MESSAGE_BODY,PROPERTY_EMAIL,PROPERTY_ATTRIBUTE_KIT,PROPERTY_VENDOR_ERROR_CODE,PROPERTY_ORDR_DETAILS,PROPERTY_DATE_TO_SEND_GIFT,PROPERTY_RECIPIENT_NAME,PROPERTY_ATTRIBUTE_FACEBOOK_ORDER_RETAILER_ID,PROPERTY_SENDER_S_NAME,PROPERTY_ATTRIBUTE_KITID,PROPERTY_CONVERSATION_ID,PROPERTY_CONVERSATION_LINK,PROPERTY_CONVERSATION_CHANNEL,PROPERTY_RATING,PROPERTY_SCORE,PROPERTY_FROM_PHONE_REGION,PROPERTY_SHIPMENT_CARRIER,PROPERTY_SHIPMENT_STATUS,PROPERTY_TO_PHONE_REGION -3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,"brand new chopstcks",,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, -3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43.000,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,"brand new shorts",,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file +3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,brand new chopstcks,,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, +3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,brand new shorts,,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/models/shopify_holistic_reporting.yml b/models/shopify_holistic_reporting.yml index a8a0cc4..662b8f1 100644 --- a/models/shopify_holistic_reporting.yml +++ b/models/shopify_holistic_reporting.yml @@ -36,8 +36,14 @@ models: description: The date and time when the customer account first associated with this email was created. - name: shopify_customer_last_created_at description: The date and time when the customer account first associated with this email was created. + - name: klaviyo_person_first_created_at + description: Timestamp of when the person's profile was first created. + - name: klaviyo_person_last_created_at + description: Timestamp of when the person's profile was last created. - name: shopify_customer_last_updated_at - description: Timestamp of when the shopify customer was last updated. + description: Timestamp of when the shopify customer was last updated. + - name: last_updated_at + description: Timestamp of when the person profile was last updated. - name: shopify_default_address_id description: The default address for the customer. - name: email @@ -62,6 +68,18 @@ models: description: The timestamp the customer completed their first order. - name: shopify_last_order_at description: The timestamp the customer completed their most recent order. + - name: customer_tags + description: A string aggregated list of all tags associated with a customer. + - name: note + description: A note about the customer. + - name: currency + description: The three-letter code for the shop currency. + - name: marketing_consent_state + description: > + Field indicating if the customer has consented to receive marketing material via email. Coalescing of the new `email_marketing_consent_state` + field and the deprecated `accepts_marketing` field. Records with the old field will be marked with '(legacy)'. + - name: marketing_consent_updated_at + description: The date and time when the customer consented to receive marketing material by email. If no date is provided, then the date and time when the consent information was sent is used. - name: shopify_average_order_value description: The average order value for the customer. - name: shopify_lifetime_total_spent @@ -70,7 +88,7 @@ models: description: The total amount of money that the customer has been refunded on orders across their order history. - name: shopify_lifetime_total_amount description: The total amount of money (minus refunds) that the customer has spent across their order history. - - name: klaviyo_person_id + - name: klaviyo_persons_id description: > Unique ID of the user if you use your own unique identifier. Otherwise, Klaviyo recommends using the email as the primary key. From 22a592fdeafc9d7542201b001f20e95114c420ad Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 9 Nov 2023 17:38:33 -0500 Subject: [PATCH 15/21] update requirements --- integration_tests/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 56a7f6d..810bae1 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,7 +1,7 @@ dbt-snowflake>=1.3.0,<2.0.0 dbt-bigquery>=1.3.0,<2.0.0 -dbt-redshift>=1.3.0,<1.5.0 +dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.6.0,<2.0.0 \ No newline at end of file From 32d13475ccf4cf5c4b9ae7e92bc35293ea1823f0 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 9 Nov 2023 17:48:13 -0500 Subject: [PATCH 16/21] update docs --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82cad42..a160b57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,10 @@ For more information on the changes in the underlying upstream pacakges, refer to the changelogs for [Shopify](https://github.com/fivetran/dbt_shopify/compare/v0.8.1...v0.10.0) and [Klaviyo](https://github.com/fivetran/dbt_klaviyo/compare/v0.5.0...v0.7.1). ## Additions -- Adds field `sum_subtotal` to `int__daily_shopify_customer_orders` - Adds field `last_touch_integration_id` to `shopify_holistic_reporting__orders_attribution` ## Under the Hood: +- Replace seed files with the new ones from the respective upstream packages. - Renamed all Klaviyo seed files so that they start with prefix `klaviyo` in order to differentiate which seed files come from Shopify versus Klaviyo. - Removed the flow_snowflake seed file now that `trigger` has been removed from the Klaviyo flow object and as such will not cause run errors in Snowflake. - Populate the order_line_refund seed file as it was empty previously. From eda30867f82e28a5d642f73ef402b8f6d82bd439 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Fri, 10 Nov 2023 14:08:45 -0500 Subject: [PATCH 17/21] add identifers --- README.md | 10 +++++----- integration_tests/dbt_project.yml | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d65416d..4cc1dda 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Include the following shopify_holistic_reporting package version in your `packag ```yml packages: - package: fivetran/shopify_holistic_reporting - version: [">=0.4.0", "<0.5.0"] # we recommend using ranges to capture non-breaking changes automatically + version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically ``` Do **NOT** include the `shopify`, `shopify_source`, `klaviyo`, or `klaviyo_source` packages in this file. The combo package itself has a dependency on these and will install the transformation and source packages as well. @@ -130,16 +130,16 @@ This dbt package is dependent on the following dbt packages. Please be aware tha ```yml packages: - package: fivetran/shopify - version: [">=0.8.0", "<0.9.0"] + version: [">=0.10.0", "<0.11.0"] - package: fivetran/shopify_source - version: [">=0.8.0", "<0.9.0"] + version: [">=0.10.0", "<0.11.0"] - package: fivetran/klaviyo - version: [">=0.5.0", "<0.6.0"] + version: [">=0.7.0", "<0.8.0"] - package: fivetran/klaviyo_source - version: [">=0.5.0", "<0.6.0"] + version: [">=0.7.0", "<0.8.0"] - package: fivetran/fivetran_utils version: [">=0.4.0", "<0.5.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 9c61e15..6f24a2e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -18,6 +18,11 @@ vars: klaviyo_schema: 'shopify_holistic_reporting_integration_tests_2' klaviyo__event_pass_through_columns: [] klaviyo__person_pass_through_columns: [] + klaviyo_campaign_identifier: "klaviyo_campaign" + klaviyo_event_identifier: "klaviyo_event" + klaviyo_flow_identifier: "klaviyo_flow" + klaviyo_person_identifier: "klaviyo_person" + klaviyo_metric_identifier: "klaviyo_metric" klaviyo: klaviyo__eligible_attribution_events: # default in klaviyo -> https://help.klaviyo.com/hc/en-us/articles/115005248128 From 0bcfa7aa2613269a079f8eaf8fb34b238ffdb0ac Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Fri, 10 Nov 2023 15:17:29 -0500 Subject: [PATCH 18/21] try new schema --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 4f3b815..5603408 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_holistic_reporting_integration_tests_2 + schema: shopify_holistic_reporting_integration_tests_3 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_holistic_reporting_integration_tests_2 + schema: shopify_holistic_reporting_integration_tests_3 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_holistic_reporting_integration_tests_2 + schema: shopify_holistic_reporting_integration_tests_3 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_holistic_reporting_integration_tests_2 + schema: shopify_holistic_reporting_integration_tests_3 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_holistic_reporting_integration_tests_2 + schema: shopify_holistic_reporting_integration_tests_3 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6f24a2e..a94f05a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -11,11 +11,11 @@ clean-targets: # directories to be removed by `dbt clean` vars: shopify_source: shopify_using_fulfillment_event: true - shopify_schema: 'shopify_holistic_reporting_integration_tests_2' + shopify_schema: 'shopify_holistic_reporting_integration_tests_3' shopify_order_identifier: 'order_table' # reserved keyword in snowflake klaviyo_source: - klaviyo_schema: 'shopify_holistic_reporting_integration_tests_2' + klaviyo_schema: 'shopify_holistic_reporting_integration_tests_3' klaviyo__event_pass_through_columns: [] klaviyo__person_pass_through_columns: [] klaviyo_campaign_identifier: "klaviyo_campaign" From 0dc23a46933f989de496e00e46873f99ee26b164 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 13 Nov 2023 10:34:47 -0500 Subject: [PATCH 19/21] try again --- integration_tests/seeds/klaviyo_campaign.csv | 2 +- integration_tests/seeds/klaviyo_event.csv | 2 +- integration_tests/seeds/klaviyo_flow.csv | 2 +- integration_tests/seeds/klaviyo_metric.csv | 2 +- integration_tests/seeds/klaviyo_person.csv | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/seeds/klaviyo_campaign.csv b/integration_tests/seeds/klaviyo_campaign.csv index 97c128f..7851aee 100644 --- a/integration_tests/seeds/klaviyo_campaign.csv +++ b/integration_tests/seeds/klaviyo_campaign.csv @@ -1,3 +1,3 @@ -ID,FROM_NAME,STATUS_ID,CAMPAIGN_TYPE,STATUS_LABEL,FROM_EMAIL,SUBJECT,NAME,STATUS,CREATED,UPDATED,SEND_TIME,SENT_AT,_FIVETRAN_DELETED,EMAIL_TEMPLATE_ID,_FIVETRAN_SYNCED,ARCHIVED,SCHEDULED +id,from_name,status_id,campaign_type,status_label,from_email,subject,name,status,created,updated,send_time,sent_at,_fivetran_deleted,email_template_id,_fivetran_synced,archived,scheduled WMddDe,friend,2,Regular,Draft,hello@gmail.com,Ready to do stuff?,Jan-2 Email,draft,2021-04-12 23:42:35.000,2021-04-12 23:42:44.000,,,FALSE,YzD79H,2021-04-14 07:45:40.981,false,2021-04-22 23:42:44.000 WViD3c,friend,1,Regular,Sent,hello@gmail.com,say hello,camp,sent,2021-03-16 19:16:37.000,2021-03-17 15:38:59.000,2021-03-17 15:38:28.000,2021-03-17 15:38:59.000,FALSE,Y8hHH4,2021-04-14 07:45:41.005,false,2021-03-22 15:38:59.000 \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_event.csv b/integration_tests/seeds/klaviyo_event.csv index b0018f5..17a9c62 100644 --- a/integration_tests/seeds/klaviyo_event.csv +++ b/integration_tests/seeds/klaviyo_event.csv @@ -1,3 +1,3 @@ -ID,PERSON_ID,CAMPAIGN_ID,FLOW_ID,FLOW_MESSAGE_ID,UUID,DATETIME,TIMESTAMP,_FIVETRAN_DELETED,METRIC_ID,TYPE,_VARIATION,PROPERTY_VALUE,PROPERTY_SOURCE_NAME,PROPERTY_EXTRA,PROPERTY_SHIPPING_RATE,PROPERTY_ITEMS,PROPERTY_TAGS,PROPERTY_ITEM_COUNT,PROPERTY_COLLECTIONS,_FIVETRAN_SYNCED,PROPERTY_CURRENCY_CODE,PROPERTY_PRODUCT_ID,PROPERTY_QUANTITY,PROPERTY_NAME,PROPERTY_VARIANT_OPTION_TITLE,PROPERTY_SKU,PROPERTY_VARIANT_NAME,PROPERTY_VENDOR,PROPERTY_EVENT_ID,PROPERTY_TOTAL_DISCOUNTS,PROPERTY_ATTRIBUTION,PROPERTY_DISCOUNT_CODES,PROPERTY_FULFILLMENT_HOURS,PROPERTY_FULFILLMENT_STATUS,PROPERTY_SUBJECT,PROPERTY_TITLE,PROPERTY_MESSAGE_INTERACTION,PROPERTY_URL,PROPERTY_SHIPMENT_TYPE,PROPERTY_COURIER_NAME,PROPERTY_CURRENT_STATUS,PROPERTY_TRACKING_SHIP_DATE,PROPERTY_TRACKING_POSTAL_CODE,PROPERTY_SHIPMENT_PACKAGE_COUNT,PROPERTY_CAMPAIGN_NAME,PROPERTY_COHORT_MESSAGE_SEND_COHORT,PROPERTY_EMAIL_DOMAIN,PROPERTY_BOUNCE_TYPE,PROPERTY_CLIENT_TYPE,PROPERTY_CLIENT_OS,PROPERTY_CLIENT_NAME,PROPERTY_CLIENT_OS_FAMILY,PROPERTY_CLIENT_CANONICAL,PROPERTY_PAGE,PROPERTY_IS_SESSION_ACTIVITY,PROPERTY_OS,PROPERTY_SESSION_END,PROPERTY_BROWSER,PROPERTY_LIST,PROPERTY_COMPARE_AT_PRICE,PROPERTY_PRICE,PROPERTY_IMAGE_URL,PROPERTY_BRAND,PROPERTY_CATEGORIES,PROPERTY_VARIANT_OPTION_SIZE,PROPERTY_ATTRIBUTE_WORKOUT,PROPERTY_ATTRIBUTE_AGE,PROPERTY_ATTRIBUTE_FITNESS_GOAL,PROPERTY_ATTRIBUTE_BREAKFAST,PROPERTY_VARIANT_OPTION_TYPE,PROPERTY_COHORT_VARIATION_SEND_COHORT,PROPERTY_METHOD,PROPERTY_TO_NUMBER,PROPERTY_MESSAGE_TYPE,PROPERTY_FROM_NUMBER,PROPERTY_MESSAGE_FORMAT,PROPERTY_MESSAGE_NAME,PROPERTY_CARRIER_DELIVERY_STATUS,PROPERTY_FAILURE_TYPE,PROPERTY_FAILURE_SOURCE,PROPERTY_MESSAGE_BODY,PROPERTY_EMAIL,PROPERTY_ATTRIBUTE_KIT,PROPERTY_VENDOR_ERROR_CODE,PROPERTY_ORDR_DETAILS,PROPERTY_DATE_TO_SEND_GIFT,PROPERTY_RECIPIENT_NAME,PROPERTY_ATTRIBUTE_FACEBOOK_ORDER_RETAILER_ID,PROPERTY_SENDER_S_NAME,PROPERTY_ATTRIBUTE_KITID,PROPERTY_CONVERSATION_ID,PROPERTY_CONVERSATION_LINK,PROPERTY_CONVERSATION_CHANNEL,PROPERTY_RATING,PROPERTY_SCORE,PROPERTY_FROM_PHONE_REGION,PROPERTY_SHIPMENT_CARRIER,PROPERTY_SHIPMENT_STATUS,PROPERTY_TO_PHONE_REGION +id,person_id,campaign_id,flow_id,flow_message_id,uuid,datetime,timestamp,_fivetran_deleted,metric_id,type,_variation,property_value,property_source_name,property_extra,property_shipping_rate,property_items,property_tags,property_item_count,property_collections,_fivetran_synced,property_currency_code,property_product_id,property_quantity,property_name,property_variant_option_title,property_sku,property_variant_name,property_vendor,property_event_id,property_total_discounts,property_attribution,property_discount_codes,property_fulfillment_hours,property_fulfillment_status,property_subject,property_title,property_message_interaction,property_url,property_shipment_type,property_courier_name,property_current_status,property_tracking_ship_date,property_tracking_postal_code,property_shipment_package_count,property_campaign_name,property_cohort_message_send_cohort,property_email_domain,property_bounce_type,property_client_type,property_client_os,property_client_name,property_client_os_family,property_client_canonical,property_page,property_is_session_activity,property_os,property_session_end,property_browser,property_list,property_compare_at_price,property_price,property_image_url,property_brand,property_categories,property_variant_option_size,property_attribute_workout,property_attribute_age,property_attribute_fitness_goal,property_attribute_breakfast,property_variant_option_type,property_cohort_variation_send_cohort,property_method,property_to_number,property_message_type,property_from_number,property_message_format,property_message_name,property_carrier_delivery_status,property_failure_type,property_failure_source,property_message_body,property_email,property_attribute_kit,property_vendor_error_code,property_ordr_details,property_date_to_send_gift,property_recipient_name,property_attribute_facebook_order_retailer_id,property_sender_s_name,property_attribute_kitid,property_conversation_id,property_conversation_link,property_conversation_channel,property_rating,property_score,property_from_phone_region,property_shipment_carrier,property_shipment_status,property_to_phone_region 3gBNHPGp,MCX88q,LyZ7re,,,6a372080-7778-11ea-8001-74013c544d9e,2020-04-05 20:02:45.000,2020-04-05 20:02:45,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:41.959,,,,,,,,,LyZ7re:325688435042320941348463595497466121544,,,,,,brand new chopstcks,,,,,,,,,,Instagram Live,1586116963:LyZ7re,gmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116963:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, 3gBNHPJn,LRqEsZ,LyZ7re,,,6905f380-7778-11ea-8001-75169aac60b8,2020-04-05 20:02:43.000,2020-04-05 20:02:43,FALSE,Q4q6PC,Received Email,H2Qz6F,,,,,,,,,2021-04-09 18:37:34.733,,,,,,,,,LyZ7re:325682810555835891208873492218887818568,,,,,,brand new shorts,,,,,,,,,,Instagram Live,1586116962:LyZ7re,hotmail.com,,,,,,,,,,,,,,,,,,,,,,,,1586116962:H2Qz6F,,,,,,,,,,,,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_flow.csv b/integration_tests/seeds/klaviyo_flow.csv index 3d20a68..b49f459 100644 --- a/integration_tests/seeds/klaviyo_flow.csv +++ b/integration_tests/seeds/klaviyo_flow.csv @@ -1,3 +1,3 @@ -ID,NAME,STATUS,CREATED,UPDATED,CUSTOMER_FILTER,_FIVETRAN_DELETED,_FIVETRAN_SYNCED,ARCHIVED,TRIGGER_TYPE +id,name,status,created,updated,customer_filter,_fivetran_deleted,_fivetran_synced,archived,trigger_type X839zt,Welcome SMS - Redesign,live,2020-10-12 22:56:59.000,2021-02-10 18:34:39.000,"{""stanzas"": [ {""criteria"": [{""timeframe"": ""alltime"",""type"": ""customer-in-flow""}]}]}",FALSE,2021-04-14 07:46:13.280,false,Metric S6uajk,Abandoned Cart - NEW VERSION - Katelyn & DW Working,draft,2020-07-08 20:55:56.000,2020-07-09 16:27:58.000,"{""stanzas"": [{""criteria"": [{""operator"": ""nin"",""region_id"": ""EUROPEAN_UNION"",""type"": ""customer-location""}]}]}",FALSE,2021-04-14 07:46:13.285,false,Added to List \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_metric.csv b/integration_tests/seeds/klaviyo_metric.csv index a13bd79..7bf5285 100644 --- a/integration_tests/seeds/klaviyo_metric.csv +++ b/integration_tests/seeds/klaviyo_metric.csv @@ -1,3 +1,3 @@ -ID,NAME,INTEGRATION_ID,CREATED,UPDATED,_FIVETRAN_DELETED,_FIVETRAN_SYNCED,INTEGRATION_CATEGORY,INTEGRATION_NAME +id,name,integration_id,created,updated,_fivetran_deleted,_fivetran_synced,integration_category,integration_name MNdFpi,Cancelled Order,0eMvjm,2019-06-19 19:10:26.000,2019-07-01 15:16:51.000,FALSE,2021-04-14 07:46:12.973,Email,Mailchimp Vx2d6i,Received SMS,0rG4eQ,2020-04-15 15:15:18.000,2020-04-15 15:15:18.000,FALSE,2021-04-14 07:46:12.974,ecommerce,Shopify \ No newline at end of file diff --git a/integration_tests/seeds/klaviyo_person.csv b/integration_tests/seeds/klaviyo_person.csv index b78a8da..7a6572a 100644 --- a/integration_tests/seeds/klaviyo_person.csv +++ b/integration_tests/seeds/klaviyo_person.csv @@ -1,3 +1,3 @@ -ID,UPDATED,CREATED,_FIVETRAN_DELETED,FIRST_NAME,LAST_NAME,ADDRESS_1,ADDRESS_2,TITLE,TIMEZONE,ORGANIZATION,REGION,LONGITUDE,LATITUDE,PHONE_NUMBER,COUNTRY,ZIP,CITY,EMAIL,CUSTOM_OBJECT,CUSTOM_EMAIL,CUSTOM_ACCEPTS_MARKETING,CUSTOM_SHOPIFY_TAGS,_FIVETRAN_SYNCED,CUSTOM_CONSENT_FORM_ID,CUSTOM_SOURCE,CUSTOM_CONSENT_METHOD,CUSTOM_CONSENT,CUSTOM_GIFT_GIVER_NAME,CUSTOM_STATE,CUSTOM_FLOW,CUSTOM_GIFT_RECIPIENT_NAME,CUSTOM_LAST_SIGN_IN,CUSTOM_CONSENT_FORM_VERSION,CUSTOM_CONSENT_TIMESTAMP,CUSTOM_MAIL_CHIMP_RATING,CUSTOM_GIFT_RECIPIENT_EMAIL,CUSTOM_GIFT_OPTIONS,CUSTOM_ADDRESS,CUSTOM_SUBSCRIPTION_EXPIRATION,CUSTOM_CITY,CUSTOM_ADDRESS_LINE_2,CUSTOM_ZIPCODE,CUSTOM_EXPECTED_DATE_OF_NEXT_ORDER,CUSTOM_SMS_ATTENTIVE_SIGNUP,CUSTOM_PHONE,CUSTOM_AFTERPAY_ORDER,CUSTOM_PHONE_NUMBER_REGION,CUSTOM_NAME,CUSTOM_REFERRER_NAME,CUSTOM_REFERRER_EMAIL,CUSTOM_BIRTHDAY,CUSTOM_FIRST_PURCHASE_DATE_,CUSTOM_UNENGAGED,CUSTOM_LANDING_PAGE_TAG,CUSTOM_FITNESS_GOAL,CUSTOM_AGE,CUSTOM_WORKOUT,CUSTOM_QUIZ,CUSTOM_BREAKFAST,CUSTOM_QUIZ_DATA,CUSTOM_RAF_SUBSCRIBE,CUSTOM_SUPPRESS,CUSTOM_SMS_CONSENT,CUSTOM_GIFT_OPTION_DREAMBELT,CUSTOM_GIFT_OPTION_STARTER,CUSTOM_GIFT_OPTION_MACHINE,CUSTOM_LANDINGPAGE_TAG,LAST_EVENT_DATE +id,updated,created,_fivetran_deleted,first_name,last_name,address_1,address_2,title,timezone,organization,region,longitude,latitude,phone_number,country,zip,city,email,custom_object,custom_email,custom_accepts_marketing,custom_shopify_tags,_fivetran_synced,custom_consent_form_id,custom_source,custom_consent_method,custom_consent,custom_gift_giver_name,custom_state,custom_flow,custom_gift_recipient_name,custom_last_sign_in,custom_consent_form_version,custom_consent_timestamp,custom_mail_chimp_rating,custom_gift_recipient_email,custom_gift_options,custom_address,custom_subscription_expiration,custom_city,custom_address_line_2,custom_zipcode,custom_expected_date_of_next_order,custom_sms_attentive_signup,custom_phone,custom_afterpay_order,custom_phone_number_region,custom_name,custom_referrer_name,custom_referrer_email,custom_birthday,custom_first_purchase_date_,custom_unengaged,custom_landing_page_tag,custom_fitness_goal,custom_age,custom_workout,custom_quiz,custom_breakfast,custom_quiz_data,custom_raf_subscribe,custom_suppress,custom_sms_consent,custom_gift_option_dreambelt,custom_gift_option_starter,custom_gift_option_machine,custom_landingpage_tag,last_event_date 01F35SYVPX6MS88ARK37864GJX,2021-04-13 14:44:06.000,2021-04-13 14:25:27.000,FALSE,,,,,,America/New_York,,New York,-12.0851,421.6117,5165555555,United States,,Brooklyn,nobody@hotmail.com,person,nobody@hotmail.com,,,2021-04-14 07:47:10.440,X2GkGj,Redesign - Mobile Email + SMS,Klaviyo Form,,,,,,,190980,,,,,,,,,,,,,,US,,,,,,,,,,,,,,,,,,,,,2023-07-18 12:43:52 UTC 01F366M1ZERT4HM2Z9R2V9G7BY,2021-04-13 18:09:10.000,2021-04-13 18:06:44.000,FALSE,MArio,Lopez,1600 Penn,,,America/Chicago,,Kansas,,,5165555555,United States,64224,Jefferson City,mrgoogle@gmail.com,person,mrgoogle@gmail.com,false,[],2021-04-14 07:47:45.376,,,,,,,,,,,,,,,,,,,,05/16/2021,,,,,,,,,2021-04-13,,,,,,,,,,,,,,,,2023-09-08 09:19:46 UTC \ No newline at end of file From 3e408a88b02a913c23e050863428304101edafcb Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 13 Nov 2023 10:51:38 -0500 Subject: [PATCH 20/21] fix --- integration_tests/dbt_project.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index a94f05a..2b4582f 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -203,7 +203,7 @@ seeds: klaviyo_person: +tags: klaviyo +column_types: - PHONE_NUMBER: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + phone_number: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" klaviyo_flow: +tags: klaviyo +enabled: "{{ true if target.type != 'snowflake' else false }}" @@ -211,10 +211,10 @@ seeds: klaviyo_event: +tags: klaviyo +column_types: - FLOW_ID: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - CAMPAIGN_ID: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - DATETIME: timestamp - TIMESTAMP: timestamp + flow_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + campaign_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" + datetime: timestamp + timestamp: timestamp klaviyo_metric: +tags: klaviyo klaviyo_campaign: From 2f00dd687797dcb313ab4c3ac6397da455bf3589 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 13 Nov 2023 16:05:26 -0500 Subject: [PATCH 21/21] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a160b57..a2e8c78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 🚨 Breaking Changes 🚨: -- This package now points to the following ranges of the upstream packages: +- This package now points to the following ranges of the upstream packages. We recommend a `dbt run --full-refresh` to capture all the latest data within incremental models. ``` - package: fivetran/shopify