Merge pull request #11 from fivetran/MagicBot/dbt-utils-cross-db-migr…

…ation Updates for dbt-utils to dbt-core cross-db macro migration
fivetran · Dec 20, 2022 · 32f1915 · 32f1915
2 parents 037a882 + 862bb6e
commit 32f1915
Show file tree

Hide file tree

Showing 23 changed files with 289 additions and 183 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Export secrets for Docker containers.
+# Restrict exposing secrets only to the steps that need them
+export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -0,0 +1,73 @@
+steps:
+  - label: ":postgres: Run Tests - Postgres"
+    key: "run-dbt-postgres"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_POSTGRES_DBT_DBNAME"
+            - "CI_POSTGRES_DBT_HOST"
+            - "CI_POSTGRES_DBT_PASS"
+            - "CI_POSTGRES_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh postgres 
+
+  - label: ":snowflake-db: Run Tests - Snowflake"
+    key: "run_dbt_snowflake"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_SNOWFLAKE_DBT_ACCOUNT"
+            - "CI_SNOWFLAKE_DBT_DATABASE"
+            - "CI_SNOWFLAKE_DBT_PASS"
+            - "CI_SNOWFLAKE_DBT_ROLE"
+            - "CI_SNOWFLAKE_DBT_USER"
+            - "CI_SNOWFLAKE_DBT_WAREHOUSE"
+    commands: |
+      bash .buildkite/scripts/run_models.sh snowflake
+
+  - label: ":gcloud: Run Tests - BigQuery"
+    key: "run_dbt_bigquery"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "GCLOUD_SERVICE_KEY"
+    commands: |
+      bash .buildkite/scripts/run_models.sh bigquery
+
+  - label: ":amazon-redshift: Run Tests - Redshift"
+    key: "run_dbt_redshift"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_REDSHIFT_DBT_DBNAME"
+            - "CI_REDSHIFT_DBT_HOST"
+            - "CI_REDSHIFT_DBT_PASS"
+            - "CI_REDSHIFT_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":bricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euo pipefail
+
+apt-get update
+apt-get install libsasl2-dev
+
+python3 -m venv venv
+. venv/bin/activate
+pip install --upgrade pip setuptools
+pip install -r integration_tests/requirements.txt
+mkdir -p ~/.dbt
+cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+
+db=$1
+echo `pwd`
+cd integration_tests
+dbt deps
+dbt seed --target "$db" --full-refresh
+dbt run --target "$db" --full-refresh
+dbt test --target "$db"
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -28,9 +28,9 @@
 
 **How did you test the PR changes?** 
 <!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] CircleCi). --> 
+<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] CircleCi <!--- CircleCi testing is only applicable to Fivetran employees. --> 
+- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
 - [ ] Local (please provide additional testing details below)
 
 **Select which warehouse(s) were used to test the PR**

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,43 @@
+# dbt_shopify_holistic_reporting v0.3.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #11](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/11/) includes the following breaking changes:
+- Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:
+    - `any_value`
+    - `bool_or`
+    - `cast_bool_to_text`
+    - `concat`
+    - `date_trunc`
+    - `dateadd`
+    - `datediff`
+    - `escape_single_quotes`
+    - `except`
+    - `hash`
+    - `intersect`
+    - `last_day`
+    - `length`
+    - `listagg`
+    - `position`
+    - `replace`
+    - `right`
+    - `safe_cast`
+    - `split_part`
+    - `string_literal`
+    - `type_bigint`
+    - `type_float`
+    - `type_int`
+    - `type_numeric`
+    - `type_string`
+    - `type_timestamp`
+    - `array_append`
+    - `array_concat`
+    - `array_construct`
+- For `current_timestamp` and `current_timestamp_in_utc` macros, the dispatch AND the macro names have been updated to the below, respectively:
+    - `dbt.current_timestamp_backcompat`
+    - `dbt.current_timestamp_in_utc_backcompat`
+- `dbt_utils.surrogate_key` has also been updated to `dbt_utils.generate_surrogate_key`. Since the method for creating surrogate keys differ, we suggest all users do a `full-refresh` for the most accurate data. For more information, please refer to dbt-utils [release notes](https://github.com/dbt-labs/dbt-utils/releases) for this update.
+- Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+
 # dbt_shopify_holistic_reporting v0.2.0
 ## Bug Fixes
 - Adjusts the incremental logic in the `shopify_holistic_reporting__orders_attribution` model. Previously, on incremental runs, this model transformed only newly-_created_ orders, comparing each order's `created_timestamp` to the `max(created_timestamp)` in the model. Now, the model will also transform newly-_updated_ orders and use `updated_timestamp` instead of `created_timestamp` to determine if an order should be included in an incremental run ([#9](https://github.com/fivetran/dbt_shopify_holistic_reporting/pull/9)).

diff --git a/README.md b/README.md
@@ -1,4 +1,15 @@
-[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+<p align="center">
+    <a alt="License"
+        href="https://github.com/fivetran/dbt_shopify_holistic_reporting/blob/main/LICENSE">
+        <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
+    <a alt="dbt-core">
+        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_<2.0.0-orange.svg" /></a>
+    <a alt="Maintained?">
+        <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
+    <a alt="PRs">
+        <img src="https://img.shields.io/badge/Contributions-welcome-blueviolet" /></a>
+</p>
+
 # Shopify Holistic Reporting
 
 This package builds off of the [Shopify dbt package](https://github.com/fivetran/dbt_shopify) to weave together your Shopify e-commerce data with insights from marketing connectors. Currently, this package supports combining Shopify with email and SMS marketing data from Fivetran's [Klaviyo dbt package](https://github.com/fivetran/dbt_klaviyo).
@@ -31,7 +42,7 @@ Include in your `packages.yml`
 ```yaml
 packages:
   - package: fivetran/shopify_holistic_reporting
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.3.0", "<0.4.0"]
 ```
 
 ## Configurations
@@ -76,6 +87,33 @@ models:
 
 Models from the individual [Shopify](https://github.com/fivetran/dbt_shopify/#changing-the-build-schema) and [Klaviyo](https://github.com/fivetran/dbt_klaviyo/#changing-the-build-schema) packages will be written their respective schemas. 
 
+# 🔍 Does this package have dependencies?
+This dbt package is dependent on the following dbt packages. Please be aware that these dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site.
+> IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts.
+```yml
+packages:
+    - package: fivetran/shopify
+      version: [">=0.7.0", "<0.8.0"]
+
+    - package: fivetran/shopify_source
+      version: [">=0.7.0", "<0.8.0"]
+
+    - package: fivetran/klaviyo
+      version: [">=0.5.0", "<0.6.0"]
+
+    - package: fivetran/klaviyo_source
+      version: [">=0.5.0", "<0.6.0"]
+
+    - package: fivetran/fivetran_utils
+      version: [">=0.4.0", "<0.5.0"]
+
+    - package: dbt-labs/dbt_utils
+      version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
+```
+
 
 ## Contributions
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,46 +1,41 @@
 name: 'shopify_holistic_reporting'
-version: '0.2.0'
+version: '0.3.0'
 config-version: 2
-
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
+require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   shopify_holistic_reporting:
     +materialized: table
     +schema: shopify_holistic
     intermediate:
-      +materialized: view # can't be ephemeral due to the use of dbt_utils.star()
-
-
+      +materialized: view
 vars:
   shopify_holistic_reporting:
-    klaviyo__email_attribution_lookback: 120 # in hours - 5 days -> https://help.klaviyo.com/hc/en-us/articles/115005248128#conversion-tracking1
-    klaviyo__sms_attribution_lookback: 24 # in hours -> https://help.klaviyo.com/hc/en-us/articles/115005248128#sms-conversion-tracking7
-
+    klaviyo__email_attribution_lookback: 120
+    klaviyo__sms_attribution_lookback: 24
     klaviyo__eligible_attribution_events: # default in klaviyo -> https://help.klaviyo.com/hc/en-us/articles/115005248128 
-      - 'opened email'
-      - 'clicked email'
-      - 'clicked sms'
+    - 'opened email'
+    - 'clicked email'
+    - 'clicked sms'
 
-    klaviyo__count_metrics: 
-      - 'Active on Site' # default API metrics
-      - 'Viewed Product'
-      - 'Ordered Product' # some default Shopify metrics
-      - 'Placed Order' # there are X 'ordered product' events for every `placed order` purchase event of X products
-      - 'Refunded Order' # note: klaviyo only tracks FULL refunds (not partial)
-      - 'Received Email' # default email and sms conversion metrics in Klaviyo
-      - 'Clicked Email'
-      - 'Opened Email'
-      - 'Marked Email as Spam'
-      - 'Unsubscribed'
-      - 'Received SMS'
-      - 'Clicked SMS'
-      - 'Sent SMS'
-      - 'Unsubscribed from SMS'
+    klaviyo__count_metrics:
+    - 'Active on Site'   # default API metrics
+    - 'Viewed Product'
+    - 'Ordered Product'   # some default Shopify metrics
+    - 'Placed Order'   # there are X 'ordered product' events for every `placed order` purchase event of X products
+    - 'Refunded Order'   # note: klaviyo only tracks FULL refunds (not partial)
+    - 'Received Email'   # default email and sms conversion metrics in Klaviyo
+    - 'Clicked Email'
+    - 'Opened Email'
+    - 'Marked Email as Spam'
+    - 'Unsubscribed'
+    - 'Received SMS'
+    - 'Clicked SMS'
+    - 'Sent SMS'
+    - 'Unsubscribed from SMS'
 
     klaviyo__sum_revenue_metrics:
-      - 'Refunded Order'
-      - 'Placed Order'
-      - 'Ordered Product'
-      - 'checkout started'
-      - 'cancelled order'
+    - 'Refunded Order'
+    - 'Placed Order'
+    - 'Ordered Product'
+    - 'checkout started'
+    - 'cancelled order'
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/graph.gpickle b/docs/graph.gpickle