Release/v0.12.0 (#82)

* feature/performance-updates

* update profile

* feature/performance-updates

* update materialization

* update for redshift

* surrogate key for redshift

* surrogate key for redshift

* update materializations

* update materializations

* update materializations

* update materializations

* update changelog, readme, regen docs

* update yml

* update materializations

* update changelog

* update changelog

* update changelog && regen docs

* update changelog

* update packages

* add databricks sql warehouse

* update sql warehouse

* update sql warehouse

* Remove order discount code filter, add index in (#81)

* Remove order discount code filter, add index in

* PR fixes

* CHANGELOG

* CHANGELOG

* regen docs

* Update packages.yml

* Update CHANGELOG.md

* Update CHANGELOG.md

---------

Co-authored-by: Avinash Kunnath <[email protected]>

Author: fivetran-catfritz

.buildkite/hooks/pre-command

@@ -22,4 +22,6 @@ export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secr
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -71,4 +71,19 @@ steps:
             - "CI_DATABRICKS_DBT_TOKEN"
             - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
-      bash .buildkite/scripts/run_models.sh databricks
+      bash .buildkite/scripts/run_models.sh databricks
+
+  - label: ":databricks: :database: Run Tests - Databricks SQL Warehouse"
+    key: "run_dbt_databricks_sql"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_SQL_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_SQL_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks-sql

.buildkite/scripts/run_models.sh

@@ -16,9 +16,20 @@ db=$1
 echo `pwd`
 cd integration_tests
 dbt deps
+
+if [ "$db" = "databricks-sql" ]; then
+dbt seed --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt test --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db"
+dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true}' --target "$db" --full-refresh
+dbt test --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db"
+dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
+
+else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
+fi

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -4,48 +4,27 @@
 **This PR will result in the following new package version:**
 <!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
 
-**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
+<!--- Copy/paste the CHANGELOG for this version below. -->
 
 ## PR Checklist
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt compile
-- [ ] dbt run –full-refresh
-- [ ] dbt run
-- [ ] dbt test
-- [ ] dbt run –vars (if applicable)
+- [ ] dbt run –full-refresh && dbt test
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked and tagged
-- [ ] You are assigned to the corresponding issue and this PR
+- [ ] The appropriate issue has been linked, tagged, and properly assigned
+- [ ] All necessary documentation and version upgrades have been applied
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
 - [ ] BuildKite integration tests are passing
+- [ ] Detailed validation steps have been provided below
 
 ### Detailed Validation
-Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
-- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
-- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
-- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
-<!--- Provide the steps you took to validate your changes below. -->
-
-### Standard Updates
-Please acknowledge that your PR contains the following standard updates:
-- Package versioning has been appropriately indexed in the following locations:
-    - [ ] indexed within dbt_project.yml
-    - [ ] indexed within integration_tests/dbt_project.yml
-- [ ] CHANGELOG has individual entries for each respective change in this PR
-    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
-- [ ] README updates have been applied (if applicable)
-    <!--- Remember to check the following README locations for common updates. →
-        <!--- Suggested install range (needed for breaking changes) →
-        <!--- Dependency matrix is appropriately updated (if applicable) →
-        <!--- New variable documentation (if applicable) -->
-- [ ] DECISIONLOG updates have been updated (if applicable)
-- [ ] Appropriate yml documentation has been added (if applicable)
+Please share any and all of your validation steps:
 
-### dbt Docs
-Please acknowledge that after the above were all completed the below were applied to your branch:
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
+<!--- Provide the steps you took to validate your changes below. -->
 
 ### If you had to summarize this PR in an emoji, which would it be?
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+:dancer:

CHANGELOG.md

@@ -1,3 +1,30 @@
+# dbt_shopify_source v0.12.0
+
+[PR #79](https://github.com/fivetran/dbt_shopify_source/pull/79) introduces the following changes: 
+## 🚨 Breaking Changes 🚨
+- To reduce storage, updated default materialization of staging models from tables to views. 
+  - Note that `stg_shopify__metafield` will still be materialized as a table for downstream use.
+>  ⚠️ Running a `--full-refresh` will be required if you have previously run these staging models as tables and get the following error: 
+> ```
+> Trying to create view <model path> but it currently exists as a table. Either drop <model path> manually, or run dbt with `--full-refresh` and dbt will drop it for you.
+> ```
+
+## Under the Hood
+- Updated the maintainer PR template to the current format.
+- Added integration testing pipeline for Databricks SQL Warehouse.
+
+[PR #81](https://github.com/fivetran/dbt_shopify_source/pull/81) introduces the following changes: 
+## 🪲 Bug Fixes 🪛
+- Removed the `index` filter in `stg_shopify__order_discount_code`, as we were erroneously filtering out multiple discounts for an order since `index` is meant to pair with `order_id` as the unique identifier for this source.
+- Added `index` as a field in `stg_shopify__order_discount_code`, as it is part of the primary key.
+
+## 📝 Documentation Updates 📝
+- Added `index` documentation to our `src_shopify.yml` and `stg_shopify.yml`.
+- Updated the `unique_combination_of_columns` test on `stg_shopify__order_discount_code` to correctly check on `index` with `order_id` and `source_relation` rather than `code`.
+
+## 🔧 Under the Hood 🔩
+- Updated the pull request templates.
+
 # dbt_shopify_source v0.11.0
 [PR #78](https://github.com/fivetran/dbt_shopify_source/pull/78) introduces the following changes: 
 

README.md

@@ -44,7 +44,7 @@ If you  are **not** using the [Shopify transformation package](https://github.co
 ```yml
 packages:
   - package: fivetran/shopify_source
-    version: [">=0.11.0", "<0.12.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.12.0", "<0.13.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ## Step 3: Define database and schema variables

dbt_project.yml

@@ -1,13 +1,14 @@
 name: 'shopify_source'
-version: '0.11.0'
+version: '0.12.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   shopify_source:
     +schema: stg_shopify
-    +materialized: table
-    tmp:
-      +materialized: view
+    +materialized: view
+    stg_shopify__metafield:
+      +materialized: table
+
 vars:
   shopify_source:
     customer_source: "{{ source('shopify','customer') }}"

docs/catalog.json

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_source_integration_tests_9
+      schema: shopify_source_integration_tests_10
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_source_integration_tests_9
+      schema: shopify_source_integration_tests_10
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_source_integration_tests_9
+      schema: shopify_source_integration_tests_10
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,21 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_source_integration_tests_8
+      schema: shopify_source_integration_tests_10
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_source_integration_tests_9
-      threads: 2
+      schema: shopify_source_integration_tests_10
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
+    databricks-sql:
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
+      schema: shopify_source_integrations_tests_sqlw 
+      threads: 8
+      token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
+      type: databricks

docs/manifest.json

@@ -1,20 +1,21 @@
 name: 'shopify_source_integration_tests'
-version: '0.11.0'
+version: '0.12.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  shopify_schema: shopify_source_integration_tests_9
+  shopify_schema: shopify_source_integration_tests_10
+  # shopify_using_fulfillment_event: true # enable when regenerating docs
   shopify_source:
-    shopify_customer_identifier:          "shopify_customer_data"
+    shopify_customer_identifier: "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"
-    shopify_order_line_identifier:        "shopify_order_line_data"
-    shopify_order_identifier:             "shopify_order_data"
-    shopify_product_identifier:           "shopify_product_data"
-    shopify_order_adjustment_identifier:  "shopify_order_adjustment_data"
-    shopify_product_variant_identifier:   "shopify_product_variant_data"
-    shopify_refund_identifier:            "shopify_refund_data"
-    shopify_transaction_identifier:       "shopify_transaction_data"
+    shopify_order_line_identifier: "shopify_order_line_data"
+    shopify_order_identifier: "shopify_order_data"
+    shopify_product_identifier: "shopify_product_data"
+    shopify_order_adjustment_identifier: "shopify_order_adjustment_data"
+    shopify_product_variant_identifier: "shopify_product_variant_data"
+    shopify_refund_identifier: "shopify_refund_data"
+    shopify_transaction_identifier: "shopify_transaction_data"
     shopify_abandoned_checkout_identifier: "shopify_abandoned_checkout_data"
     shopify_collection_product_identifier: "shopify_collection_product_data"
     shopify_collection_identifier: "shopify_collection_data"
@@ -45,6 +46,10 @@ dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
 
+models:
+  shopify_source:
+    +schema: "{{ 'shopify_source_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'shopify_source' }}"
+
 seeds:
   shopify_source_integration_tests:
     +quote_columns: "{{ true if target.type in ('redshift', 'postgres') else false }}"
@@ -130,6 +135,7 @@ seeds:
         usage_count: float
     shopify_order_discount_code_data:
       +column_types:
+        order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         amount: float
     shopify_order_shipping_tax_line_data:
       +column_types:

docs/run_results.json

@@ -1,4 +1,6 @@
 index,order_id,_fivetran_synced,amount,code,type
-1,368347,2022-11-20 08:14:52.957000,11.0,GIFTCARD,percentage
-1,423483,2022-11-19 11:59:50.040000,0.0,SHIPPING2022,shipping
-1,365403,2022-11-20 10:22:23.877000,2.0,GIFTCARD,percentage
+1,2674098602081,2022-11-20 08:14:52.957000,11.0,GIFTCARD,percentage
+2,2674098602081,2022-11-20 08:14:52.957000,5.0,SHIPPING2022,shipping
+3,2674098602081,2022-11-20 08:14:52.957000,1.0,FIXED,fixed_amount
+1,2669516488801,2022-11-19 11:59:50.040000,0.0,SHIPPING2022,shipping
+1,2669509541985,2022-11-20 10:22:23.877000,2.0,GIFTCARD,percentage

integration_tests/ci/sample.profiles.yml

@@ -1320,6 +1320,8 @@ sources:
             description: Associated order ID.
           - name: type
             description: The type of discount - `fixed_amount`, `percentage`, or `shipping`.
+          - name: index
+            description: Pairs with `order_id` to provide unique identifier for order discount code.
 
       - name: abandoned_checkout_shipping_line
         description: Shipping lines associated with abandoned checkouts.

integration_tests/dbt_project.yml

@@ -1542,7 +1542,7 @@ models:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - order_id
-            - code
+            - index
             - source_relation
     columns:
       - name: _fivetran_synced
@@ -1561,6 +1561,9 @@ models:
         description: The type of discount - `fixed_amount`, `percentage`, or `shipping`.
       - name: source_relation
         description: "{{ doc('source_relation') }}"
+      - name: index
+        description: Pairs with `order_id` to provide unique identifier for order discount code.
+      
 
   - name: stg_shopify__abandoned_checkout_shipping_line
     description: Shipping lines associated with abandoned checkouts.

integration_tests/seeds/shopify_order_discount_code_data.csv

@@ -27,14 +27,14 @@ final as (
 
     select 
         order_id,
+        index,
         upper(code) as code,
         type,
         amount,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         source_relation
 
     from fields
-    where index = 1 -- Sanity check. index should not > 1 but open an issue if that's not the case in your data
 )
 
 select *

models/src_shopify.yml

@@ -4,4 +4,4 @@ packages:
 - package: dbt-labs/spark_utils
   version: [">=0.3.0", "<0.4.0"]
 - package: calogica/dbt_date
-  version: [">=0.9.0", "<1.0.0"]
+  version: [">=0.9.0", "<1.0.0"]