Merge pull request #15 from fivetran/feature/databricks-compatibility

Feature/databricks compatibility

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -57,3 +57,18 @@ steps:
             - "CI_REDSHIFT_DBT_USER"
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":databricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

.buildkite/scripts/run_models.sh

@@ -21,3 +21,5 @@ dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run --vars '{mailchimp_using_automations: false, mailchimp_using_segments: false}' --target "$db" --full-refresh
 dbt test --target "$db"
+
+dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -0,0 +1,51 @@
+## PR Overview
+**This PR will address the following Issue/Feature:**
+
+**This PR will result in the following new package version:**
+<!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
+
+**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+
+## PR Checklist
+### Basic Validation
+Please acknowledge that you have successfully performed the following commands locally:
+- [ ] dbt compile
+- [ ] dbt run –full-refresh
+- [ ] dbt run
+- [ ] dbt test
+- [ ] dbt run –vars (if applicable)
+
+Before marking this PR as "ready for review" the following have been applied:
+- [ ] The appropriate issue has been linked and tagged
+- [ ] You are assigned to the corresponding issue and this PR
+- [ ] BuildKite integration tests are passing
+
+### Detailed Validation
+Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
+- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
+- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
+- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+<!--- Provide the steps you took to validate your changes below. -->
+
+### Standard Updates
+Please acknowledge that your PR contains the following standard updates:
+- Package versioning has been appropriately indexed in the following locations:
+    - [ ] indexed within dbt_project.yml
+    - [ ] indexed within integration_tests/dbt_project.yml
+- [ ] CHANGELOG has individual entries for each respective change in this PR
+    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
+- [ ] README updates have been applied (if applicable)
+    <!--- Remember to check the following README locations for common updates. →
+        <!--- Suggested install range (needed for breaking changes) →
+        <!--- Dependency matrix is appropriately updated (if applicable) →
+        <!--- New variable documentation (if applicable) -->
+- [ ] DECISIONLOG updates have been updated (if applicable)
+- [ ] Appropriate yml documentation has been added (if applicable)
+
+### dbt Docs
+Please acknowledge that after the above were all completed the below were applied to your branch:
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
+
+### If you had to summarize this PR in an emoji, which would it be?
+<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
+:dancer:

.github/pull_request_template.md

@@ -1,52 +1,33 @@
-**Are you a current Fivetran customer?** 
-<!--- Please tell us your name, title and company -->
+**Please provide your name and company**
 
-**What change(s) does this PR introduce?** 
-<!--- Describe what changes your PR introduces to the package and how to leverage this new feature. -->
+**Link the issue/feature request which this PR is meant to address**
+<!--- If an issue was not created, please create one first so we may discuss the PR prior to opening one. -->
 
-**Did you update the CHANGELOG?** 
+**Detail what changes this PR introduces and how this addresses the issue/feature request linked above.**
+
+**How did you validate the changes introduced within this PR?**
+
+**Which warehouse did you use to develop these changes?**
+
+**Did you update the CHANGELOG?**
 <!--- Please update the new package version’s CHANGELOG entry detailing the changes included in this PR. -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Does this PR introduce a breaking change?**
-<!--- Does this PR introduce changes that will cause current package users' jobs to fail or require a `--full-refresh`? -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes (please provide breaking change details below.)
-- [ ] No  (please provide an explanation as to how the change is non-breaking below.)
-
-**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)** 
+**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)**
 <!--- The dbt_project.yml and the integration_tests/dbt_project.yml files contain the version number. Be sure to upgrade it accordingly -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Is this PR in response to a previously created Bug or Feature Request**
-<!--- If an Issue was created it is helpful to track the progress by linking it in the PR. -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes, Issue/Feature [link bug/feature number here]
-- [ ] No 
-
-**How did you test the PR changes?** 
-<!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
-- [ ] Local (please provide additional testing details below)
-
-**Select which warehouse(s) were used to test the PR**
-<!--- To check a warehouse remove the space and insert an x in the box (eg. [x] Bigquery). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] BigQuery
-- [ ] Redshift
-- [ ] Snowflake
-- [ ] Postgres
-- [ ] Databricks
-- [ ] Other (provide details below)
-
 **Provide an emoji that best describes your current mood**
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
 :dancer:
 
 **Feedback**
 
 We are so excited you decided to contribute to the Fivetran community dbt package! We continue to work to improve the packages and would greatly appreciate your [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next.
+
+**PR Template** 
+- [Community Pull Request Template](?expand=1&template=pull_request_template.md) (default)
+
+- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)

CHANGELOG.md

@@ -1,3 +1,10 @@
+# dbt_mailchimp_source v0.5.0
+## 🎉 Feature Update
+- Databricks compatibility! ([#15](https://github.com/fivetran/dbt_mailchimp_source/pull/15))
+## Under the Hood:
+- Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job. ([#14](https://github.com/fivetran/dbt_mailchimp_source/pull/14))
+- Updated the pull request [templates](/.github). ([#14](https://github.com/fivetran/dbt_mailchimp_source/pull/14))
+
 # dbt_mailchimp_source v0.4.0
 [PR #12](https://github.com/fivetran/dbt_mailchimp_source/pull/12) includes the following breaking changes:
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -23,15 +23,23 @@
 ## Step 1: Prerequisites
 To use this dbt package, you must have the following:
 - At least one Fivetran Mailchimp connector syncing data into your destination. 
-- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination.
+- A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
 
-## Step 2: Install the package
-Include the following mailchimp_source package version in your `packages.yml` file.
+### Databricks dispatch configuration
+If you are using a Databricks destination with this package, you must add the following (or a variation of the following) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+```yml
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']
+```
+
+## Step 2: Install the package (skip if also using the `mailchimp` transformation package)
+If you  are **NOT** using the [Mailchimp transformation package](https://github.com/fivetran/dbt_mailchimp), include the following package version in your `packages.yml` file. If you are installing the transform package, the source package is automatically installed as a dependency.
 > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 ```yml
 packages:
   - package: fivetran/mailchimp_source
-    version: [">=0.4.0", "<0.5.0"]
+    version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ## Step 3: Define database and schema variables

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'mailchimp_source'
-version: '0.4.0'
+version: '0.5.0'
 config-version: 2
 models:
   mailchimp_source:

integration_tests/ci/sample.profiles.yml

@@ -45,7 +45,7 @@ integration_tests:
       schema: mailchimp_source_integration_tests
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: mailchimp_source_integration_tests

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'mailchimp_source_integration_tests'
-version: '0.4.0'
+version: '0.5.0'
 config-version: 2
 profile: 'integration_tests'
 vars:
@@ -61,3 +61,7 @@ seeds:
     unsubscribe:
       +column_types:
         timestamp: timestamp
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['spark_utils', 'dbt_utils']

models/src_mailchimp.yml

@@ -2,7 +2,7 @@ version: 2
 sources:
   - name: mailchimp
     schema: "{{ var('mailchimp_schema', 'mailchimp') }}"
-    database: "{{ var('mailchimp_database', target.database) }}"
+    database: "{% if target.type != 'spark' %}{{ var('mailchimp_database', target.database) }}{% endif %}"
     loader: fivetran
     loaded_at_field: _fivetran_synced
     freshness: