Merge pull request #12 from fivetran/dependency-upgrades

Dependency upgrades

Author: DylanBaker

.circleci/config.yml

@@ -15,12 +15,23 @@ jobs:
       - run:
           name: "Setup dbt"
           command: |
+            sudo apt install libsasl2-dev
             python3 -m venv venv
             . venv/bin/activate
             pip install --upgrade pip setuptools
             pip install -r integration_tests/requirements.txt
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+      - run:
+          name: "Run Tests - Spark"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target spark --full-refresh
+            dbt run --target spark --full-refresh
+            dbt test --target spark
       - run:
           name: "Run Tests - Redshift"
           command: |

README.md

@@ -170,7 +170,7 @@ models:
 
 ## Database Support
 
-This package has been tested on BigQuery, Snowflake, and Redshift.
+This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Spark.
 
 ## Contributions
 
@@ -180,12 +180,14 @@ or open PRs against `master`. Check out
 on the best workflow for contributing to a package.
 
 ## Resources:
+- Provide [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next
+- Have questions, feedback, or need help? Book a time during our office hours [using Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at [email protected]
 - Find all of Fivetran's pre-built dbt packages in our [dbt hub](https://hub.getdbt.com/fivetran/)
-- Learn more about Fivetran [in the Fivetran docs](https://fivetran.com/docs)
+- Learn how to orchestrate [dbt transformations with Fivetran](https://fivetran.com/docs/transformations/dbt)
+- Learn more about Fivetran overall [in our docs](https://fivetran.com/docs)
 - Check out [Fivetran's blog](https://fivetran.com/blog)
 - Learn more about dbt [in the dbt docs](https://docs.getdbt.com/docs/introduction)
 - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
 - Join the [chat](http://slack.getdbt.com/) on Slack for live discussions and support
 - Find [dbt events](https://events.getdbt.com) near you
-- Find [dbt events](https://events.getdbt.com) near you
-- Check out [dbt's blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
+- Check out [the dbt blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

dbt_project.yml

@@ -2,7 +2,7 @@ name: 'ad_reporting'
 version: '1.0.0'
 config-version: 2
 
-require-dbt-version: [">=0.18.0", "<0.20.0"]
+require-dbt-version: [">=0.19.0", "<0.20.0"]
 
 vars:
   ad_reporting__pinterest_enabled: true
@@ -12,6 +12,8 @@ vars:
   ad_reporting__twitter_ads_enabled: true
   ad_reporting__facebook_ads_enabled: true
   ad_reporting__snapchat_ads_enabled: true
+  dbt_utils_dispatch_list: ['spark_utils', 'fivetran_utils']
+  fivetran_utils_dispatch_list: ['spark_utils']
 
 models:
   ad_reporting:

integration_tests/ci/sample.profiles.yml

@@ -34,4 +34,16 @@ integration_tests:
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
       schema: ad_reporting_integration_tests
-      threads: 8
+      threads: 8
+    spark:
+      type: spark
+      method: http
+      schema: ad_reporting_integration_tests
+      host: "{{ env_var('CI_SPARK_DBT_HOST') }}"
+      organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}"
+      token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}"
+      cluster: "{{ env_var('CI_SPARK_DBT_CLUSTER') }}"
+      port: 443
+      connect_timeout: 60
+      connect_retries: 5
+      threads: 4

integration_tests/dbt_project.yml

@@ -158,9 +158,9 @@ seeds:
     ### Facebook Seeds
 
     facebook_ads_creative_history_data:
-      +column_types:
-        page_link: "{{ 'string' if target.name == 'bigquery' else 'varchar' }}"
-        template_page_link: "{{ 'string' if target.name == 'bigquery' else 'varchar' }}"
+      +column_types:        
+        page_link: "{{ 'string' if target.name in ['bigquery','spark'] else 'varchar' }}"
+        template_page_link: "{{ 'string' if target.name in ['bigquery','spark'] else 'varchar' }}"
         id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}"
         account_id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}"
     facebook_ads_account_history_data:

integration_tests/requirements.txt

@@ -1 +1,3 @@
-dbt==0.19.0
+dbt==0.19.1
+dbt-spark==0.19.1
+dbt-spark[PyHive]==0.19.1

packages.yml

@@ -1,22 +1,22 @@
 packages:
 
   - package: fivetran/pinterest
-    version: [">=0.2.0", "<0.3.0"]
+    version: [">=0.3.0", "<0.4.0"]
 
   - package: fivetran/microsoft_ads
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 
   - package: fivetran/linkedin
     version: [">=0.2.0", "<0.3.0"]
 
   - package: fivetran/google_ads
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 
   - package: fivetran/twitter_ads
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 
   - package: fivetran/facebook_ads
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 
   - package: fivetran/snapchat_ads
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.1.1", "<0.2.0"]