Add tox support for testing redshift, bigquery and snowflake

.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+# **what?**
+# Run tests for dbt-date against supported adapters
+
+# **why?**
+# To ensure that dbt-date works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ workflow_dispatch:
+
+jobs:
+ run-tests:
+ uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+ with:
+ # redshift
+ REDSHIFT_HOST: ${{ vars.REDSHIFT_TEST_HOST }}
+ REDSHIFT_USER: ${{ vars.REDSHIFT_TEST_USER }}
+ REDSHIFT_DATABASE: ${{ vars.REDSHIFT_TEST_DBNAME }}
+ REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}"
+ REDSHIFT_PORT: ${{ vars.REDSHIFT_TEST_PORT }}
+ # snowflake
+ SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_TEST_USER }}
+ SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_TEST_ROLE }}
+ SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_TEST_DBNAME }}
+ SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_WHNAME }}
+ SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
+ secrets:
+ DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+ SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+ DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_TEST_PASS }}

.github/workflows/integration_tests.yml

@@ -41,19 +41,25 @@ jobs:
  dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target ${{ matrix.data-platform }}
  dbt run-operation dbt_external_tables.stage_external_sources --target ${{ matrix.data-platform }}
  dbt test --target ${{ matrix.data-platform }}
- env: 
- REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
- REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
- REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
- REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
- REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+ env:
+ # redshift
+ REDSHIFT_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+ REDSHIFT_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+ DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+ REDSHIFT_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+ REDSHIFT_DATABASE: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+ # TODO: below value - does it need to be secret or could we hard code it in tox and here?
  REDSHIFT_SPECTRUM_IAM_ROLE: ${{ secrets.REDSHIFT_SPECTRUM_IAM_ROLE }}
- SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
- SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
- SNOWFLAKE_TEST_PASS: ${{ secrets.SNOWFLAKE_TEST_PASS }}
- SNOWFLAKE_TEST_WHNAME: ${{ secrets.SNOWFLAKE_TEST_WHNAME }}
- SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
- SNOWFLAKE_TEST_DBNAME: ${{ secrets.SNOWFLAKE_TEST_DBNAME }}
+ REDSHIFT_SCHEMA: "dbt_external_tables_integration_tests_redshift"
+ #snowflake
+ SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+ SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+ DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_TEST_PASS }}
+ SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WHNAME }}
+ SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+ SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DBNAME }}
+ SNOWFLAKE_SCHEMA: "dbt_external_tables_integration_tests_snowflake"
+ # bigquery
  BIGQUERY_TEST_PROJECT: ${{ secrets.BIGQUERY_TEST_PROJECT }}
  BIGQUERY_PRIVATE_KEY: ${{ secrets.BIGQUERY_PRIVATE_KEY }}
  BIGQUERY_PRIVATE_KEY_ID: ${{ secrets.BIGQUERY_PRIVATE_KEY_ID }}

integration_tests/profiles.yml

@@ -11,25 +11,25 @@ integration_tests:
  outputs:
 
  redshift:
- type: redshift
- host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
- user: "{{ env_var('REDSHIFT_TEST_USER') }}"
- pass: "{{ env_var('REDSHIFT_TEST_PASS') }}"
- dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
- port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
- schema: dbt_external_tables_integration_tests_redshift
- threads: 1
+ type: "redshift"
+ host: "{{ env_var('REDSHIFT_HOST') }}"
+ user: "{{ env_var('REDSHIFT_USER') }}"
+ pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
+ dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
+ port: "{{ env_var('REDSHIFT_PORT') | as_number }}"
+ schema: "{{ env_var('REDSHIFT_SCHEMA') }}"
+ threads: 5
 
  snowflake:
- type: snowflake
- account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
- user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
- password: "{{ env_var('SNOWFLAKE_TEST_PASS') }}"
- role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
- database: "{{ env_var('SNOWFLAKE_TEST_DBNAME') }}"
- warehouse: "{{ env_var('SNOWFLAKE_TEST_WHNAME') }}"
- schema: dbt_external_tables_integration_tests_snowflake
- threads: 1
+ type: "snowflake"
+ account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+ user: "{{ env_var('SNOWFLAKE_USER') }}"
+ password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
+ role: "{{ env_var('SNOWFLAKE_ROLE') }}"
+ database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+ warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
+ schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
+ threads: 10
 
  bigquery:
  type: bigquery

integration_tests/test.env.sample

@@ -1,21 +1,21 @@
 # gh secret set -f integration_tests/test.env -e ci_testing
 
 # redshift
-REDSHIFT_TEST_HOST=
-REDSHIFT_TEST_USER=
-REDSHIFT_TEST_PASS=
-REDSHIFT_TEST_DBNAME=
-REDSHIFT_TEST_PORT=
+REDSHIFT_HOST=
+REDSHIFT_USER=
+DBT_ENV_SECRET_REDSHIFT_PASS=
+REDSHIFT_DATABASE=
+REDSHIFT_PORT=
 REDSHIFT_SPECTRUM_IAM_ROLE=
 
 # snowflake
 
-SNOWFLAKE_TEST_ACCOUNT=
-SNOWFLAKE_TEST_USER=
-SNOWFLAKE_TEST_PASS=
-SNOWFLAKE_TEST_ROLE=
-SNOWFLAKE_TEST_DBNAME=
-SNOWFLAKE_TEST_WHNAME=
+SNOWFLAKE_ACCOUNT=
+SNOWFLAKE_USER=
+DBT_ENV_SECRET_SNOWFLAKE_PASS=
+SNOWFLAKE_ROLE=
+SNOWFLAKE_DATABASE=
+SNOWFLAKE_WAREHOUSE=
 
 # bigquery
 BIGQUERY_PRIVATE_KEY=

supported_adapters.env

@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=snowflake,redshift

tox.ini

@@ -0,0 +1,53 @@
+[tox]
+skipsdist = True
+envlist = lint_all, testenv
+
+[testenv]
+passenv =
+ # redshift
+ REDSHIFT_HOST
+ REDSHIFT_USER
+ DBT_ENV_SECRET_REDSHIFT_PASS
+ REDSHIFT_DATABASE
+ REDSHIFT_SCHEMA
+ REDSHIFT_PORT
+ # snowflake
+ SNOWFLAKE_ACCOUNT
+ SNOWFLAKE_USER
+ DBT_ENV_SECRET_SNOWFLAKE_PASS
+ SNOWFLAKE_ROLE
+ SNOWFLAKE_DATABASE
+ SNOWFLAKE_WAREHOUSE
+ SNOWFLAKE_SCHEMA
+
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_redshift]
+changedir = integration_tests
+allowlist_externals = 
+ dbt
+skip_install = true
+commands =
+ dbt deps --target redshift
+ dbt deps --target redshift
+ dbt seed --full-refresh --target redshift
+ dbt run --target redshift
+ dbt run-operation prep_external --target redshift
+ dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --targetredshift
+ dbt run-operation dbt_external_tables.stage_external_sources --target redshift
+ dbt test --target redshift
+
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_snowflake]
+changedir = integration_tests
+allowlist_externals = 
+ dbt
+skip_install = true
+commands =
+ dbt deps --target snowflake
+ dbt deps --target snowflake
+ dbt seed --full-refresh --target snowflake
+ dbt run --target snowflake
+ dbt run-operation prep_external --target snowflake
+ dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --targetsnowflake
+ dbt run-operation dbt_external_tables.stage_external_sources --target snowflake
+ dbt test --target snowflake