add in bigquery, use vars instead of secrets

.github/workflows/ci.yml

@@ -21,18 +21,23 @@ jobs:
  uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
  with:
  # redshift
- REDSHIFT_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
- REDSHIFT_USER: ${{ secrets.REDSHIFT_TEST_USER }}
- REDSHIFT_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
- REDSHIFT_DATABASE: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+ REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+ REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+ REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }}
+ REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
  REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}"
-  # snowflake
- SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
- SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WHNAME }}
- SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
- SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DBNAME }}
+ # snowflake
+ SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
+ SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
+ SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
+ SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
  SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
+ # bigquery
+ BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+ BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}"
+
  secrets:
- DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
- SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
- DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_TEST_PASS }}
+ DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }}
+ SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+ DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.DBT_ENV_SECRET_SNOWFLAKE_PASS }}
+ BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}

.github/workflows/integration_tests.yml

@@ -48,7 +48,7 @@ jobs:
  DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
  REDSHIFT_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
  REDSHIFT_DATABASE: ${{ secrets.REDSHIFT_TEST_DBNAME }}
- # TODO: below value - does it need to be secret or could we hard code it in tox and here?
+ # TODO: below value - does it need to be env-var or could we hard code it in tox and here?
  REDSHIFT_SPECTRUM_IAM_ROLE: ${{ secrets.REDSHIFT_SPECTRUM_IAM_ROLE }}
  REDSHIFT_SCHEMA: "dbt_external_tables_integration_tests_redshift"
  #snowflake
@@ -60,8 +60,6 @@ jobs:
  SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DBNAME }}
  SNOWFLAKE_SCHEMA: "dbt_external_tables_integration_tests_snowflake"
  # bigquery
- BIGQUERY_TEST_PROJECT: ${{ secrets.BIGQUERY_TEST_PROJECT }}
- BIGQUERY_PRIVATE_KEY: ${{ secrets.BIGQUERY_PRIVATE_KEY }}
- BIGQUERY_PRIVATE_KEY_ID: ${{ secrets.BIGQUERY_PRIVATE_KEY_ID }}
- BIGQUERY_CLIENT_EMAIL: ${{ secrets.BIGQUERY_CLIENT_EMAIL }}
- BIGQUERY_CLIENT_ID: ${{ secrets.BIGQUERY_CLIENT_ID }}
+ BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+ BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}
+ BIGQUERY_SCHEMA: "dbt_external_tables_integration_tests_bigquery"

integration_tests/profiles.yml

@@ -28,22 +28,14 @@ integration_tests:
  threads: 10
 
  bigquery:
- type: bigquery
- method: service-account-json
+ type: "bigquery"
+ method: "service-account-json"
+ project: "{{ env_var('BIGQUERY_PROJECT') }}"
+ dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
+ threads: 10
  keyfile_json:
- type: "service_account"
- project_id: "{{ env_var('BIGQUERY_TEST_PROJECT') }}"
- private_key: "{{ env_var('BIGQUERY_PRIVATE_KEY') }}"
- private_key_id: "{{ env_var('BIGQUERY_PRIVATE_KEY_ID') }}"
- client_email: "{{ env_var('BIGQUERY_CLIENT_EMAIL') }}"
- client_id: "{{ env_var('BIGQUERY_CLIENT_ID') }}"
- auth_uri: "https://accounts.google.com/o/oauth2/auth"
- token_uri: "https://oauth2.googleapis.com/token"
- auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs"
- client_x509_cert_url: https://www.googleapis.com/robot/v1/metadata/x509/{{ env_var('BIGQUERY_CLIENT_EMAIL') | urlencode }}"
- project: "{{ env_var('BIGQUERY_TEST_PROJECT') }}"
- schema: dbt_external_tables_integration_tests_bigquery
- threads: 1
+ "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}"
+ job_retries: 3
 
  databricks:
  type: spark

supported_adapters.env

@@ -1 +1 @@
-SUPPORTED_ADAPTERS=snowflake,redshift
+SUPPORTED_ADAPTERS=snowflake,redshift,bigquery

tox.ini

@@ -32,7 +32,7 @@ commands =
  dbt seed --full-refresh --target redshift
  dbt run --target redshift
  dbt run-operation prep_external --target redshift
- dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --targetredshift
+ dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target redshift
  dbt run-operation dbt_external_tables.stage_external_sources --target redshift
  dbt test --target redshift
 
@@ -48,6 +48,22 @@ commands =
  dbt seed --full-refresh --target snowflake
  dbt run --target snowflake
  dbt run-operation prep_external --target snowflake
- dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --targetsnowflake
+ dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target snowflake
  dbt run-operation dbt_external_tables.stage_external_sources --target snowflake
  dbt test --target snowflake
+
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_bigquery]
+changedir = integration_tests
+allowlist_externals = 
+ dbt
+skip_install = true
+commands =
+ dbt deps --target bigquery
+ dbt deps --target bigquery
+ dbt seed --full-refresh --target bigquery
+ dbt run --target bigquery
+ dbt run-operation prep_external --target bigquery
+ dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target bigquery
+ dbt run-operation dbt_external_tables.stage_external_sources --target bigquery
+ dbt test --target bigquery