From 9fbfa896f864c98190e9839fc672148e89e1db82 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 18 Oct 2024 14:54:59 -0500 Subject: [PATCH] add in bigquery, use vars instead of secrets --- .github/workflows/ci.yml | 29 +++++++++++++++---------- .github/workflows/integration_tests.yml | 10 ++++----- integration_tests/profiles.yml | 22 ++++++------------- supported_adapters.env | 2 +- tox.ini | 20 +++++++++++++++-- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4927b8e..1dfc2c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,18 +21,23 @@ jobs: uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 with: # redshift - REDSHIFT_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - REDSHIFT_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_DATABASE: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} + REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" - # snowflake - SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} - SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WHNAME }} - SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} - SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DBNAME }} + # snowflake + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}" + # bigquery + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}" + secrets: - DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} - DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_TEST_PASS }} + DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.DBT_ENV_SECRET_SNOWFLAKE_PASS }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ae1fbe0..2a13095 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -48,7 +48,7 @@ jobs: DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} REDSHIFT_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} REDSHIFT_DATABASE: ${{ secrets.REDSHIFT_TEST_DBNAME }} - # TODO: below value - does it need to be secret or could we hard code it in tox and here? + # TODO: below value - does it need to be env-var or could we hard code it in tox and here? REDSHIFT_SPECTRUM_IAM_ROLE: ${{ secrets.REDSHIFT_SPECTRUM_IAM_ROLE }} REDSHIFT_SCHEMA: "dbt_external_tables_integration_tests_redshift" #snowflake @@ -60,8 +60,6 @@ jobs: SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DBNAME }} SNOWFLAKE_SCHEMA: "dbt_external_tables_integration_tests_snowflake" # bigquery - BIGQUERY_TEST_PROJECT: ${{ secrets.BIGQUERY_TEST_PROJECT }} - BIGQUERY_PRIVATE_KEY: ${{ secrets.BIGQUERY_PRIVATE_KEY }} - BIGQUERY_PRIVATE_KEY_ID: ${{ secrets.BIGQUERY_PRIVATE_KEY_ID }} - BIGQUERY_CLIENT_EMAIL: ${{ secrets.BIGQUERY_CLIENT_EMAIL }} - BIGQUERY_CLIENT_ID: ${{ secrets.BIGQUERY_CLIENT_ID }} \ No newline at end of file + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} + BIGQUERY_SCHEMA: "dbt_external_tables_integration_tests_bigquery" \ No newline at end of file diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 60ddeb7..432e08b 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -28,22 +28,14 @@ integration_tests: threads: 10 bigquery: - type: bigquery - method: service-account-json + type: "bigquery" + method: "service-account-json" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" + threads: 10 keyfile_json: - type: "service_account" - project_id: "{{ env_var('BIGQUERY_TEST_PROJECT') }}" - private_key: "{{ env_var('BIGQUERY_PRIVATE_KEY') }}" - private_key_id: "{{ env_var('BIGQUERY_PRIVATE_KEY_ID') }}" - client_email: "{{ env_var('BIGQUERY_CLIENT_EMAIL') }}" - client_id: "{{ env_var('BIGQUERY_CLIENT_ID') }}" - auth_uri: "https://accounts.google.com/o/oauth2/auth" - token_uri: "https://oauth2.googleapis.com/token" - auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs" - client_x509_cert_url: https://www.googleapis.com/robot/v1/metadata/x509/{{ env_var('BIGQUERY_CLIENT_EMAIL') | urlencode }}" - project: "{{ env_var('BIGQUERY_TEST_PROJECT') }}" - schema: dbt_external_tables_integration_tests_bigquery - threads: 1 + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}" + job_retries: 3 databricks: type: spark diff --git a/supported_adapters.env b/supported_adapters.env index 23db802..9df6d74 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=snowflake,redshift +SUPPORTED_ADAPTERS=snowflake,redshift,bigquery diff --git a/tox.ini b/tox.ini index a95de79..a4583b0 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ commands = dbt seed --full-refresh --target redshift dbt run --target redshift dbt run-operation prep_external --target redshift - dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --targetredshift + dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target redshift dbt run-operation dbt_external_tables.stage_external_sources --target redshift dbt test --target redshift @@ -48,6 +48,22 @@ commands = dbt seed --full-refresh --target snowflake dbt run --target snowflake dbt run-operation prep_external --target snowflake - dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --targetsnowflake + dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target snowflake dbt run-operation dbt_external_tables.stage_external_sources --target snowflake dbt test --target snowflake + +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_bigquery] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt deps --target bigquery + dbt deps --target bigquery + dbt seed --full-refresh --target bigquery + dbt run --target bigquery + dbt run-operation prep_external --target bigquery + dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target bigquery + dbt run-operation dbt_external_tables.stage_external_sources --target bigquery + dbt test --target bigquery \ No newline at end of file