Skip to content

Commit 2450b5e

Browse files
davidzajac1davidzajac1
authored
Update to dbt-core 1.7 compatibility (#119)
* Update packages.yml * fix gh actions
1 parent df5064c commit 2450b5e

File tree

9 files changed

+42
-34
lines changed

9 files changed

+42
-34
lines changed

‎.github/workflows/generate-docs.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ on: [push]
44

55
env:
66
DBT_PROFILES_DIR: ./
7+
DBT_VERSION: 1.7
78

89
jobs:
910
generate-docs:
@@ -27,14 +28,13 @@ jobs:
2728
- name: Check out
2829
uses: actions/checkout@v2
2930

30-
- uses: actions/setup-python@v1
31+
- uses: actions/setup-python@v4
3132
with:
32-
python-version: "3.7.x"
33+
python-version: "3.8.x"
3334

3435
- name: Install dependencies and run
3536
run: |
36-
pip install dbt-core==1.3
37-
pip install dbt-postgres==1.3
37+
pip install dbt-postgres==$DBT_VERSION
3838
dbt deps
3939
dbt run
4040
@@ -48,7 +48,7 @@ jobs:
4848

4949
- name: Deploy
5050
uses: peaceiris/actions-gh-pages@v3
51-
if: github.ref == 'refs/heads/main'
51+
if: github.ref == 'refs/heads/main' && !env.ACT
5252
with:
5353
github_token: ${{ secrets.GITHUB_TOKEN }}
5454
publish_dir: ./docs

‎.github/workflows/run-db-tests.yml

+19-20
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,17 @@ env:
88
RE_DATA_TESTING_USER: ${{ secrets.RE_DATA_TESTING_USER }}
99
RE_DATA_TESTING_PASSWORD: ${{ secrets.RE_DATA_TESTING_PASSWORD }}
1010
REDSHIFT_RE_DATA_TESTING_HOST: ${{ secrets.REDSHIFT_RE_DATA_TESTING_HOST }}
11+
BIGQUERY_TESTING_TYPE: ${{ secrets.BIGQUERY_TESTING_TYPE }}
1112
BIGQUERY_TESTING_PROJECT_ID: ${{ secrets.BIGQUERY_TESTING_PROJECT_ID }}
12-
BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64: ${{ secrets.BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64 }}
13-
BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH: ${{ github.workspace }}/bigquery-testing-service-account.json
13+
BIGQUERY_TESTING_PRIVATE_KEY_ID: ${{ secrets.BIGQUERY_TESTING_PRIVATE_KEY_ID }}
14+
BIGQUERY_TESTING_PRIVATE_KEY: ${{ secrets.BIGQUERY_TESTING_PRIVATE_KEY }}
15+
BIGQUERY_TESTING_CLIENT_EMAIL: ${{ secrets.BIGQUERY_TESTING_CLIENT_EMAIL }}
16+
BIGQUERY_TESTING_CLIENT_ID: ${{ secrets.BIGQUERY_TESTING_CLIENT_ID }}
17+
BIGQUERY_TESTING_AUTH_URI: ${{ secrets.BIGQUERY_TESTING_AUTH_URI }}
18+
BIGQUERY_TESTING_TOKEN_URI: ${{ secrets.BIGQUERY_TESTING_TOKEN_URI }}
19+
BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL }}
20+
BIGQUERY_TESTING_CLIENT_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_CLIENT_X509_CERT_URL }}
21+
DBT_VERSION: 1.7
1422

1523
jobs:
1624
test-other-dbs:
@@ -24,9 +32,9 @@ jobs:
2432
- name: Check out
2533
uses: actions/checkout@v2
2634

27-
- uses: actions/setup-python@v1
35+
- uses: actions/setup-python@v4
2836
with:
29-
python-version: "3.7.x"
37+
python-version: "3.8.x"
3038

3139
- name: Inject slug/short variables
3240
uses: rlespinasse/[email protected]
@@ -40,16 +48,11 @@ jobs:
4048
run: |
4149
echo $DQ_SCHEMA
4250
43-
- name: Decode service account for BigQuery
44-
if: matrix.database == 'bigquery'
45-
run: 'echo $BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64 | base64 --decode > $BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH'
46-
shell: bash
47-
4851
- name: Install dependencies
4952
working-directory: ./integration_tests
5053
run: |
5154
pip install -r requirements.txt
52-
pip install dbt-${{ matrix.database }}==1.4.0
55+
pip install dbt-${{ matrix.database }}==$DBT_VERSION
5356
dbt deps
5457
5558
- name: Drop schemas
@@ -89,14 +92,15 @@ jobs:
8992
- name: Check out
9093
uses: actions/checkout@v2
9194

92-
- uses: actions/setup-python@v1
95+
- uses: actions/setup-python@v4
9396
with:
94-
python-version: "3.7.x"
97+
python-version: "3.8.x"
9598

9699
- name: Install dependencies
97100
working-directory: ./integration_tests
98101
run: |
99102
pip install -r requirements.txt
103+
pip install dbt-postgres==$DBT_VERSION
100104
dbt deps
101105
102106
- name: Test DB
@@ -114,9 +118,9 @@ jobs:
114118
- name: Check out
115119
uses: actions/checkout@v2
116120

117-
- uses: actions/setup-python@v1
121+
- uses: actions/setup-python@v4
118122
with:
119-
python-version: "3.7.x"
123+
python-version: "3.8.x"
120124

121125
- name: Inject slug/short variables
122126
uses: rlespinasse/[email protected]
@@ -130,15 +134,10 @@ jobs:
130134
run: |
131135
echo $DQ_SCHEMA
132136
133-
- name: Decode service account for BigQuery
134-
if: matrix.database == 'bigquery'
135-
run: 'echo $BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64 | base64 --decode > $BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH'
136-
shell: bash
137-
138137
- name: Install dependencies and drop branch schema
139138
working-directory: ./integration_tests
140139
run: |
141140
pip install -r requirements.txt
142-
pip install dbt-${{ matrix.database }}==1.5.0
141+
pip install dbt-${{ matrix.database }}==$DBT_VERSION
143142
dbt deps
144143
dbt run-operation drop_all_schemas --args "{ schema_name: ${{ env.DQ_SCHEMA }} }" --profile re_data_${{ matrix.database }} --vars "{ source_schema: ${{ env.DQ_SCHEMA }} }"

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ logs/*
99
dbt_modules/*
1010
dbt_packages/*
1111
target/*
12+
.env

‎dbt_project.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: "re_data"
2-
version: "0.10.8"
2+
version: "0.10.9"
33
config-version: 2
44

55
require-dbt-version: [">=1.0.0", "<2.0.0"]

‎integration_tests/packages.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ packages:
22
- local: ../
33

44
- package: dbt-labs/dbt_utils
5-
version: [">=1.0.0", "<1.1.0"]
5+
version: [">=1.0.0", "<1.2.0"]

‎integration_tests/requirements.txt

-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
11
pytest==6.2.5
22
pyyaml==6.0
3-
dbt-core==1.4.0
4-
dbt-postgres==1.4.0

‎macros/public/store/generate_overview.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@
8686
{% endset %}
8787

8888
{% set overview_result = run_query(overview_query) %}
89-
{% set overview_file_path = overview_path or 'target/re_data/overview.json' %}
89+
{% set overview_file_path = overview_path or '../target/re_data/overview.json' %}
9090
{% do overview_result.to_json(overview_file_path) %}
9191
{{ save_monitored(monitored_path) }}
9292

‎packages.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11

22
packages:
33
- package: dbt-labs/dbt_utils
4-
version: [">=1.0.0", "<1.1.0"]
4+
version: [">=1.0.0", "<1.2.0"]

‎profiles.yml

+13-3
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,22 @@ re_data_bigquery:
3939
outputs:
4040
dev:
4141
type: bigquery
42-
method: service-account
42+
method: service-account-json
4343
project: "{{ env_var('BIGQUERY_TESTING_PROJECT_ID') }}"
4444
dataset: "{{ env_var('DQ_SCHEMA') }}"
4545
threads: 4
46-
keyfile: "{{ env_var('BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH') }}"
47-
location: EU
46+
keyfile_json:
47+
type: "{{ env_var('BIGQUERY_TESTING_TYPE') }}"
48+
project_id: "{{ env_var('BIGQUERY_TESTING_PROJECT_ID') }}"
49+
private_key_id: "{{ env_var('BIGQUERY_TESTING_PRIVATE_KEY_ID') }}"
50+
private_key: "{{ env_var('BIGQUERY_TESTING_PRIVATE_KEY') }}"
51+
client_email: "{{ env_var('BIGQUERY_TESTING_CLIENT_EMAIL') }}"
52+
client_id: "{{ env_var('BIGQUERY_TESTING_CLIENT_ID') }}"
53+
auth_uri: "{{ env_var('BIGQUERY_TESTING_AUTH_URI') }}"
54+
token_uri: "{{ env_var('BIGQUERY_TESTING_TOKEN_URI') }}"
55+
auth_provider_x509_cert_url: "{{ env_var('BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL') }}"
56+
client_x509_cert_url: "{{ env_var('BIGQUERY_TESTING_CLIENT_X509_CERT_URL') }}"
57+
location: US
4858
timeout_seconds: 300
4959
priority: interactive
5060
retries: 1

0 commit comments

Comments
 (0)