From 92431f7e82f5191c15537f49c40a707aadce5202 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Tue, 16 Jul 2024 14:49:49 -0500 Subject: [PATCH 1/5] create bigquery version of test as it has different default behavior for case sensitive columns --- dev-requirements.txt | 4 +- .../test_incremental_on_schema_change.py | 58 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index cf7b1b87c..d4073504b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,8 +2,10 @@ # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-adapters.git +git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819 +# -e /Users/matthewmcknight/git/dbt-adapters git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +# -e /Users/matthewmcknight/git/dbt-adapters/dbt-tests-adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor black>=24.3 diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index 65e855241..fd23f93d4 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -4,7 +4,7 @@ BaseIncrementalOnSchemaChangeSetup, BaseIncrementalOnSchemaChange, ) - +from dbt.tests.util import run_dbt, run_dbt_and_capture from dbt.tests.adapter.incremental.fixtures import ( _MODELS__A, _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, @@ -251,3 +251,59 @@ def test_run_incremental_sync_all_columns_time_ingestion_partitioning(self, proj compare_source = "incremental_sync_all_columns_time_ingestion_partitioning" compare_target = "incremental_sync_all_columns_time_ingestion_partitioning_target" self.run_twice_and_assert(select, compare_source, compare_target, project) + + +_MODELS__SRC_ARTISTS = """ +{{ + config( + materialized='table', + ) +}} +{% if var("version", 0) == 0 %} + select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name +{% else %} + -- add a non-zero version to the end of the command to get a different version: + -- --vars "{'version': 1}" + select {{ dbt.current_timestamp() }} as inserted_at, 'engineer' as Job, +{% endif %} +""" + +_MODELS__DIM_ARTISTS = """ +{{ + config( + materialized='incremental', + on_schema_change='append_new_columns', + ) +}} +select * from {{ ref("src_artists") }} +""" + + +class BaseBigQueryIncrementalCaseSenstivityOnSchemaChange: + @pytest.fixture(scope="class") + def models(self): + return { + "src_artists.sql": _MODELS__SRC_ARTISTS, + "dim_artists.sql": _MODELS__DIM_ARTISTS, + } + + def test_incremental_case_sensitivity(self, project): + select = "src_artists dim_artists" + run_dbt(["run", "--models", select, "--full-refresh"]) + run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + res, logs = run_dbt_and_capture( + ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ) + assert "Job" not in logs + run_dbt(["run", "--vars", "{'version': 1}"]) + run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) + res, logs = run_dbt_and_capture( + ["show", "--inline", "select * from {{ ref('dim_artists') }}"] + ) + assert "Job" in logs + + +class TestBigQueryIncrementalCaseSenstivityOnSchemaChange( + BaseBigQueryIncrementalCaseSenstivityOnSchemaChange +): + pass From 6ef9eda590ae24b73af03cf3b5e20e645fe4389a Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 17 Jul 2024 09:46:28 -0500 Subject: [PATCH 2/5] update dev-requirements --- dev-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index d4073504b..7b70b9246 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,9 +3,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819 -# -e /Users/matthewmcknight/git/dbt-adapters -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -# -e /Users/matthewmcknight/git/dbt-adapters/dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor black>=24.3 From 435b2af1faa2ab3bf21b40d00c3c123bace8c8a8 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 17 Jul 2024 10:16:08 -0500 Subject: [PATCH 3/5] add changelog --- .changes/unreleased/Under the Hood-20240717-101601.yaml | 6 ++++++ dev-requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20240717-101601.yaml diff --git a/.changes/unreleased/Under the Hood-20240717-101601.yaml b/.changes/unreleased/Under the Hood-20240717-101601.yaml new file mode 100644 index 000000000..c40cc0c8f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240717-101601.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: create test for checking case sensitivity on incrmeneal models that add new columns +time: 2024-07-17T10:16:01.119077-05:00 +custom: + Author: McKnight-42 + Issue: "1277" diff --git a/dev-requirements.txt b/dev-requirements.txt index 7b70b9246..e0a18d0c7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,7 +3,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819 -git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819#subdirectory=dbt-tests-adapterch # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor black>=24.3 From 2ecd49e1bd5de77bb1a2eb46ef48a1ce6668da9f Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Wed, 17 Jul 2024 10:22:26 -0500 Subject: [PATCH 4/5] typo --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index e0a18d0c7..7b70b9246 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,7 +3,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819 -git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819#subdirectory=dbt-tests-adapterch +git+https://github.com/dbt-labs/dbt-adapters.git@mcknight/ct-2819#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor black>=24.3 From 99347fd1a7c973118e970a5e8f42bf6f81a4c338 Mon Sep 17 00:00:00 2001 From: McKnight-42 Date: Thu, 1 Aug 2024 15:46:42 -0500 Subject: [PATCH 5/5] use universal version of test, remove bigquery version --- .../test_incremental_on_schema_change.py | 56 +------------------ 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index fd23f93d4..f664e6428 100644 --- a/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -3,8 +3,8 @@ from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChangeSetup, BaseIncrementalOnSchemaChange, + BaseIncrementalCaseSenstivityOnSchemaChange, ) -from dbt.tests.util import run_dbt, run_dbt_and_capture from dbt.tests.adapter.incremental.fixtures import ( _MODELS__A, _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, @@ -253,57 +253,5 @@ def test_run_incremental_sync_all_columns_time_ingestion_partitioning(self, proj self.run_twice_and_assert(select, compare_source, compare_target, project) -_MODELS__SRC_ARTISTS = """ -{{ - config( - materialized='table', - ) -}} -{% if var("version", 0) == 0 %} - select {{ dbt.current_timestamp() }} as inserted_at, 'john' as name -{% else %} - -- add a non-zero version to the end of the command to get a different version: - -- --vars "{'version': 1}" - select {{ dbt.current_timestamp() }} as inserted_at, 'engineer' as Job, -{% endif %} -""" - -_MODELS__DIM_ARTISTS = """ -{{ - config( - materialized='incremental', - on_schema_change='append_new_columns', - ) -}} -select * from {{ ref("src_artists") }} -""" - - -class BaseBigQueryIncrementalCaseSenstivityOnSchemaChange: - @pytest.fixture(scope="class") - def models(self): - return { - "src_artists.sql": _MODELS__SRC_ARTISTS, - "dim_artists.sql": _MODELS__DIM_ARTISTS, - } - - def test_incremental_case_sensitivity(self, project): - select = "src_artists dim_artists" - run_dbt(["run", "--models", select, "--full-refresh"]) - run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_artists') }}"] - ) - assert "Job" not in logs - run_dbt(["run", "--vars", "{'version': 1}"]) - run_dbt(["show", "--inline", "select * from {{ ref('dim_artists') }}"]) - res, logs = run_dbt_and_capture( - ["show", "--inline", "select * from {{ ref('dim_artists') }}"] - ) - assert "Job" in logs - - -class TestBigQueryIncrementalCaseSenstivityOnSchemaChange( - BaseBigQueryIncrementalCaseSenstivityOnSchemaChange -): +class TestIncrementalCaseSenstivityOnSchemaChange(BaseIncrementalCaseSenstivityOnSchemaChange): pass