From 4df6e54b8214da49f60aa0486ede477de55359d9 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 10 Oct 2024 15:17:20 -0400 Subject: [PATCH 1/7] Enable setting current value of dbt_valid_to (#321) --- .../unreleased/Features-20240927-134248.yaml | 6 ++++++ .../materializations/snapshots/helpers.sql | 18 +++++++++++++++--- .../snapshots/snapshot_merge.sql | 7 ++++++- 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Features-20240927-134248.yaml diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml new file mode 100644 index 00000000..0b456244 --- /dev/null +++ b/.changes/unreleased/Features-20240927-134248.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable setting current value of dbt_valid_to +time: 2024-09-27T13:42:48.654556-04:00 +custom: + Author: gshank + Issue: "320" diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 8d982855..52fdb9bf 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,7 +53,13 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ columns.dbt_valid_to }} is null + where + {% if config.get('dbt_valid_to_current') %} + {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} + ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} ), @@ -64,7 +70,7 @@ {{ strategy.unique_key }} as dbt_unique_key, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} from snapshot_query @@ -166,7 +172,7 @@ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }} + {{ get_dbt_valid_to_current(strategy, columns) }} from ( {{ sql }} ) sbq @@ -210,3 +216,9 @@ {% endif %} {% endif %} {% endmacro %} + +{% macro get_dbt_valid_to_current(strategy, columns) %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} + coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) + as {{ columns.dbt_valid_to }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 74494ed2..cf787e4f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -14,7 +14,12 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current") %} + and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} From 7d187c4dbe6aa159b7a28472af869204ef96cb94 Mon Sep 17 00:00:00 2001 From: FishtownBuildBot <77737458+FishtownBuildBot@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:35:24 -0400 Subject: [PATCH 2/7] [Automated] Update pre-commit hooks (#297) Co-authored-by: Mike Alfare Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f2a03f7..aa942447 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: dbt-core-in-adapters-check - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.8.0 hooks: - id: black args: @@ -30,7 +30,7 @@ repos: - --force-exclude=dbt/adapters/events/adapter_types_pb2.py - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/ @@ -41,7 +41,7 @@ repos: - --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.11.2 hooks: - id: mypy exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py From 3b6ad1f93e731b5046651ddecb635113a5c1cbe7 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 15 Oct 2024 23:03:35 +0000 Subject: [PATCH 3/7] Bumping version to 1.7.1 and generate changelog --- .changes/1.7.1.md | 5 +++++ .changes/unreleased/Features-20240927-134248.yaml | 6 ------ CHANGELOG.md | 10 ++++++++-- dbt/adapters/__about__.py | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 .changes/1.7.1.md delete mode 100644 .changes/unreleased/Features-20240927-134248.yaml diff --git a/.changes/1.7.1.md b/.changes/1.7.1.md new file mode 100644 index 00000000..4acda7f8 --- /dev/null +++ b/.changes/1.7.1.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml deleted file mode 100644 index 0b456244..00000000 --- a/.changes/unreleased/Features-20240927-134248.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Enable setting current value of dbt_valid_to -time: 2024-09-27T13:42:48.654556-04:00 -custom: - Author: gshank - Issue: "320" diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb37ad5..8e9f64fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,18 @@ and is generated by [Changie](https://github.com/miniscruff/changie). - dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315)) - - ## dbt-adapters 1.10.1 - September 16, 2024 ## dbt-adapters 1.10.0 - September 12, 2024 +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) + + + ## dbt-adapters 1.7.0 - September 19, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index a55413d1..116d5667 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.7.0" +version = "1.7.1" From 1029b2d5c55a840fcba86eb0b1d3819b26dc7e57 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:06:34 -0400 Subject: [PATCH 4/7] Drop support for Python 3.8 (#332) --- .changes/unreleased/Breaking Changes-20241016-180629.yaml | 6 ++++++ .github/actions/publish-results/action.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- .pre-commit-config.yaml | 2 +- dbt-tests-adapter/pyproject.toml | 3 +-- pyproject.toml | 3 +-- 6 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-180629.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-180629.yaml b/.changes/unreleased/Breaking Changes-20241016-180629.yaml new file mode 100644 index 00000000..40f103c0 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-180629.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:06:29.535761-04:00 +custom: + Author: mikealfare + Issue: "332" diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml index 0d5cb7e6..7c73a94f 100644 --- a/.github/actions/publish-results/action.yml +++ b/.github/actions/publish-results/action.yml @@ -5,7 +5,7 @@ inputs: description: File type for file name stub (e.g. "unit-tests") required: true python-version: - description: Python version for the file name stub (e.g. "3.8") + description: Python version for the file name stub (e.g. "3.9") required: true source-file: description: File to be uploaded diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b61c83d7..b4ac615d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Check out repository diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa942447..b7835274 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,10 +23,10 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 + - --target-version=py312 - --force-exclude=dbt/adapters/events/adapter_types_pb2.py - repo: https://github.com/pycqa/flake8 diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index c9082d43..d2f732b7 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -4,7 +4,7 @@ name = "dbt-tests-adapter" description = "The set of reusable tests and test fixtures used to test common functionality" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/pyproject.toml b/pyproject.toml index 76ca3dee..52550fb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ name = "dbt-adapters" description = "The set of adapter protocols and base functionality that supports integration with dbt-core" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 937c8c7d257055ed6cd3eaf7f7e5e01026d7b7d7 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 17 Oct 2024 20:45:23 -0700 Subject: [PATCH 5/7] Always ensure valid incremental strategy (#331) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Fixes-20241016-160412.yaml | 6 +++ .../dbt/tests/adapter/basic/files.py | 7 ++++ .../tests/adapter/basic/test_incremental.py | 39 +++++++++++++++++++ .../models/incremental/incremental.sql | 5 ++- 4 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241016-160412.yaml diff --git a/.changes/unreleased/Fixes-20241016-160412.yaml b/.changes/unreleased/Fixes-20241016-160412.yaml new file mode 100644 index 00000000..00a1982c --- /dev/null +++ b/.changes/unreleased/Fixes-20241016-160412.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Always validate an incremental model's `incremental_strategy` +time: 2024-10-16T16:04:12.58581-05:00 +custom: + Author: QMalcolm + Issue: "330" diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/files.py b/dbt-tests-adapter/dbt/tests/adapter/basic/files.py index 751b01a0..d0253a53 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/basic/files.py +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/files.py @@ -186,6 +186,10 @@ {{ config(materialized="incremental") }} """ +config_materialized_incremental_invalid_strategy = """ + {{ config(materialized="incremental", incremental_strategy="bad_strategy") }} +""" + config_materialized_var = """ {{ config(materialized=var("materialized_var", "table"))}} """ @@ -217,3 +221,6 @@ ephemeral_view_sql = config_materialized_view + model_ephemeral ephemeral_table_sql = config_materialized_table + model_ephemeral incremental_sql = config_materialized_incremental + model_incremental +incremental_invalid_strategy_sql = ( + config_materialized_incremental_invalid_strategy + model_incremental +) diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py b/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py index fe04a5a1..57cc4db9 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py @@ -86,6 +86,45 @@ def test_incremental_not_schema_change(self, project): assert run_result == RunStatus.Success +class BaseIncrementalBadStrategy: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "incremental"} + + @pytest.fixture(scope="class") + def models(self): + return { + "incremental.sql": files.incremental_invalid_strategy_sql, + "schema.yml": files.schema_base_yml, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"base.csv": files.seeds_base_csv, "added.csv": files.seeds_added_csv} + + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + def test_incremental_invalid_strategy(self, project): + # seed command + results = run_dbt(["seed"]) + assert len(results) == 2 + + # try to run the incremental model, it should fail on the first attempt + results = run_dbt(["run"], expect_pass=False) + assert len(results.results) == 1 + assert ( + 'dbt could not find an incremental strategy macro with the name "get_incremental_bad_strategy_sql"' + in results.results[0].message + ) + + class Testincremental(BaseIncremental): pass diff --git a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql index f932751a..41d2de26 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql @@ -32,6 +32,9 @@ {% set to_drop = [] %} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% if existing_relation is none %} {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %} {% elif full_refresh_mode %} @@ -52,9 +55,7 @@ {% endif %} {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} - {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} - {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} From 5fd5467da1df8c568423869136356ea192a95498 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 18 Oct 2024 20:31:17 -0400 Subject: [PATCH 6/7] Allow use of multiple column unique keys in snapshots (#326) Co-authored-by: Antonio Papa Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Features-20240422-081302.yaml | 6 ++ dbt/adapters/base/relation.py | 11 +++ .../materializations/snapshots/helpers.sql | 79 +++++++++++++------ .../materializations/snapshots/snapshot.sql | 18 +++-- .../materializations/snapshots/strategies.sql | 6 +- pyproject.toml | 2 +- 6 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 .changes/unreleased/Features-20240422-081302.yaml diff --git a/.changes/unreleased/Features-20240422-081302.yaml b/.changes/unreleased/Features-20240422-081302.yaml new file mode 100644 index 00000000..c58e471e --- /dev/null +++ b/.changes/unreleased/Features-20240422-081302.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allows unique_key for snapshots to take a list +time: 2024-04-22T08:13:02.937534-04:00 +custom: + Author: agpapa + Issue: "181" diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 80dbd34b..ecd87345 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -6,6 +6,7 @@ Dict, FrozenSet, Iterator, + List, Optional, Set, Tuple, @@ -341,6 +342,16 @@ def create( ) return cls.from_dict(kwargs) + @classmethod + def scd_args(cls: Type[Self], primary_key: Union[str, List[str]], updated_at) -> List[str]: + scd_args = [] + if isinstance(primary_key, list): + scd_args.extend(primary_key) + else: + scd_args.append(primary_key) + scd_args.append(updated_at) + return scd_args + @property def can_be_renamed(self) -> bool: return self.type in self.renameable_relations diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 52fdb9bf..5c2bad99 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -49,9 +49,7 @@ snapshotted_data as ( - select *, - {{ strategy.unique_key }} as dbt_unique_key - + select *, {{ unique_key_fields(strategy.unique_key) }} from {{ target_relation }} where {% if config.get('dbt_valid_to_current') %} @@ -65,9 +63,7 @@ insertions_source_data as ( - select - *, - {{ strategy.unique_key }} as dbt_unique_key, + select *, {{ unique_key_fields(strategy.unique_key) }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, {{ get_dbt_valid_to_current(strategy, columns) }}, @@ -78,9 +74,7 @@ updates_source_data as ( - select - *, - {{ strategy.unique_key }} as dbt_unique_key, + select *, {{ unique_key_fields(strategy.unique_key) }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_to }} @@ -92,9 +86,7 @@ deletes_source_data as ( - select - *, - {{ strategy.unique_key }} as dbt_unique_key + select *, {{ unique_key_fields(strategy.unique_key) }} from snapshot_query ), {% endif %} @@ -106,13 +98,11 @@ source_data.* from insertions_source_data as source_data - left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key - where snapshotted_data.dbt_unique_key is null - or ( - snapshotted_data.dbt_unique_key is not null - and ( - {{ strategy.row_changed }} - ) + left outer join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and {{ strategy.row_changed }}) + ) ), @@ -125,7 +115,8 @@ snapshotted_data.{{ columns.dbt_scd_id }} from updates_source_data as source_data - join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key + join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where ( {{ strategy.row_changed }} ) @@ -145,8 +136,9 @@ snapshotted_data.{{ columns.dbt_scd_id }} from snapshotted_data - left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key - where source_data.dbt_unique_key is null + left join deletes_source_data as source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} ) {%- endif %} @@ -217,8 +209,51 @@ {% endif %} {% endmacro %} + {% macro get_dbt_valid_to_current(strategy, columns) %} {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) as {{ columns.dbt_valid_to }} {% endmacro %} + + +{% macro unique_key_fields(unique_key) %} + {% if unique_key | is_list %} + {% for key in unique_key %} + {{ key }} as dbt_unique_key_{{ loop.index }} + {%- if not loop.last %} , {%- endif %} + {% endfor %} + {% else %} + {{ unique_key }} as dbt_unique_key + {% endif %} +{% endmacro %} + + +{% macro unique_key_join_on(unique_key, identifier, from_identifier) %} + {% if strategy.unique_key | is_list %} + {% for key in strategy.unique_key %} + {{ identifier }}.dbt_unique_key_{{ loop.index }} = {{ from_identifier }}.dbt_unique_key_{{ loop.index }} + {%- if not loop.last %} and {%- endif %} + {% endfor %} + {% else %} + {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key + {% endif %} +{% endmacro %} + + +{% macro unique_key_is_null(unique_key, identifier) %} + {% if unique_key | is_list %} + {{ identifier }}.dbt_unique_key_1 is null + {% else %} + {{ identifer }}.dbt_unique_key is null + {% endif %} +{% endmacro %} + + +{% macro unique_key_is_not_null(unique_key, identifier) %} + {% if unique_key | is_list %} + {{ identifier }}.dbt_unique_key_1 is not null + {% else %} + {{ identifer }}.dbt_unique_key is not null + {% endif %} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index 5daead4c..0c9590b6 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -46,20 +46,22 @@ {% do adapter.expand_target_column_types(from_relation=staging_table, to_relation=target_relation) %} + {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %} + {% if unique_key | is_list %} + {% for key in strategy.unique_key %} + {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }} + {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }} + {% endfor %} + {% endif %} + {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | rejectattr('name', 'in', remove_columns) | list %} {% do create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | rejectattr('name', 'in', remove_columns) | list %} {% set quoted_source_columns = [] %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 8c086182..f9f5afbd 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -70,7 +70,8 @@ ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) {%- endset %} - {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} + {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %} + {% set scd_id_expr = snapshot_hash_arguments(scd_args) %} {% do return({ "unique_key": primary_key, @@ -166,7 +167,8 @@ ) {%- endset %} - {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} + {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %} + {% set scd_id_expr = snapshot_hash_arguments(scd_args) %} {% do return({ "unique_key": primary_key, diff --git a/pyproject.toml b/pyproject.toml index 52550fb5..7a8d1a50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common>=1.10,<2.0", + "dbt-common>=1.11,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", From 2bf38087bc7225dd2ffadc09fcfdc7fc781287fa Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 21 Oct 2024 10:26:55 -0400 Subject: [PATCH 7/7] Fix some typos in multiple unique_key (#335) --- .../macros/materializations/snapshots/helpers.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 5c2bad99..b4cd7c14 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -101,7 +101,7 @@ left outer join snapshotted_data on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} - or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and {{ strategy.row_changed }}) + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }}) ) @@ -230,8 +230,8 @@ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %} - {% if strategy.unique_key | is_list %} - {% for key in strategy.unique_key %} + {% if unique_key | is_list %} + {% for key in unique_key %} {{ identifier }}.dbt_unique_key_{{ loop.index }} = {{ from_identifier }}.dbt_unique_key_{{ loop.index }} {%- if not loop.last %} and {%- endif %} {% endfor %} @@ -245,7 +245,7 @@ {% if unique_key | is_list %} {{ identifier }}.dbt_unique_key_1 is null {% else %} - {{ identifer }}.dbt_unique_key is null + {{ identifier }}.dbt_unique_key is null {% endif %} {% endmacro %} @@ -254,6 +254,6 @@ {% if unique_key | is_list %} {{ identifier }}.dbt_unique_key_1 is not null {% else %} - {{ identifer }}.dbt_unique_key is not null + {{ identifier }}.dbt_unique_key is not null {% endif %} {% endmacro %}