add user passthrough column to users model and update dependencies

.buildkite/scripts/run_models.sh

@@ -21,7 +21,9 @@ dbt compile --target "$db"
 dbt run --target "$db" --full-refresh
 dbt run --target "$db"
 dbt test --target "$db"
-dbt run --vars '{iterable__using_campaign_label_history: false, iterable__using_user_unsubscribed_message_type_history: false, iterable__using_campaign_suppression_list_history: false}' --target "$db" --full-refresh
+dbt run --vars '{iterable_user_history_pass_through_columns: [{name: "phone_number_updated_at"}], iterable_event_extension_pass_through_columns: [{name: "web_push_message"}]}' --target "$db" --full-refresh
+dbt run --vars '{iterable__using_campaign_label_history: false, iterable__using_user_unsubscribed_message_type_history: false, iterable__using_campaign_suppression_list_history: false, iterable__using_event_extension: false}' --target "$db" --full-refresh
+dbt run --vars '{iterable__using_campaign_label_history: false, iterable__using_user_unsubscribed_message_type_history: false, iterable__using_campaign_suppression_list_history: false, iterable__using_event_extension: false}' --target "$db"
 dbt run --vars '{does_table_exist('user_unsubscribed_message_type'): false, does_table_exist('user_unsubscribed_channel'): false}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.quickstart/quickstart.yml

@@ -10,6 +10,8 @@ table_variables:
     - campaign_suppression_list_history
   iterable__using_user_unsubscribed_message_type:
     - user_unsubscribed_message_type
+  iterable__using_event_extension:
+    - event_extension
 
 destination_configurations:
   databricks:

CHANGELOG.md

@@ -1,8 +1,19 @@
-# dbt_iterable v0.RELEASE.RELEASE
+# dbt_iterable v0.12.0
+[PR #44](https://github.com/fivetran/dbt_iterable/pull/44) includes the following updates:
+
+## 🚨 Breaking Changes 🚨
+- Introduces variable `iterable__using_event_extension` to allow the `event_extension` table to be disabled and exclude its field, `experiment_id`, from persisting downstream. This permits the downstream models to run even if the source `event_extension` table does not exist. By default the variable is set to True. If you don't have this table, you will need to set `iterable__using_event_extension` to False. For more information on how to configure the `iterable__using_event_extension` variable, refer to the [README](https://github.com/fivetran/dbt_iterable/blob/main/README.md#step-4-enablingdisabling-models). 
+   - This will be a breaking change if you choose to disable the `event_extension` table as `experiment_id` will be removed from downstream models. Conversely, if you wish to include the `experiment_id` grain, ensure that `iterable__using_event_extension` is not explicitly set to False.
+   - Following this, the uniqueness tests in related models have been updated to account for whether `iterable__using_event_extension` is enabled/disabled by now relying on new surrogate keys:
+      - `unique_campaign_version_id`: Unique identifier for the `iterable__campaigns` model that combines `campaign_id`, `template_id`, and if available, `experiment_id`.
+      - unique_user_campaign_id: Unique identifier for the `iterable__user_campaign` model that combines `unique_user_key`, `campaign_id`, and if available, `experiment_id`.
+
+- Persists `user_history` passthrough columns, as stipulated via the `iterable_user_history_pass_through_columns` variable, through to the `iterable__users` model. For more information on how to configure the `iterable_user_history_pass_through_columns` variable, refer to the [README](https://github.com/fivetran/dbt_iterable/blob/main/README.md#passing-through-additional-fields).
 
 ## Under the Hood
-- Addition of integrity and consistency validation tests within integration tests pertaining to the `iterable__user_unsubscriptions` model.
-- Updated seed data to ensure proper testing of the latest [v0.8.1 `dbt_iterable_source` release](https://github.com/fivetran/dbt_iterable_source/releases/tag/v0.8.1).
+- Updates logic in `int_iterable__campaign_event_metrics`, `iterable__events`, and `iterable__user_campaign` to account for the `iterable__using_event_extension` variable being disabled or enabled. If disabled, `experiment_id` will not show up as a grain.
+- Addition of integrity and consistency validation tests within integration tests pertaining to the `iterable__user_unsubscriptions`, `iterable__campaigns`, `iterable__events`, `iterable__user_campaign`, and `iterable_users` models.
+- Updated seed data to ensure proper testing of the latest [v0.8.1 `dbt_iterable_source` release](https://github.com/fivetran/dbt_iterable_source/releases/tag/v0.8.1) in addition to testing of the pass_through column features.
 - Updated [pull request and issue templates](https://github.com/fivetran/dbt_iterable_source/tree/v0.8.1/.github).
 - Included auto-releaser GitHub Actions workflow to automate future releases.
 

README.md

@@ -76,7 +76,7 @@ Include the following Iterable package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/iterable
-    version: [">=0.11.0", "<0.12.0"]
+    version: [">=0.12.0", "<0.13.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `iterable` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile). If this is not where your Iterable data is located (for example, if your Iterable schema is named `iterable_fivetran`), add the following configuration to your root `dbt_project.yml` file:
@@ -97,19 +97,18 @@ vars:
     iterable__using_campaign_label_history: false                    # default is true
     iterable__using_user_unsubscribed_message_type_history: false    # default is true
     iterable__using_campaign_suppression_list_history: false         # default is true   
+    iterable__using_event_extension: false         # default is true   
 ```
 
 
 
 ## (Optional) Step 5: Additional configurations
-<details><summary>Expand for details</summary>
-<br>
 
 ### Passing Through Additional Fields
 
-This package includes fields we judged were standard across Iterable users. However, the Fivetran connector allows for additional columns to be brought through in the `event_extension` and `user_history` objects. Therefore, if you wish to bring them through, leverage our passthrough column variables. 
+This package includes fields we judged were standard across Iterable users. However, the Fivetran connector allows for additional columns to be brought through in the `event_extension` and `user_history` objects. Therefore, if you wish to bring them through, leverage our passthrough column variables. For `event_extension` columns, ensure that `iterable__using_event_extension` is set to True, which is the default.
 
-You will see these additional columns populate in the end `iterable__events` and `iterable__users` models.
+You will see these additional columns populate in the end `iterable__list_user_history`, `iterable__events`, and `iterable__users` models.
 
 **Notice**: A `dbt run --full-refresh` is required each time these variables are edited.
 
@@ -169,7 +168,6 @@ By default, this package refers to the new table (`CAMPAIGN_SUPPRESSION_LIST_HIS
 vars:
     iterable_campaign_suppression_list_history_identifier: "campaign_supression_list_history"
 ```
-</details>
 
 ## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™
 <details><summary>Expand for details</summary>
@@ -191,7 +189,7 @@ packages:
       version: [">=1.0.0", "<2.0.0"]
 
     - package: fivetran/iterable_source
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.9.0", "<0.10.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'iterable'
-version: '0.11.0'
+version: '0.12.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: iterable_integration_tests_10
+      schema: iterable_integration_tests_13
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: iterable_integration_tests_10
+      schema: iterable_integration_tests_13
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: iterable_integration_tests_10
+      schema: iterable_integration_tests_13
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: iterable_integration_tests_10
+      schema: iterable_integration_tests_13
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: iterable_integration_tests_10
+      schema: iterable_integration_tests_13
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 config-version: 2
 name: 'iterable_integration_tests'
-version: '0.11.0'
+version: '0.12.0'
 profile: 'integration_tests'
 vars:
   iterable_source:
-    iterable_schema: iterable_integration_tests_10
+    iterable_schema: iterable_integration_tests_13
     iterable_campaign_history_identifier: "campaign_history_data"
     iterable_campaign_label_history_identifier: "campaign_label_history_data"
     iterable_campaign_list_history_identifier: "campaign_list_history_data"
@@ -20,8 +20,7 @@ vars:
     iterable_user_unsubscribed_message_type_identifier: "user_unsubscribed_message_type_data"
     iterable_user_unsubscribed_channel_history_identifier: "user_unsubscribed_channel_history_data"
     iterable_user_unsubscribed_message_type_history_identifier: "user_unsubscribed_message_type_history_data"
-
-  ## For validation testing. To be commented out before release.
+
 models:
   +schema: "iterable_{{ var('directed_schema','dev') }}"
 

integration_tests/tests/consistency/consistency_campaigns.sql

@@ -0,0 +1,51 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+{% set exclude_fields = ["labels"] %}
+{% set fields = dbt_utils.star(from=ref('iterable__campaigns'), except=exclude_fields) %}
+
+-- this test ensures the iterable__campaigns end model matches the prior version
+with prod as (
+    select {{ fields }}
+    from {{ target.schema }}_iterable_prod.iterable__campaigns
+    where date(updated_at) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select {{ fields }}
+    from {{ target.schema }}_iterable_dev.iterable__campaigns
+    where date(updated_at) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/consistency_events.sql

@@ -0,0 +1,47 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+-- this test ensures the iterable__events end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_iterable_prod.iterable__events
+    where date(created_at) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_iterable_dev.iterable__events
+    where date(created_at) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/consistency_user_campaign.sql

@@ -0,0 +1,47 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+-- this test ensures the iterable__user_campaign end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_iterable_prod.iterable__user_campaign
+    where date(first_event_at) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_iterable_dev.iterable__user_campaign
+    where date(first_event_at) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/consistency_user_unsubscribes_pre_aug_2023.sql

@@ -50,6 +50,6 @@ final as (
 
 select *
 from final
-where unique_users_prod != unique_users_dev
+where unique_user_records_prod != unique_user_records_dev
     and channels_prod != channels_dev 
     and message_types_prod != message_types_dev

integration_tests/tests/consistency/consistency_users.sql

@@ -0,0 +1,47 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+-- this test ensures the iterable__users end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_iterable_prod.iterable__users
+    where date(updated_at) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_iterable_dev.iterable__users
+    where date(updated_at) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final