Merge pull request #69 from fivetran/bugfix/history-valid-to

adding field_name to lead() window fcn

.circleci/config.yml

@@ -23,20 +23,6 @@ jobs:
             pip install -r integration_tests/requirements.txt
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
-      - run:
-          name: "Run Tests - Redshift"
-          command: |
-            . venv/bin/activate
-            echo `pwd`
-            cd integration_tests
-            dbt deps
-            dbt seed --target redshift --full-refresh
-            dbt run --target redshift --full-refresh
-            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target redshift --full-refresh
-            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target redshift
-            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target redshift --full-refresh
-            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target redshift --full-refresh
-            dbt test --target redshift
       - run:
           name: "Run Tests - Postgres"
           command: |
@@ -82,3 +68,17 @@ jobs:
             dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target bigquery --full-refresh
             dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target bigquery --full-refresh
             dbt test --target bigquery
+      - run:
+          name: "Run Tests - Redshift"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target redshift --full-refresh
+            dbt run --target redshift --full-refresh
+            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target redshift --full-refresh
+            dbt run --vars '{hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target redshift
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target redshift --full-refresh
+            dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target redshift --full-refresh
+            dbt test --target redshift

CHANGELOG.md

@@ -1,6 +1,13 @@
 # dbt_hubspot v0.5.3
 ## Under the Hood
 - Added integration testing to support the new `stg_hubspot__deal_contact` model to the `dbt_hubspot_source` package.
+- Updated the below models such that the respective `lead()` functions also account for `field_name` in partitions
+  - `hubspot__contact_history`
+  - `hubspot__company_history`
+  - `hubspot__deal_history`
+- Added test cases for the above models that can be found, respectively, in: 
+  - `models/marketing/history/history.yml`
+  - `models/sales/history/history.yml`
 # dbt_hubspot v0.5.2
 ## Under the Hood
 - Updated the `engagements_joined` [macro](https://github.com/fivetran/dbt_hubspot/blob/main/macros/engagements_joined.sql) to conditionally include relevant fields if their respective variables are enabled. Previously if a variable was disabled then the `hubspot__engagement_calls` model, which depends on the `engagements_joined` macro, would error out on the missing fields that were from the disabled variables. ([#65](https://github.com/fivetran/dbt_hubspot/pull/65))

models/marketing/history/history.yml

@@ -3,7 +3,13 @@ version: 2
 models:
 
   - name: hubspot__contact_history
-    description: Each record represents a change to contact record in Hubspot.
+    description: Each record represents a change to contact record in Hubspot; the grain of this model is contact_id -> field_name -> valid_to.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - contact_id
+            - field_name
+            - valid_to
     columns:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'

models/marketing/history/hubspot__contact_history.sql

@@ -14,7 +14,7 @@ with history as (
         change_source_id,
         change_timestamp as valid_from,
         new_value,
-        lead(change_timestamp) over (partition by contact_id order by change_timestamp) as valid_to
+        lead(change_timestamp) over (partition by contact_id, field_name order by change_timestamp) as valid_to
     from history
 
 ), surrogate as (

models/sales/history/history.yml

@@ -3,7 +3,13 @@ version: 2
 models:
 
   - name: hubspot__company_history
-    description: Each record represents a change to company record in Hubspot.
+    description: Each record represents a change to company record in Hubspot; the grain of this model is company_id -> field_name -> valid_to.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - company_id
+            - field_name
+            - valid_to
     columns:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
@@ -36,7 +42,13 @@ models:
           - not_null
 
   - name: hubspot__deal_history
-    description: Each record represents a change to deal record in Hubspot.
+    description: Each record represents a change to deal record in Hubspot; the grain of this model is deal_id -> field_name -> valid_to.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - deal_id
+            - field_name
+            - valid_to
     columns:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'

models/sales/history/hubspot__company_history.sql

@@ -14,7 +14,7 @@ with history as (
         change_source_id,
         change_timestamp as valid_from,
         new_value,
-        lead(change_timestamp) over (partition by company_id order by change_timestamp) as valid_to
+        lead(change_timestamp) over (partition by company_id, field_name order by change_timestamp) as valid_to
     from history
 
 ), surrogate as (

models/sales/history/hubspot__deal_history.sql

@@ -14,7 +14,7 @@ with history as (
         change_source_id,
         change_timestamp as valid_from,
         new_value,
-        lead(change_timestamp) over (partition by deal_id order by change_timestamp) as valid_to
+        lead(change_timestamp) over (partition by deal_id, field_name order by change_timestamp) as valid_to
     from history
 
 ), surrogate as (