Merge pull request #92 from fivetran/feature/include-deleted-records

updating models
fivetran · Jan 11, 2023 · a218221 · a218221
2 parents f29e025 + f3904f7
commit a218221
Show file tree

Hide file tree

Showing 14 changed files with 132 additions and 27 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,44 @@
+# dbt_hubspot v0.8.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #92](https://github.com/fivetran/dbt_hubspot/pull/92) incorporates the following changes:
+
+- The below models/macros have new soft deleted record flags **explicitly** added to them:
+  - `is_contact_deleted` added in `macros/email_events_joined` which can affect the following downstream models found in `models/marketing/email_events/`:
+    - `hubspot__email_event_bounce`
+    - `hubspot__email_event_clicks`
+    - `hubspot__email_event_deferred`
+    - `hubspot__email_event_delivered`
+    - `hubspot__email_event_dropped`
+    - `hubspot__email_event_forward`
+    - `hubspot__email_event_open`
+    - `hubspot__email_event_print`
+    - `hubspot__email_event_sent`
+    - `hubspot__email_event_spam_report`
+    - `hubspot__email_event_status_change`
+  - `is_deal_pipeline_deleted` and `is_deal_pipeline_stage_deleted` added in `models/sales/intermediate/int_hubspot__deals_enhanced` which can affect the following downstream models found in `models/sales/`:
+    - `hubspot__deal_stages`
+    - `hubspot__deals`
+  - `is_deal_pipeline_deleted`, `is_deal_pipeline_stage_deleted` and `is_deal_deleted` added in `models/sales/hubspot__deal_stages`
+
+- Soft deleted records are also now **implicitly** inherited (e.g. via a `select *`) from the `dbt_hubspot_source` package's staging models and can affect the below final models (for more information, see [dbt_hubspot_source PR #96](https://github.com/fivetran/dbt_hubspot_source/pull/96)):
+  - Soft deleted company records (`companies.is_company_deleted`) included in the below models:
+    - `sales/hubspot__companies`
+  - Soft deleted deal records (`deals.is_deal_deleted`) included in the below models:
+    - `sales/hubspot__deal_stages`
+    - `sales/hubspot__deals`
+  - Soft deleted contact list records (`contact_lists.is_contact_list_deleted`) included in the below models:
+    - `marketing/hubspot__contact_lists`
+  - Soft deleted contact records (`contacts.is_contact_deleted`) included in the below models:
+    - `marketing/hubspot__contacts`
+  - Soft deleted email sends records (`sends.is_contact_deleted`) included in the below models:
+    - `marketing/hubspot__email_sends`
+
+- For completeness, soft deleted records are also now included for `ticket*` tables, however does not currently affect this package directly and includes the following staging models:
+    - `stg_hubspot__ticket` (`is_ticket_deleted`)
+    - `stg_hubspot__ticket_pipeline_stage`(`is_ticket_pipeline_stage_deleted`)
+    - `stg_hubspot__ticket_pipeline` (`is_ticket_pipeline_deleted`)
+
 # dbt_hubspot v0.7.0
 
 ## 🚨 Breaking Changes 🚨:

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.7.0'
+version: '0.8.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_integration_tests
+      schema: hubspot_integration_tests_1
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_integration_tests
+      schema: hubspot_integration_tests_1
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_integration_tests
+      schema: hubspot_integration_tests_1
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_integration_tests
+      schema: hubspot_integration_tests_1
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_integration_tests
+      schema: hubspot_integration_tests_1
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,9 +1,9 @@
 name: 'hubspot_integration_tests'
-version: '0.7.0'
+version: '0.8.0'
 profile: 'integration_tests'
 config-version: 2
 vars:
-  hubspot_schema: hubspot_integration_tests
+  hubspot_schema: hubspot_integration_tests_1
   hubspot_source:
     hubspot_service_enabled: true
     hubspot_company_property_history_identifier: "company_property_history_data"

diff --git a/macros/email_events_joined.sql b/macros/email_events_joined.sql
@@ -36,7 +36,8 @@ with base as (
 
     select 
         events_joined.*,
-        contacts.contact_id
+        contacts.contact_id,
+        coalesce(contacts.is_contact_deleted, false) as is_contact_deleted
     from events_joined
     left join contacts
         on events_joined.recipient_email_address = contacts.email

diff --git a/models/marketing/email_events/email_events.yml b/models/marketing/email_events/email_events.yml
@@ -8,6 +8,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: bounce_category
         description: |
               The best-guess of the type of bounce encountered. 
@@ -50,6 +53,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: browser
         description: '{{ doc("email_event_browser") }}'
 
@@ -98,6 +104,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: attempt_number
         description: The delivery attempt number.
 
@@ -134,6 +143,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: event_id
         description: The ID of the event.
         tests:
@@ -170,6 +182,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: bcc_emails
         description: The 'bcc' field of the email message.
 
@@ -221,6 +236,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: browser
         description: '{{ doc("email_event_browser") }}'
 
@@ -263,6 +281,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: browser
         description: '{{ doc("email_event_browser") }}'
 
@@ -308,6 +329,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: browser
         description: '{{ doc("email_event_browser") }}'
 
@@ -350,6 +374,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: bcc_emails
         description: The 'cc' field of the email message.
 
@@ -395,6 +422,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: event_id
         description: The ID of the event.
         tests:
@@ -431,6 +461,9 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: change_source
         description: The source of the subscription change. 
 

diff --git a/models/marketing/marketing.yml b/models/marketing/marketing.yml
@@ -7,6 +7,9 @@ models:
     columns:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
+
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: bcc_emails
         description: The 'cc' field of the email message.
@@ -206,11 +209,11 @@ models:
   - name: hubspot__contacts
     description: Each record represents a contact in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
-        description: '{{ doc("_fivetran_deleted") }}'
+        description: '{{ doc("_fivetran_synced") }}'
 
       - name: contact_id
         description: The ID of the contact.
@@ -323,11 +326,8 @@ models:
   - name: hubspot__contact_lists
     description: Each record represents a contact list in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
-
       - name: _fivetran_synced
-        description: '{{ doc("_fivetran_deleted") }}'
+        description: '{{ doc("_fivetran_synced") }}'
 
       - name: contact_list_id
         description: The ID of the contact list.
@@ -338,6 +338,9 @@ models:
       - name: contact_list_name
         description: The name of the contact list.
 
+      - name: is_contact_list_deleted
+        description: '{{ doc("is_deleted") }}'
+
       - name: created_timestamp
         description: A timestamp of the time the list was created.
 

diff --git a/models/sales/hubspot__deal_stages.sql b/models/sales/hubspot__deal_stages.sql
@@ -29,7 +29,6 @@ with deals_enhanced as (
         deal_stage._fivetran_start as date_stage_entered,
         deal_stage._fivetran_end as date_stage_exited,
         deal_stage._fivetran_active as is_stage_active,
-
         deal_stage.deal_stage_name as pipeline_stage_id,
         pipeline_stage.pipeline_stage_label,
         pipeline_stage.deal_pipeline_id as pipeline_id,
@@ -41,7 +40,10 @@ with deals_enhanced as (
         pipeline_stage.is_closed_won as is_pipeline_stage_closed_won,
         pipeline_stage.display_order as pipeline_stage_display_order,
         pipeline.display_order as pipeline_display_order,
-        pipeline_stage.probability as pipeline_stage_probability
+        pipeline_stage.probability as pipeline_stage_probability,
+        coalesce(pipeline.is_deal_pipeline_deleted, false) as is_deal_pipeline_deleted,
+        coalesce(pipeline_stage.is_deal_pipeline_stage_deleted, false) as is_deal_pipeline_stage_deleted,
+        coalesce(deals_enhanced.is_deal_deleted, false) as is_deal_deleted
 
     from deal_stage
 

diff --git a/models/sales/intermediate/int_hubspot__deals_enhanced.sql b/models/sales/intermediate/int_hubspot__deals_enhanced.sql
@@ -24,12 +24,13 @@ with deals as (
 
     select 
         deals.*,
+        coalesce(pipelines.is_deal_pipeline_deleted, false) as is_deal_pipeline_deleted,
         pipelines.pipeline_label,
         pipelines.is_active as is_pipeline_active,
+        coalesce(pipeline_stages.is_deal_pipeline_stage_deleted, false) as is_deal_pipeline_stage_deleted,
         pipeline_stages.pipeline_stage_label,
         owners.email_address as owner_email_address,
         owners.full_name as owner_full_name
-
     from deals    
     left join pipelines 
         on deals.deal_pipeline_id = pipelines.deal_pipeline_id

diff --git a/models/sales/sales.yml b/models/sales/sales.yml
@@ -29,8 +29,14 @@ models:
       - name: deal_name
         description: The name you have given this deal.
 
-      - name: is_deleted
-        description: Whether the record was deleted.
+      - name: is_deal_deleted
+        description: '{{ doc("is_deleted") }}'
+
+      - name: is_deal_pipeline_deleted
+        description: '{{ doc("is_deleted") }}'
+
+      - name: is_deal_pipeline_stage_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: portal_id
         description: '{{ doc("portal_id") }}'
@@ -88,6 +94,15 @@ models:
         tests:
           - not_null
           - unique
+
+      - name: is_deal_pipeline_deleted
+        description: '{{ doc("is_deleted") }}'
+
+      - name: is_deal_pipeline_stage_deleted
+        description: '{{ doc("is_deleted") }}'
+
+      - name: is_deal_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: pipeline_stage_id
         description: The name of the deal stage.
@@ -143,6 +158,15 @@ models:
       - name: deal_name
         description: The name you have given this deal.
 
+      - name: is_deal_pipeline_deleted
+        description: Whether the record was deleted from the deal pipeline table.
+
+      - name: is_deal_pipeline_stage_deleted
+        description: Whether the record was deleted from the deal pipeline stage table.
+
+      - name: is_deal_deleted
+        description: Whether the record was deleted from the deal table.
+
   - name: hubspot__companies
     description: Each record represents a company in Hubspot.
     columns:
@@ -155,7 +179,7 @@ models:
       - name: portal_id
         description: '{{ doc("portal_id") }}'
 
-      - name: is_deleted
+      - name: is_company_deleted
         description: '{{ doc("is_deleted") }}'
 
       - name: count_engagement_notes

diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,3 @@
 packages:
-- package: fivetran/hubspot_source
-  version: [">=0.7.0", "<0.8.0"]
+  - package: fivetran/hubspot_source
+    version: [">=0.8.0", "<0.9.0"]