introduce the hubspot engagement table to adjust the joins in int_hub…

CHANGELOG.md

@@ -1,3 +1,8 @@
+# dbt_unified_rag v0.1.0-a3
+
+- Added `stg_rag_hubspot__engagement` model as part of the hubspot staging models.
+- Updates `int_rag_hubspot__deal_document` to adjust the method that `hubspot_engagement_*` models are joined by leveraging the `hubspot__engagement` table as the intermediary joining table for the `engagement_contact` and `engagement_company` tables.
+
 # dbt_unified_rag v0.1.0-a3
 [PR #9](https://github.com/fivetran/dbt_unified_rag/pull/9) includes the following updates: 
 

README.md

@@ -47,6 +47,7 @@ Include the following package_display_name package version in your `packages.yml
 packages:
   - package: fivetran/unified_rag
     version: 0.1.0-a3
+    version: 0.2.0
 ```
 
 ### Step 3: Define database and schema variables

dbt_project.yml

@@ -2,7 +2,7 @@ config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 name: 'unified_rag'
-version: '0.1.0'
+version: '0.2.0'
 
 vars:
   unified_rag:
@@ -19,6 +19,7 @@ vars:
     jira_priority: "{{ source('rag_jira', 'priority') }}"
 
     # Hubspot Sources
+    hubspot_engagement: "{{ source('rag_hubspot', 'engagement') }}"
     hubspot_engagement_note: "{{ source('rag_hubspot', 'engagement_note') }}"
     hubspot_engagement_email: "{{ source('rag_hubspot', 'engagement_email') }}"
     hubspot_engagement_company: "{{ source('rag_hubspot', 'engagement_company') }}"

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: "rag_integration_tests"
-version: '0.1.0'
+version: '0.2.0'
 
 profile: "integration_tests"
 
@@ -14,6 +14,7 @@ vars:
   rag__using_zendesk: True
   rag__using_hubspot: True
 
+  rag_hubspot_engagement_identifier: "hubspot_engagement"
   rag_hubspot_engagement_note_identifier: "hubspot_engagement_note"
   rag_hubspot_engagement_email_identifier: "hubspot_engagement_email"
   rag_hubspot_engagement_company_identifier: "hubspot_engagement_company"
@@ -98,6 +99,9 @@ seeds:
         _fivetran_synced: timestamp
         property_closedate: timestamp
         property_createdate: timestamp
+    hubspot_engagement:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     hubspot_engagement_company:
       +column_types:
         engagement_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"

integration_tests/seeds/hubspot_engagement.csv

@@ -0,0 +1,2 @@
+id,type,_fivetran_synced,portal_id
+19732910159,CALL,2023-06-08 23:22:38.270000,4703379

macros/staging/hubspot/get_hubspot_engagement_columns.sql

@@ -0,0 +1,16 @@
+{% macro get_hubspot_engagement_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "active", "datatype": "boolean", "alias": "is_active"},
+    {"name": "created_at", "datatype": dbt.type_timestamp(), "alias": "created_timestamp"},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "owner_id", "datatype": dbt.type_int()},
+    {"name": "portal_id", "datatype": dbt.type_int()},
+    {"name": "timestamp", "datatype": dbt.type_timestamp(), "alias": "occurred_timestamp"},
+    {"name": "type", "datatype": dbt.type_string(), "alias": "engagement_type"}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/intermediate/hubspot/int_rag_hubspot__deal_document.sql

 {{ unified_rag.coalesce_cast(["engagement_emails.engagement_type", "engagement_notes.engagement_type", "'UNKNOWN'"], dbt.type_string()) }} as engagement_type, 
 {{ unified_rag.coalesce_cast(["engagement_emails.engagement_type", "engagement_notes.engagement_type", "'UNKNOWN'"], dbt.type_string()) }} as engagement_type, 
@@ -18,6 +18,11 @@ companies as (
     from {{ ref('stg_rag_hubspot__company') }}
 ), 
 
+engagements as (
+    select *
+    from {{ ref('stg_rag_hubspot__engagement') }}
+),
+
 engagement_companies as (
 
     select *
@@ -64,15 +69,18 @@ engagement_detail_prep as (
     left join engagement_deals
         on deals.deal_id = engagement_deals.deal_id
         and deals.source_relation = engagement_deals.source_relation
+    left join engagements
+        on engagement_deals.engagement_id = engagement.engagement_id
+        and engagement_deals.source_relation = engagement.source_relation
     left join engagement_contacts
-        on engagement_deals.engagement_id = engagement_contacts.engagement_id 
-        and engagement_deals.source_relation = engagement_contacts.source_relation
+        on engagement.engagement_id = engagement_contacts.engagement_id 
+        and engagement.source_relation = engagement_contacts.source_relation
+    left join engagement_companies
+        on engagement.engagement_id = engagement_companies.engagement_id 
+        and engagement.source_relation = engagement_companies.source_relation
     left join contacts 
         on engagement_contacts.contact_id = contacts.contact_id
         and engagement_contacts.source_relation = contacts.source_relation
-    left join engagement_companies
-        on engagement_deals.engagement_id = engagement_companies.engagement_id 
-        and engagement_deals.source_relation = engagement_companies.source_relation
     left join companies
         on engagement_companies.company_id = companies.company_id
         and engagement_companies.source_relation = companies.source_relation

models/staging/hubspot_staging/src_rag_hubspot.yml

@@ -6,6 +6,41 @@ sources:
     loaded_at_field: _fivetran_synced
 
     tables:
+      - name: engagement
+        identifier: "{{ var('rag_hubspot_engagement_identifier', 'engagement')}}"
+        description: Each record represents an engagement
+        config:
+          enabled: "{{ var('rag_hubspot_sales_enabled', true) and var('rag_hubspot_engagement_enabled', true) }}"
+        columns:
+          - name: _fivetran_synced
+            description: '{{ doc("_fivetran_synced") }}'
+          - name: active
+            description: >
+              Whether the engagement is currently being shown in the UI.
+
+              PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release.
+          - name: created_at
+            description: >
+              A timestamp representing when the engagement was created.
+
+              PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release.
+          - name: id
+            description: The ID of the engagement.
+          - name: owner_id
+            description: >
+              The ID of the engagement's owner.
+
+              PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release.
+          - name: portal_id
+            description: '{{ doc("portal_id") }}'
+          - name: timestamp
+            description: >
+              A timestamp in representing the time that the engagement should appear in the timeline.
+
+              PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release.
+          - name: type
+            description: One of NOTE, EMAIL, TASK, MEETING, or CALL, the type of the engagement.
+
       - name: engagement_note
         identifier: "{{ var('rag_hubspot_engagement_note_identifier', 'engagement_note')}}"
         description: Each record represents a NOTE engagement event.

models/staging/hubspot_staging/stg_rag_hubspot.yml

@@ -96,6 +96,39 @@ models:
       - name: source_relation
         description: The source of the record if the unioning functionality is being used. If it is not this field will be empty.
 
+  - name: stg_rag_hubspot__engagement
+    description: Each record represents an engagement 
+    columns:
+      - name: engagement_id
+        description: The ID of the engagement.
+        tests:
+          - not_null
+          - unique
+      - name: engagement_type
+        description: One of NOTE, EMAIL, TASK, MEETING, or CALL, the type of the engagement.
+      - name: is_active
+        description: >
+          Whether the engagement is currently being shown in the UI.
+
+          PLEASE NOTE - This field will only be populated for pre HubSpot v3 API versions. This field is only included to allow for backwards compatibility between HubSpot API versions. This field will be deprecated in the near future.
+      - name: occurred_timestamp
+        description: >
+          A timestamp in representing the time that the engagement should appear in the timeline.
+
+          PLEASE NOTE - This field will only be populated for pre HubSpot v3 API versions. This field is only included to allow for backwards compatibility between HubSpot API versions. This field will be deprecated in the near future.
+      - name: created_timestamp
+        description: >
+          This field marks the call's time of creation and determines where the call sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. 
+
+          PLEASE NOTE: This field will only be populated for pre HubSpot v3 API versions. This field is only included to allow for backwards compatibility between HubSpot API versions. This field will be deprecated in the near future.
+      - name: owner_id
+        description: >
+          The ID of the engagement's owner.
+
+          PLEASE NOTE - This field will only be populated for pre HubSpot v3 API versions. This field is only included to allow for backwards compatibility between HubSpot API versions. This field will be deprecated in the near future.
+      - name: portal_id
+        description: '{{ doc("portal_id") }}'
+
   - name: stg_rag_hubspot__engagement_company
     description: Each record represents a 'link' between a company and an engagement.
     columns:

models/staging/hubspot_staging/stg_rag_hubspot__engagement.sql

@@ -0,0 +1,50 @@
+{{ config(enabled=var('rag__using_hubspot', True)) }}
+
+with base as (
+
+    {{
+        fivetran_utils.union_data(
+            table_identifier='engagement', 
+            database_variable='rag_hubspot_database', 
+            schema_variable='rag_hubspot_schema', 
+            default_database=target.database,
+            default_schema='rag_hubspot',
+            default_variable='hubspot_engagement',
+            union_schema_variable='rag_hubspot_union_schemas',
+            union_database_variable='rag_hubspot_union_databases'
+        )
+    }}
+),
+
+fields as (
+
+    select 
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(source('rag_hubspot','engagement')),
+                staging_columns=get_hubspot_engagement_columns()
+            )
+        }}
+
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='rag_hubspot_union_schemas', 
+            union_database_variable='rag_hubspot_union_databases') 
+        }}
+    from base
+),
+
+final as (
+
+    select
+        id as engagement_id,
+        created_timestamp,
+        occurred_timestamp,
+        owner_id,
+        source_relation,
+        portal_id,
+        is_active
+    from fields  
+)  
+
+select *
+from final