From 34940d952d2c60f94c2a4d097b868d3ea01d78ff Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 10 Aug 2022 11:39:12 +0100 Subject: [PATCH 1/5] Adding the connector_type table to the package as well as other package updates --- CHANGELOG.md | 5 +++ dbt_project.yml | 3 +- integration_tests/dbt_project.yml | 3 +- macros/get_brand_columns.sql | 3 +- models/fivetran_log.yml | 14 +++++++- models/fivetran_log__connector_status.sql | 29 +++++++++++++--- models/staging/src_fivetran_log.yml | 32 ++++++++++++++++++ models/staging/stg_fivetran_log.yml | 33 ++++++++++++++++++- .../staging/stg_fivetran_log__connector.sql | 4 ++- .../stg_fivetran_log__connector_type.sql | 26 +++++++++++++++ models/staging/stg_fivetran_log__log.sql | 3 +- models/staging/stg_fivetran_log__user.sql | 3 +- 12 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 models/staging/stg_fivetran_log__connector_type.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index bd6a1b65..671bd979 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_fivetran_log v0.6.3 +## Feature - connector_type table added +- Added the connector_type table to the package and incorporated some of it's fields to the `fivetran_log__connector_status` model. +- Also made some changes to the staging connector, user and log tables as per the connector release notes: https://fivetran.com/docs/logs/fivetran-log/changelog + # dbt_fivetran_log v0.6.2 ## Fixes - Extend model disablement with `config: is_enabled` setting in sources to avoid running source freshness when a model is disabled. ([#58](https://github.com/fivetran/dbt_fivetran_log/pull/58)) diff --git a/dbt_project.yml b/dbt_project.yml index 5575940d..9fb70e03 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'fivetran_log' -version: '0.6.2' +version: '0.6.3' require-dbt-version: [">=1.0.0", "<2.0.0"] @@ -21,6 +21,7 @@ vars: account_membership: "{{ source('fivetran_log', 'account_membership') }}" active_volume: "{{ source('fivetran_log', 'active_volume') }}" connector: "{{ source('fivetran_log', 'connector') }}" + connector_type: "{{ source('fivetran_log', 'connector_type') }}" credits_used: "{{ source('fivetran_log', 'credits_used') }}" destination: "{{ source('fivetran_log', 'destination') }}" destination_membership: "{{ source('fivetran_log', 'destination_membership') }}" diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c2743316..0fb83721 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'fivetran_log_integration_tests' -version: '0.6.2' +version: '0.6.3' config-version: 2 profile: 'integration_tests' @@ -15,6 +15,7 @@ vars: fivetran_log_account_membership_identifier: "account_membership" fivetran_log_active_volume_identifier: "active_volume" fivetran_log_connector_identifier: "connector" + fivetran_log_connector_type_identifier: "connector_type" fivetran_log_credits_used_identifier: "credits_used" fivetran_log_usage_cost_identifier: "usage_cost" fivetran_log_destination_identifier: "destination" diff --git a/macros/get_brand_columns.sql b/macros/get_brand_columns.sql index 49eda8ce..3f2147fd 100644 --- a/macros/get_brand_columns.sql +++ b/macros/get_brand_columns.sql @@ -11,7 +11,8 @@ {"name": "destination_id", "datatype": dbt_utils.type_string()}, {"name": "paused", "datatype": "boolean"}, {"name": "service_version", "datatype": dbt_utils.type_int()}, - {"name": "signed_up", "datatype": dbt_utils.type_timestamp()} + {"name": "signed_up", "datatype": dbt_utils.type_timestamp()}, + {"name": "sync_frequency", "datatype": dbt_utils.type_int()} ] %} {{ return(columns) }} diff --git a/models/fivetran_log.yml b/models/fivetran_log.yml index 256b4e83..7372a574 100644 --- a/models/fivetran_log.yml +++ b/models/fivetran_log.yml @@ -20,8 +20,14 @@ models: - name: connector_name description: Name of the connector. + - name: official_connector_name + description: The official name of the connector. + - name: connector_type - description: The kind of connector (ie google sheets, webhooks). + description: The type of connector (e.g. Productivity, Finance). + + - name: availability + description: The general availability of the connector. - name: destination_id description: > @@ -49,6 +55,12 @@ models: - name: set_up_at description: Timestamp of when the connector was set up. + - name: is_deleted + description: Boolean indicating if the connector has been deleted. + + - name: is_broken + description: Boolean indicating if the connector is broken. + - name: number_of_schema_changes_last_month description: > The number of schema changes (creating tables or schemas and altering tables) in the past 30 days. diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql index e859139f..49c77592 100644 --- a/models/fivetran_log__connector_status.sql +++ b/models/fivetran_log__connector_status.sql @@ -50,6 +50,13 @@ connector as ( ), +connector_type as ( + select + * + from + {{ ref('stg_fivetran_log__connector_type') }} +), + destination as ( select * @@ -61,10 +68,14 @@ connector_metrics as ( select connector.connector_id, connector.connector_name, - connector.connector_type, + connector_type.connector as official_connector_name, + connector_type.connector_type, + connector_type.availability, connector.destination_id, connector.is_paused, connector.set_up_at, + connector_type.is_deleted, + connector_type.is_broken, max(case when connector_log.event_subtype = 'sync_start' then connector_log.created_at else null end) as last_sync_started_at, max(case when connector_log.event_subtype = 'sync_end' @@ -90,9 +101,11 @@ connector_metrics as ( max(case when event_type = 'WARNING' then connector_log.created_at else null end) as last_warning_at from connector + left join connector_type + on connector_type.connector_type_id = connector.connector_type left join connector_log on connector_log.connector_id = connector.connector_id - {{ dbt_utils.group_by(n=6) }} + {{ dbt_utils.group_by(n=10) }} ), @@ -133,13 +146,17 @@ connector_recent_logs as ( select connector_health.connector_id, connector_health.connector_name, + connector_health.official_connector_name, connector_health.connector_type, + connector_health.availability, connector_health.destination_id, connector_health.connector_health, connector_health.last_successful_sync_completed_at, connector_health.last_sync_started_at, connector_health.last_sync_completed_at, connector_health.set_up_at, + connector_health.is_deleted, + connector_health.is_broken, connector_log.event_subtype, connector_log.event_type, connector_log.message_data @@ -156,7 +173,7 @@ connector_recent_logs as ( and {{ fivetran_utils.json_parse(string="connector_log.message_data", string_path=["status"]) }} ='RESCHEDULED' and {{ fivetran_utils.json_parse(string="connector_log.message_data", string_path=["reason"]) }} like '%intended behavior%') - {{ dbt_utils.group_by(n=12) }} -- de-duping error messages + {{ dbt_utils.group_by(n=16) }} -- de-duping error messages ), @@ -166,7 +183,9 @@ final as ( select connector_recent_logs.connector_id, connector_recent_logs.connector_name, + connector_recent_logs.official_connector_name, connector_recent_logs.connector_type, + connector_recent_logs.availability, connector_recent_logs.destination_id, destination.destination_name, connector_recent_logs.connector_health, @@ -174,6 +193,8 @@ final as ( connector_recent_logs.last_sync_started_at, connector_recent_logs.last_sync_completed_at, connector_recent_logs.set_up_at, + connector_recent_logs.is_deleted, + connector_recent_logs.is_broken, coalesce(schema_changes.number_of_schema_changes_last_month, 0) as number_of_schema_changes_last_month {% if var('fivetran_log_using_sync_alert_messages', true) %} @@ -186,7 +207,7 @@ final as ( on connector_recent_logs.connector_id = schema_changes.connector_id join destination on destination.destination_id = connector_recent_logs.destination_id - {{ dbt_utils.group_by(n=11) }} + {{ dbt_utils.group_by(n=15) }} ) select * from final \ No newline at end of file diff --git a/models/staging/src_fivetran_log.yml b/models/staging/src_fivetran_log.yml index c4e4c506..9b93743f 100644 --- a/models/staging/src_fivetran_log.yml +++ b/models/staging/src_fivetran_log.yml @@ -56,6 +56,34 @@ sources: description: Boolean that is true if the connector's sync is currently paused. - name: signed_up description: Timestamp of when the connection was set up. + - name: sync_frequency + description: The frequency of how often each connector runs (in minutes). + - name: _fivetran_deleted + description: Boolean that tracks when a connector has been deleted. + + - name: connector_type + identifier: "{{ var('fivetran_log_connector_identifier', 'connector_type')}}" + description: > + Table of all connectors specified by their type to capture a connector's release phase. + columns: + - name: id + description: The name of the connector. + - name: official_connector_name + description: The official name of the connector. + - name: type + description: The type of the connector (e.g. Productivity). + - name: availability + description: The general availability of the connector. + - name: created_at + description: Timestamp indicating when the connector was created. + - name: public_beta_at + description: Timestamp indicating when the connector was released to the public for beta testing. + - name: release_at + description: Timestamp indicating when the connector was released. + - name: deleted + description: Boolean indicating if the connector has been deleted. + - name: broken + description: Boolean indicating if the connector is broken. - name: credits_used identifier: "{{ var('fivetran_log_credits_used_identifier', 'credits_used')}}" @@ -116,6 +144,8 @@ sources: description: The routine involved in the log, defined [here](https://fivetran.com/docs/logs#logevents). - name: transformation_id description: Foreign key referencing the `transformation` if the event refers to a transformation run. + - name: sync_id + description: The ID of each sync per connector. - name: account identifier: "{{ var('fivetran_log_account_identifier', 'account')}}" @@ -188,6 +218,8 @@ sources: description: Phone number associated with user. - name: verified description: Boolean that indicates whether the user has verified their email address in the account creation process. + - name: _fivetran_deleted + description: Boolean that tracks when a user has been deleted. - name: trigger_table identifier: "{{ var('fivetran_log_trigger_table_identifier', 'trigger_table')}}" diff --git a/models/staging/stg_fivetran_log.yml b/models/staging/stg_fivetran_log.yml index 7a995012..5a628936 100644 --- a/models/staging/stg_fivetran_log.yml +++ b/models/staging/stg_fivetran_log.yml @@ -52,6 +52,33 @@ models: description: Boolean that is true if the connector's sync is currently paused. - name: set_up_at description: Timestamp of when the connection was set up. + - name: sync_frequency_minutes + description: The frequency of how often each connector runs (in minutes). + - name: is_deleted + description: Boolean that tracks when a connector has been deleted. + + - name: stg_fivetran_log__connector_type + description: Table of all connectors specified by their type to capture a connector's release phase. + columns: + - name: connector_type_id + description: The ID and name of the connector. + - name: connector + description: The official name of the connector. + - name: connector_type + description: The type of the connector (e.g. Productivity). + - name: availability + description: The general availability of the connector. + - name: created_at + description: Timestamp indicating when the connector was created. + - name: public_beta_at + description: Timestamp indicating when the connector was released to the public for beta testing. + - name: release_at + description: Timestamp indicating when the connector was released. + - name: is_deleted + description: Boolean indicating if the connector has been deleted. + - name: is_broken + description: Boolean indicating if the connector is broken. + - name: stg_fivetran_log__credits_used description: Table of the credits used by the customer per month for each destination. @@ -126,6 +153,8 @@ models: description: The routine involved in the log, defined [here](https://fivetran.com/docs/logs#logevents). - name: transformation_id description: Foreign key referencing the `transformation` if the event refers to a transformation run. + - name: sync_id + description: The ID of each sync per connector. - name: stg_fivetran_log__account description: Table of information on this Fivetran account. @@ -250,4 +279,6 @@ models: - name: phone description: Phone number associated with user. - name: is_verified - description: Boolean that indicates whether the user has verified their email address in the account creation process. \ No newline at end of file + description: Boolean that indicates whether the user has verified their email address in the account creation process. + - name: is_deleted + description: Boolean that tracks when a user has been deleted. \ No newline at end of file diff --git a/models/staging/stg_fivetran_log__connector.sql b/models/staging/stg_fivetran_log__connector.sql index 1da865b6..25b20661 100644 --- a/models/staging/stg_fivetran_log__connector.sql +++ b/models/staging/stg_fivetran_log__connector.sql @@ -25,7 +25,9 @@ final as ( destination_id, connecting_user_id, paused as is_paused, - signed_up as set_up_at + signed_up as set_up_at, + sync_frequency as sync_frequency_minutes, + _fivetran_deleted as is_deleted from fields -- Only look at the most recent one diff --git a/models/staging/stg_fivetran_log__connector_type.sql b/models/staging/stg_fivetran_log__connector_type.sql new file mode 100644 index 00000000..7bf48bd2 --- /dev/null +++ b/models/staging/stg_fivetran_log__connector_type.sql @@ -0,0 +1,26 @@ + +with connector_type as ( + select + * + from + {{ var('connector_type') }} + ) + +, fields as ( + + select + id as connector_type_id + , official_connector_name as connector + , type as connector_type + , availability + , created_at + , public_beta_at + , release_at + , deleted as is_deleted + , broken as is_broken + from + connector_type + + ) + +select * from fields \ No newline at end of file diff --git a/models/staging/stg_fivetran_log__log.sql b/models/staging/stg_fivetran_log__log.sql index 8da61ae2..d3f0ecfc 100644 --- a/models/staging/stg_fivetran_log__log.sql +++ b/models/staging/stg_fivetran_log__log.sql @@ -17,7 +17,8 @@ fields as ( when transformation_id is not null and message_data like '%has succeeded%' then 'transformation run success' when transformation_id is not null and message_data like '%has failed%' then 'transformation run failed' else message_event end as event_subtype, - transformation_id + transformation_id, + sync_id from log ) diff --git a/models/staging/stg_fivetran_log__user.sql b/models/staging/stg_fivetran_log__user.sql index ac21c124..f9f2e692 100644 --- a/models/staging/stg_fivetran_log__user.sql +++ b/models/staging/stg_fivetran_log__user.sql @@ -16,7 +16,8 @@ fields as ( family_name as last_name, given_name as first_name, phone, - verified as is_verified + verified as is_verified, + _fivetran_deleted as is_deleted from fivetran_user ) From a148a0fe33b0ee5fa362ca7c46e4fe29738156f3 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:30:56 -0700 Subject: [PATCH 2/5] connector_type seed file --- integration_tests/seeds/connector_type.csv | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 integration_tests/seeds/connector_type.csv diff --git a/integration_tests/seeds/connector_type.csv b/integration_tests/seeds/connector_type.csv new file mode 100644 index 00000000..3d8386f4 --- /dev/null +++ b/integration_tests/seeds/connector_type.csv @@ -0,0 +1,11 @@ +id,_fivetran_synced,availability,broken,created_at,deleted,official_connector_name,public_beta_at,release_at,type +adwords,2022-07-05T10:59:37.158,DEVELOPMENT,false,2017-07-07T13:52:12,false,Google Ads (AdWords),,,Marketing +google_sheets,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-03-08T23:55:49,false,Google Sheets,2018-03-09T19:09:28,2018-03-09T19:09:28,File +fivetran_log,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-07-31T16:08:02,false,Fivetran Log,2020-03-13T10:44:09,2020-04-06T05:46:33,Free +shopify,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-02-15T03:06:56,false,Shopify,2017-04-10T17:15:05,2017-12-14T01:37:29,Sales +amplitude,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-12-10T10:54:16,false,Amplitude,2019-01-07T10:04:58,2019-07-02T17:23:16,Events +stripe,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2021-09-29T09:01:54.237,false,Stripe,2021-09-29T09:01:54.237,2021-09-29T09:01:54.237,Finance +quickbooks,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-07T08:29:42,false,QuickBooks,2017-12-07T08:29:42,2017-12-07T08:29:42,Finance +recurly,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-05-28T09:59:07,true,Recurly,2019-05-28T09:59:07,2019-09-05T23:09:17,Finance +qualtrics,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2020-09-29T19:00:45.287,false,Qualtrics,2022-01-11T18:01:21.651,2022-07-27T19:10:30.269,Support +zendesk,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-06T21:45:17,false,Zendesk Support,2017-12-06T21:45:17,2018-03-28T02:05:35,Support \ No newline at end of file From b271659cb5b089298069afdbb97a02e3aee28786 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:40:59 -0700 Subject: [PATCH 3/5] circleci --- integration_tests/seeds/connector_type.csv | 22 +++++----- integration_tests/seeds/log.csv | 50 +++++++++++----------- integration_tests/seeds/user.csv | 6 +-- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/integration_tests/seeds/connector_type.csv b/integration_tests/seeds/connector_type.csv index 3d8386f4..5b270f66 100644 --- a/integration_tests/seeds/connector_type.csv +++ b/integration_tests/seeds/connector_type.csv @@ -1,11 +1,11 @@ -id,_fivetran_synced,availability,broken,created_at,deleted,official_connector_name,public_beta_at,release_at,type -adwords,2022-07-05T10:59:37.158,DEVELOPMENT,false,2017-07-07T13:52:12,false,Google Ads (AdWords),,,Marketing -google_sheets,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-03-08T23:55:49,false,Google Sheets,2018-03-09T19:09:28,2018-03-09T19:09:28,File -fivetran_log,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-07-31T16:08:02,false,Fivetran Log,2020-03-13T10:44:09,2020-04-06T05:46:33,Free -shopify,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-02-15T03:06:56,false,Shopify,2017-04-10T17:15:05,2017-12-14T01:37:29,Sales -amplitude,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-12-10T10:54:16,false,Amplitude,2019-01-07T10:04:58,2019-07-02T17:23:16,Events -stripe,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2021-09-29T09:01:54.237,false,Stripe,2021-09-29T09:01:54.237,2021-09-29T09:01:54.237,Finance -quickbooks,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-07T08:29:42,false,QuickBooks,2017-12-07T08:29:42,2017-12-07T08:29:42,Finance -recurly,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-05-28T09:59:07,true,Recurly,2019-05-28T09:59:07,2019-09-05T23:09:17,Finance -qualtrics,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2020-09-29T19:00:45.287,false,Qualtrics,2022-01-11T18:01:21.651,2022-07-27T19:10:30.269,Support -zendesk,2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-06T21:45:17,false,Zendesk Support,2017-12-06T21:45:17,2018-03-28T02:05:35,Support \ No newline at end of file +_fivetran_synced,availability,broken,created_at,deleted,official_connector_name,public_beta_at,release_at,type,id +2022-07-05T10:59:37.158,DEVELOPMENT,false,2017-07-07T13:52:12,false,Google Ads (AdWords),,,Marketing,adwords +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-03-08T23:55:49,false,Google Sheets,2018-03-09T19:09:28,2018-03-09T19:09:28,File,google_sheets +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-07-31T16:08:02,false,Fivetran Log,2020-03-13T10:44:09,2020-04-06T05:46:33,Free,fivetran_log +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-02-15T03:06:56,false,Shopify,2017-04-10T17:15:05,2017-12-14T01:37:29,Sales,shopify +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2018-12-10T10:54:16,false,Amplitude,2019-01-07T10:04:58,2019-07-02T17:23:16,Events,amplitude +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2021-09-29T09:01:54.237,false,Stripe,2021-09-29T09:01:54.237,2021-09-29T09:01:54.237,Finance,stripe +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-07T08:29:42,false,QuickBooks,2017-12-07T08:29:42,2017-12-07T08:29:42,Finance,quickbooks +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2019-05-28T09:59:07,true,Recurly,2019-05-28T09:59:07,2019-09-05T23:09:17,Finance,recurly +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2020-09-29T19:00:45.287,false,Qualtrics,2022-01-11T18:01:21.651,2022-07-27T19:10:30.269,Support,qualtrics +2022-09-23T15:37:14.347,GENERAL_AVAILABILITY,false,2017-12-06T21:45:17,false,Zendesk Support,2017-12-06T21:45:17,2018-03-28T02:05:35,Support,zendesk \ No newline at end of file diff --git a/integration_tests/seeds/log.csv b/integration_tests/seeds/log.csv index e5bd6d96..ebb1201d 100644 --- a/integration_tests/seeds/log.csv +++ b/integration_tests/seeds/log.csv @@ -1,25 +1,25 @@ -id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id -legislative_lazy,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning, -paint_wedges,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call, -intrinsic_departed,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end, -intrinsic_departed,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified, -intrinsic_departed,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start, -intrinsic_departed,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified, -intrinsic_departed,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified, -intrinsic_departed,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified, -intrinsic_departed,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified, -intrinsic_departed,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end, -intrinsic_departed,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start, -intrinsic_departed,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start, -intrinsic_departed,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start, -intrinsic_departed,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start, -intrinsic_departed,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start, \ No newline at end of file +id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id,sync_id +legislative_lazy,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning,,25 +paint_wedges,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call,,24 +intrinsic_departed,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end,,23 +intrinsic_departed,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,,22 +intrinsic_departed,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,,21 +intrinsic_departed,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,,20 +intrinsic_departed,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,,19 +intrinsic_departed,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,,18 +intrinsic_departed,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,,17 +intrinsic_departed,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,,16 +intrinsic_departed,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,,15 +intrinsic_departed,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,,14 +intrinsic_departed,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,,13 +intrinsic_departed,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,,12 +intrinsic_departed,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,,11 +intrinsic_departed,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,,9 +intrinsic_departed,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,,8 +intrinsic_departed,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,,7 +intrinsic_departed,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,,6 +intrinsic_departed,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start,,5 +intrinsic_departed,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start,,4 +intrinsic_departed,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start,,3 +intrinsic_departed,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,,2 +intrinsic_departed,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start,,1 \ No newline at end of file diff --git a/integration_tests/seeds/user.csv b/integration_tests/seeds/user.csv index 0577244c..3ff46595 100644 --- a/integration_tests/seeds/user.csv +++ b/integration_tests/seeds/user.csv @@ -1,3 +1,3 @@ -id,_fivetran_synced,created_at,email,email_disabled,family_name,given_name,phone,verified -barista_fibula,2021-03-12 21:03:46.208,2020-05-19 20:09:23.765024,me@me.com,false,Squarepants,Spngebob,,false -backlog_jack,2021-03-12 21:03:45.888,2020-08-19 15:12:41.254224,you@you.com,false,McGee,Knucklehead,,true +id,_fivetran_synced,created_at,email,email_disabled,family_name,given_name,phone,verified,_fivetran_deleted +barista_fibula,2021-03-12 21:03:46.208,2020-05-19 20:09:23.765024,me@me.com,false,Squarepants,Spngebob,,false,false +backlog_jack,2021-03-12 21:03:45.888,2020-08-19 15:12:41.254224,you@you.com,false,McGee,Knucklehead,,true,false From 53c8c783ae89b3453b92a592258e8946b0b57fdf Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:46:26 -0700 Subject: [PATCH 4/5] circleci being weird on spark" --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 9821bb4a..13903e19 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,14 +16,14 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: fivetrans_logs_integration_tests + schema: fivetrans_logs_integration_tests_1 threads: 8 bigquery: type: bigquery method: service-account keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}" project: 'dbt-package-testing' - schema: fivetrans_logs_integration_tests + schema: fivetrans_logs_integration_tests_1 threads: 8 snowflake: type: snowflake @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: fivetrans_logs_integration_tests + schema: fivetrans_logs_integration_tests_1 threads: 8 postgres: type: postgres @@ -42,12 +42,12 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: fivetrans_logs_integration_tests + schema: fivetrans_logs_integration_tests_1 threads: 8 spark: type: spark method: http - schema: fivetrans_logs_integration_tests + schema: fivetrans_logs_integration_tests_1 host: "{{ env_var('CI_SPARK_DBT_HOST') }}" organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}" token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}" diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 0cec1c14..e04b2bc5 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -10,7 +10,7 @@ dispatch: vars: fivetran_log: - fivetran_log_schema: fivetrans_logs_integration_tests + fivetran_log_schema: fivetrans_logs_integration_tests_1 fivetran_log_account_identifier: "account" fivetran_log_account_membership_identifier: "account_membership" fivetran_log_active_volume_identifier: "active_volume" From 4948e9a7d736d2b73db300073585d8e59b541e7a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 23 Sep 2022 13:48:28 -0700 Subject: [PATCH 5/5] fix identifier var --- models/staging/src_fivetran_log.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/src_fivetran_log.yml b/models/staging/src_fivetran_log.yml index 9b93743f..3b3c6ada 100644 --- a/models/staging/src_fivetran_log.yml +++ b/models/staging/src_fivetran_log.yml @@ -62,7 +62,7 @@ sources: description: Boolean that tracks when a connector has been deleted. - name: connector_type - identifier: "{{ var('fivetran_log_connector_identifier', 'connector_type')}}" + identifier: "{{ var('fivetran_log_connector_type_identifier', 'connector_type')}}" description: > Table of all connectors specified by their type to capture a connector's release phase. columns: