From 67118166d530d3efb4fbc57dd1becd8ba96f1c8a Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:07:32 -0500 Subject: [PATCH 1/2] bugfix/postgres-performance-updates --- dbt_project.yml | 2 + integration_tests/dbt_project.yml | 2 +- models/apple_store__app_version_report.sql | 24 +-------- models/apple_store__device_report.sql | 24 +-------- models/apple_store__overview_report.sql | 7 +-- .../apple_store__platform_version_report.sql | 25 +--------- models/apple_store__source_type_report.sql | 8 +-- models/apple_store__subscription_report.sql | 32 +----------- models/apple_store__territory_report.sql | 9 +--- .../int_apple_store__app_store_report.sql | 17 +++++++ .../int_apple_store__device_crash_report.sql | 39 +++++++++++++++ ...int_apple_store__platform_crash_report.sql | 39 +++++++++++++++ .../int_apple_store__source_type_report.sql | 18 +++++++ .../int_apple_store__subscription_report.sql | 49 +++++++++++++++++++ ...nt_apple_store__territory_report_grain.sql | 18 +++++++ .../int_apple_store__usage_crash_report.sql | 39 +++++++++++++++ 16 files changed, 236 insertions(+), 116 deletions(-) create mode 100644 models/intermediate/reporting_grain/int_apple_store__app_store_report.sql create mode 100644 models/intermediate/reporting_grain/int_apple_store__device_crash_report.sql create mode 100644 models/intermediate/reporting_grain/int_apple_store__platform_crash_report.sql create mode 100644 models/intermediate/reporting_grain/int_apple_store__source_type_report.sql create mode 100644 models/intermediate/reporting_grain/int_apple_store__subscription_report.sql create mode 100644 models/intermediate/reporting_grain/int_apple_store__territory_report_grain.sql create mode 100644 models/intermediate/reporting_grain/int_apple_store__usage_crash_report.sql diff --git a/dbt_project.yml b/dbt_project.yml index 34ea672..f942502 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -32,3 +32,5 @@ models: +schema: apple_store intermediate: +materialized: ephemeral + reporting_grain: + +materialized: table diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index da1893a..073c4b6 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,7 +6,7 @@ version: '0.4.0' profile: 'integration_tests' vars: - # apple_store__using_subscriptions: True # un-comment this line when generating docs! + apple_store__using_subscriptions: True # un-comment this line when generating docs! apple_store_schema: apple_store_integration_tests_7 apple_store_source: apple_store_app_identifier: "app" diff --git a/models/apple_store__app_version_report.sql b/models/apple_store__app_version_report.sql index f0917b9..ba7a195 100644 --- a/models/apple_store__app_version_report.sql +++ b/models/apple_store__app_version_report.sql @@ -16,30 +16,10 @@ usage_app_version_report as ( from {{ var('usage_app_version') }} ), -reporting_grain_combined as ( - - select - source_relation, - date_day, - app_id, - source_type, - app_version - from usage_app_version_report - union all - select - source_relation, - date_day, - app_id, - source_type, - app_version - from crashes_app_version_report -), - reporting_grain as ( - select - distinct * - from reporting_grain_combined + select * + from {{ ref('int_apple_store__usage_crash_report') }} ), joined as ( diff --git a/models/apple_store__device_report.sql b/models/apple_store__device_report.sql index b3e478c..8033a9b 100644 --- a/models/apple_store__device_report.sql +++ b/models/apple_store__device_report.sql @@ -36,30 +36,10 @@ subscription_device as ( ), {% endif %} -reporting_grain_combined as ( - - select - source_relation, - date_day, - app_id, - source_type, - device - from app_store_device - union all - select - source_relation, - date_day, - app_id, - source_type, - device - from crashes_device -), - reporting_grain as ( - select - distinct * - from reporting_grain_combined + select * + from {{ ref('int_apple_store__device_crash_report') }} ), joined as ( diff --git a/models/apple_store__overview_report.sql b/models/apple_store__overview_report.sql index 0c55bb5..197d6cd 100644 --- a/models/apple_store__overview_report.sql +++ b/models/apple_store__overview_report.sql @@ -38,11 +38,8 @@ usage as ( reporting_grain as ( - select distinct - source_relation, - date_day, - app_id - from app_store + select * + from {{ ref('int_apple_store__app_store_report') }} ), joined as ( diff --git a/models/apple_store__platform_version_report.sql b/models/apple_store__platform_version_report.sql index a9e2395..ce479c8 100644 --- a/models/apple_store__platform_version_report.sql +++ b/models/apple_store__platform_version_report.sql @@ -28,31 +28,10 @@ usage_platform_version as ( from {{ var('usage_platform_version') }} ), -reporting_grain_combined as ( - - select - source_relation, - date_day, - app_id, - source_type, - platform_version - from app_store_platform_version - union all - select - source_relation, - date_day, - app_id, - source_type, - platform_version - from crashes_platform_version -), - reporting_grain as ( - select - distinct * - from reporting_grain_combined - + select * + from {{ ref('int_apple_store__platform_crash_report') }} ), joined as ( diff --git a/models/apple_store__source_type_report.sql b/models/apple_store__source_type_report.sql index 04310f4..945c217 100644 --- a/models/apple_store__source_type_report.sql +++ b/models/apple_store__source_type_report.sql @@ -24,12 +24,8 @@ usage_source_type as ( reporting_grain as ( - select distinct - source_relation, - date_day, - app_id, - source_type - from app_store_source_type + select * + from {{ ref('int_apple_store__source_type_report') }} ), joined as ( diff --git a/models/apple_store__subscription_report.sql b/models/apple_store__subscription_report.sql index 4a27d56..a7042df 100644 --- a/models/apple_store__subscription_report.sql +++ b/models/apple_store__subscription_report.sql @@ -18,38 +18,10 @@ country_codes as ( from {{ var('apple_store_country_codes') }} ), -reporting_grain_combined as ( - - select - source_relation, - cast(date_day as date) as date_day, - account_id, - account_name, - app_name, - app_id, - subscription_name, - country, - state - from subscription_summary - union all - select - source_relation, - cast(date_day as date) as date_day, - account_id, - account_name, - app_name, - app_id, - subscription_name, - country, - state - from subscription_events -), - reporting_grain as ( - select - distinct * - from reporting_grain_combined + select * + from {{ ref('int_apple_store__subscription_report') }} ), joined as ( diff --git a/models/apple_store__territory_report.sql b/models/apple_store__territory_report.sql index 8830812..1914a64 100644 --- a/models/apple_store__territory_report.sql +++ b/models/apple_store__territory_report.sql @@ -30,13 +30,8 @@ usage_territory as ( reporting_grain as ( - select distinct - source_relation, - date_day, - app_id, - source_type, - territory - from app_store_territory + select * + from {{ ref('int_apple_store__territory_report_grain') }} ), joined as ( diff --git a/models/intermediate/reporting_grain/int_apple_store__app_store_report.sql b/models/intermediate/reporting_grain/int_apple_store__app_store_report.sql new file mode 100644 index 0000000..a9a496d --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__app_store_report.sql @@ -0,0 +1,17 @@ +with app_store as ( + + select * + from {{ ref('int_apple_store__app_store_overview') }} +), + +final as ( + + select distinct + source_relation, + date_day, + app_id + from app_store +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/reporting_grain/int_apple_store__device_crash_report.sql b/models/intermediate/reporting_grain/int_apple_store__device_crash_report.sql new file mode 100644 index 0000000..c928132 --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__device_crash_report.sql @@ -0,0 +1,39 @@ +with app_store_device as ( + + select * + from {{ var('app_store_device') }} +), + +crashes_device as ( + + select * + from {{ ref('int_apple_store__crashes_device') }} +), + +reporting_grain_combined as ( + + select + source_relation, + date_day, + app_id, + source_type, + device + from app_store_device + union all + select + source_relation, + date_day, + app_id, + source_type, + device + from crashes_device +), + +final as ( + + select distinct * + from reporting_grain_combined +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/reporting_grain/int_apple_store__platform_crash_report.sql b/models/intermediate/reporting_grain/int_apple_store__platform_crash_report.sql new file mode 100644 index 0000000..494ba62 --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__platform_crash_report.sql @@ -0,0 +1,39 @@ +with app_store_platform_version as ( + + select * + from {{ var('app_store_platform_version') }} +), + +crashes_platform_version as ( + + select * + from {{ ref('int_apple_store__platform_version') }} +), + +reporting_grain_combined as ( + + select + source_relation, + date_day, + app_id, + source_type, + platform_version + from app_store_platform_version + union all + select + source_relation, + date_day, + app_id, + source_type, + platform_version + from crashes_platform_version +), + +final as ( + + select distinct * + from reporting_grain_combined +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/reporting_grain/int_apple_store__source_type_report.sql b/models/intermediate/reporting_grain/int_apple_store__source_type_report.sql new file mode 100644 index 0000000..f4cf1f7 --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__source_type_report.sql @@ -0,0 +1,18 @@ +with app_store_source_type as ( + + select * + from {{ ref('int_apple_store__app_store_source_type') }} +), + +final as ( + + select distinct + source_relation, + date_day, + app_id, + source_type + from app_store_source_type +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/reporting_grain/int_apple_store__subscription_report.sql b/models/intermediate/reporting_grain/int_apple_store__subscription_report.sql new file mode 100644 index 0000000..9cda823 --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__subscription_report.sql @@ -0,0 +1,49 @@ +{{ config(enabled=var('apple_store__using_subscriptions', False)) }} + +with subscription_summary as ( + + select * + from {{ ref('int_apple_store__sales_subscription_summary') }} +), + +subscription_events as ( + + select * + from {{ ref('int_apple_store__sales_subscription_events') }} +), + +reporting_grain_combined as ( + + select + source_relation, + cast(date_day as date) as date_day, + account_id, + account_name, + app_name, + app_id, + subscription_name, + country, + state + from subscription_summary + union all + select + source_relation, + cast(date_day as date) as date_day, + account_id, + account_name, + app_name, + app_id, + subscription_name, + country, + state + from subscription_events +), + +final as ( + + select distinct * + from reporting_grain_combined +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/reporting_grain/int_apple_store__territory_report_grain.sql b/models/intermediate/reporting_grain/int_apple_store__territory_report_grain.sql new file mode 100644 index 0000000..9556401 --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__territory_report_grain.sql @@ -0,0 +1,18 @@ +with app_store_territory as ( + + select * + from {{ var('app_store_territory') }} +), + +final as ( + select distinct + source_relation, + date_day, + app_id, + source_type, + territory + from app_store_territory +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/reporting_grain/int_apple_store__usage_crash_report.sql b/models/intermediate/reporting_grain/int_apple_store__usage_crash_report.sql new file mode 100644 index 0000000..8420b6e --- /dev/null +++ b/models/intermediate/reporting_grain/int_apple_store__usage_crash_report.sql @@ -0,0 +1,39 @@ +with crashes_app_version_report as ( + + select * + from {{ ref('int_apple_store__crashes_app_version') }} +), + +usage_app_version_report as ( + + select * + from {{ var('usage_app_version') }} +), + +reporting_grain_combined as ( + + select + source_relation, + date_day, + app_id, + source_type, + app_version + from usage_app_version_report + union all + select + source_relation, + date_day, + app_id, + source_type, + app_version + from crashes_app_version_report +), + +final as ( + + select distinct * + from reporting_grain_combined +) + +select * +from final \ No newline at end of file From 389e7d623d09396357843c07adf7bd768060619d Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:08:20 -0500 Subject: [PATCH 2/2] gitignore update --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6a57f9f..3901b13 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ target/ dbt_packages/ logs/ -package-lock.yml \ No newline at end of file +package-lock.yml +env/ +.DS_Store \ No newline at end of file