Nov 2024 schema

CHANGELOG.md

@@ -1,4 +1,12 @@
-# dbt_apple_store version.version
+# dbt_apple_store v0.5.0
+
+## Breaking Changes: Schema Change
+- Following the connector's [Nov 2024 Update](https://fivetran.com/docs/connectors/applications/apple-app-store/changelog#november2024) to sync from the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi), we've updated this dbt package to reflect the new schema which includes the following changes:
+
+# Breaking Changes
+- The `account_id` and `account_name` fields have been removed.
+
+## to be complete
 
 ## Documentation
 - Added Quickstart model counts to README. ([#31](https://github.com/fivetran/dbt_apple_store/pull/31))

README.md

@@ -33,7 +33,7 @@ The following table provides a detailed list of all tables materialized within t
 | [apple_store__overview_report](https://fivetran.github.io/dbt_apple_store/#!/model/model.apple_store.apple_store__overview_report)     | Each record represents daily metrics for each app_id. |
 | [apple_store__platform_version_report](https://fivetran.github.io/dbt_apple_store/#!/model/model.apple_store.apple_store__platform_version_report)    | Each record represents daily metrics for each by app_id, source_type and platform version. |
 | [apple_store__source_type_report](https://fivetran.github.io/dbt_apple_store/#!/model/model.apple_store.apple_store__source_type_report)   | Each record represents daily metrics by app_id and source_type. |
-| [apple_store__subscription_report](https://fivetran.github.io/dbt_apple_store/#!/model/model.apple_store.apple_store__subscription_report) | Each record represents daily subscription metrics by account, app, subscription name, country and state. |
+| [apple_store__subscription_report](https://fivetran.github.io/dbt_apple_store/#!/model/model.apple_store.apple_store__subscription_report) | Each record represents daily subscription metrics by app, subscription name, country and state. |
 | [apple_store__territory_report](https://fivetran.github.io/dbt_apple_store/#!/model/model.apple_store.apple_store__source_type_report) | Each record represents daily subscription metrics by app_id, source_type and territory. |
 
 ### Materialized Models
@@ -54,7 +54,7 @@ Include the following apple_store package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/apple_store
-    version: [">=0.4.0", "<0.5.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do NOT include the `apple_store_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -146,7 +146,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/apple_store_source
-      version: [">=0.4.0", "<0.5.0"]
+      version: [">=0.5.0", "<0.6.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,26 +1,19 @@
 name: 'apple_store'
-version: '0.4.0'
+version: '0.5.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
   apple_store:
-    app: "{{ ref('stg_apple_store__app') }}"
-    app_store_device: "{{ ref('stg_apple_store__app_store_device') }}"
-    usage_device: "{{ ref('stg_apple_store__usage_device') }}"
-    downloads_device: "{{ ref('stg_apple_store__downloads_device') }}"
-    app_store_territory: "{{ ref('stg_apple_store__app_store_territory') }}"
-    downloads_territory: "{{ ref('stg_apple_store__downloads_territory') }}"
-    usage_territory: "{{ ref('stg_apple_store__usage_territory') }}"
-    app_store_platform_version: "{{ ref('stg_apple_store__app_store_platform_version') }}"
-    crashes_platform_version: "{{ ref('stg_apple_store__crashes_platform_version') }}"
-    downloads_platform_version: "{{ ref('stg_apple_store__downloads_platform_version') }}"
-    usage_platform_version: "{{ ref('stg_apple_store__usage_platform_version') }}"
-    crashes_app_version: "{{ ref('stg_apple_store__crashes_app_version') }}"
-    usage_app_version: "{{ ref('stg_apple_store__usage_app_version') }}"
+    app_store_app: "{{ ref('stg_apple_store__app_store_app') }}"
     sales_account: "{{ ref('stg_apple_store__sales_account') }}"
     sales_subscription_events: "{{ ref('stg_apple_store__sales_subscription_events') }}"
     sales_subscription_summary: "{{ ref('stg_apple_store__sales_subscription_summary') }}"
     apple_store_country_codes: "{{ ref('apple_store_country_codes') }}"
+    app_store_discovery_and_engagement_detailed_daily: "{{ ref('stg_apple_store__app_store_discovery_and_engagement_daily')}}"
+    app_crash_daily: "{{ ref('stg_apple_store__app_crash_daily')}}"
+    app_store_download_detailed_daily: "{{ ref('stg_apple_store__app_store_download_daily')}}"
+    app_session_detailed_daily: "{{ ref('stg_apple_store__app_session_daily')}}"
+    app_store_installation_and_deletion_detailed_daily: "{{ ref('stg_apple_store__app_store_installation_and_deletion_daily')}}"
     apple_store__subscription_events:
     - 'Renew'
     - 'Cancel'

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: apple_store_integration_tests_7
+      schema: apple_store_integration_tests_8
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: apple_store_integration_tests_7
+      schema: apple_store_integration_tests_8
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: apple_store_integration_tests_7
+      schema: apple_store_integration_tests_8
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: apple_store_integration_tests_7
+      schema: apple_store_integration_tests_8
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: apple_store_integration_tests_7
+      schema: apple_store_integration_tests_8
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,30 +1,22 @@
 config-version: 2
 
 name: 'apple_store_integration_tests'
-version: '0.4.0'
+version: '0.5.0'
 
 profile: 'integration_tests'
 
 vars:
   # apple_store__using_subscriptions: True # un-comment this line when generating docs!
-  apple_store_schema: apple_store_integration_tests_7
+  apple_store_schema: apple_store_integration_tests_8
   apple_store_source:
-    apple_store_app_identifier: "app"
-    apple_store_app_store_platform_version_source_type_report_identifier: "app_store_platform_version_source_type"
-    apple_store_app_store_source_type_device_report_identifier: "app_store_source_type_device"
-    apple_store_app_store_territory_source_type_report_identifier: "app_store_territory_source_type"
-    apple_store_crashes_app_version_device_report_identifier: "crashes_app_version"
-    apple_store_crashes_platform_version_device_report_identifier: "crashes_platform_version"
-    apple_store_downloads_platform_version_source_type_report_identifier: "downloads_platform_version_source_type"
-    apple_store_downloads_source_type_device_report_identifier: "downloads_source_type_device"
-    apple_store_downloads_territory_source_type_report_identifier: "downloads_territory_source_type"
-    apple_store_sales_account_identifier: "sales_account"
-    apple_store_sales_subscription_event_summary_identifier: "sales_subscription_events"
+    apple_store_app_identifier: "app_store_app"
+    apple_store_sales_subscription_event_summary_identifier: "sales_subscription_event_summary"
     apple_store_sales_subscription_summary_identifier: "sales_subscription_summary"
-    apple_store_usage_app_version_source_type_report_identifier: "usage_app_version_source_type"
-    apple_store_usage_platform_version_source_type_report_identifier: "usage_platform_version_source_type"
-    apple_store_usage_source_type_device_report_identifier: "usage_source_type_device"
-    apple_store_usage_territory_source_type_report_identifier: "usage_territory_source_type"
+    apple_store_discovery_and_engagement_detailed_daily_identifier: "app_store_discovery_and_engagement_detailed_daily"
+    apple_store_crash_daily_identifier: "app_crash_daily"
+    apple_store_download_detailed_daily_identifier: "app_store_download_detailed_daily"
+    apple_store_session_detailed_daily_identifier: "app_session_detailed_daily"
+    apple_store_installation_and_deletion_detailed_daily_identifier: "app_store_installation_and_deletion_detailed_daily"
 
     apple_store__subscription_events:
       - 'Renew'
@@ -42,7 +34,7 @@ seeds:
       +quote_columns: "{{ true if target.type == 'redshift' else false }}"
       +column_types:
         _fivetran_synced: timestamp
-        date: timestamp
+        date: date
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/seeds/app_crash_daily.csv

@@ -0,0 +1,11 @@
+_fivetran_id,app_id,date,app_version,device,platform_version,crashes,unique_devices,_fivetran_synced
+mATdTIo1WH34/utbwQZkvLNs6wU=,239587236,2024-12-04,120614,iPhone,iOS 16.4,1,1,2024-12-11 05:13:33.444 +00:00
+PYNoHVNnmMuAKBkHqvpL2G6462E=,239587236,2024-12-05,1.3.4,Apple Vision,visionOS 1.0,1,1,2024-12-12 05:14:16.162 +00:00
+YKv85Suuzv2jkvVhAyg2F8WM7SY=,239587236,2024-12-05,1242228,iPhone,iOS 17.1,1,1,2024-12-12 05:14:16.173 +00:00
+fyzwmTKUa2Rcwyzp7iKJVNwa004=,239587236,2024-12-05,1206422,iPhone,iOS 15.1,1,1,2024-12-12 05:14:16.170 +00:00
+rI4nrxlzrGiShj+7lnkVNLA4DdU=,239587236,2024-12-05,120222,iPhone,iOS 17.4,1,1,2024-12-12 05:14:16.166 +00:00
+iBFL/ZX+cw+LEb9QzQ0EIDmIsqg=,239587236,2024-12-05,120222.1,iPhone,iOS 17.2,1,1,2024-12-12 05:14:16.189 +00:00
+RN7svEING9NuyLcOQkgRgcyPcGc=,239587236,2024-12-05,55555,iPhone,iOS 14.2,1,1,2024-12-12 05:14:16.175 +00:00
+WIP6VVo9+T0ptzvWfSChZ+oSK1w=,239587236,2024-12-05,55555.5,iPhone,iOS 17.4,1,1,2024-12-12 05:14:16.157 +00:00
+9liOcvJVocUOsGJdywFyCnwiIlg=,239587236,2024-12-06,55554,iPhone,iOS 18.1,1,1,2024-12-13 05:12:19.497 +00:00
+ow5NCW34KSFvzijb+vJaiqeZr9c=,239587236,2024-12-06,1,iPhone,iOS 17.5,1,1,2024-12-13 05:12:19.527 +00:00

integration_tests/seeds/app_session_detailed_daily.csv

@@ -0,0 +1,11 @@
+_fivetran_id,app_id,date,app_version,device,platform_version,source_type,page_type,app_download_date,territory,sessions,total_session_duration,unique_devices,source_info,page_title,_fivetran_synced
+o5wEoLRdDH/NskmQrUaaZBKLaTM=,239587236,2024-11-08,329372.0,iPhone,iOS 17.6,App referrer,Product page,,US,14,797,5,spotify.com,Default Custom Product Page,2024-11-13 17:10:45.370 +00:00
+z5uCMEdxXVl3h0n+kYepEiVUtvo=,239587236,2024-11-09,120652.0,iPhone,iOS 16.7,App Store search,Product page,,IT,9,898,5,,Default Custom Product Page,2024-11-14 17:10:29.921 +00:00
+VbvXuJhvMfNLeK0uN1kG/9H1K3c=,239587236,2024-11-09,329372.0,iPhone,iOS 18.1,Web referrer,Product page,,TW,23,752,5,google.com.tw,Default Custom Product Page,2024-11-14 17:10:32.333 +00:00
+xq+QCXmwomsV+hoAVUXQ53pVevY=,239587236,2024-11-10,329372.0,iPhone,iOS 17.6,App Store search,No page,2024-11-02,PT,18,306,5,,Default No Page,2024-11-15 17:15:52.331 +00:00
+Xa6PaV9l8ni0U6VGuNwSNMm7KdU=,239587236,2024-11-10,329372.1,iPhone,iOS 17.4,App Store search,No page,,HK,15,3198,7,,Default No Page,2024-11-15 17:15:53.906 +00:00
+GF8h2QWP830nj8JLncVlc3ijoqA=,239587236,2024-11-10,329372.1,iPhone,iOS 17.6,App Store search,No page,2024-11-10,IL,20,2873,7,,Default No Page,2024-11-15 17:15:53.993 +00:00
+kGwZC0OxQTTlblE1X/H4KjUpFv8=,239587236,2024-11-10,120658.0,iPhone,iOS 18.0,App Store search,Product page,,IT,6,76,6,,Default Custom Product Page,2024-11-15 17:15:51.026 +00:00
+NgrBbeUJS4ydIChu1HbkUteIoM4=,239587236,2024-11-10,329372.0,iPhone,iOS 17.5,Web referrer,Product page,,SE,6,358,5,cnet.com,Default Custom Product Page,2024-11-15 17:15:52.045 +00:00
+iSC24SA4YvjP88OUma5VAuqbAIk=,239587236,2024-11-10,120654.0,iPhone,iOS 16.7,App Store search,No page,,LB,85,15485,5,,Default No Page,2024-11-15 17:15:50.794 +00:00
+NSjp+2R/xirT0vO4JQMQfWDwEHk=,239587236,2024-11-10,329372.1,iPhone,iOS 17.4,App Store search,No page,,GB,11,385,5,,Default No Page,2024-11-15 17:15:53.906 +00:00

integration_tests/seeds/app_store_app.csv

@@ -0,0 +1,2 @@
+id,name,primary_locale,content_rights_declaration,was_made_for_kids,subscription_status_url,subscription_status_url_version,subscription_status_url_for_sandbox,subscription_status_url_version_for_sandbox,_fivetran_synced
+239587236,Sample,en-US,DOES_NOT_USE_THIRD_PARTY_CONTENT,,https://example.com/subscription_status,v1,https://sandbox.example.com/subscription_status,v2,2022-01-02 12:34:56.789000+00:00

integration_tests/seeds/app_store_discovery_and_engagement_detailed_daily.csv

@@ -0,0 +1,11 @@
+_fivetran_id,app_id,date,event,page_type,source_type,engagement_type,device,platform_version,territory,counts,unique_counts,page_title,source_info,_fivetran_synced
+5SJIE4ZfUINJ3AI1T1A5AzRUqLc=,239587236,2024-11-04,Page view,Store sheet,App referrer,,iPhone,iOS 17.4,US,7,5,Default product page,com.wordle,2024-11-07 17:10:21.652 +00:00
+fTN+30viu9DOGf7xi0alJ3h3HMs=,239587236,2024-11-04,Page view,Store sheet,App Store browse,,iPhone,iOS 17.3,US,6,6,Default product page,,2024-11-07 17:10:18.235 +00:00
+9zmUs3grlpd8K7mhYs6t0P7GGBc=,239587236,2024-11-04,Page view,Store sheet,App referrer,,iPhone,iOS 18.0,US,5,5,Default product page,com.tradle.us.ios,2024-11-07 17:10:21.376 +00:00
+Ar8iylQfK9915AfMCqvslNeqbco=,239587236,2024-11-05,Page view,Store sheet,App referrer,,iPad,iOS 16.7,FR,5,5,Default product page,com.g2g.g2g.ios,2024-11-08 17:11:38.625 +00:00
+vRqeI3eSZtOtBqzLpWvtQXheFDI=,239587236,2024-11-05,Tap,Store sheet,App Store browse,Open,iPhone,iOS 17.6,CA,6,6,Default product page,,2024-11-08 17:11:38.572 +00:00
+pvCxArKHnaAFxuZf63/1PYKkiR4=,239587236,2024-11-05,Tap,Store sheet,App Store browse,Open,iPhone,iOS 18.0,GR,5,5,Default product page,,2024-11-08 17:11:40.033 +00:00
+6lXCI/W8NqhA3UsQQFvU58cuTZw=,239587236,2024-11-05,Impression,No page,App Store search,,iPhone,iOS 16.6,FR,5,5,League Pass FY25 (ASA),,2024-11-08 17:11:31.470 +00:00
+32rZ60OOHhVps86YhkBS3Z8+BiE=,239587236,2024-11-05,Page view,Store sheet,App referrer,,iPhone,iOS 18.1,FR,5,5,Default product page,com.whoo.hehe,2024-11-08 17:11:31.699 +00:00
+Gm1Bl6Omn5JN2deWlmUAYpfjc/w=,239587236,2024-11-05,Page view,Store sheet,App referrer,,iPhone,iOS 16.7,FR,11,5,Default product page,com.squigle.woo,2024-11-08 17:11:34.334 +00:00
+pK52DqMhmHqf7bp6bWbAV159zDQ=,239587236,2024-11-05,Page view,Store sheet,App referrer,,iPhone,iOS 18.0,FR,5,5,Default product page,com.conventino.hmm,2024-11-08 17:11:31.056 +00:00

integration_tests/seeds/app_store_download_detailed_daily.csv

@@ -0,0 +1,11 @@
+_fivetran_id,app_id,date,download_type,app_version,device,platform_version,source_type,page_type,pre_order,territory,counts,source_info,page_title,_fivetran_synced
+4wA7BEsAKZf8NT1FwxQRAi/GfSI=,239587236,2024-10-31,Auto-update,329372.0,iPhone,iOS 18.1,Unavailable,No page,,DE,5,,No page,2024-11-02 17:08:20.343 +00:00
+eKdeGwYA7mc5y+dG/KIypR37d6U=,239587236,2024-10-31,Auto-update,120652.0,iPad,iOS 18.0,Unavailable,No page,,JP,5,,No page,2024-11-02 17:08:18.687 +00:00
+BkesX890oPBVMhTXQ/hiDdx6qtI=,239587236,2024-10-31,Auto-update,329372.0,iPhone,iOS 17.6,Web referrer,Product page,,SI,5,chegg.com,Default custom product page,2024-11-02 17:08:18.158 +00:00
+Ropru4fq66wJDBlw8uX7S3Y8C3c=,239587236,2024-10-31,Auto-update,329372.0,Apple TV,tvOS 17.2,App Store search,No page,,AU,6,,No page,2024-11-02 17:08:23.535 +00:00
+O7UP8N94zIg8GGMIx9B+G1NdIso=,239587236,2024-10-31,Auto-update,329372.1,iPad,iOS 16.3,App Store search,No page,,MY,8,,No page,2024-11-02 17:08:20.909 +00:00
+0thg2HpfH+pyt51xfqsX2gjBqng=,239587236,2024-10-31,Auto-update,329372.1,Apple TV,tvOS 18.0,Unavailable,Product page,,AU,5,,Default custom product page,2024-11-02 17:08:23.298 +00:00
+rHAiOrf6uCyTLOQI83Sp/4U7I3w=,239587236,2024-10-31,Auto-update,120658.0,iPhone,iOS 18.1,App Store browse,No page,,HK,9,,No page,2024-11-02 17:08:25.364 +00:00
+lueVOUt20qfGZTy7okfwofpHWEw=,239587236,2024-10-31,Auto-update,329372.0,iPad,iOS 17.7,App referrer,Store sheet,,AU,5,com.apple.Spotlight,Default custom product page,2024-11-02 17:08:21.313 +00:00
+QP/giakN+TvGdJeYYUri1dZ9eAU=,239587236,2024-10-31,Manual update,120654.0,iPhone,iOS 17.5,Unavailable,No page,,MY,5,,No page,2024-11-02 17:08:20.541 +00:00
+jB997hDHmq8fhclBbRLme9x+S2I=,239587236,2024-10-31,Auto-update,329372.1,iPhone,iOS 17.6,Unavailable,Product page,,PT,5,,Default custom product page,2024-11-02 17:08:24.920 +00:00

integration_tests/seeds/app_store_installation_and_deletion_detailed_daily.csv

@@ -0,0 +1,11 @@
+_fivetran_id,app_id,date,event,download_type,app_version,device,platform_version,source_type,page_type,app_download_date,territory,counts,unique_devices,source_info,page_title,_fivetran_synced
+rLTCNO6J9D59i7ffRhp+E5EyleQ=,239587236,2024-11-11,Install,Manual update,329372.0,iPhone,iOS 17.5,Web referrer,Product page,,AU,5,5,walmart.com,Default Custom Product Page,2024-11-16 17:09:33.790 +00:00
+4NMQBTa2qQSIR9OAJEoekOzqANM=,239587236,2024-11-11,Install,Manual update,329372.1,iPhone,iOS 18.1,App Store browse,No page,2024-10-24,MX,6,6,,Default No Page,2024-11-16 17:09:34.269 +00:00
+TGwYHBcBQrDPz5kyrHqwRquf2Pc=,239587236,2024-11-12,Install,Manual update,329372.0,iPhone,iOS 18.0,App Store search,No page,,CR,5,5,,Default No Page,2024-11-18 05:10:16.445 +00:00
+QnICsNy0tjs++YD8Jd/gKnkVqr8=,239587236,2024-11-12,Install,Redownload,329372.1,iPhone,iOS 18.0,App Store browse,No page,2024-11-11,US,12,5,,Default No Page,2024-11-18 05:10:17.222 +00:00
+4tfCNvQ77h0QLLhbIfilFbxr4Oo=,239587236,2024-11-12,Install,Manual update,329372.1,iPad,iOS 17.7,App Store search,No page,,KR,8,6,,Default No Page,2024-11-18 05:10:16.576 +00:00
+Vki6Z8OPqh87H8+vc0ISkQ5S9aU=,239587236,2024-11-12,Install,Manual update,329372.0,iPhone,iOS 17.6,Unavailable,No page,,SG,7,7,,Default No Page,2024-11-18 05:10:16.424 +00:00
+uYwjMGDUKeWHsgF0G6BBREsc8gg=,239587236,2024-11-12,Install,Manual update,329372.1,iPad,iOS 18.0,App Store search,Product page,,US,50,45,,Default Custom Product Page,2024-11-18 05:10:16.605 +00:00
+53TtP8bqpftYRBqkTsZLXsnjG2A=,239587236,2024-11-12,Install,Manual update,329372.1,iPhone,iOS 18.1,Unavailable,No page,,TR,23,20,,Default No Page,2024-11-18 05:10:17.173 +00:00
+hpZC6khrDuFKOt5BHC8E4yxeunY=,239587236,2024-11-12,Install,Manual update,329372.1,iPad,iOS 17.6,App Store browse,No page,,AU,16,16,,Default No Page,2024-11-18 05:10:16.529 +00:00
+DjTfw5IHcig1bhl3leRb5SExOis=,239587236,2025-01-27,Delete,,120670.0,iPhone,iOS 18.2,App Store browse,No page,,US,6,6,,Default No Page,2025-01-28 17:09:31.693 +00:00