Merge pull request #141 from fivetran/feature/add_transformation_runs

Feature/add transformation runs

.buildkite/scripts/run_models.sh

@@ -37,20 +37,20 @@ echo `pwd`
 cd integration_tests
 dbt deps
 if [ "$db" = "databricks-sql" ]; then
-dbt seed --vars '{fivetran_platform_schema: sqlw_tests}' --target "$db" --full-refresh
-dbt compile --vars '{fivetran_platform_schema: sqlw_tests}' --target "$db"
-dbt run --vars '{fivetran_platform_schema: sqlw_tests}' --target "$db" --full-refresh
-dbt run --vars '{fivetran_platform_schema: sqlw_tests}' --target "$db"
-dbt test --vars '{fivetran_platform_schema: sqlw_tests}' --target "$db"
-dbt run --vars '{fivetran_platform_schema: sqlw_tests, fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
-dbt run --vars '{fivetran_platform_schema: sqlw_tests, fivetran_platform__usage_pricing: true}' --target "$db"
-dbt test --vars '{fivetran_platform_schema: sqlw_tests}' --target "$db"
-dbt run --vars '{fivetran_platform_schema: sqlw_tests, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
-dbt run --vars '{fivetran_platform_schema: sqlw_tests, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true}' --target "$db"
-dbt test --vars '{fivetran_platform_schema: sqlw_tests}'  --target "$db"
-dbt run --vars '{fivetran_platform_schema: sqlw_tests, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false}' --target "$db" --full-refresh
-dbt run --vars '{fivetran_platform_schema: sqlw_tests, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false}' --target "$db"
-dbt test --vars '{fivetran_platform_schema: sqlw_tests}'  --target "$db"
+dbt seed --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh
+dbt compile --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db"
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db"
+dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db"
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: true}' --target "$db"
+dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db"
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db"
+dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}'  --target "$db"
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db"
+dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}'  --target "$db"
 else
 dbt seed --target "$db" --full-refresh
 dbt compile --target "$db"
@@ -60,11 +60,11 @@ dbt test --target "$db"
 dbt run --vars '{fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
 dbt run --vars '{fivetran_platform__usage_pricing: true}' --target "$db"
 dbt test --target "$db"
-dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
-dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true}' --target "$db"
+dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db"
 dbt test --target "$db"
-dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false}' --target "$db" --full-refresh
-dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false}' --target "$db"
+dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db"
 dbt test --target "$db"
 fi
 if [ "$1" != "sqlserver" ]; then

CHANGELOG.md

@@ -1,9 +1,27 @@
-# dbt_fivetran_log version.version
+# dbt_fivetran_log v1.11.0
+[PR #141](https://github.com/fivetran/dbt_fivetran_log/pull/141) includes the following updates:
 
-## Documentation
+## Schema Changes: Adding the Transformation Runs Table
+- This package now accounts for the `transformation_runs` source table. Therefore, a new staging model [`stg_fivetran_platform__transformation_runs`](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.stg_fivetran_platform__transformation_runs) has been added. Note that not all customers have the `transformation_runs` source table, particularly if they are not using Fivetran Transformations. If the table doesn't exist, `stg_fivetran_platform__transformation_runs` will persist as an empty model and respective downstream fields will be null. 
+
+- In addition, the following fields have been added to the [`fivetran_platform__usage_mar_destination_history`](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__usage_mar_destination_history) end model:
+    - `paid_model_runs`
+    - `free_model_runs`
+    - `total_model_runs`
+
+## Documentation Updates
+- Included documentation about the `transformation_runs` source table and the aggregated `*_model_runs` fields.
+- Added information about manually configuring the `fivetran_platform_using_transformations` variable in the [DECISION LOG.](https://github.com/fivetran/dbt_fivetran_log/blob/main/DECISIONLOG.md)
 - Added Quickstart model counts to README. ([#145](https://github.com/fivetran/dbt_fivetran_log/pull/145))
 - Corrected references to connectors and connections in the README. ([#145](https://github.com/fivetran/dbt_fivetran_log/pull/145))
 
+## Under the Hood
+- Introduced the variable `fivetran_platform_using_transformations` to control the `stg_fivetran_platform__transformation_runs` output. It is configured based on whether the `transformation_runs` table exists. For more information, refer to the [DECISION LOG.](https://github.com/fivetran/dbt_fivetran_log/blob/main/DECISIONLOG.md)
+- Added the `get_transformation_runs_columns()` macro to ensure all required columns are present.
+- Added `transformation_runs` seed data in `integration_tests/seeds/`.
+- Added a `run_count__usage_mar_destination_history` validation test to check model run counts across staging and end model.
+- (Redshift only) Updates to use limit 1 instead of limit 0 for empty tables. This ensures that Redshift will respect the package's datatype casts.
+
 # dbt_fivetran_log v1.10.0
 [PR #140](https://github.com/fivetran/dbt_fivetran_log/pull/140) includes the following updates:
 
@@ -16,6 +34,7 @@
 ## Under the Hood (Maintainers Only)
 - Enhanced seed data for integration testing to include the different spellings and ensure compatibility with Redshift.
 
+
 # dbt_fivetran_log v1.9.1
 [PR #138](https://github.com/fivetran/dbt_fivetran_log/pull/138) includes the following updates:
 

DECISIONLOG.md

@@ -9,4 +9,14 @@ The below variables control the respective behaviors and may be overridden if de
 vars:
   fivetran_platform__usage_pricing: true ## Dynamically checks the source at runtime to set as either true or false. May be overridden using this variable if desired.
   fivetran_platform__credits_pricing: false ## Dynamically checks the source at runtime to set as either true or false. May be overridden using this variable if desired.
+```
+
+## Transformation Runs
+Not all customers have the `transformation_runs` source table, particularly if they are not using Fivetran Transformations. Therefore, we leverage a new variable `fivetran_platform_using_transformations`, which automatically checks for the table. If it exists, the variable is set to True, which then persists the `transformation_runs` source table and related models and downstream fields. If the table doesn't exist, the staging `stg_fivetran_platform__transformation_runs` model will persist as an empty model and respective downstream fields will be null. 
+
+In the case you have the `transformation_runs` source table but still wish to disable it to prevent it from being populated in the package, you may set `fivetran_platform_using_transformations` to False in your project.yml:
+
+```yml
+vars:
+  fivetran_platform_using_transformations: false ## Dynamically checks the source at runtime to set as either true or false. May be overridden using this variable if desired.
 ```

README.md

@@ -90,12 +90,12 @@ vars:
 ```
 
 ### Step 4: Disable Models for Non Existent Sources
-If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` sources. It's also possible you might not have any To disable the corresponding functionality in the package, you must add the following variable(s) to your root `dbt_project.yml` file. By default, all variables are assumed to be `true`:
+If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` source tables. The `user` and `destination_membership` are enabled by default. Therefore in order to switch the default configurations, you must add the following variable(s) to your root `dbt_project.yml` file for the respective source tables you wish to disable:
 
 ```yml
 vars:
-    fivetran_platform_using_destination_membership: false # this will disable only the destination membership logic
-    fivetran_platform_using_user: false # this will disable only the user logic
+    fivetran_platform_using_destination_membership: false # Default is true. This will disable only the destination membership logic
+    fivetran_platform_using_user: false # Default is true. This will disable only the user logic
 ```
 
 ### (Optional) Step 5: Additional Configurations

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.10.0'
+version: '1.11.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:
@@ -22,4 +22,5 @@ vars:
     destination_membership: "{{ source('fivetran_platform', 'destination_membership') }}"
     log: "{{ source('fivetran_platform', 'log') }}"
     user: "{{ source('fivetran_platform', 'user') }}"
-    usage_cost: "{{ source('fivetran_platform', 'usage_cost') }}"
+    usage_cost: "{{ source('fivetran_platform', 'usage_cost') }}"
+    transformation_runs: "{{ source('fivetran_platform','transformation_runs') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_13 
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: fivetran_platform_integration_tests_5
+      schema: fivetran_platform_integration_tests_13
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: fivetran_platform_integration_tests_5
+      schema: fivetran_platform_integration_tests_13
       threads: 8
     postgres:
       type: postgres
@@ -42,21 +42,21 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_13 
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_13 
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
     databricks-sql:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
-      schema: sqlw_tests 
+      schema: sqlw_tests_4 
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
       type: databricks
@@ -66,7 +66,7 @@ integration_tests:
       server: "{{ env_var('CI_SQLSERVER_DBT_SERVER') }}"
       port: 1433
       database: "{{ env_var('CI_SQLSERVER_DBT_DATABASE') }}"
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_13 
       user: "{{ env_var('CI_SQLSERVER_DBT_USER') }}"
       password: "{{ env_var('CI_SQLSERVER_DBT_PASS') }}"
       threads: 8

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.10.0'
+version: '1.11.0'
 
 config-version: 2
 profile: 'integration_tests'
@@ -10,7 +10,7 @@ dispatch:
 
 vars:
   fivetran_log:
-    fivetran_platform_schema: "fivetran_platform_integration_tests_5"
+    fivetran_platform_schema: "fivetran_platform_integration_tests_13"
     fivetran_platform_account_identifier: "account"
     fivetran_platform_incremental_mar_identifier: "incremental_mar"
     fivetran_platform_connector_identifier: "connector"
@@ -20,10 +20,12 @@ vars:
     fivetran_platform_destination_membership_identifier: "destination_membership"
     fivetran_platform_log_identifier: "log"
     fivetran_platform_user_identifier: "user"
+    fivetran_platform_transformation_runs_identifier: "transformation_runs"
+
 
 models:
   fivetran_log:
-    +schema: "{{ 'sqlw_tests' if target.name == 'databricks-sql' else 'fivetran_platform' }}"
+    +schema: "{{ 'sqlw_tests_4' if target.name == 'databricks-sql' else 'fivetran_platform' }}"
     # +schema: "fivetran_platform_{{ var('directed_schema','dev') }}"
 
 seeds:

integration_tests/seeds/transformation_runs.csv

@@ -0,0 +1,3 @@
+destination_id,job_id,measured_date,project_type,free_type,job_name,updated_at,model_runs,_fivetran_synced
+egg_shell,formosa,2024-12-15 00:00:00.000000,QUICKSTART,PAID,Github/staging,2024-12-15 18:13:59.840000,6,2024-12-16 00:11:52.792000
+egg_shell,macau,2024-12-08 00:00:00.000000,QUICKSTART,PROMOTION,Github/staging,2024-12-09 15:21:02.278000,6,2024-12-09 18:11:46.905000

integration_tests/tests/integrity/run_count__usage_mar_destination_history.sql

@@ -0,0 +1,44 @@
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with end_model as (
+    select
+        measured_month,
+        destination_id,
+        sum(total_model_runs) as total_model_runs,
+        sum(paid_model_runs) as paid_model_runs,
+        sum(free_model_runs) as free_model_runs
+    from {{ ref('fivetran_platform__usage_mar_destination_history') }}
+    group by 1,2
+),
+
+staging_model as (
+    select
+        measured_month,
+        destination_id,
+        sum(model_runs) as total_model_runs, 
+        sum(case when free_type = 'PAID' then model_runs else 0 end) as paid_model_runs, 
+        sum(case when free_type != 'PAID' then model_runs else 0 end) as free_model_runs
+    from {{ ref('stg_fivetran_platform__transformation_runs') }}
+    group by 1,2
+)
+
+select 
+    end_model.measured_month,
+    end_model.destination_id,
+    end_model.total_model_runs,
+    end_model.paid_model_runs,
+    end_model.free_model_runs,
+    staging_model.total_model_runs,
+    staging_model.paid_model_runs,
+    staging_model.free_model_runs
+from end_model
+left join staging_model
+    on end_model.destination_id = staging_model.destination_id
+    and end_model.measured_month = staging_model.measured_month
+where staging_model.total_model_runs != end_model.total_model_runs
+or staging_model.paid_model_runs != end_model.paid_model_runs
+or staging_model.free_model_runs != end_model.free_model_runs

macros/get_transformation_runs_columns.sql

@@ -0,0 +1,17 @@
+{% macro get_transformation_runs_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "destination_id", "datatype": dbt.type_string()},
+    {"name": "free_type", "datatype": dbt.type_string()},
+    {"name": "job_id", "datatype": dbt.type_string()},
+    {"name": "job_name", "datatype": dbt.type_string()},
+    {"name": "measured_date", "datatype": dbt.type_timestamp()},
+    {"name": "model_runs", "datatype": dbt.type_int()},
+    {"name": "project_type", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/fivetran_platform.yml

@@ -163,6 +163,15 @@ models:
       - name: dollars_spent
         description: The dollar amount used by the destination for the given month.
 
+      - name: paid_model_runs
+        description: The number of paid model runs for the destination for the given month.
+
+      - name: free_model_runs
+        description: The number of free model runs for the destination for the given month.
+
+      - name: total_model_runs
+        description: The total number of model runs for the destination for the given month.
+
   - name: fivetran_platform__connector_daily_events
     description: >
       Table of each connector's daily history of the quantity of api calls, schema changes, and records modified,