Merge pull request #24 from fivetran/bugfix/datespine-timestamp-error

Bug Fix: Date spine timestamp error

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_sage_intacct v0.5.0
+[PR #24](https://github.com/fivetran/dbt_sage_intacct/pull/24) includes the following updates.
+
+## 🚨 Breaking Changes: Bug Fixes 🚨
+- Updated the structure of the `int_sage_intacct__general_ledger_date_spine` model for improved performance and maintainability.
+    - Modified the date spine logic so that the model will take the maximum `entry_date_at` from the `sage_intacct__general_ledger` for the last date of the spine if it is available, rather than the current date. This will help capture future-dated transactions as well beyond the current date. 
+    - This is a breaking change, as changing the behavior of the date spine which will adjust the transaction records in the  `sage_intacct__general_ledger_by_period` model to be more accurate.
+
+# Bug Fixes
+- Updated the `int_sage_intacct__general_ledger_date_spine` model to accommodate for the cases when the compiled `sage_intacct__general_ledger` model has no transactions. In this case, the model now defaults to a range of one month leading up to the current date.
+
+## Under The Hood
+- Added `flags.WHICH in ('run', 'build')` as a condition in `int_sage_intacct__general_ledger_date_spine` to prevent call statements from querying the staging models during a `dbt compile`.
+- Addition of integrity and consistency validation tests within integration tests for the `sage_intacct__general_ledger` and `sage_intacct__general_ledger_by_period` model.
+
 # dbt_sage_intacct v0.4.0
 [PR #22](https://github.com/fivetran/dbt_sage_intacct/pull/22) includes the following updates.
 

README.md

@@ -54,7 +54,7 @@ Include the following sage_intacct package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/sage_intacct
-    version: [">=0.4.0", "<0.5.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do NOT include the `sage_intacct_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'sage_intacct'
-version: '0.4.0'
+version: '0.5.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   sage_intacct:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: sage_intacct_integration_tests_4
+      schema: sage_intacct_integration_tests_6
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: sage_intacct_integration_tests_4
+      schema: sage_intacct_integration_tests_6
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: sage_intacct_integration_tests_4
+      schema: sage_intacct_integration_tests_6
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: sage_intacct_integration_tests_4
+      schema: sage_intacct_integration_tests_6
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: sage_intacct_integration_tests_4
+      schema: sage_intacct_integration_tests_6
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,27 +1,27 @@
 name: 'sage_intacct_integration_tests'
-version: '0.4.0'
+version: '0.5.0'
 config-version: 2
 
 profile: 'integration_tests'
 
-vars:
+vars:  
   sage_intacct_gl_detail_identifier: "sage_intacct__gl_detail"
   sage_intacct_gl_batch_identifier: "sage_intacct__gl_batch"
   sage_intacct_gl_account_identifier: "sage_intacct__gl_account"
   sage_intacct_ap_bill_identifier: "sage_intacct__ap_bill"
   sage_intacct_ap_bill_item_identifier: "sage_intacct__ap_bill_item"
   sage_intacct_ar_invoice_identifier: "sage_intacct__ar_invoice"
   sage_intacct_ar_invoice_item_identifier: "sage_intacct__ar_invoice_item"
-  sage_intacct_schema: sage_intacct_integration_tests_4
+  sage_intacct_schema: sage_intacct_integration_tests_6
 
-  # ## For validation testing. To be commented out before release.
-  # sage_intacct_database: <see_corresponding_height_ticket>
-  # sage_intacct_schema: <see_corresponding_height_ticket>
+  # For validation testing. To be commented out before release.
+  # sage_intacct_database: <insert database>
+  # sage_intacct_schema: <insert schema>
   # fivetran_consistency_ap_ar_enhanced_exclusion_documents: () ## -- Variable for filtering out documents if inconsistency is expected. 
   # fivetran_integrity_ap_ar_enhanced_exclusion_documents: () ## -- Variable for filtering out documents if inconsistency is expected. 
 
 models:
-  +schema: "sage_intacct_{{ var('directed_schema','dev') }}"
+  +schema: "sage_intacct_{{ var('directed_schema','dev') }}" 
 
 seeds:
     +quote_columns: "{{ true if target.type == 'redshift' else false }}"
@@ -90,6 +90,18 @@ seeds:
           vendorname: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
           warehouseid: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
           warehousename: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
+      sage_intacct__gl_account:
+        +column_types:
+          category: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
+          normalbalance: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
+          accounttype: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
+      sage_intacct__gl_batch:
+        +column_types:
+          _fivetran_deleted: boolean
+      sage_intacct__gl_detail:
+        +column_types:
+          entry_date: date
+          accounttitle: "{{ 'varchar(100)' if target.name in ('redshift', 'postgres') else 'string' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/tests/consistency/consistency_general_ledger.sql

@@ -0,0 +1,48 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the general ledger end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_sage_intacct_prod.sage_intacct__general_ledger
+    where date(entry_date_at) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select * 
+    from {{ target.schema }}_sage_intacct_dev.sage_intacct__general_ledger
+    where date(entry_date_at) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/consistency_general_ledger_by_period.sql

@@ -0,0 +1,48 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the general ledger end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_sage_intacct_prod.sage_intacct__general_ledger_by_period
+    where date(period_last_day) < date({{ dbt.current_timestamp() }})
+),
+
+dev as (
+    select * 
+    from {{ target.schema }}_sage_intacct_dev.sage_intacct__general_ledger_by_period
+    where date(period_last_day) < date({{ dbt.current_timestamp() }})
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/integrity/integrity_general_ledger_by_period.sql

@@ -0,0 +1,24 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure there is no fanout between the spine and the general_ledger_by_period
+
+with spine as (
+
+    select count(*) as spine_count
+    from {{ ref('int_sage_intacct__general_ledger_date_spine') }}
+),
+
+general_ledger_by_period as (
+
+    select count(*) as glp_count
+    from {{ ref('sage_intacct__general_ledger_by_period') }}
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from spine
+join general_ledger_by_period
+    on spine.spine_count != general_ledger_by_period.glp_count

models/intermediate/int_sage_intacct__general_ledger_date_spine.sql

@@ -1,51 +1,44 @@
 with spine as (
 
-    {% if execute %}
+    {% if execute and flags.WHICH in ('run', 'build') %}
+
     {% set first_date_query %}
-        select  min( entry_date_at ) as min_date from {{ ref('sage_intacct__general_ledger') }}
+        select 
+            coalesce(
+                min(cast(entry_date_at as date)), 
+                cast({{ dbt.dateadd("month", -1, "current_date") }} as date)
+                ) as min_date 
+        from {{ ref('sage_intacct__general_ledger') }}
     {% endset %}
-    {% set first_date = run_query(first_date_query).columns[0][0]|string %}
-
-        {% if target.type == 'postgres' %}
-            {% set first_date_adjust = "cast('" ~ first_date[0:10] ~ "' as date)" %}
-
-        {% else %}
-            {% set first_date_adjust = "'" ~ first_date[0:10] ~ "'" %}
-
-        {% endif %}
 
-    {% else %} {% set first_date_adjust = "'2015-01-01'" %}
-    {% endif %}
-
-    {% if execute %}
     {% set last_date_query %}
-        select  max( entry_date_at ) as max_date from {{ ref('sage_intacct__general_ledger') }}
-    {% endset %}
-
-    {% set current_date_query %}
-        select current_date
+        select 
+            coalesce(
+                max(cast(entry_date_at as date)), 
+                cast(current_date as date)
+                ) as max_date 
+        from {{ ref('sage_intacct__general_ledger') }}
     {% endset %}
+
+    {% else %}
 
-    {% if run_query(current_date_query).columns[0][0]|string < run_query(last_date_query).columns[0][0]|string %}
+    {%- set first_date_query%}
+        select cast({{ dbt.dateadd("month", -1, "current_date") }} as date)
+    {% endset -%}
 
-    {% set last_date = run_query(last_date_query).columns[0][0]|string %}
+    {% set last_date_query %}
+        select cast({{ dbt.current_timestamp() }} as date)
+    {% endset -%}
 
-    {% else %} {% set last_date = run_query(current_date_query).columns[0][0]|string %}
     {% endif %}
-
-    {% if target.type == 'postgres' %}
-        {% set last_date_adjust = "cast('" ~ last_date[0:10] ~ "' as date)" %}
 
-    {% else %}
-        {% set last_date_adjust = "'" ~ last_date[0:10] ~ "'" %}
-
-    {% endif %}
-    {% endif %}
+    {%- set first_date = dbt_utils.get_single_value(first_date_query) %}
+    {%- set last_date = dbt_utils.get_single_value(last_date_query) %}
 
     {{ dbt_utils.date_spine(
         datepart="month",
-        start_date=first_date_adjust,
-        end_date=dbt.dateadd("month", 1, last_date_adjust)
+        start_date="cast('" ~ first_date ~ "' as date)",
+        end_date=dbt.dateadd("month", 1, "cast('" ~ last_date  ~ "' as date)")
         )
     }}
 ),