Merge pull request #10 from fivetran/bug/multi-fixes

Bug/multi fixes
fivetran · Apr 21, 2022 · a1a0512 · a1a0512
2 parents 1c75283 + aa08091
commit a1a0512
Show file tree

Hide file tree

Showing 11 changed files with 238 additions and 75 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -33,6 +33,8 @@ jobs:
             dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
             dbt test --target redshift
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target redshift
+            dbt test --target redshift
       - run:
           name: "Run Tests - Postgres"
           command: |
@@ -43,6 +45,8 @@ jobs:
             dbt seed --target postgres --full-refresh
             dbt run --target postgres --full-refresh
             dbt test --target postgres
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target postgres
+            dbt test --target postgres
       - run:
           name: "Run Tests - Spark"
           command: |
@@ -53,6 +57,8 @@ jobs:
             dbt seed --target spark --full-refresh
             dbt run --target spark --full-refresh
             dbt test --target spark
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target spark
+            dbt test --target spark
       - run:
           name: "Run Tests - Snowflake"
           command: |
@@ -63,6 +69,8 @@ jobs:
             dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
             dbt test --target snowflake
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target snowflake
+            dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
           environment:
@@ -75,4 +83,6 @@ jobs:
             dbt deps
             dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
+            dbt test --target bigquery
+            dbt run --vars '{sage_intacct__using_invoices: false, sage_intacct__using_bills: false}' --full-refresh --target bigquery
             dbt test --target bigquery
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+# dbt_sage_intacct v0.1.1
+## 🚨 Breaking Changes 🚨
+- Allow for pass-through columns from the  `gl_detail` and `gl_account` source tables. In the gl_detail staging model, added additional fields and enabled/disabled configs for the AP Bill and AR Invoice tables which may not be present in the customer schema.
+
+([#6](https://github.com/fivetran/dbt_sage_intacct/issues/6))
+([#7](https://github.com/fivetran/dbt_sage_intacct/issues/7))
+## Contributors
+Thank you @santi95 for raising these to our attention! ([#9](https://github.com/fivetran/dbt_sage_intacct/pull/9))
+
+
 # dbt_sage_intacct v0.1.0
 
 ## 🎉 Initial Release of the Fivetran Sage Intacct Ads dbt package 🎉

diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ vars:
     sage_intacct_schema: your_schema_name
 ```
 ### Passthrough Columns
-This package allows users to add additional columns to the `stg_sage_intacct__gl_account` table. 
+This package allows users to add additional columns to the `stg_sage_intacct__gl_account` and `stg_sage_intacct__gl_detail` table. 
 Columns passed through must be present in the upstream source tables. See below for an example of how the passthrough columns should be configured within your `dbt_project.yml` file.
 
 ```yml
@@ -55,10 +55,27 @@ Columns passed through must be present in the upstream source tables. See below
 ...
 vars:
   sage_account_pass_through_columns: ['new_custom_field', 'custom_field_2']
+  sage_gl_pass_through_columns: ['custom_field_3', 'custom_field_4']
 ```
 ### Custom Account Classification
 Accounts roll up into different accounting classes based on their category. The categories are brought in from the `gl_account` table. We created a variable for each accounting class (`Asset`, `Liability`, `Equity`, `Revenue`, `Expense`) that can be modified to include different categories based on your business. You can modify the variables within your root `dbt_project.yml` file. The default values for the respective variables are as follows:
 
+### Disabling and Enabling Models
+
+When setting up your Sage Intacct (Sage) connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Sage or have actively decided to not sync some tables. In order to disable the relevant functionality in the package, you will need to add the relevant variables.
+
+By default, all variables are assumed to be `true`. You only need to add variables for the tables you would like to disable:
+
+```yml
+# dbt_project.yml
+
+config-version: 2
+
+vars:
+    sage_intacct__using_invoices: false                 # default is true
+    sage_intacct__using_bills: false                    # default is true
+```
+
 
 ```yml
 # dbt_project.yml

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'sage_intacct'
-version: '0.1.0'
+version: '0.1.1'
 
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 
@@ -13,6 +13,9 @@ models:
       +materialized: view 
 
 vars:
+  sage_account_pass_through_columns: []
+  sage_gl_pass_through_columns: []
+
   sage_intacct:
     gl_detail: "{{ ref('stg_sage_intacct__gl_detail') }}"
     gl_account: "{{ ref('stg_sage_intacct__gl_account') }}"
@@ -21,6 +24,7 @@ vars:
     ar_invoice_item: "{{ ref('stg_sage_intacct__ar_invoice_item') }}"
     ar_invoice: "{{ ref('stg_sage_intacct__ar_invoice') }}"
 
+    sage_gl_pass_through_columns: [] 
     sage_account_pass_through_columns: [] 
 
     sage_intacct_category_asset: ('Inventory','Fixed Assets','Other Current Assets','Cash and Cash Equivalents','Intercompany Receivable','Accounts Receivable','Deposits and Prepayments','Goodwill','Intangible Assets','Short-Term Investments','Inventory','Accumulated Depreciation','Other Assets','Unrealized Currency Gain/Loss','Patents','Investment in Subsidiary','Escrows and Reserves','Long Term Investments')
@@ -29,5 +33,7 @@ vars:
     sage_intacct_category_liability: ('Accounts Payable','Other Current Liabilities','Accrued Liabilities','Note Payable - Current','Deferred Taxes Liabilities - Long Term','Note Payable - Long Term','Other Liabilities','Deferred Revenue - Current')
     sage_intacct_category_revenue: ('Revenue','Revenue - Sales','Dividend Income','Revenue - Other','Other Income','Revenue - Services','Revenue - Products')
 
+    sage_intacct__using_invoices: true
+    sage_intacct__using_bills: true
 
 analysis-paths: ["analysis"]
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'sage_intacct_integration_tests'
-version: '0.1.0'
+version: '0.1.1'
 config-version: 2
 
 profile: 'integration_tests'

diff --git a/models/intermediate/int_sage_intacct__general_ledger_balances.sql b/models/intermediate/int_sage_intacct__general_ledger_balances.sql
@@ -8,7 +8,35 @@ gl_accounting_periods as (
     from {{ ref('int_sage_intacct__general_ledger_date_spine') }}
 ), 
 
-gl_period_balances as (    
+
+gl_period_balances_is as (
+    select 
+        account_no,
+        account_title,
+        book_id,
+        category,
+        classification,
+        currency, 
+        entry_state,
+        account_type,
+        cast({{ dbt_utils.date_trunc("month", "entry_date_at") }} as date) as date_month, 
+        cast({{ dbt_utils.date_trunc("year", "entry_date_at") }} as date) as date_year
+
+        {% if var('sage_account_pass_through_columns') %} 
+        , 
+        {{ var('sage_account_pass_through_columns') | join (", ")}}
+
+        {% endif %}
+        , 
+        sum(amount) as period_amount
+    from general_ledger
+    where account_type = 'incomestatement'
+
+    {{ dbt_utils.group_by(10 + var('sage_account_pass_through_columns')|length) }}
+
+), 
+
+gl_period_balances_bs as (
     select 
         account_no,
         account_title,
@@ -19,19 +47,45 @@ gl_period_balances as (
         entry_state,
         account_type,
         cast({{ dbt_utils.date_trunc("month", "entry_date_at") }} as date) as date_month, 
-        cast({{ dbt_utils.date_trunc("year", "entry_date_at") }} as date) as date_year, 
+        cast({{ dbt_utils.date_trunc("year", "entry_date_at") }} as date) as date_year
+
+        {% if var('sage_account_pass_through_columns') %} 
+        , 
+        {{ var('sage_account_pass_through_columns') | join (", ")}}
+
+        {% endif %}
+        ,
         sum(amount) as period_amount
     from general_ledger
+    where account_type = 'balancesheet'
 
-    {{ dbt_utils.group_by(10) }} 
+    {{ dbt_utils.group_by(10 + var('sage_account_pass_through_columns')|length) }}
 
 ), 
 
+gl_period_balances as (
+    select *
+    from gl_period_balances_bs
+
+    union all
+
+    select *
+    from gl_period_balances_is
+
+),
+
 gl_cumulative_balances as (
     select 
         *,
         case
-            when account_type = 'balancesheet' then sum(period_amount) over (partition by account_no, account_title, book_id, entry_state order by date_month, account_no rows unbounded preceding)
+            when account_type = 'balancesheet' then sum(period_amount) over (partition by account_no, account_title, book_id, entry_state 
+                {% if var('sage_account_pass_through_columns') %} 
+                , 
+                {{ var('sage_account_pass_through_columns') | join (", ")}}
+
+                {% endif %}
+
+                order by date_month, account_no rows unbounded preceding)
             else 0 
         end as cumulative_amount   
     from gl_period_balances
@@ -61,7 +115,14 @@ gl_patch as (
         coalesce(gl_beginning_balance.currency, gl_accounting_periods.currency) as currency,
         coalesce(gl_beginning_balance.entry_state, gl_accounting_periods.entry_state) as entry_state,
         coalesce(gl_beginning_balance.account_type, gl_accounting_periods.account_type) as account_type,
-        coalesce(gl_beginning_balance.date_year, gl_accounting_periods.date_year) as date_year,
+        coalesce(gl_beginning_balance.date_year, gl_accounting_periods.date_year) as date_year
+
+        {% if var('sage_account_pass_through_columns') %} 
+        , 
+        {{ var('sage_account_pass_through_columns') | join (", gl_beginning_balance.")}}
+
+        {% endif %}
+        ,
         gl_accounting_periods.period_first_day,
         gl_accounting_periods.period_last_day,
         gl_accounting_periods.period_index,
@@ -113,6 +174,12 @@ final as (
             first_value(period_ending_amount_starter) over (partition by gl_partition order by period_last_day rows unbounded preceding)) as period_beg_amount,
         coalesce(period_ending_amount_starter,
             first_value(period_ending_amount_starter) over (partition by gl_partition order by period_last_day rows unbounded preceding)) as period_ending_amount
+        {% if var('sage_account_pass_through_columns') %} 
+        , 
+        {{ var('sage_account_pass_through_columns') | join (", ")}}
+
+        {% endif %}
+
     from gl_value_partition
 )
 

diff --git a/models/sage_intacct__ap_ar_enhanced.sql b/models/sage_intacct__ap_ar_enhanced.sql
@@ -1,23 +1,34 @@
-with ap_bill as (
+
+
+{{ config(enabled=fivetran_utils.enabled_vars_one_true(vars=["sage_intacct__using_bills", "sage_intacct__using_invoices"])) }}
+
+with
+
+{% if var('sage_intacct__using_bills', True) %}
+ap_bill as (
     select * 
     from {{ ref('stg_sage_intacct__ap_bill') }} 
 ), 
 
 ap_bill_item as (
     select * 
     from {{ ref('stg_sage_intacct__ap_bill_item') }} 
-), 
+),
+{% endif %}
 
-ar_invoice as (    
+{% if var('sage_intacct__using_invoices', True) %}
+ar_invoice as (
     select * 
     from {{ ref('stg_sage_intacct__ar_invoice') }} 
-), 
+),
 
 ar_invoice_item as (
     select * 
     from {{ ref('stg_sage_intacct__ar_invoice_item') }} 
-), 
+),
+{% endif %}
 
+{% if var('sage_intacct__using_bills', True) %}
 ap_bill_enhanced as (
     select
         ap_bill_item.bill_id,
@@ -60,8 +71,10 @@ ap_bill_enhanced as (
     left join ap_bill
         on ap_bill_item.bill_id = ap_bill.bill_id
 ), 
+{% endif %}
 
-ar_invoice_enhanced as (
+{% if var('sage_intacct__using_invoices', True) %}
+    ar_invoice_enhanced as (
     select 
         cast(null as {{ dbt_utils.type_string() }}) as bill_id,
         cast(null as {{ dbt_utils.type_string() }}) as bill_item_id,
@@ -97,34 +110,62 @@ ar_invoice_enhanced as (
         ar_invoice.total_entered,
         ar_invoice.total_paid,
         ar_invoice.record_id,
-        count(*) over (partition by ar_invoice_item.invoice_id) as number_of_items  
-    from ar_invoice_item
-
-    left join ar_invoice
-        on ar_invoice_item.invoice_id = ar_invoice.invoice_id
-), 
+        count(*) over (partition by ar_invoice_item.invoice_id) as number_of_items
+
+        from ar_invoice_item
+        left join ar_invoice
+            on ar_invoice_item.invoice_id = ar_invoice.invoice_id
+    ),
+{% endif %}
+
 
 ap_ar_enhanced as (
+    {% if var('sage_intacct__using_bills', True) %}
     select * 
     from ap_bill_enhanced
+    {% endif %}
 
+
+    {% if fivetran_utils.enabled_vars(vars=["sage_intacct__using_bills", "sage_intacct__using_invoices"]) %}
     union all
+    {% endif %}
 
+    {% if var('sage_intacct__using_invoices', True) %}
     select * 
     from ar_invoice_enhanced
+    {% endif %}
+
 ), 
 
+
 final as (
     select 
-        coalesce(bill_id, invoice_id) as document_id,
-        coalesce(bill_item_id, invoice_item_id) as document_item_id,
+        coalesce(
+                {% if var('sage_intacct__using_bills', True) %} bill_id {% endif %}
+
+                {% if fivetran_utils.enabled_vars(vars=["sage_intacct__using_bills", "sage_intacct__using_invoices"]) %}
+                ,
+                {% endif %}
+
+                {% if var('sage_intacct__using_invoices', True) %} invoice_id {% endif %}
+        ) as document_id,
+        coalesce(
+                {% if var('sage_intacct__using_bills', True) %} bill_item_id {% endif %}
+
+                {% if fivetran_utils.enabled_vars(vars=["sage_intacct__using_bills", "sage_intacct__using_invoices"]) %}
+                , 
+                {% endif %}
+
+                {% if var('sage_intacct__using_invoices', True) %}  invoice_item_id {% endif %} 
+        ) as document_item_id,
         case 
-            when bill_id is not null then 'bill' 
-            when invoice_id is not null then 'invoice'
+            {% if var('sage_intacct__using_bills', True) %} when bill_id is not null then 'bill' {% endif %} 
+            {% if var('sage_intacct__using_invoices', True) %} when invoice_id is not null then 'invoice' {% endif %} 
         end as document_type,
         entry_date_at,
         entry_description,
         amount,
+        currency,
         due_in_days,
         item_id,
         item_name,

diff --git a/models/sage_intacct__balance_sheet.sql b/models/sage_intacct__balance_sheet.sql
@@ -21,4 +21,3 @@ final as (
 
 select *
 from final
-