Multivalue metafield macro support

CHANGELOG.md

@@ -1,3 +1,14 @@
+# dbt_shopify v0.16.1
+[PR #99](https://github.com/fivetran/dbt_shopify/pull/99) includes the following updates:
+
+## Bug Fixes
+- Updated the `get_metafields` macro to support multiple reference values, ensuring compatibility with both the Shopify GraphQL API (ex: 'PRODUCTVARIANT' from the `product_variant` source) and the deprecated REST API (previously 'variant' for `product_variant`). [See the Shopify API docs for more information](https://shopify.dev/docs/api/admin-graphql/2025-01/objects/metafield).
+  - Updated `shopify__product_variant_metafields` and `shopify__product_image_metafields` to ensure all metafields from `stg_shopify__metafield` are properly included, preventing loss of existing records.
+  - Introduced `id_column_override` in `get_metafields` macros to to handle non-standard primary key column names (`variant_id` from `stg_shopify__product_variant`, `product_image_id` from `stg_shopify__product_image` where `image` is the default reference value). The override ensures models explicitly call the correct column names.
+
+## Under the Hood
+- Updated the `shopify_metafield_data` seed to validate the functionality of the `get_metafields` macro, ensuring it correctly retrieves metafield data for all supported reference values.
+
 # dbt_shopify v0.16.0
 This release includes the following updates:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.16.0'
+version: '0.16.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_integration_tests_15
+      schema: shopify_integration_tests_16
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.16.0'
+version: '0.16.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -13,7 +13,8 @@ vars:
   # shopify_using_fulfillment_event: true
   # shopify_using_all_metafields: true
   # shopify__standardized_billing_model_enabled: true
-  shopify_schema: shopify_integration_tests_15
+
+  shopify_schema: shopify_integration_tests_16
 
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
@@ -181,6 +182,8 @@ seeds:
       +column_types:
         value_type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
         type: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        owner_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         _fivetran_synced: timestamp
         created_at: timestamp
         updated_at: timestamp

integration_tests/seeds/shopify_metafield_data.csv

@@ -1,6 +1,25 @@
 id,_fivetran_synced,created_at,description,key,namespace,owner_id,owner_resource,updated_at,value,value_type,type
 5445055,2022-11-19 10:06:09.531000,2019-10-28 20:06:39.000000,,returnAuthorizations,blade_runner,390244,order,2019-10-28 20:06:39.000000,"[{""id"":""ce95-49e4-9daf-41f29bbbb799"",""totalValue"":44444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":4444,""validReturnItems"":[{""UPC"":""19073825552"",""Quantity"":""1"",""Reason"":""changed-mind"",""LineItem"":""40055558892132""}]},""createdAt"":""2019-10-28T20:06:39.569Z"",""modifiedAt"":""2019-10-28T20:06:39.569Z""}]",,json_string
 6337647,2022-11-21 01:57:33.851000,2020-06-17 11:35:28.000000,,returnAuthorizations,blade_runner,254671,order,2020-06-17 11:35:28.000000,"[{""id"":""557ece73-658b-cf694dcd3f7e"",""totalValue"":4444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":444.77,""validReturnItems"":[{""UPC"":""19055550468"",""Quantity"":""1"",""Reason"":""fit-issues"",""LineItem"":""4935555579471""}]},""createdAt"":""2020-06-17T11:35:28.469Z"",""modifiedAt"":""2020-06-17T11:35:28.470Z""}]",,json_string
-576111,2022-11-21 03:19:59.064000,2020-06-10 18:35:44.000000,,returnAuthorizations,blade_runner,22527,order,2020-06-10 18:35:44.000000,"[{""id"":""e461c20a-9dc7-d38de1c9012a"",""totalValue"":4444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":444,""validReturnItems"":[{""UPC"":""190735551121"",""Quantity"":""1"",""Reason"":""too-big"",""LineItem"":""4925555231""}]},""createdAt"":""2020-06-10T18:35:44.043Z"",""modifiedAt"":""2020-06-10T18:35:44.043Z""}]",,json_string
-55241839,2022-11-21 01:29:09.347000,2020-07-15 21:24:16.000000,,returnAuthorizations,blade_runner,2335775,order,2020-07-15 21:24:16.000000,"[{""id"":""0c79163e-f55b56f50aff"",""totalValue"":44478.000000000004,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":4444.78000000000003,""validReturnItems"":[{""UPC"":""190555325"",""Quantity"":""1"",""Reason"":""fit-issues"",""LineItem"":""5555599407""}]},""createdAt"":""2020-07-15T21:24:16.210Z"",""modifiedAt"":""2020-07-15T21:24:16.210Z""}]",,json_string
-4575,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,220655,order,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+576111,2022-11-21 03:19:59.064000,2020-06-10 18:35:44.000000,,returnAuthorizations,blade_runner,2674098602081,order,2020-06-10 18:35:44.000000,"[{""id"":""e461c20a-9dc7-d38de1c9012a"",""totalValue"":4444,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":444,""validReturnItems"":[{""UPC"":""190735551121"",""Quantity"":""1"",""Reason"":""too-big"",""LineItem"":""4925555231""}]},""createdAt"":""2020-06-10T18:35:44.043Z"",""modifiedAt"":""2020-06-10T18:35:44.043Z""}]",,json_string
+55241839,2022-11-21 01:29:09.347000,2020-07-15 21:24:16.000000,,returnAuthorizations,blade_runner,2669516488801,order,2020-07-15 21:24:16.000000,"[{""id"":""0c79163e-f55b56f50aff"",""totalValue"":44478.000000000004,""status"":""RECEIVED"",""payload"":{""totalReturnValue"":4444.78000000000003,""validReturnItems"":[{""UPC"":""190555325"",""Quantity"":""1"",""Reason"":""fit-issues"",""LineItem"":""5555599407""}]},""createdAt"":""2020-07-15T21:24:16.210Z"",""modifiedAt"":""2020-07-15T21:24:16.210Z""}]",,json_string
+4575,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,2674098602081,order,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+323013,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,41928883470372,variant,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+4135254,2022-11-21 03:07:20.669000,2020-06-24 17:23:12.000000,,returnAuthorizations,blade_runner,41928967323684,productvariant,2020-06-24 17:23:12.000000,"[{""id"":""3679-4811-94fd-555bf9846753"",""totalValue"":44581,""status"":""BACKEND_GENERATED"",""payload"":{""totalReturnValue"":4444.81,""validReturnItems"":[{""UPC"":""190735558"",""Quantity"":1,""Reason"":""Changed My Mind"",""LineItem"":""455555711""}]},""createdAt"":""2020-06-24T17:23:12.272Z"",""modifiedAt"":""2020-06-24T17:23:12.272Z""}]",,json_string
+4131434,2022-11-21 03:08:20.669000,2020-06-24 17:23:12.000000,,isInSpace,star_wars,2669516488801,ORDER,2020-06-24 17:23:12.000000,"true",,boolean
+9310039,2022-11-21 03:09:20.669000,2020-06-24 17:23:12.000000,,whenInSpace,star_wars,41928967323684,variant,2020-06-24 17:23:12.000000,1977-05-25,,date
+40032,2022-11-21 03:10:20.669000,2020-06-24 17:23:12.000000,,whereAreThey,avengers,41928967323684,PRODUCTVARIANT,2020-06-24 17:23:12.000000,"assembling",,string
+21408,2022-11-21 03:10:20.669000,2020-06-24 13:29:22.000000,,howMany,avengers,2674098602081,order,2020-06-24 17:23:12.000000,6,,integer
+2142325,2022-11-21 04:10:20.669000,2020-06-24 13:30:22.000000,,numberSequels,star_wars,2674098602081,order,2020-06-24 17:23:12.000000,9,,integer
+30000,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,fullName,star_wars,3584045351010,CUSTOMER,2020-06-24 17:23:12.000000,"Boba Fett",,string
+30001,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,firstName,star_wars,3584045351010,CUSTOMER,2020-06-24 17:23:12.000000,"Boba",,string
+30002,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,lastName,star_wars,3584045351010,CUSTOMER,2020-06-24 17:23:12.000000,"Fett",,string
+3000520,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,fullName,star_wars,3589760876641,customer,2020-06-24 17:23:12.000000,"Darth Vader",,string
+3000521,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,firstName,star_wars,3589760876641,customer,2020-06-24 17:23:12.000000,"Darth",,string
+3000522,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,lastName,star_wars,3589760876641,customer,2020-06-24 17:23:12.000000,"Vader",,string
+3000523,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isJedi,star_wars,3589760876641,CUSTOMER,2020-06-24 17:23:12.000000,true,,boolean
+3000524,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isPhoto,star_wars,25974778232868,image,2020-06-24 17:23:12.000000,true,,boolean
+4009,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,fullName,star_wars,3584045351009,CUSTOMER,2020-06-24 17:23:12.000000,"Chewbacca",,string
+4010,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,firstName,star_wars,3584045351009,CUSTOMER,2020-06-24 17:23:12.000000,"Chewbacca",,string
+4011,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isJedi,star_wars,3584045351009,CUSTOMER,2020-06-24 17:23:12.000000,false,,boolean
+4012,2022-11-21 04:20:20.669000,2020-06-24 13:31:22.000000,,isPhoto,star_wars,25974777741348,productimage,2020-06-24 17:23:12.000000,true,,boolean

macros/get_metafields.sql

@@ -1,6 +1,51 @@
-{% macro get_metafields(source_object, reference_value, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource") %}
+{% macro get_metafields(source_object, reference_values=None, reference_value=None, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource", id_column_override=None) %}
+    {{ adapter.dispatch('get_metafields', 'shopify')(
+        source_object=source_object,
+        reference_values=reference_values,
+        reference_value=reference_value,
+        lookup_object=lookup_object,
+        key_field=key_field,
+        key_value=key_value,
+        reference_field=reference_field,
+        id_column_override=id_column_override
+    ) }}
+{% endmacro %}
 
-{% set pivot_fields = dbt_utils.get_column_values(table=ref(lookup_object), column=key_field, where="lower(" ~ reference_field ~ ") = lower('" ~ reference_value ~ "')") %}
+
+{% macro default__get_metafields(source_object, reference_values=None, reference_value=None, lookup_object="stg_shopify__metafield", key_field="metafield_reference", key_value="value", reference_field="owner_resource", id_column_override=None) %}
+
+{# Handle backward compatibility for reference_value #}
+{% if reference_values is none and reference_value is not none %}
+    {% set reference_values = [reference_value] %}
+{% endif %}
+
+{# Ensure reference_values is defined #}
+{% if reference_values is none %}
+    {% do exceptions.raise_compiler_error("Either reference_values or reference_value must be provided.") %}
+{% endif %}
+
+{# Determine the _id column to use (respect id_column_override) #}
+{% if id_column_override is not none %}
+    {% set id_column = id_column_override %}
+    {% do log("Using id_column_override: " ~ id_column, info=true) %}
+{% else %}
+    {% set id_column = reference_values[0] ~ '_id' %}
+    {% do log("Dynamically resolved id_column: " ~ id_column, info=true) %}
+{% endif %}
+
+{# Manually quote and join reference values #}
+{% set quoted_values = [] %}
+{% for value in reference_values %}
+    {% do quoted_values.append("'" ~ value | lower ~ "'") %}
+{% endfor %}
+{% set reference_values_clause = quoted_values | join(", ") %}
+
+{# Get the pivot fields dynamically based on the reference values #}
+{% set pivot_fields = dbt_utils.get_column_values(
+    table=ref(lookup_object),
+    column=key_field,
+    where="lower(" ~ reference_field ~ ") IN (" ~ reference_values_clause ~ ")"
+) %}
 
 {% set source_columns = adapter.get_columns_in_relation(ref(source_object)) %}
 {% set source_column_count = source_columns | length %}
@@ -37,8 +82,8 @@ final as (
         {% endfor %}
     from source_table
     left join lookup_object 
-        on lookup_object.{{ reference_field }}_id = source_table.{{ reference_value }}_id
-        and lookup_object.{{ reference_field }} = '{{ reference_value }}'
+        on lookup_object.owner_resource_id = source_table.{{ id_column }}
+        and lower(lookup_object.{{ reference_field }}) IN ({{ reference_values_clause }})
     {{ dbt_utils.group_by(source_column_count) }}
 )
 

models/metafields/shopify__collection_metafields.sql

@@ -1,6 +1,6 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__collection", 
     reference_value = 'collection') 
 }}

models/metafields/shopify__customer_metafields.sql

@@ -1,6 +1,6 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_customer_metafields', False)) )}}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__customer", 
     reference_value = 'customer') 
 }}

models/metafields/shopify__order_metafields.sql

@@ -1,6 +1,6 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_order_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__order", 
     reference_value = 'order') 
 }}

models/metafields/shopify__product_image_metafields.sql

@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_image_metafields', False)) ) }}
 
-{{ get_metafields( 
-    source_object = "stg_shopify__product_image", 
-    reference_value = 'image') 
-}}
+{{ shopify.get_metafields(
+    source_object = "stg_shopify__product_image",
+    id_column_override = 'product_image_id', 
+    reference_values = ['image', 'productimage']
+) }}

models/metafields/shopify__product_metafields.sql

@@ -1,6 +1,6 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__product", 
     reference_value = 'product') 
 }}

models/metafields/shopify__product_variant_metafields.sql

@@ -1,6 +1,7 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_variant_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__product_variant", 
-    reference_value = 'variant') 
+    id_column_override = 'variant_id',
+    reference_values = ['variant', 'productvariant']) 
 }}

models/metafields/shopify__shop_metafields.sql

@@ -1,6 +1,6 @@
 {{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_shop_metafields', False)) ) }}
 
-{{ get_metafields( 
+{{ shopify.get_metafields( 
     source_object = "stg_shopify__shop", 
     reference_value = 'shop') 
 }}