Teradata · septimit · Aug 16, 2024
diff --git a/macros/dbt-external-tables/macros/plugins/teradata/create_external_schema.sql b/macros/dbt-external-tables/macros/plugins/teradata/create_external_schema.sql
@@ -0,0 +1,5 @@
+{%- macro teradata__create_external_schema(source_node) -%}
+    {{ exceptions.raise_compiler_error(
+        "Creating external schema is not implemented for the Teradata adapter"
+    ) }}
+{%- endmacro -%}
diff --git a/macros/dbt-external-tables/macros/plugins/teradata/create_external_table.sql b/macros/dbt-external-tables/macros/plugins/teradata/create_external_table.sql
@@ -0,0 +1,29 @@
+{% macro teradata__create_external_table(source_node) %}
+    {# https://docs.teradata.com/r/Enterprise_IntelliFlex_VMware/SQL-Data-Definition-Language-Syntax-and-Examples/Table-Statements/CREATE-FOREIGN-TABLE/CREATE-FOREIGN-TABLE-Syntax #}
+    {%- set columns = source_node.columns.values() -%}
+    {%- set external = source_node.external -%}
+    {%- set partitions = external.partitions -%}
+
+    CREATE FOREIGN TABLE {{source(source_node.source_name, source_node.name)}}
+        {{ ', ' + external.tbl_properties if external.tbl_properties }}
+        {% if columns -%}
+        (
+            {% for column in columns %}
+                {%- set column_quoted = adapter.quote(column.name) if column.quote else column.name %}
+                {{column_quoted}} {{column.data_type}} {{- ',' if not loop.last -}}
+            {%- endfor %}
+        )
+        {%- endif %}
+    USING (
+        LOCATION('{{ external.location }}')
+        {{ external.using }}
+    )
+    NO PRIMARY INDEX
+    {% if partitions -%}
+    PARTITION BY (
+        {%- for partition in partitions -%}
+            {{ partition.name }} {{ partition.data_type if partition.data_type }} {{', ' if not loop.last}}
+        {%- endfor -%}
+    )
+    {%- endif %}
+{% endmacro %}
diff --git a/macros/dbt-external-tables/macros/plugins/teradata/get_external_build_plan.sql b/macros/dbt-external-tables/macros/plugins/teradata/get_external_build_plan.sql
@@ -0,0 +1,24 @@
+{% macro teradata__get_external_build_plan(source_node) %}
+
+    {% set build_plan = [] %}
+
+    {% set old_relation = adapter.get_relation(
+        database = source_node.database,
+        schema = source_node.schema,
+        identifier = source_node.identifier
+    ) %}
+
+    {% set create_or_replace = (old_relation is none or var('ext_full_refresh', false)) %}
+
+    {% if create_or_replace %}
+        {% set build_plan = build_plan + [
+            dbt_external_tables.dropif(source_node),
+            dbt_external_tables.create_external_table(source_node)
+        ] %}
+    {% else %}
+        {% set build_plan = build_plan + dbt_external_tables.refresh_external_table(source_node) %}
+    {% endif %}
+
+    {% do return(build_plan) %}
+
+{% endmacro %}
diff --git a/macros/dbt-external-tables/macros/plugins/teradata/helpers/dropif.sql b/macros/dbt-external-tables/macros/plugins/teradata/helpers/dropif.sql
@@ -0,0 +1,20 @@
+{% macro teradata__dropif(node) %}
+
+    {% set old_relation = adapter.get_relation(
+        database = none,
+        schema = node.schema,
+        identifier = node.identifier
+    ) %}
+
+    {% set drop_relation = old_relation is not none %}
+
+    {% set ddl = '' %}
+    {% if drop_relation %}
+        {% set ddl %}
+          DROP TABLE {{ old_relation.include(database=False) }}
+        {% endset %}
+    {% endif %}
+
+    {{ return(ddl) }}
+
+{% endmacro %}
diff --git a/test/dbt-external-tables/Makefile b/test/dbt-external-tables/Makefile
@@ -0,0 +1,10 @@
+test-teradata:
+	dbt deps
+	dbt seed --full-refresh
+	dbt run-operation prep_external
+	dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true'
+	dbt run-operation dbt_external_tables.stage_external_sources
+	dbt test
+
+test-all: test-teradata
+	echo "Completed successfully"
diff --git a/test/dbt-external-tables/README.txt b/test/dbt-external-tables/README.txt
@@ -0,0 +1,7 @@
+dbt-external-tables is based on https://github.com/dbt-labs/dbt-external-tables
+
+Testing instructions:
+1. create a teradata dbt profile called integration_tests
+2. create python venv with dbt-teradata
+3. change directory to  integration_tests
+4. run make -f ../Makefile test-all
diff --git a/test/dbt-external-tables/integration_tests/.gitignore b/test/dbt-external-tables/integration_tests/.gitignore
@@ -0,0 +1,7 @@
+
+target/
+dbt_packages/
+logs/
+.env/
+profiles.yml
+package-lock.yml
diff --git a/test/dbt-external-tables/integration_tests/dbt_project.yml b/test/dbt-external-tables/integration_tests/dbt_project.yml
@@ -0,0 +1,38 @@
+
+name: 'dbt_external_tables_integration_tests'
+version: '1.0'
+
+profile: 'integration_tests'
+
+config-version: 2
+
+model-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+dispatch:
+  - macro_namespace: dbt_external_tables
+    search_order: ['teradata_utils', 'dbt_external_tables', 'dbt_external_tables_integration_tests']
+  - macro_namespace: dbt_utils
+    search_order: ['teradata_utils', 'dbt_utils']
+
+seeds:
+  +quote_columns: false
+
+#models:
+
+sources:
+  dbt_external_tables_integration_tests:
+    plugins:
+      teradata:
+        +enabled: "{{ target.type == 'teradata' }}"
+
+#tests:
+
diff --git a/test/dbt-external-tables/integration_tests/macros/common/prep_external.sql b/test/dbt-external-tables/integration_tests/macros/common/prep_external.sql
@@ -0,0 +1,8 @@
+{% macro prep_external() %}
+    {{ return(adapter.dispatch('prep_external', 'dbt_external_tables')()) }}
+{% endmacro %}
+
+{% macro default__prep_external() %}
+    {% do log('No prep necessary, skipping', info = true) %}
+    {# noop #}
+{% endmacro %}
diff --git a/test/dbt-external-tables/integration_tests/models/common/control.yml b/test/dbt-external-tables/integration_tests/models/common/control.yml
@@ -0,0 +1,10 @@
+version: 2
+
+sources:
+  - name: not_external
+    tables:
+      - name: take_no_action
+        columns:
+          - name: id
+            data_type: varchar(255)
+            description: "primary key"
diff --git a/test/dbt-external-tables/integration_tests/models/plugins/teradata/teradata_external.yml b/test/dbt-external-tables/integration_tests/models/plugins/teradata/teradata_external.yml
@@ -0,0 +1,64 @@
+version: 2
+
+sources:
+  - name: teradata_external
+    schema: "{{ target.schema }}"
+    loader: S3
+
+    tables:
+      - name: people_csv_unpartitioned
+        external: &csv-people
+          location: "/s3/s3.amazonaws.com/dbt-external-tables-testing/csv/"
+          using: |
+            STOREDAS('TEXTFILE')
+            -- PATHPATTERN  ('$var1/$section/$var3')
+          tbl_properties: |
+            MAP = TD_MAP1
+            -- ,EXTERNAL SECURITY  MyAuthObj
+        columns: &cols-of-the-people
+          - name: id
+            data_type: int
+          - name: first_name
+            data_type: varchar(64)
+          - name: last_name
+            data_type: varchar(64)
+          - name: email
+            data_type: varchar(64)
+        data_tests: &equal-to-the-people
+          - dbt_utils.equality:
+              compare_model: ref('people')
+              compare_columns:
+                - id
+                - first_name
+                - last_name
+                - email
+
+      - name: people_csv_partitioned
+        external:
+          <<: *csv-people
+          partitions:
+            - name: section
+              data_type: CHAR(1)
+        columns: *cols-of-the-people
+        data_tests:
+          - <<: *equal-to-the-people
+          - dbt_utils.at_least_one:
+              column_name: section
+
+
+      - name: people_pq_unpartitioned
+        external:
+          location: "/s3/s3.amazonaws.com/dbt-external-tables-testing/parquet/"
+        data_tests: *equal-to-the-people
+
+      - name: people_pq_partitioned
+        external:
+          location: "/s3/s3.amazonaws.com/dbt-external-tables-testing/parquet/"
+          partitions:
+            - name: COLUMN
+            - name: section
+              data_type: CHAR(1)
+        data_tests:
+          - <<: *equal-to-the-people
+          - dbt_utils.at_least_one:
+              column_name: section
diff --git a/test/dbt-external-tables/integration_tests/packages.yml b/test/dbt-external-tables/integration_tests/packages.yml
@@ -0,0 +1,7 @@
+
+packages:
+  - local: ../../../
+  - package: dbt-labs/dbt_utils
+    version: [">0.9.0", "<2.0.0"]
+  - package: dbt-labs/dbt_external_tables
+    version: [">=0.9.0", "<1.0.0"]
diff --git a/test/dbt-external-tables/integration_tests/public_data/csv/section=a/people_a.csv b/test/dbt-external-tables/integration_tests/public_data/csv/section=a/people_a.csv
@@ -0,0 +1,51 @@
+id,first_name,last_name,email
+1,Jack,Hunter,jhunter0@pbs.org
+2,Kathryn,Walker,kwalker1@ezinearticles.com
+3,Gerald,Ryan,gryan2@com.com
+4,Bonnie,Spencer,bspencer3@ameblo.jp
+5,Harold,Taylor,htaylor4@people.com.cn
+6,Jacqueline,Griffin,jgriffin5@t.co
+7,Wanda,Arnold,warnold6@google.nl
+8,Craig,Ortiz,cortiz7@sciencedaily.com
+9,Gary,Day,gday8@nih.gov
+10,Rose,Wright,rwright9@yahoo.co.jp
+11,Raymond,Kelley,rkelleya@fc2.com
+12,Gerald,Robinson,grobinsonb@disqus.com
+13,Mildred,Martinez,mmartinezc@samsung.com
+14,Dennis,Arnold,darnoldd@google.com
+15,Judy,Gray,jgraye@opensource.org
+16,Theresa,Garza,tgarzaf@epa.gov
+17,Gerald,Robertson,grobertsong@csmonitor.com
+18,Philip,Hernandez,phernandezh@adobe.com
+19,Julia,Gonzalez,jgonzalezi@cam.ac.uk
+20,Andrew,Davis,adavisj@patch.com
+21,Kimberly,Harper,kharperk@foxnews.com
+22,Mark,Martin,mmartinl@marketwatch.com
+23,Cynthia,Ruiz,cruizm@google.fr
+24,Samuel,Carroll,scarrolln@youtu.be
+25,Jennifer,Larson,jlarsono@vinaora.com
+26,Ashley,Perry,aperryp@rakuten.co.jp
+27,Howard,Rodriguez,hrodriguezq@shutterfly.com
+28,Amy,Brooks,abrooksr@theatlantic.com
+29,Louise,Warren,lwarrens@adobe.com
+30,Tina,Watson,twatsont@myspace.com
+31,Janice,Kelley,jkelleyu@creativecommons.org
+32,Terry,Mccoy,tmccoyv@bravesites.com
+33,Jeffrey,Morgan,jmorganw@surveymonkey.com
+34,Louis,Harvey,lharveyx@sina.com.cn
+35,Philip,Miller,pmillery@samsung.com
+36,Willie,Marshall,wmarshallz@ow.ly
+37,Patrick,Lopez,plopez10@redcross.org
+38,Adam,Jenkins,ajenkins11@harvard.edu
+39,Benjamin,Cruz,bcruz12@linkedin.com
+40,Ruby,Hawkins,rhawkins13@gmpg.org
+41,Carlos,Barnes,cbarnes14@a8.net
+42,Ruby,Griffin,rgriffin15@bravesites.com
+43,Sean,Mason,smason16@icq.com
+44,Anthony,Payne,apayne17@utexas.edu
+45,Steve,Cruz,scruz18@pcworld.com
+46,Anthony,Garcia,agarcia19@flavors.me
+47,Doris,Lopez,dlopez1a@sphinn.com
+48,Susan,Nichols,snichols1b@freewebs.com
+49,Wanda,Ferguson,wferguson1c@yahoo.co.jp
+50,Andrea,Pierce,apierce1d@google.co.uk
diff --git a/test/dbt-external-tables/integration_tests/public_data/csv/section=b/people_b.csv b/test/dbt-external-tables/integration_tests/public_data/csv/section=b/people_b.csv
@@ -0,0 +1,51 @@
+id,first_name,last_name,email
+51,Lawrence,Phillips,lphillips1e@jugem.jp
+52,Judy,Gilbert,jgilbert1f@multiply.com
+53,Eric,Williams,ewilliams1g@joomla.org
+54,Ralph,Romero,rromero1h@sogou.com
+55,Jean,Wilson,jwilson1i@ocn.ne.jp
+56,Lori,Reynolds,lreynolds1j@illinois.edu
+57,Donald,Moreno,dmoreno1k@bbc.co.uk
+58,Steven,Berry,sberry1l@eepurl.com
+59,Theresa,Shaw,tshaw1m@people.com.cn
+60,John,Stephens,jstephens1n@nationalgeographic.com
+61,Richard,Jacobs,rjacobs1o@state.tx.us
+62,Andrew,Lawson,alawson1p@over-blog.com
+63,Peter,Morgan,pmorgan1q@rambler.ru
+64,Nicole,Garrett,ngarrett1r@zimbio.com
+65,Joshua,Kim,jkim1s@edublogs.org
+66,Ralph,Roberts,rroberts1t@people.com.cn
+67,George,Montgomery,gmontgomery1u@smugmug.com
+68,Gerald,Alvarez,galvarez1v@flavors.me
+69,Donald,Olson,dolson1w@whitehouse.gov
+70,Carlos,Morgan,cmorgan1x@pbs.org
+71,Aaron,Stanley,astanley1y@webnode.com
+72,Virginia,Long,vlong1z@spiegel.de
+73,Robert,Berry,rberry20@tripadvisor.com
+74,Antonio,Brooks,abrooks21@unesco.org
+75,Ruby,Garcia,rgarcia22@ovh.net
+76,Jack,Hanson,jhanson23@blogtalkradio.com
+77,Kathryn,Nelson,knelson24@walmart.com
+78,Jason,Reed,jreed25@printfriendly.com
+79,George,Coleman,gcoleman26@people.com.cn
+80,Rose,King,rking27@ucoz.com
+81,Johnny,Holmes,jholmes28@boston.com
+82,Katherine,Gilbert,kgilbert29@altervista.org
+83,Joshua,Thomas,jthomas2a@ustream.tv
+84,Julie,Perry,jperry2b@opensource.org
+85,Richard,Perry,rperry2c@oracle.com
+86,Kenneth,Ruiz,kruiz2d@wikimedia.org
+87,Jose,Morgan,jmorgan2e@webnode.com
+88,Donald,Campbell,dcampbell2f@goo.ne.jp
+89,Debra,Collins,dcollins2g@uol.com.br
+90,Jesse,Johnson,jjohnson2h@stumbleupon.com
+91,Elizabeth,Stone,estone2i@histats.com
+92,Angela,Rogers,arogers2j@goodreads.com
+93,Emily,Dixon,edixon2k@mlb.com
+94,Albert,Scott,ascott2l@tinypic.com
+95,Barbara,Peterson,bpeterson2m@ow.ly
+96,Adam,Greene,agreene2n@fastcompany.com
+97,Earl,Sanders,esanders2o@hc360.com
+98,Angela,Brooks,abrooks2p@mtv.com
+99,Harold,Foster,hfoster2q@privacy.gov.au
+100,Carl,Meyer,cmeyer2r@disqus.com
diff --git a/test/dbt-external-tables/integration_tests/public_data/csv/section=c/people_c.csv b/test/dbt-external-tables/integration_tests/public_data/csv/section=c/people_c.csv
@@ -0,0 +1,51 @@
+id,first_name,last_name,email
+101,Michael,Perez,mperez0@chronoengine.com
+102,Shawn,Mccoy,smccoy1@reddit.com
+103,Kathleen,Payne,kpayne2@cargocollective.com
+104,Jimmy,Cooper,jcooper3@cargocollective.com
+105,Katherine,Rice,krice4@typepad.com
+106,Sarah,Ryan,sryan5@gnu.org
+107,Martin,Mcdonald,mmcdonald6@opera.com
+108,Frank,Robinson,frobinson7@wunderground.com
+109,Jennifer,Franklin,jfranklin8@mail.ru
+110,Henry,Welch,hwelch9@list-manage.com
+111,Fred,Snyder,fsnydera@reddit.com
+112,Amy,Dunn,adunnb@nba.com
+113,Kathleen,Meyer,kmeyerc@cdc.gov
+114,Steve,Ferguson,sfergusond@reverbnation.com
+115,Teresa,Hill,thille@dion.ne.jp
+116,Amanda,Harper,aharperf@mail.ru
+117,Kimberly,Ray,krayg@xing.com
+118,Johnny,Knight,jknighth@jalbum.net
+119,Virginia,Freeman,vfreemani@tiny.cc
+120,Anna,Austin,aaustinj@diigo.com
+121,Willie,Hill,whillk@mail.ru
+122,Sean,Harris,sharrisl@zdnet.com
+123,Mildred,Adams,madamsm@usatoday.com
+124,David,Graham,dgrahamn@zimbio.com
+125,Victor,Hunter,vhuntero@ehow.com
+126,Aaron,Ruiz,aruizp@weebly.com
+127,Benjamin,Brooks,bbrooksq@jalbum.net
+128,Lisa,Wilson,lwilsonr@japanpost.jp
+129,Benjamin,King,bkings@comsenz.com
+130,Christina,Williamson,cwilliamsont@boston.com
+131,Jane,Gonzalez,jgonzalezu@networksolutions.com
+132,Thomas,Owens,towensv@psu.edu
+133,Katherine,Moore,kmoorew@naver.com
+134,Jennifer,Stewart,jstewartx@yahoo.com
+135,Sara,Tucker,stuckery@topsy.com
+136,Harold,Ortiz,hortizz@vkontakte.ru
+137,Shirley,James,sjames10@yelp.com
+138,Dennis,Johnson,djohnson11@slate.com
+139,Louise,Weaver,lweaver12@china.com.cn
+140,Maria,Armstrong,marmstrong13@prweb.com
+141,Gloria,Cruz,gcruz14@odnoklassniki.ru
+142,Diana,Spencer,dspencer15@ifeng.com
+143,Kelly,Nguyen,knguyen16@altervista.org
+144,Jane,Rodriguez,jrodriguez17@biblegateway.com
+145,Scott,Brown,sbrown18@geocities.jp
+146,Norma,Cruz,ncruz19@si.edu
+147,Marie,Peters,mpeters1a@mlb.com
+148,Lillian,Carr,lcarr1b@typepad.com
+149,Judy,Nichols,jnichols1c@t-online.de
+150,Billy,Long,blong1d@yahoo.com