Merge pull request #574 from accorvin/fix-tekton

Update trino partition updater to use volume for tables list
AICoE · Dec 5, 2023 · 49ca6c5 · 49ca6c5
2 parents 218874b + f95e615
commit 49ca6c5
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 11 deletions.
diff --git a/trino-partition-updater/base/get-all-partitioned-tables-task.yaml b/trino-partition-updater/base/get-all-partitioned-tables-task.yaml
@@ -3,9 +3,8 @@ kind: Task
 metadata:
   name: get-all-partitioned-tables
 spec:
-  results:
+  workspaces:
     - name: tables-list
-      description: The list of trino tables that are partitioned
   steps:
     - name: get-tables-list
       image: quay.io/opendatahub/trino:356
@@ -84,11 +83,11 @@ spec:
                 print("Error adding {}.{} to list of partitioned tables. Caused by:".format(row[0], row[1]))
                 print(e)
 
-        with open('$(results.tables-list.path)','w') as f:
+        with open('$(workspaces.tables-list.path)','w') as f:
             print("Dumping table list as json data...")
             json.dump(schemas, f)
 
-        with open('$(results.tables-list.path)') as f:
+        with open('$(workspaces.tables-list.path)') as f:
             print(f.read())
 
         if failures_detected:

diff --git a/trino-partition-updater/base/trino-partition-updater-pipeline.yaml b/trino-partition-updater/base/trino-partition-updater-pipeline.yaml
@@ -3,14 +3,17 @@ kind: Pipeline
 metadata:
   name: trino-partition-updater
 spec:
+  workspaces:
+    - name: tables-list
   tasks:
     - name: get-all-partitioned-tables
       taskRef:
         name: get-all-partitioned-tables
+      workspaces:
+        - name: tables-list
     - name: update-partition-metadata
       timeout: 10h0m0s
       taskRef:
         name: update-partition-metadata
-      params:
-        - name: tables_list
-          value: $(tasks.get-all-partitioned-tables.results.tables-list)
+      workspaces:
+        - name: tables-list
diff --git a/trino-partition-updater/base/update-partition-metadata-task.yaml b/trino-partition-updater/base/update-partition-metadata-task.yaml
@@ -3,9 +3,8 @@ kind: Task
 metadata:
   name: update-partition-metadata
 spec:
-  params:
-    - name: tables_list
-      type: string
+  workspaces:
+    - name: tables-list
   steps:
     - name: update-partition-metadata
       image: quay.io/opendatahub/trino:356
@@ -36,7 +35,8 @@ spec:
         # Disable warnings from urllib3
         urllib3.disable_warnings()
 
-        TABLE_DEFS = '$(params.tables_list)'
+        with open('$(workspaces.messages.path)') as f:
+            TABLE_DEFS = f.read()
 
         UPDATE_PARTITION = "call hive.system.sync_partition_metadata('{}','{}','FULL')"