Mount Taxonomy CA Cert and use if provided

- Mount teacher-server ConfigMap as volume to git clone task - Provide mount path as env var to git clone task - Check if 'taxonomy-ca.crt' exists in mounted vol - Use TLS if cert exists, otherwise use standard git operations Signed-off-by: Giulio Frasca <[email protected]>
opendatahub-io · Jan 29, 2025 · d076d30 · d076d30
1 parent 7a1abfb
commit d076d30
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 30 deletions.
diff --git a/pipeline.py b/pipeline.py
@@ -49,6 +49,10 @@
 DEFAULT_REPO_URL = "https://github.com/instructlab/taxonomy.git"
 
 # Model Serving SSL connection
+TAXONOMY_CA_CERT_CM_KEY = "taxonomy-ca.crt"
+TAXONOMY_CA_CERT_ENV_VAR_NAME = "TAXONOMY_CA_CERT_PATH"
+TAXONOMY_CA_CERT_PATH = "/tmp/cert"
+
 SDG_CA_CERT_CM_KEY = "ca.crt"
 SDG_CA_CERT_ENV_VAR_NAME = "SDG_CA_CERT_PATH"
 SDG_CA_CERT_PATH = "/tmp/cert"
@@ -148,7 +152,13 @@ def ilab_pipeline(
         repo_branch=sdg_repo_branch,
         repo_pr=sdg_repo_pr if sdg_repo_pr and sdg_repo_pr > 0 else None,
         repo_url=sdg_repo_url,
-        ca_cert_path="",  # TODO(gfrasca)
+    )
+    use_config_map_as_volume(
+        git_clone_task, TEACHER_CONFIG_MAP, mount_path=TAXONOMY_CA_CERT_PATH
+    )
+    git_clone_task.set_env_variable(
+        TAXONOMY_CA_CERT_ENV_VAR_NAME,
+        os.path.join(TAXONOMY_CA_CERT_PATH, TAXONOMY_CA_CERT_CM_KEY),
     )
     mount_pvc(
         task=git_clone_task,

diff --git a/pipeline.yaml b/pipeline.yaml
@@ -270,9 +270,6 @@ components:
     executorLabel: exec-git-clone-op
     inputDefinitions:
       parameters:
-        ca_cert_path:
-          isOptional: true
-          parameterType: STRING
         repo_branch:
           parameterType: STRING
         repo_pr:
@@ -696,16 +693,31 @@ deploymentSpec:
     exec-git-clone-op:
       container:
         args:
-        - 'git clone  {{$.inputs.parameters[''repo_url'']}} {{$.inputs.parameters[''taxonomy_path'']}}
-          && cd {{$.inputs.parameters[''taxonomy_path'']}} && if [ -n "{{$.inputs.parameters[''repo_branch'']}}"
-          ]; then git fetch origin {{$.inputs.parameters[''repo_branch'']}} && git
-          checkout {{$.inputs.parameters[''repo_branch'']}}; elif [ -n "{{$.inputs.parameters[''repo_pr'']}}"
-          ] && [ {{$.inputs.parameters[''repo_pr'']}} -gt 0 ]; then git fetch origin
-          pull/{{$.inputs.parameters[''repo_pr'']}}/head:{{$.inputs.parameters[''repo_pr'']}}
-          && git checkout {{$.inputs.parameters[''repo_pr'']}}; fi '
+        - "\n            # Increase logging verbosity\n            set -x &&\n\n \
+          \           # Add TLS Parameters if CA Cert exists and is non-zero size\n\
+          \            ADDITIONAL_CLONE_PARAMS=\"\"\n            if [ -s \"$TAXONOMY_CA_CERT_PATH\"\
+          \ ]; then\n                ADDITIONAL_CLONE_PARAMS=\"-c http.sslVerify=true\
+          \ -c http.sslCAInfo=$TAXONOMY_CA_CERT_PATH\"\n            fi\n\n       \
+          \     # Clone Taxonomy Repo\n            git clone $ADDITIONAL_CLONE_PARAMS\
+          \ {{$.inputs.parameters['repo_url']}} {{$.inputs.parameters['taxonomy_path']}}\
+          \ &&\n            cd {{$.inputs.parameters['taxonomy_path']}} &&\n\n   \
+          \         # Run additional configuration if TLS certs provided\n       \
+          \     if [ -s \"$TAXONOMY_CA_CERT_PATH\" ]; then\n                git config\
+          \ http.sslVerify true &&\n                git config http.sslCAInfo $TAXONOMY_CA_CERT_PATH\n\
+          \            fi &&\n\n            # Checkout and use taxonomy repo branch\
+          \ or PR if specified\n            if [ -n \"{{$.inputs.parameters['repo_branch']}}\"\
+          \ ]; then\n                git fetch origin {{$.inputs.parameters['repo_branch']}}\
+          \ && git checkout {{$.inputs.parameters['repo_branch']}};\n            elif\
+          \ [ -n \"{{$.inputs.parameters['repo_pr']}}\" ] && [ {{$.inputs.parameters['repo_pr']}}\
+          \ -gt 0 ]; then\n                git fetch origin pull/{{$.inputs.parameters['repo_pr']}}/head:{{$.inputs.parameters['repo_pr']}}\
+          \ && git checkout {{$.inputs.parameters['repo_pr']}}; \n            fi\n\
+          \            "
         command:
         - /bin/sh
         - -c
+        env:
+        - name: TAXONOMY_CA_CERT_PATH
+          value: /tmp/cert/taxonomy-ca.crt
         image: registry.redhat.io/ubi9/toolbox@sha256:da31dee8904a535d12689346e65e5b00d11a6179abf1fa69b548dbd755fa2770
     exec-importer:
       importer:
@@ -1669,9 +1681,6 @@ root:
         - createpvc
         inputs:
           parameters:
-            ca_cert_path:
-              runtimeValue:
-                constant: ''
             repo_branch:
               componentInputParameter: sdg_repo_branch
             repo_pr:
@@ -2140,6 +2149,10 @@ platforms:
               outputParameterKey: name
               producerTask: createpvc
         exec-git-clone-op:
+          configMapAsVolume:
+          - configMapName: teacher-server
+            mountPath: /tmp/cert
+            optional: false
           pvcMount:
           - mountPath: /data
             taskOutputParameter:

diff --git a/sdg/components.py b/sdg/components.py
@@ -13,27 +13,38 @@ def git_clone_op(
     repo_pr: Optional[int],
     repo_url: Optional[str],
     taxonomy_path: str = "/data/taxonomy",
-    ca_cert_path: Optional[str] = None,
 ):
-    import os
-
-    additional_clone_params = ""
-    additional_config_cmds = ""
-
-    if ca_cert_path and os.path.exists(f"{ca_cert_path}") and (os.path.getsize(f"{ca_cert_path}") > 0):
-        full_ca_path = os.path.abspath(f"{ca_cert_path}")
-        additional_clone_params = f"-c http.sslVerify=true -c http.sslCAInfo={full_ca_path}"
-        additional_config_cmds = f"git config http.sslVerify true && git config http.sslCAInfo {full_ca_path} &&"
-
     return dsl.ContainerSpec(
         TOOLBOX_IMAGE,
         ["/bin/sh", "-c"],
         [
-            f"git clone {additional_clone_params} {repo_url} {taxonomy_path} && cd {taxonomy_path} && {additional_config_cmds}"
-            + f'if [ -n "{repo_branch}" ]; then '
-            + f"git fetch origin {repo_branch} && git checkout {repo_branch}; "
-            + f'elif [ -n "{repo_pr}" ] && [ {repo_pr} -gt 0 ]; then '
-            + f"git fetch origin pull/{repo_pr}/head:{repo_pr} && git checkout {repo_pr}; fi "
+            f'''
+            # Increase logging verbosity
+            set -x &&
+
+            # Add TLS Parameters if CA Cert exists and is non-zero size
+            ADDITIONAL_CLONE_PARAMS=""
+            if [ -s "$TAXONOMY_CA_CERT_PATH" ]; then
+                ADDITIONAL_CLONE_PARAMS="-c http.sslVerify=true -c http.sslCAInfo=$TAXONOMY_CA_CERT_PATH"
+            fi
+
+            # Clone Taxonomy Repo
+            git clone $ADDITIONAL_CLONE_PARAMS {repo_url} {taxonomy_path} &&
+            cd {taxonomy_path} &&
+
+            # Run additional configuration if TLS certs provided
+            if [ -s "$TAXONOMY_CA_CERT_PATH" ]; then
+                git config http.sslVerify true &&
+                git config http.sslCAInfo $TAXONOMY_CA_CERT_PATH
+            fi &&
+
+            # Checkout and use taxonomy repo branch or PR if specified
+            if [ -n "{repo_branch}" ]; then
+                git fetch origin {repo_branch} && git checkout {repo_branch};
+            elif [ -n "{repo_pr}" ] && [ {repo_pr} -gt 0 ]; then
+                git fetch origin pull/{repo_pr}/head:{repo_pr} && git checkout {repo_pr}; 
+            fi
+            '''
         ],
     )