From d076d302432649d1d4182a3e2ae6d3a53f31edb3 Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Tue, 28 Jan 2025 17:38:41 -0500 Subject: [PATCH] Mount Taxonomy CA Cert and use if provided - Mount teacher-server ConfigMap as volume to git clone task - Provide mount path as env var to git clone task - Check if 'taxonomy-ca.crt' exists in mounted vol - Use TLS if cert exists, otherwise use standard git operations Signed-off-by: Giulio Frasca --- pipeline.py | 12 +++++++++++- pipeline.yaml | 39 ++++++++++++++++++++++++++------------- sdg/components.py | 43 +++++++++++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 30 deletions(-) diff --git a/pipeline.py b/pipeline.py index 7dfd28a..e219896 100644 --- a/pipeline.py +++ b/pipeline.py @@ -49,6 +49,10 @@ DEFAULT_REPO_URL = "https://github.com/instructlab/taxonomy.git" # Model Serving SSL connection +TAXONOMY_CA_CERT_CM_KEY = "taxonomy-ca.crt" +TAXONOMY_CA_CERT_ENV_VAR_NAME = "TAXONOMY_CA_CERT_PATH" +TAXONOMY_CA_CERT_PATH = "/tmp/cert" + SDG_CA_CERT_CM_KEY = "ca.crt" SDG_CA_CERT_ENV_VAR_NAME = "SDG_CA_CERT_PATH" SDG_CA_CERT_PATH = "/tmp/cert" @@ -148,7 +152,13 @@ def ilab_pipeline( repo_branch=sdg_repo_branch, repo_pr=sdg_repo_pr if sdg_repo_pr and sdg_repo_pr > 0 else None, repo_url=sdg_repo_url, - ca_cert_path="", # TODO(gfrasca) + ) + use_config_map_as_volume( + git_clone_task, TEACHER_CONFIG_MAP, mount_path=TAXONOMY_CA_CERT_PATH + ) + git_clone_task.set_env_variable( + TAXONOMY_CA_CERT_ENV_VAR_NAME, + os.path.join(TAXONOMY_CA_CERT_PATH, TAXONOMY_CA_CERT_CM_KEY), ) mount_pvc( task=git_clone_task, diff --git a/pipeline.yaml b/pipeline.yaml index 09d5e49..bd0dbca 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -270,9 +270,6 @@ components: executorLabel: exec-git-clone-op inputDefinitions: parameters: - ca_cert_path: - isOptional: true - parameterType: STRING repo_branch: parameterType: STRING repo_pr: @@ -696,16 +693,31 @@ deploymentSpec: exec-git-clone-op: container: args: - - 'git clone {{$.inputs.parameters[''repo_url'']}} {{$.inputs.parameters[''taxonomy_path'']}} - && cd {{$.inputs.parameters[''taxonomy_path'']}} && if [ -n "{{$.inputs.parameters[''repo_branch'']}}" - ]; then git fetch origin {{$.inputs.parameters[''repo_branch'']}} && git - checkout {{$.inputs.parameters[''repo_branch'']}}; elif [ -n "{{$.inputs.parameters[''repo_pr'']}}" - ] && [ {{$.inputs.parameters[''repo_pr'']}} -gt 0 ]; then git fetch origin - pull/{{$.inputs.parameters[''repo_pr'']}}/head:{{$.inputs.parameters[''repo_pr'']}} - && git checkout {{$.inputs.parameters[''repo_pr'']}}; fi ' + - "\n # Increase logging verbosity\n set -x &&\n\n \ + \ # Add TLS Parameters if CA Cert exists and is non-zero size\n\ + \ ADDITIONAL_CLONE_PARAMS=\"\"\n if [ -s \"$TAXONOMY_CA_CERT_PATH\"\ + \ ]; then\n ADDITIONAL_CLONE_PARAMS=\"-c http.sslVerify=true\ + \ -c http.sslCAInfo=$TAXONOMY_CA_CERT_PATH\"\n fi\n\n \ + \ # Clone Taxonomy Repo\n git clone $ADDITIONAL_CLONE_PARAMS\ + \ {{$.inputs.parameters['repo_url']}} {{$.inputs.parameters['taxonomy_path']}}\ + \ &&\n cd {{$.inputs.parameters['taxonomy_path']}} &&\n\n \ + \ # Run additional configuration if TLS certs provided\n \ + \ if [ -s \"$TAXONOMY_CA_CERT_PATH\" ]; then\n git config\ + \ http.sslVerify true &&\n git config http.sslCAInfo $TAXONOMY_CA_CERT_PATH\n\ + \ fi &&\n\n # Checkout and use taxonomy repo branch\ + \ or PR if specified\n if [ -n \"{{$.inputs.parameters['repo_branch']}}\"\ + \ ]; then\n git fetch origin {{$.inputs.parameters['repo_branch']}}\ + \ && git checkout {{$.inputs.parameters['repo_branch']}};\n elif\ + \ [ -n \"{{$.inputs.parameters['repo_pr']}}\" ] && [ {{$.inputs.parameters['repo_pr']}}\ + \ -gt 0 ]; then\n git fetch origin pull/{{$.inputs.parameters['repo_pr']}}/head:{{$.inputs.parameters['repo_pr']}}\ + \ && git checkout {{$.inputs.parameters['repo_pr']}}; \n fi\n\ + \ " command: - /bin/sh - -c + env: + - name: TAXONOMY_CA_CERT_PATH + value: /tmp/cert/taxonomy-ca.crt image: registry.redhat.io/ubi9/toolbox@sha256:da31dee8904a535d12689346e65e5b00d11a6179abf1fa69b548dbd755fa2770 exec-importer: importer: @@ -1669,9 +1681,6 @@ root: - createpvc inputs: parameters: - ca_cert_path: - runtimeValue: - constant: '' repo_branch: componentInputParameter: sdg_repo_branch repo_pr: @@ -2140,6 +2149,10 @@ platforms: outputParameterKey: name producerTask: createpvc exec-git-clone-op: + configMapAsVolume: + - configMapName: teacher-server + mountPath: /tmp/cert + optional: false pvcMount: - mountPath: /data taskOutputParameter: diff --git a/sdg/components.py b/sdg/components.py index 658ed7b..505975d 100644 --- a/sdg/components.py +++ b/sdg/components.py @@ -13,27 +13,38 @@ def git_clone_op( repo_pr: Optional[int], repo_url: Optional[str], taxonomy_path: str = "/data/taxonomy", - ca_cert_path: Optional[str] = None, ): - import os - - additional_clone_params = "" - additional_config_cmds = "" - - if ca_cert_path and os.path.exists(f"{ca_cert_path}") and (os.path.getsize(f"{ca_cert_path}") > 0): - full_ca_path = os.path.abspath(f"{ca_cert_path}") - additional_clone_params = f"-c http.sslVerify=true -c http.sslCAInfo={full_ca_path}" - additional_config_cmds = f"git config http.sslVerify true && git config http.sslCAInfo {full_ca_path} &&" - return dsl.ContainerSpec( TOOLBOX_IMAGE, ["/bin/sh", "-c"], [ - f"git clone {additional_clone_params} {repo_url} {taxonomy_path} && cd {taxonomy_path} && {additional_config_cmds}" - + f'if [ -n "{repo_branch}" ]; then ' - + f"git fetch origin {repo_branch} && git checkout {repo_branch}; " - + f'elif [ -n "{repo_pr}" ] && [ {repo_pr} -gt 0 ]; then ' - + f"git fetch origin pull/{repo_pr}/head:{repo_pr} && git checkout {repo_pr}; fi " + f''' + # Increase logging verbosity + set -x && + + # Add TLS Parameters if CA Cert exists and is non-zero size + ADDITIONAL_CLONE_PARAMS="" + if [ -s "$TAXONOMY_CA_CERT_PATH" ]; then + ADDITIONAL_CLONE_PARAMS="-c http.sslVerify=true -c http.sslCAInfo=$TAXONOMY_CA_CERT_PATH" + fi + + # Clone Taxonomy Repo + git clone $ADDITIONAL_CLONE_PARAMS {repo_url} {taxonomy_path} && + cd {taxonomy_path} && + + # Run additional configuration if TLS certs provided + if [ -s "$TAXONOMY_CA_CERT_PATH" ]; then + git config http.sslVerify true && + git config http.sslCAInfo $TAXONOMY_CA_CERT_PATH + fi && + + # Checkout and use taxonomy repo branch or PR if specified + if [ -n "{repo_branch}" ]; then + git fetch origin {repo_branch} && git checkout {repo_branch}; + elif [ -n "{repo_pr}" ] && [ {repo_pr} -gt 0 ]; then + git fetch origin pull/{repo_pr}/head:{repo_pr} && git checkout {repo_pr}; + fi + ''' ], )