.cloudbuild.yaml

# Copyright 2018 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Test before submit:
# gcloud builds submit --config=.cloudbuild.yaml --substitutions=COMMIT_SHA="$(git rev-parse HEAD)" --project=ml-pipeline-test

steps:

# Build the Python SDK
- name: 'python:3-alpine'
  entrypoint: '/bin/sh'
  args: ['-c', 'cd /workspace/sdk/python/; python3 setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp.tar.gz']
  id:   'preparePythonSDK'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz']
  id:   'copyPythonSDK'
  waitFor: ['preparePythonSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz']
  id:   'copyPythonSDKToLatest'
  waitFor: ['preparePythonSDK']

# Build the Python Component SDK
- name: 'python:2-alpine'
  entrypoint: '/bin/sh'
  args: ['-c', 'cd /workspace/components/gcp/container/component_sdk/python;python setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp-component.tar.gz']
  id:   'preparePythonComponentSDK'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz']
  id:   'copyPythonComponentSDK'
  waitFor: ['preparePythonComponentSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp-component.tar.gz']
  id:   'copyPythonComponentSDKToLatest'
  waitFor: ['preparePythonComponentSDK']

# Build the pipeline system images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: /bin/bash
  args:
    - -ceux
    - |
      sed -i -e "s/ARG DATE/ENV DATE \"$(date -u)\"/" /workspace/frontend/Dockerfile
      docker build -t gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA \
        --build-arg COMMIT_HASH=$COMMIT_SHA \
        --build-arg TAG_NAME="$(cat /workspace/VERSION)" \
        -f /workspace/frontend/Dockerfile \
        /workspace
  id:   'buildFrontend'
  waitFor: ['copyPythonSDK']
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: /bin/bash
  args:
    - -ceux
    - |
      docker build -t gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA \
        --build-arg COMMIT_SHA=$COMMIT_SHA \
        --build-arg TAG_NAME="$(cat /workspace/VERSION)" \
        -f /workspace/backend/Dockerfile /workspace
  id:   'buildApiServer'
  waitFor: ['-']

- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.scheduledworkflow', '/workspace']
  id:   'buildScheduledWorkflow'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.viewercontroller', '/workspace']
  id:   'buildViewerCrdController'
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.persistenceagent', '/workspace']
  id:   'buildPersistenceAgent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', '-f',
         '/workspace/proxy/Dockerfile', '/workspace/proxy']
  id:   'buildInverseProxyAgent'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.visualization', '/workspace']
  id: 'buildVisualizationServer'
  waitFor: ["-"]
- id: 'buildMetadataWriter'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/backend/metadata_writer/Dockerfile', '/workspace']
  waitFor: ["-"]
- id: 'buildCacheServer'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA',
          '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
          '/workspace/backend/Dockerfile.cacheserver', '/workspace']
  waitFor: ["-"]
- id: 'buildCacheDeployer'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA',
          '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
          '/workspace/backend/src/cache/deployer/Dockerfile', '/workspace']
  waitFor: ["-"]

# Build marketplace deployer
- id: 'buildMarketplaceDeployer'
  name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/manifests/gcp_marketplace/deployer/Dockerfile', '/workspace/manifests/gcp_marketplace']
  waitFor: ["-"]

# Build the Kubeflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA',
         '/workspace/components/kubeflow/deployer']
  id: 'buildDeployer'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/launcher && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTFJobLauncher'
  waitFor: ["-"]
- id: 'buildCpuTrainer'
  name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer -b 2.3.0']
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer-gpu -b 2.3.0-gpu']
  id: 'buildGpuTrainer'
  waitFor: ["-"]

# Build the Generic GCP component image
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/gcp/container/ && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildGcpGenericComponent'
  waitFor: ["-"]

# Build the local pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildConfusionMatrix'
  waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildROC'
  waitFor: ["-"]

# Build third_party images
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', '-f',
         '/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
  id:   'buildMetadataEnvoy'

# Pull third_party images
# ! Sync to the same MLMD version:
# * backend/metadata_writer/requirements.in and requirements.txt
# * @kubeflow/frontend/src/mlmd/generated
# * .cloudbuild.yaml and .release.cloudbuild.yaml
# * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
# * test/tag_for_hosted.sh
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0']
  id:   'pullMetadataServer'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
  id:   'pullMinio'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/mysql:5.7-debian']
  id:   'pullMysql'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0']
  id:   'pullCloudsqlProxy'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.3.8-license-compliance']
  id:   'pullArgoExecutor'
- name: 'gcr.io/cloud-builders/docker'
  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.3.8-license-compliance']
  id:   'pullArgoWorkflowController'

# Tag for Hosted - SemVersion to Major.Minor parsing
- id: "parseMajorMinorVersion"
  waitFor: ["-"]
  name: gcr.io/cloud-builders/docker
  entrypoint: /bin/bash
  args:
    - -ceux
    - |
      # Parse major minor version and save to a file for reusing in other steps.
      # e.g. 1.0.0-rc.1 and 1.0.1 are parsed as 1.0
      cat /workspace/VERSION | sed -e "s#\([0-9]\+[.][0-9]\+\)[.].*#\1#" > /workspace/mm.ver

# Tag for Hosted - Tag to hosted folder with MKP friendly name
- id:   'tagForHosted'
  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',
            'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
            'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
            'buildMarketplaceDeployer', 'pullMetadataServer', 'pullMinio', 'pullMysql', 'pullCloudsqlProxy',
            'pullArgoExecutor', 'pullArgoWorkflowController']
  name: 'gcr.io/cloud-builders/docker'
  entrypoint: bash
  args:
  - -ceux
  - |
    /workspace/test/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)

images:
# Images for the pipeline system itself
- 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA'

# Images for Marketplace
- 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA'

# Images for the Kubeflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tfjob:$COMMIT_SHA'

# Images for the local components
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'

# Images for the third_party components
- 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA'

timeout: '3600s'
options:
 diskSizeGb: 300
 machineType: 'N1_HIGHCPU_8'
tags:
- build-each-commit