Skip to content

Commit

Permalink
Refactor README and Makefile
Browse files Browse the repository at this point in the history
Signed-off-by: Andrey Velichkevich <[email protected]>
  • Loading branch information
andreyvelich committed Jan 16, 2025
1 parent e7414ba commit 04c0a95
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 204 deletions.
137 changes: 50 additions & 87 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
# Image URL to use all building/pushing image targets
IMG ?= kubeflow/training-operator:latest
# CRD generation options
CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true,maxDescLen=400"

# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
Expand All @@ -16,8 +11,6 @@ endif
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec

all: build

##@ General

# The help target prints out all targets with their descriptions organized
Expand All @@ -36,108 +29,78 @@ help: ## Display this help.

##@ Development

PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))


## Tool Binaries
LOCALBIN ?= $(PROJECT_DIR)/bin
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
ENVTEST ?= $(LOCALBIN)/setup-envtest

ENVTEST_K8S_VERSION ?= 1.31

.PHONY: envtest
envtest: ## Download the setup-envtest binary if required.
test -s $(ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/[email protected]

.PHONY: controller-gen
controller-gen: ## Download the controller-gen binary if required.
test -s $(CONTROLLER_GEN) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/[email protected]

# Download external CRDs for the integration testings.
EXTERNAL_CRDS_DIR ?= $(PROJECT_DIR)/manifests/external-crds

JOBSET_ROOT = $(shell go list -m -mod=readonly -f "{{.Dir}}" sigs.k8s.io/jobset)
.PHONY: jobset-operator-crd
jobset-operator-crd: ## Copy the CRDs from the JobSet repository to the manifests/external-crds directory.
mkdir -p $(EXTERNAL_CRDS_DIR)/jobset-operator/
cp -f $(JOBSET_ROOT)/config/components/crd/bases/* $(EXTERNAL_CRDS_DIR)/jobset-operator/

SCHEDULER_PLUGINS_ROOT = $(shell go list -m -f "{{.Dir}}" sigs.k8s.io/scheduler-plugins)
.PHONY: scheduler-plugins-crd
scheduler-plugins-crd: ## Copy the CRDs from the Scheduler Plugins repository to the manifests/external-crds directory.
mkdir -p $(EXTERNAL_CRDS_DIR)/scheduler-plugins/
cp -f $(SCHEDULER_PLUGINS_ROOT)/manifests/coscheduling/* $(EXTERNAL_CRDS_DIR)/scheduler-plugins

.PHONY: manifests
manifests: controller-gen ## Generate manifests.
$(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" rbac:roleName=training-operator-v2 webhook \
paths="./pkg/apis/kubeflow.org/v2alpha1/...;./pkg/controller.v2/...;./pkg/runtime.v2/...;./pkg/webhooks.v2/...;./pkg/cert/..." \
output:crd:artifacts:config=manifests/v2/base/crds \
output:rbac:artifacts:config=manifests/v2/base/rbac \
output:webhook:artifacts:config=manifests/v2/base/webhook

.PHONY: generate
generate: go-mod-download manifests ## Generate APIs and SDK.
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.go.txt" paths="./pkg/apis/..."
hack/update-codegen.sh
hack/python-sdk-v2/gen-sdk.sh

fmt: ## Run go fmt against code.
.PHONY: go-mod-download
go-mod-download: ## Run go mod download to download modules.
go mod download

.PHONY: fmt
fmt: ## Run go fmt against the code.
go fmt ./...

vet: ## Run go vet against code.
.PHONY: vet
vet: ## Run go vet against the code.
go vet ./...

GOLANGCI_LINT=$(shell which golangci-lint)
golangci-lint:
.PHONY: golangci-lint
golangci-lint: ## Run golangci-lint to verify Go files.
ifeq ($(GOLANGCI_LINT),)
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.61.0
$(info golangci-lint has been installed)
endif
golangci-lint run --timeout 5m --go 1.23 ./...

ENVTEST_K8S_VERSION ?= 1.31
HAS_SETUP_ENVTEST := $(shell command -v setup-envtest;)

testall: manifests generate fmt vet golangci-lint test ## Run tests.

test: envtest
KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" \
go test ./pkg/apis/kubeflow.org/v1/... ./pkg/cert/... ./pkg/common/... ./pkg/config/... ./pkg/controller.v1/... ./pkg/core/... ./pkg/util/... ./pkg/webhooks/... -coverprofile cover.out

.PHONY: test-integrationv2
test-integrationv2: envtest jobset-operator-crd scheduler-plugins-crd
KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/... -coverprofile cover.out

.PHONY: testv2
testv2:
.PHONY: test
test: ## Run Go unit test.
go test ./pkg/apis/kubeflow.org/v2alpha1/... ./pkg/controller.v2/... ./pkg/runtime.v2/... ./pkg/webhooks.v2/... ./pkg/util.v2/... -coverprofile cover.out

envtest:
ifndef HAS_SETUP_ENVTEST
go install sigs.k8s.io/controller-runtime/tools/[email protected]
@echo "setup-envtest has been installed"
endif
@echo "setup-envtest has already installed"

build: generate fmt vet ## Build manager binary.
go build -o bin/manager cmd/training-operator.v1/main.go

run: manifests generate fmt vet ## Run a controller from your host.
go run ./cmd/training-operator.v1/main.go

docker-build: test ## Build docker image with the manager.
docker build -t ${IMG} -f build/images/training-operator/Dockerfile .

docker-push: ## Push docker image with the manager.
docker push ${IMG}

##@ Deployment

install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build manifests/base/crds | kubectl apply --server-side -f -

uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build manifests/base/crds | kubectl delete -f -

deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
cd manifests/overlays/standalone && $(KUSTOMIZE) edit set image kubeflow/training-operator=${IMG}
$(KUSTOMIZE) build manifests/overlays/standalone | kubectl apply -f -

undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build manifests/overlays/standalone | kubectl delete -f -

PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))

.PHONY: go-mod-download
go-mod-download:
go mod download

CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
controller-gen: ## Download controller-gen locally if necessary.
GOBIN=$(PROJECT_DIR)/bin go install sigs.k8s.io/controller-tools/cmd/[email protected]

KUSTOMIZE = $(shell pwd)/bin/kustomize
kustomize: ## Download kustomize locally if necessary.
GOBIN=$(PROJECT_DIR)/bin go install sigs.k8s.io/kustomize/kustomize/[email protected]

## Download external CRDs for the integration testings.
EXTERNAL_CRDS_DIR ?= $(PROJECT_DIR)/manifests/external-crds

JOBSET_ROOT = $(shell go list -m -mod=readonly -f "{{.Dir}}" sigs.k8s.io/jobset)
.PHONY: jobset-operator-crd
jobset-operator-crd: ## Copy the CRDs from the jobset-operator to the manifests/external-crds directory.
mkdir -p $(EXTERNAL_CRDS_DIR)/jobset-operator/
cp -f $(JOBSET_ROOT)/config/components/crd/bases/* $(EXTERNAL_CRDS_DIR)/jobset-operator/

SCHEDULER_PLUGINS_ROOT = $(shell go list -m -f "{{.Dir}}" sigs.k8s.io/scheduler-plugins)
.PHONY: scheduler-plugins-crd
scheduler-plugins-crd:
mkdir -p $(EXTERNAL_CRDS_DIR)/scheduler-plugins/
cp -f $(SCHEDULER_PLUGINS_ROOT)/manifests/coscheduling/* $(EXTERNAL_CRDS_DIR)/scheduler-plugins
.PHONY: test-integration
test-integration: envtest jobset-operator-crd scheduler-plugins-crd ## Run Go integration test.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/... -coverprofile cover.out
100 changes: 25 additions & 75 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,112 +1,62 @@
# Kubeflow Training Operator
# Kubeflow Trainer

[![Build Status](https://github.com/kubeflow/training-operator/actions/workflows/test-go.yaml/badge.svg?branch=master)](https://github.com/kubeflow/training-operator/actions/workflows/test-go.yaml?branch=master)
[![Coverage Status](https://coveralls.io/repos/github/kubeflow/training-operator/badge.svg?branch=master)](https://coveralls.io/github/kubeflow/training-operator?branch=master)
[![Go Report Card](https://goreportcard.com/badge/github.com/kubeflow/training-operator)](https://goreportcard.com/report/github.com/kubeflow/training-operator)

## Overview

Kubeflow Training Operator is a Kubernetes-native project for fine-tuning and
scalable distributed training of machine learning (ML) models created with various ML frameworks
such as PyTorch, TensorFlow, HuggingFace, [JAX](https://jax.readthedocs.io/en/latest/), DeepSpeed, XGBoost, PaddlePaddle and others.

You can run high-performance computing (HPC) tasks with the Training Operator and `MPIJob` since it
supports running Message Passing Interface (MPI) on Kubernetes which is heavily used for HPC.
The Training Operator implements the V1 API version of MPI Operator. For the MPI Operator V2 version,
please follow [this guide](https://www.kubeflow.org/docs/components/training/user-guides/mpi/) to
install MPI Operator V2.

The Training Operator allows you to use Kubernetes workloads to effectively train your large models
via [Kubernetes Custom Resources APIs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)
or using the Training Operator Python SDK.

## Prerequisites

Please check [the official Kubeflow documentation](https://www.kubeflow.org/docs/components/training/installation/#prerequisites)
for prerequisites to install the Training Operator.

## Installation

Please follow [the Kubeflow Training Operator guide](https://www.kubeflow.org/docs/components/training/installation/#installing-the-training-operator)
for the detailed instructions on how to install Training Operator.

### Installing the Control Plane
TODO (andreyvelich): Add logo

Run the following command to install the latest stable release of the Training Operator control plane: `v1.8.0`.
TODO (andreyvelich): Add diagram

```bash
kubectl apply --server-side -k "github.com/kubeflow/training-operator.git/manifests/overlays/standalone?ref=v1.8.0"
```
## Overview

Run the following command to install the latest changes of the Training Operator control plane:
Kubeflow Trainer is a Kubernetes-native project designed for large language models (LLMs)
fine-tuning and enabling scalable, distributed training of machine learning (ML) models across
various frameworks, including PyTorch, JAX, TensorFlow, and others.

```bash
kubectl apply --server-side -k "github.com/kubeflow/training-operator/manifests/overlays/standalone"
```
You can integrate other ML libraries such as [HuggingFace](https://huggingface.co),
[DeepSpeed](https://github.com/microsoft/DeepSpeed), or [Megatron-LM](https://github.com/NVIDIA/Megatron-LM)
with Kubeflow Training to orchestrate their ML training on Kubernetes.

### Installing the Python SDK
Kubeflow Trainer allows you effortlessly develop your LLMs with the Kubeflow Python SDK and
build Kubernetes-native Training Runtimes with Kubernetes Custom Resources APIs.

The Training Operator [implements a Python SDK](https://pypi.org/project/kubeflow-training/)
to simplify creation of distributed training and fine-tuning jobs for Data Scientists.
## Kubeflow Trainer Introduction

Run the following command to install the latest stable release of the Training SDK:
The following KubeCon + CloudNativeCon 2024 talk provides an overview of Kubeflow Trainer capabilities:

```
pip install -U kubeflow-training
```
[![Kubeflow Trainer](https://img.youtube.com/vi/Lgy4ir1AhYw/0.jpg)](https://www.youtube.com/watch?v=Lgy4ir1AhYw)

## Getting Started

Please refer to [the getting started guide](https://www.kubeflow.org/docs/components/training/getting-started/#getting-started-with-pytorchjob)
to quickly create your first distributed training job using the Python SDK.

If you want to work directly with Kubernetes Custom Resources provided by Training Operator,
follow [the PyTorchJob MNIST guide](https://www.kubeflow.org/docs/components/training/pytorch/#creating-a-pytorch-training-job).
Please check [the official Kubeflow documentation](https://www.kubeflow.org/docs/components/training/getting-started)
to install and get started with Kubeflow Trainer.

## Community

The following links provide information on how to get involved in the community:

- Attend [the bi-weekly AutoML and Training Working Group](https://bit.ly/2PWVCkV) community meeting.
- Join our [`#kubeflow-training` Slack channel](https://www.kubeflow.org/docs/about/community/#kubeflow-slack).
- Attend [the bi-weekly AutoML and Training Working Group](https://bit.ly/2PWVCkV) community meeting.
- Check out [who is using the Training Operator](ADOPTERS.md).

This is a part of Kubeflow, so please see [readme in kubeflow/kubeflow](https://github.com/kubeflow/kubeflow#get-involved) to get in touch with the community.

## Contributing

Please refer to the [CONTRIBUTING guide](CONTRIBUTING.md).

## Change Log
## Changelog

Please refer to the [CHANGELOG](CHANGELOG.md).

## Version Matrix

The following table lists the most recent few versions of the operator.

| Operator Version | API Version | Kubernetes Version |
| ---------------------- | ----------- | ------------------ |
| `v1.4.x` | `v1` | 1.23+ |
| `v1.5.x` | `v1` | 1.23+ |
| `v1.6.x` | `v1` | 1.23+ |
| `v1.7.x` | `v1` | 1.25+ |
| `v1.8.x` | `v1` | 1.27+ |
| `latest` (master HEAD) | `v1` | 1.27+ |

## Reference
## Kubeflow Training Operator V1

For a complete reference of the custom resource definitions, please refer to the API Definition.
Kubeflow Trainer project is currently in <strong>alpha</strong> status, and APIs may change.
If you are using Kubeflow Training Operator V1, please refer [to this migration document](/docs/components/training/operator-guides/migration).

- [TensorFlow API Definition](pkg/apis/kubeflow.org/v1/tensorflow_types.go)
- [PyTorch API Definition](pkg/apis/kubeflow.org/v1/pytorch_types.go)
- [XGBoost API Definition](pkg/apis/kubeflow.org/v1/xgboost_types.go)
- [MPI API Definition](pkg/apis/kubeflow.org/v1/mpi_types.go)
- [PaddlePaddle API Definition](pkg/apis/kubeflow.org/v1/paddlepaddle_types.go)
- [JAX API Definition](pkg/apis/kubeflow.org/v1/jax_types.go)
Kubeflow Community will maintain the Training Operator V1 source code at
[the `release-1.9` branch](https://github.com/kubeflow/training-operator/tree/release-1.9).

For details on the Training Operator custom resources APIs, refer to
[the following API documentation](docs/api/kubeflow.org_v1_generated.asciidoc)
You can find the documentation for Kubeflow Training V1 in [these guides](https://www.kubeflow.org/docs/components/training/legacy-v1).

## Acknowledgement

Expand Down
1 change: 1 addition & 0 deletions generate
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
test
8 changes: 2 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@ require (
github.com/onsi/ginkgo/v2 v2.20.1
github.com/onsi/gomega v1.35.1
github.com/open-policy-agent/cert-controller v0.12.0
github.com/prometheus/client_golang v1.20.2
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
go.uber.org/zap v1.27.0
k8s.io/api v0.31.3
k8s.io/apimachinery v0.31.3
Expand All @@ -24,8 +21,6 @@ require (
sigs.k8s.io/kueue v0.6.3
sigs.k8s.io/scheduler-plugins v0.28.9
sigs.k8s.io/structured-merge-diff/v4 v4.4.1
sigs.k8s.io/yaml v1.4.0
volcano.sh/apis v1.9.0
)

require (
Expand Down Expand Up @@ -58,7 +53,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.20.2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
Expand All @@ -85,4 +80,5 @@ require (
k8s.io/apiextensions-apiserver v0.31.2 // indirect
k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
7 changes: 0 additions & 7 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -94,13 +94,10 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
Expand Down Expand Up @@ -140,7 +137,6 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
Expand Down Expand Up @@ -175,7 +171,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8=
Expand Down Expand Up @@ -212,5 +207,3 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
volcano.sh/apis v1.9.0 h1:e+9yEbQOi6HvgaayAxYULT6n+59mkYvmqjKhp9Z06sY=
volcano.sh/apis v1.9.0/go.mod h1:yXNfsZRzAOq6EUyPJYFrlMorh1XsYQGonGWyr4IiznM=
Loading

0 comments on commit 04c0a95

Please sign in to comment.