From 867d34f7974d42b73cd340f55a49dad78753c4c1 Mon Sep 17 00:00:00 2001
From: Paco Xu <paco.xu@daocloud.io>
Date: Tue, 24 May 2022 21:22:50 +0800
Subject: [PATCH] initial all commands

---
 .vscode/configurationCache.log                |   1 -
 .vscode/dryrun.log                            |  10 -
 .vscode/settings.json                         |   3 -
 .vscode/targets.log                           | 365 ------------------
 Makefile                                      |   2 +-
 api/v1alpha1/command_descriptor_types.go      |  29 +-
 api/v1alpha1/zz_generated.deepcopy.go         |  21 +-
 commands/kubeadm_renew_certificate.go         |  19 +
 commands/kubeadm_upgrade_node.go              |  21 +-
 commands/kubectl_drain.go                     |  16 +-
 commands/kubectl_uncordon.go                  |  16 +-
 commands/preflight.go                         |   3 +-
 commands/upgrade_kubeadm.go                   |  33 +-
 commands/upgrade_kubectlkubelet.go            |  36 +-
 .../operator.kubeadm.x-k8s.io_operations.yaml |  43 +++
 ...or.kubeadm.x-k8s.io_runtimetaskgroups.yaml |  37 ++
 ...perator.kubeadm.x-k8s.io_runtimetasks.yaml |  35 ++
 config/manager/kustomization.yaml             |   2 +-
 config/manager/manager.yaml                   |   2 +-
 19 files changed, 300 insertions(+), 394 deletions(-)
 delete mode 100644 .vscode/configurationCache.log
 delete mode 100644 .vscode/dryrun.log
 delete mode 100644 .vscode/settings.json
 delete mode 100644 .vscode/targets.log

diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log
deleted file mode 100644
index 0ee3133..0000000
--- a/.vscode/configurationCache.log
+++ /dev/null
@@ -1 +0,0 @@
-{"buildTargets":["all","controller-gen","deploy","docker-build","docker-push","fmt","generate","install","manager","manifests","release","run","test","undeploy","vet"],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}}
\ No newline at end of file
diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log
deleted file mode 100644
index 7805d68..0000000
--- a/.vscode/dryrun.log
+++ /dev/null
@@ -1,10 +0,0 @@
-make --dry-run --always-make --keep-going --print-directory
-make: Entering directory `/Users/pacoxu/git/gopath/src/k8s.io/kubeadm-operator'
-go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0
- 
-/Users/pacoxu/go/bin/controller-gen object:headerFile=./hack/boilerplate.go.txt paths="./..."
-go fmt ./...
-go vet ./...
-go build -o bin/manager main.go
-make: Leaving directory `/Users/pacoxu/git/gopath/src/k8s.io/kubeadm-operator'
- 
diff --git a/.vscode/settings.json b/.vscode/settings.json
deleted file mode 100644
index 65e1ec0..0000000
--- a/.vscode/settings.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "makefile.extensionOutputFolder": "./.vscode"
-}
\ No newline at end of file
diff --git a/.vscode/targets.log b/.vscode/targets.log
deleted file mode 100644
index bad887a..0000000
--- a/.vscode/targets.log
+++ /dev/null
@@ -1,365 +0,0 @@
-make all --print-data-base --no-builtin-variables --no-builtin-rules --question
-# GNU Make 3.81
-# Copyright (C) 2006  Free Software Foundation, Inc.
-# This is free software; see the source for copying conditions.
-# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE.
-
-# This program built for i386-apple-darwin11.3.0
- 
-
-# Make data base, printed on Thu May 19 22:39:00 2022
-
-# Variables
-
-# automatic
-<D = $(patsubst %/,%,$(dir $<))
-# automatic
-?F = $(notdir $?)
-# environment
-VSCODE_LOG_NATIVE = false
-# automatic
-?D = $(patsubst %/,%,$(dir $?))
-# automatic
-@D = $(patsubst %/,%,$(dir $@))
-# automatic
-@F = $(notdir $@)
-# makefile
-CURDIR := /Users/pacoxu/git/gopath/src/k8s.io/kubeadm-operator
-# makefile
-SHELL = /bin/sh
-# environment
-VSCODE_NLS_CONFIG = {"locale":"zh-cn","availableLanguages":{"*":"zh-cn"},"_languagePackId":"42e84e0914b17800d483f69e299e1b00.zh-cn","_translationsConfigFile":"/Users/pacoxu/Library/Application Support/Code/clp/42e84e0914b17800d483f69e299e1b00.zh-cn/tcf.json","_cacheRoot":"/Users/pacoxu/Library/Application Support/Code/clp/42e84e0914b17800d483f69e299e1b00.zh-cn","_resolvedLanguagePackCoreLocation":"/Users/pacoxu/Library/Application Support/Code/clp/42e84e0914b17800d483f69e299e1b00.zh-cn/c3511e6c69bb39013c4a4b7b9566ec1ca73fc4d5","_corruptedFile":"/Users/pacoxu/Library/Application Support/Code/clp/42e84e0914b17800d483f69e299e1b00.zh-cn/corrupted.info","_languagePackSupport":true}
-# environment
-_ = /usr/bin/make
-# makefile (from `Makefile', line 1)
-MAKEFILE_LIST :=  Makefile
-# environment
-VSCODE_AMD_ENTRYPOINT = vs/workbench/api/node/extensionHostProcess
-# environment
-VSCODE_VERBOSE_LOGGING = true
-# environment
-__CFBundleIdentifier = com.microsoft.VSCode
-# environment
-VSCODE_IPC_HOOK_EXTHOST = /var/folders/g7/ywncky4109zfc_6v5ww1fc2w0000gn/T/vscode-ipc-01d6ffe0-54e6-46b8-9010-0d5b07226452.sock
-# environment
-VSCODE_CWD = /
-# environment
-GOPROXY = https://proxy.golang.org,direct
-# environment
-PATH = /usr/local/opt/go@1.18/bin:/Users/pacoxu/git/gopath/src/k8s.io/kubernetes/third_party/etcd:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin
-# environment
-LSCOLORS = Gxfxcxdxbxegedabagacad
-# environment
-GOPATH = /Users/pacoxu/go
-# environment
-VSCODE_LOG_STACK = false
-# environment
-ELECTRON_RUN_AS_NODE = 1
-# default
-.FEATURES := target-specific order-only second-expansion else-if archives jobserver check-symlink
-# makefile (from `Makefile', line 72)
-CONTROLLER_GEN = $(GOBIN)/controller-gen
-# environment
-SSH_AUTH_SOCK = /private/tmp/com.apple.launchd.FlDNrnFngH/Listeners
-# automatic
-%F = $(notdir $%)
-# makefile (from `Makefile', line 9)
-GOBIN = $(shell go env GOPATH)/bin
-# environment
-VSCODE_PIPE_LOGGING = true
-# environment
-PWD = /Users/pacoxu/git/gopath/src/k8s.io/kubeadm-operator
-# environment
-ORIGINAL_XDG_CURRENT_DESKTOP = undefined
-# environment
-GOMODCACHE = /Users/pacoxu/go/pkg/mod
-# environment
-HOME = /Users/pacoxu
-# default
-MAKEFILEPATH = $(shell /usr/bin/xcode-select -print-path 2>/dev/null || echo /Developer)/Makefiles
-# environment
-VSCODE_CODE_CACHE_PATH = /Users/pacoxu/Library/Application Support/Code/CachedData/c3511e6c69bb39013c4a4b7b9566ec1ca73fc4d5
-# environment
-LOGNAME = pacoxu
-# environment
-APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true
-# environment
-ZSH = /Users/pacoxu/.oh-my-zsh
-# environment
-VSCODE_HANDLES_UNCAUGHT_ERRORS = true
-# automatic
-^D = $(patsubst %/,%,$(dir $^))
-# environment
-XPC_FLAGS = 0x0
-# makefile (from `Makefile', line 5)
-CRD_OPTIONS = "crd"
-# default
-MAKE = $(MAKE_COMMAND)
-# default
-MAKECMDGOALS := all
-# environment
-SHLVL = 1
-# default
-MAKE_VERSION := 3.81
-# environment
-USER = pacoxu
-# makefile
-.DEFAULT_GOAL := all
-# environment
-LESS = -R
-# automatic
-%D = $(patsubst %/,%,$(dir $%))
-# default
-MAKE_COMMAND := /Library/Developer/CommandLineTools/usr/bin/make
-# default
-.VARIABLES := 
-# environment
-TMPDIR = /var/folders/g7/ywncky4109zfc_6v5ww1fc2w0000gn/T/
-# automatic
-*F = $(notdir $*)
-# makefile (from `Makefile', line 3)
-IMG = daocloud.io/daocloud/kubeadm-operator:v0.0.3-dev
-# environment
-VSCODE_IPC_HOOK = /Users/pacoxu/Library/Application Support/Code/1.67.2-main.sock
-# makefile
-MAKEFLAGS = Rrqp
-# environment
-MFLAGS = -Rrqp
-# automatic
-*D = $(patsubst %/,%,$(dir $*))
-# environment
-XPC_SERVICE_NAME = application.com.microsoft.VSCode.58980345.58980351
-# automatic
-+D = $(patsubst %/,%,$(dir $+))
-# automatic
-+F = $(notdir $+)
-# environment
-__CF_USER_TEXT_ENCODING = 0x1F5:0x19:0x34
-# environment
-COMMAND_MODE = unix2003
-# environment
-GITHUB_USER = pacoxu
-# default
-MAKEFILES := 
-# automatic
-<F = $(notdir $<)
-# environment
-PAGER = less
-# environment
-LC_ALL = C
-# automatic
-^F = $(notdir $^)
-# default
-SUFFIXES := 
-# default
-.INCLUDE_DIRS = /usr/local/include
-# environment
-MAKELEVEL := 0
-# environment
-LANG = C
-# environment
-VSCODE_PID = 4957
-# variable set hash-table stats:
-# Load=74/1024=7%, Rehash=0, Collisions=2/106=2%
-
-# Pattern-specific Variable Values
-
-# No pattern-specific variable values.
-
-# Directories
-
-# . (device 16777221, inode 39096290): 22 files, no impossibilities.
-
-# 22 files, no impossibilities in 1 directories.
-
-# Implicit Rules
-
-# No implicit rules.
-
-# Files
-
-all: manager
-#  Command-line target.
-#  Implicit rule search has been done.
-#  File does not exist.
-#  File has been updated.
-#  Needs to be updated (-q is set).
-# variable set hash-table stats:
-# Load=0/32=0%, Rehash=0, Collisions=0/2=0%
-
-docker-build: test
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 59):
-	docker build . -t ${IMG}
-	
-
-# Not a target:
-.SUFFIXES:
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-
-manager: generate fmt vet
-#  Implicit rule search has not been done.
-#  File does not exist.
-#  File has been updated.
-#  Needs to be updated (-q is set).
-# variable set hash-table stats:
-# Load=0/32=0%, Rehash=0, Collisions=0/2=0%
-#  commands to execute (from `Makefile', line 22):
-	go build -o bin/manager main.go
-	
-
-# Not a target:
-Makefile:
-#  Implicit rule search has been done.
-#  Last modified 2022-05-07 16:51:36
-#  File has been updated.
-#  Successfully updated.
-# variable set hash-table stats:
-# Load=0/32=0%, Rehash=0, Collisions=0/0=0%
-
-test: generate fmt vet manifests
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 18):
-	go test ./... -coverprofile cover.out
-	
-
-vet:
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 51):
-	go vet ./...
-	
-
-generate: controller-gen
- 
-#  Implicit rule search has not been done.
-#  File does not exist.
-#  File has been updated.
-#  Needs to be updated (-q is set).
-# variable set hash-table stats:
-# Load=0/32=0%, Rehash=0, Collisions=0/2=0%
-#  commands to execute (from `Makefile', line 55):
-	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
-	
-
-undeploy: manifests
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 38):
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl delete -f -
-	
-
-manifests: controller-gen
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 43):
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	
-
-release: docker-build docker-push
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-
-# Not a target:
-.DEFAULT:
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-
-run: generate fmt vet manifests
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 26):
-	go run ./main.go
-	
-
-fmt:
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 47):
-	go fmt ./...
-	
-
-deploy: manifests
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 34):
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl apply -f -
-	
-
-controller-gen:
-#  Implicit rule search has not been done.
-#  Implicit/static pattern stem: `'
-#  File does not exist.
-#  File has been updated.
-#  Needs to be updated (-q is set).
-# automatic
-# @ := controller-gen
-# automatic
-# % := 
-# automatic
-# * := 
-# automatic
-# + := 
-# automatic
-# | := 
-# automatic
-# < := 
-# automatic
-# ^ := 
-# automatic
-# ? := 
-# variable set hash-table stats:
-# Load=8/32=25%, Rehash=0, Collisions=1/10=10%
-#  commands to execute (from `Makefile', line 71):
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0
-	
-
-docker-push:
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 63):
-	docker push ${IMG}
-	
-
-install: manifests
-#  Implicit rule search has not been done.
-#  Modification time never checked.
-#  File has not been updated.
-#  commands to execute (from `Makefile', line 30):
-	kustomize build config/crd | kubectl apply -f -
-	
-
-# files hash-table stats:
-# Load=18/1024=2%, Rehash=0, Collisions=0/52=0%
-# VPATH Search Paths
-
-# No `vpath' search paths.
-
-# No general (`VPATH' variable) search path.
-
-# # of strings in strcache: 1
-# # of strcache buffers: 1
-# strcache size: total = 4096 / max = 4096 / min = 4096 / avg = 4096
-# strcache free: total = 4087 / max = 4087 / min = 4087 / avg = 4087
-
-# Finished Make data base on Thu May 19 22:39:00 2022
-
- 
diff --git a/Makefile b/Makefile
index dae37ed..47ba129 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= daocloud.io/daocloud/kubeadm-operator:v0.0.3
+IMG ?= daocloud.io/daocloud/kubeadm-operator:v0.0.4-dev
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd"
 
diff --git a/api/v1alpha1/command_descriptor_types.go b/api/v1alpha1/command_descriptor_types.go
index 027ff0d..e92b304 100644
--- a/api/v1alpha1/command_descriptor_types.go
+++ b/api/v1alpha1/command_descriptor_types.go
@@ -70,6 +70,12 @@ type UpgradeKubeadmCommandSpec struct {
 	// If the version is empty, we will skip this command.
 	KubernetesVersion string `json:"kubernetesVersion"`
 
+	// +optional
+	// Local is by default true.
+	// If true, operator will use the binary in /usr/bin
+	// If not, operator will download the binary from official websites
+	Local bool `json:"local,omitempty"`
+
 	// INSERT ADDITIONAL SPEC FIELDS -
 	// Important: Run "make" to regenerate code after modifying this file
 }
@@ -87,12 +93,18 @@ type KubeadmUpgradeApplyCommandSpec struct {
 // TODO download the specified version bin and replace it in the node
 // KubeadmUpgradeNodeCommandSpec provides...
 type KubeadmUpgradeNodeCommandSpec struct {
-	// INSERT ADDITIONAL SPEC FIELDS -
-	// Important: Run "make" to regenerate code after modifying this file
+	// for dry run mode
+	DryRun bool `json:"dryRun,omitempty"`
 }
 
 // KubectlDrainCommandSpec provides...
 type KubectlDrainCommandSpec struct {
+	// +optional
+	// Nodes is a list of nodes to uncordon
+	Nodes []string `json:"nodes"`
+
+	// TODO support dry-run
+	// TODO support ignore daemonsets
 
 	// INSERT ADDITIONAL SPEC FIELDS -
 	// Important: Run "make" to regenerate code after modifying this file
@@ -100,6 +112,9 @@ type KubectlDrainCommandSpec struct {
 
 // KubectlUncordonCommandSpec provides...
 type KubectlUncordonCommandSpec struct {
+	// +optional
+	// Nodes is a list of nodes to uncordon
+	Nodes []string `json:"nodes"`
 
 	// INSERT ADDITIONAL SPEC FIELDS -
 	// Important: Run "make" to regenerate code after modifying this file
@@ -113,12 +128,22 @@ type UpgradeKubeletAndKubeactlCommandSpec struct {
 	// If the version is empty, we will skip this command.
 	KubernetesVersion string `json:"kubernetesVersion"`
 
+	// +optional
+	// Local is by default true.
+	// If true, operator will use the binary in /usr/bin
+	// If not, operator will download the binary from official websites
+	Local bool `json:"local,omitempty"`
+
 	// INSERT ADDITIONAL SPEC FIELDS -
 	// Important: Run "make" to regenerate code after modifying this file
 }
 
 // KubeadmRenewCertsCommandSpec provides...
 type KubeadmRenewCertsCommandSpec struct {
+	// +optional
+	// Commands is a list of commands to run: all, apiserver, apiserver-etcd-client, apiserver-kubelet-client,
+	// controller-manager.conf, etcd-healthcheck-client, etcd-peer, etcd-server, front-proxy-client, scheduler.conf
+	Commands []string `json:"commands"`
 
 	// INSERT ADDITIONAL SPEC FIELDS -
 	// Important: Run "make" to regenerate code after modifying this file
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 0d5f4c3..2463d35 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -31,7 +31,7 @@ func (in *CommandDescriptor) DeepCopyInto(out *CommandDescriptor) {
 	if in.KubeadmRenewCertificates != nil {
 		in, out := &in.KubeadmRenewCertificates, &out.KubeadmRenewCertificates
 		*out = new(KubeadmRenewCertsCommandSpec)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.KubeadmUpgradeApply != nil {
 		in, out := &in.KubeadmUpgradeApply, &out.KubeadmUpgradeApply
@@ -61,12 +61,12 @@ func (in *CommandDescriptor) DeepCopyInto(out *CommandDescriptor) {
 	if in.KubectlDrain != nil {
 		in, out := &in.KubectlDrain, &out.KubectlDrain
 		*out = new(KubectlDrainCommandSpec)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.KubectlUncordon != nil {
 		in, out := &in.KubectlUncordon, &out.KubectlUncordon
 		*out = new(KubectlUncordonCommandSpec)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Pass != nil {
 		in, out := &in.Pass, &out.Pass
@@ -135,6 +135,11 @@ func (in *FailCommandSpec) DeepCopy() *FailCommandSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeadmRenewCertsCommandSpec) DeepCopyInto(out *KubeadmRenewCertsCommandSpec) {
 	*out = *in
+	if in.Commands != nil {
+		in, out := &in.Commands, &out.Commands
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmRenewCertsCommandSpec.
@@ -180,6 +185,11 @@ func (in *KubeadmUpgradeNodeCommandSpec) DeepCopy() *KubeadmUpgradeNodeCommandSp
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubectlDrainCommandSpec) DeepCopyInto(out *KubectlDrainCommandSpec) {
 	*out = *in
+	if in.Nodes != nil {
+		in, out := &in.Nodes, &out.Nodes
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubectlDrainCommandSpec.
@@ -195,6 +205,11 @@ func (in *KubectlDrainCommandSpec) DeepCopy() *KubectlDrainCommandSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubectlUncordonCommandSpec) DeepCopyInto(out *KubectlUncordonCommandSpec) {
 	*out = *in
+	if in.Nodes != nil {
+		in, out := &in.Nodes, &out.Nodes
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubectlUncordonCommandSpec.
diff --git a/commands/kubeadm_renew_certificate.go b/commands/kubeadm_renew_certificate.go
index 21dc0da..6f5b312 100644
--- a/commands/kubeadm_renew_certificate.go
+++ b/commands/kubeadm_renew_certificate.go
@@ -17,11 +17,30 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 
 	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
 )
 
+// runKubeadmRenewCertificates run the command on control plane nodes
 func runKubeadmRenewCertificates(spec *operatorv1.KubeadmRenewCertsCommandSpec, log logr.Logger) error {
+	if len(spec.Commands) == 0 {
+		spec.Commands = []string{"all"}
+	}
+
+	for _, command := range spec.Commands {
+		cmd := newCmd("kubeadm", "certs", "renew", command)
+
+		lines, err := cmd.RunAndCapture()
+		if err != nil {
+			return errors.WithStack(errors.WithMessage(err, strings.Join(lines, "\n")))
+		}
+
+		log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n")))
+	}
 	return nil
 }
diff --git a/commands/kubeadm_upgrade_node.go b/commands/kubeadm_upgrade_node.go
index 4d9ba11..3f4add8 100644
--- a/commands/kubeadm_upgrade_node.go
+++ b/commands/kubeadm_upgrade_node.go
@@ -17,12 +17,31 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 
 	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
 )
 
-// TODO this is a temporary hack to get the "kubeadm upgrade node" to work
+// "kubeadm upgrade node" on current node
 func runKubeadmUpgradeNode(spec *operatorv1.KubeadmUpgradeNodeCommandSpec, log logr.Logger) error {
+	var cmd *cmd
+
+	// TODO: add real dry run support
+	cmd = newCmd("kubeadm", "upgrade", "node", "--v=4")
+	if spec.DryRun {
+		cmd = newCmd("kubeadm", "upgrade", "node", "--dry-run", "--v=4")
+	}
+
+	lines, err := cmd.RunAndCapture()
+	if err != nil {
+		return errors.WithStack(errors.WithMessage(err, strings.Join(lines, "\n")))
+	}
+
+	log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n")))
+
 	return nil
 }
diff --git a/commands/kubectl_drain.go b/commands/kubectl_drain.go
index 4905de7..970466c 100644
--- a/commands/kubectl_drain.go
+++ b/commands/kubectl_drain.go
@@ -17,12 +17,26 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 
 	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
 )
 
-// TODO this is a temporary hack to get the "kubectl drain" to work
+// Uncordon drain specified nodes
 func runKubectlDrain(spec *operatorv1.KubectlDrainCommandSpec, log logr.Logger) error {
+	for _, node := range spec.Nodes {
+		cmd := newCmd("kubectl", "drain", node)
+
+		lines, err := cmd.RunAndCapture()
+		if err != nil {
+			return errors.WithStack(errors.WithMessage(err, strings.Join(lines, "\n")))
+		}
+
+		log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n")))
+	}
 	return nil
 }
diff --git a/commands/kubectl_uncordon.go b/commands/kubectl_uncordon.go
index a7cdfab..f187a8a 100644
--- a/commands/kubectl_uncordon.go
+++ b/commands/kubectl_uncordon.go
@@ -17,12 +17,26 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 
 	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
 )
 
-// TODO this is a temporary hack to get the "kubectl uncordon" to work
+// Uncordon specified nodes
 func runKubectlUncordon(spec *operatorv1.KubectlUncordonCommandSpec, log logr.Logger) error {
+	for _, node := range spec.Nodes {
+		cmd := newCmd("kubectl", "uncordon", node)
+
+		lines, err := cmd.RunAndCapture()
+		if err != nil {
+			return errors.WithStack(errors.WithMessage(err, strings.Join(lines, "\n")))
+		}
+
+		log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n")))
+	}
 	return nil
 }
diff --git a/commands/preflight.go b/commands/preflight.go
index 517b34e..6f12512 100644
--- a/commands/preflight.go
+++ b/commands/preflight.go
@@ -31,10 +31,9 @@ func runPreflight(spec *operatorv1.PreflightCommandSpec, log logr.Logger) error
 
 	lines, err := cmd.RunAndCapture()
 	if err != nil {
-		return errors.WithStack(err)
+		return errors.WithStack(errors.WithMessage(err, strings.Join(lines, "\n")))
 	}
 
 	log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n")))
-
 	return nil
 }
diff --git a/commands/upgrade_kubeadm.go b/commands/upgrade_kubeadm.go
index 841406e..3a954c3 100644
--- a/commands/upgrade_kubeadm.go
+++ b/commands/upgrade_kubeadm.go
@@ -17,12 +17,43 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
+	"runtime"
+	"strings"
+
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 
 	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
 )
 
-// TODO this is a temporary hack to get the upgrading kubeadm to work
+// sudo curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/${ARCH}/{kubeadm,kubelet,kubectl}
+const DownloadURLTemplate = "https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/%s"
+
+// runUpgradeKubeadm will try to download the binary from official websites;
 func runUpgradeKubeadm(spec *operatorv1.UpgradeKubeadmCommandSpec, log logr.Logger) error {
+	if spec.Local {
+		return nil
+	}
+
+	return DownloadFromOfficialWebsite(spec.KubernetesVersion, "kubeadm", log)
+}
+
+func DownloadFromOfficialWebsite(version, bin string, log logr.Logger) error {
+	var cmd *cmd
+
+	cmd = newCmd("curl", "-L", "--remote-name-all", fmt.Sprintf(DownloadURLTemplate, version, runtime.GOARCH, bin), "-o", "/usr/bin/"+bin)
+	donwlod, err := cmd.RunAndCapture()
+	if err != nil {
+		return errors.WithStack(errors.WithMessage(err, strings.Join(donwlod, "\n")))
+	}
+	log.Info(fmt.Sprintf("%s", strings.Join(donwlod, "\n")))
+
+	cmd = newCmd("chmod", "+x", "/usr/bin/"+bin)
+	lines, err := cmd.RunAndCapture()
+	if err != nil {
+		return errors.WithStack(errors.WithMessage(err, strings.Join(lines, "\n")))
+	}
+	log.Info(fmt.Sprintf("%s", strings.Join(lines, "\n")))
 	return nil
 }
diff --git a/commands/upgrade_kubectlkubelet.go b/commands/upgrade_kubectlkubelet.go
index 27e4625..3b3c336 100644
--- a/commands/upgrade_kubectlkubelet.go
+++ b/commands/upgrade_kubectlkubelet.go
@@ -17,12 +17,46 @@ limitations under the License.
 package commands
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 
 	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
 )
 
-// TODO this is a temporary hack to get the kubectl & kubelet upgrade to work
+// runUpgradeKubectlAndKubelet skip download when using local.
+// If not local, download kubectl and kubelet. Kubelet replacement needs to be done after kubelet stopped.
 func runUpgradeKubectlAndKubelet(spec *operatorv1.UpgradeKubeletAndKubeactlCommandSpec, log logr.Logger) error {
+	if spec.Local {
+		return nil
+	}
+	err := DownloadFromOfficialWebsite(spec.KubernetesVersion, "kubectl", log)
+	if err != nil {
+		return err
+	}
+
+	var cmd *cmd
+
+	cmd = newCmd("systemctl", "stop", "kubelet")
+	stop, err := cmd.RunAndCapture()
+	if err != nil {
+		return errors.WithStack(errors.WithMessage(err, strings.Join(stop, "\n")))
+	}
+	log.Info(fmt.Sprintf("%s", strings.Join(stop, "\n")))
+
+	err = DownloadFromOfficialWebsite(spec.KubernetesVersion, "kubelet", log)
+	if err != nil {
+		return err
+	}
+
+	cmd = newCmd("systemctl", "start", "kubelet")
+	start, err := cmd.RunAndCapture()
+	if err != nil {
+		return errors.WithStack(errors.WithMessage(err, strings.Join(start, "\n")))
+	}
+	log.Info(fmt.Sprintf("%s", strings.Join(start, "\n")))
+
 	return nil
 }
diff --git a/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml b/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml
index bb89918..93ed3cf 100644
--- a/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml
+++ b/config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml
@@ -223,6 +223,17 @@ spec:
                                           kubeadmRenewCertificates:
                                             description: KubeadmRenewCertsCommandSpec
                                               provides...
+                                            properties:
+                                              commands:
+                                                description: 'Commands is a list of
+                                                  commands to run: all, apiserver,
+                                                  apiserver-etcd-client, apiserver-kubelet-client,
+                                                  controller-manager.conf, etcd-healthcheck-client,
+                                                  etcd-peer, etcd-server, front-proxy-client,
+                                                  scheduler.conf'
+                                                items:
+                                                  type: string
+                                                type: array
                                             type: object
                                           kubeadmUpgradeApply:
                                             description: KubeadmUpgradeApplyCommandSpec
@@ -242,13 +253,31 @@ spec:
                                             description: TODO download the specified
                                               version bin and replace it in the node
                                               KubeadmUpgradeNodeCommandSpec provides...
+                                            properties:
+                                              dryRun:
+                                                description: for dry run mode
+                                                type: boolean
                                             type: object
                                           kubectlDrain:
                                             description: KubectlDrainCommandSpec provides...
+                                            properties:
+                                              nodes:
+                                                description: Nodes is a list of nodes
+                                                  to uncordon
+                                                items:
+                                                  type: string
+                                                type: array
                                             type: object
                                           kubectlUncordon:
                                             description: KubectlUncordonCommandSpec
                                               provides...
+                                            properties:
+                                              nodes:
+                                                description: Nodes is a list of nodes
+                                                  to uncordon
+                                                items:
+                                                  type: string
+                                                type: array
                                             type: object
                                           pass:
                                             description: Pass provide a dummy command
@@ -267,6 +296,13 @@ spec:
                                                   the version is empty, we will skip
                                                   this command.
                                                 type: string
+                                              local:
+                                                description: Local is by default true.
+                                                  If true, operator will use the binary
+                                                  in /usr/bin If not, operator will
+                                                  download the binary from official
+                                                  websites
+                                                type: boolean
                                             type: object
                                           upgradeKubeletAndKubeactl:
                                             description: TODO download the specified
@@ -280,6 +316,13 @@ spec:
                                                   the version is empty, we will skip
                                                   this command.
                                                 type: string
+                                              local:
+                                                description: Local is by default true.
+                                                  If true, operator will use the binary
+                                                  in /usr/bin If not, operator will
+                                                  download the binary from official
+                                                  websites
+                                                type: boolean
                                             type: object
                                           wait:
                                             description: Wait pauses the execution
diff --git a/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml
index 55dc95a..036d858 100644
--- a/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml
+++ b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml
@@ -181,6 +181,15 @@ spec:
                               type: object
                             kubeadmRenewCertificates:
                               description: KubeadmRenewCertsCommandSpec provides...
+                              properties:
+                                commands:
+                                  description: 'Commands is a list of commands to
+                                    run: all, apiserver, apiserver-etcd-client, apiserver-kubelet-client,
+                                    controller-manager.conf, etcd-healthcheck-client,
+                                    etcd-peer, etcd-server, front-proxy-client, scheduler.conf'
+                                  items:
+                                    type: string
+                                  type: array
                               type: object
                             kubeadmUpgradeApply:
                               description: KubeadmUpgradeApplyCommandSpec provides...
@@ -198,12 +207,28 @@ spec:
                               description: TODO download the specified version bin
                                 and replace it in the node KubeadmUpgradeNodeCommandSpec
                                 provides...
+                              properties:
+                                dryRun:
+                                  description: for dry run mode
+                                  type: boolean
                               type: object
                             kubectlDrain:
                               description: KubectlDrainCommandSpec provides...
+                              properties:
+                                nodes:
+                                  description: Nodes is a list of nodes to uncordon
+                                  items:
+                                    type: string
+                                  type: array
                               type: object
                             kubectlUncordon:
                               description: KubectlUncordonCommandSpec provides...
+                              properties:
+                                nodes:
+                                  description: Nodes is a list of nodes to uncordon
+                                  items:
+                                    type: string
+                                  type: array
                               type: object
                             pass:
                               description: Pass provide a dummy command for testing
@@ -220,6 +245,12 @@ spec:
                                     kubernetes version If the version is empty, we
                                     will skip this command.
                                   type: string
+                                local:
+                                  description: Local is by default true. If true,
+                                    operator will use the binary in /usr/bin If not,
+                                    operator will download the binary from official
+                                    websites
+                                  type: boolean
                               type: object
                             upgradeKubeletAndKubeactl:
                               description: TODO download the specified version bin
@@ -231,6 +262,12 @@ spec:
                                     kubernetes version If the version is empty, we
                                     will skip this command.
                                   type: string
+                                local:
+                                  description: Local is by default true. If true,
+                                    operator will use the binary in /usr/bin If not,
+                                    operator will download the binary from official
+                                    websites
+                                  type: boolean
                               type: object
                             wait:
                               description: Wait pauses the execution on the next command
diff --git a/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml
index 6e6315f..6d4e788 100644
--- a/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml
+++ b/config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml
@@ -63,6 +63,15 @@ spec:
                       type: object
                     kubeadmRenewCertificates:
                       description: KubeadmRenewCertsCommandSpec provides...
+                      properties:
+                        commands:
+                          description: 'Commands is a list of commands to run: all,
+                            apiserver, apiserver-etcd-client, apiserver-kubelet-client,
+                            controller-manager.conf, etcd-healthcheck-client, etcd-peer,
+                            etcd-server, front-proxy-client, scheduler.conf'
+                          items:
+                            type: string
+                          type: array
                       type: object
                     kubeadmUpgradeApply:
                       description: KubeadmUpgradeApplyCommandSpec provides...
@@ -78,12 +87,28 @@ spec:
                     kubeadmUpgradeNode:
                       description: TODO download the specified version bin and replace
                         it in the node KubeadmUpgradeNodeCommandSpec provides...
+                      properties:
+                        dryRun:
+                          description: for dry run mode
+                          type: boolean
                       type: object
                     kubectlDrain:
                       description: KubectlDrainCommandSpec provides...
+                      properties:
+                        nodes:
+                          description: Nodes is a list of nodes to uncordon
+                          items:
+                            type: string
+                          type: array
                       type: object
                     kubectlUncordon:
                       description: KubectlUncordonCommandSpec provides...
+                      properties:
+                        nodes:
+                          description: Nodes is a list of nodes to uncordon
+                          items:
+                            type: string
+                          type: array
                       type: object
                     pass:
                       description: Pass provide a dummy command for testing the kubeadm-operator
@@ -99,6 +124,11 @@ spec:
                           description: KubernetesVersion specifies the target kubernetes
                             version If the version is empty, we will skip this command.
                           type: string
+                        local:
+                          description: Local is by default true. If true, operator
+                            will use the binary in /usr/bin If not, operator will
+                            download the binary from official websites
+                          type: boolean
                       type: object
                     upgradeKubeletAndKubeactl:
                       description: TODO download the specified version bin and replace
@@ -108,6 +138,11 @@ spec:
                           description: KubernetesVersion specifies the target kubernetes
                             version If the version is empty, we will skip this command.
                           type: string
+                        local:
+                          description: Local is by default true. If true, operator
+                            will use the binary in /usr/bin If not, operator will
+                            download the binary from official websites
+                          type: boolean
                       type: object
                     wait:
                       description: Wait pauses the execution on the next command for
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
index 8a4ca90..58c37b6 100644
--- a/config/manager/kustomization.yaml
+++ b/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: daocloud.io/daocloud/kubeadm-operator
-  newTag: v0.0.3
+  newTag: v0.0.4-dev
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 124339f..8af5884 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -39,7 +39,7 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
-        image: daocloud.io/daocloud/kubeadm-operator:v0.0.3
+        image: daocloud.io/daocloud/kubeadm-operator:v0.0.4-dev
         name: manager
         resources:
           limits: