Migrate kubelet service to run in docker (#47)

* Migrate kubelet service to run in docker Upstream tectonic/installer, now deprecated in favour of openshift/installer runs kubelet in docker. Support for rkt is hard to come by. https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service
utilitywarehouse · Jul 30, 2018 · f9522ff · f9522ff
1 parent 92fff17
commit f9522ff
Show file tree

Hide file tree

Showing 2 changed files with 89 additions and 70 deletions.
diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service
@@ -1,19 +1,9 @@
+# https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service
 [Unit]
-Description=kubelet-on-rkt service
-After=systemd-resolved.service
+Description=Kubernetes Kubelet
+Requires=docker.service
+After=docker.service
 [Service]
-Environment=KUBELET_IMAGE_URL=docker://${kubelet_image_url}
-Environment=KUBELET_IMAGE_TAG=${kubelet_image_tag}
-Environment="RKT_RUN_ARGS=\
-  --insecure-options=image \
-  --uuid-file-save=/var/run/kubelet-pod.uuid \
-  --volume var-log,kind=host,source=/var/log --mount volume=var-log,target=/var/log \
-  --volume cni-bin,kind=host,source=/opt/cni/bin --mount volume=cni-bin,target=/opt/cni/bin \
-  --volume var-lib-cni,kind=host,source=/var/lib/cni --mount volume=var-lib-cni,target=/var/lib/cni \
-  --volume etc-cni-netd,kind=host,source=/etc/cni/net.d --mount volume=etc-cni-netd,target=/etc/cni/net.d \
-  --volume dns,kind=host,source=/etc/resolv.conf --mount volume=dns,target=/etc/resolv.conf \
-  --volume var-run-calico,kind=host,source=/var/run/calico --mount volume=var-run-calico,target=/var/run/calico \
-  --volume var-lib-calico,kind=host,source=/var/lib/calico --mount volume=var-lib-calico,target=/var/lib/calico"
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
 ExecStartPre=/usr/bin/mkdir -p /var/log/containers
 ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
@@ -28,25 +18,45 @@ ExecStartPre=/opt/bin/cfssl-keys-and-certs-get
 ExecStartPre=/opt/bin/cfssl-new-cert
 ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-controller-manager' | awk '{ print $1; }')"
 ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-apiserver' | awk '{ print $1; }')"
-ExecStartPre=-/usr/bin/rkt gc --grace-period=0s
-ExecStart=/usr/lib/coreos/kubelet-wrapper \
-  --kubeconfig=/var/lib/kubelet/kubeconfig \
-  --node-labels=role=master,node-role.kubernetes.io/master="" \
-  --register-node=true \
-  --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
-  --container-runtime=docker \
-  --network-plugin=cni \
-  --cni-bin-dir=/opt/cni/bin \
-  --cni-conf-dir=/etc/cni/net.d \
-  --allow-privileged=true \
-  --pod-manifest-path=/etc/kubernetes/manifests \
-  ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
-  --cluster-dns=${cluster_dns} \
-  --cluster-domain=cluster.local \
-  --v=0
-ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+ExecStart=/usr/bin/docker \
+  run \
+    --rm \
+    --net host \
+    --pid host \
+    --privileged \
+    --volume /dev:/dev:rw \
+    --volume /sys:/sys:ro \
+    --volume /var/run:/var/run:rw \
+    --volume /var/lib/cni/:/var/lib/cni:rw \
+    --volume /var/lib/docker/:/var/lib/docker:rw \
+    --volume /var/lib/kubelet/:/var/lib/kubelet:shared \
+    --volume /var/log:/var/log:shared \
+    --volume /etc/kubernetes:/etc/kubernetes:ro \
+    --volume /etc/cni/net.d:/etc/cni/net.d:rw \
+    --volume /etc/resolv.conf:/etc/resolv.conf:ro \
+    --volume /opt/cni/bin:/opt/cni/bin:rw \
+    --volume /var/run/calico:/var/run/calico:rw \
+    --volume /var/lib/calico:/var/lib/calico:rw \
+    --entrypoint /usr/local/bin/kubelet \
+  "${kubelet_image_url}:${kubelet_image_tag}" \
+    --kubeconfig=/var/lib/kubelet/kubeconfig \
+    --node-labels=role=master,node-role.kubernetes.io/master="" \
+    --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
+    --container-runtime=docker \
+    --network-plugin=cni \
+    --cni-bin-dir=/opt/cni/bin \
+    --cni-conf-dir=/etc/cni/net.d \
+    --allow-privileged \
+    --pod-manifest-path=/etc/kubernetes/manifests \
+    --minimum-container-ttl-duration=6m0s \
+    --cluster-dns=${cluster_dns} \
+    --cluster_domain=cluster.local \
+    ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
+    --serialize-image-pulls=false \
+    --lock-file=/var/run/lock/kubelet.lock \
+    --exit-on-lock-contention \
+    --v=0
 Restart=always
 RestartSec=10
-TimeoutStartSec=3m
 [Install]
 WantedBy=multi-user.target
diff --git a/resources/worker-kubelet.service b/resources/worker-kubelet.service
@@ -1,21 +1,9 @@
+# https://github.com/openshift/installer/blob/master/modules/ignition/resources/services/kubelet.service
 [Unit]
-Description=kubelet-on-rkt service
-After=systemd-resolved.service
+Description=Kubernetes Kubelet
+Requires=docker.service
+After=docker.service
 [Service]
-Environment=KUBELET_IMAGE_URL=docker://${kubelet_image_url}
-Environment=KUBELET_IMAGE_TAG=${kubelet_image_tag}
-Environment="RKT_RUN_ARGS=\
-  --insecure-options=image \
-  --uuid-file-save=/var/run/kubelet-pod.uuid \
-  --volume var-log,kind=host,source=/var/log --mount volume=var-log,target=/var/log \
-  --volume cni-bin,kind=host,source=/opt/cni/bin --mount volume=cni-bin,target=/opt/cni/bin \
-  --volume var-lib-cni,kind=host,source=/var/lib/cni --mount volume=var-lib-cni,target=/var/lib/cni \
-  --volume etc-cni-netd,kind=host,source=/etc/cni/net.d --mount volume=etc-cni-netd,target=/etc/cni/net.d \
-  --volume dns,kind=host,source=/etc/resolv.conf --mount volume=dns,target=/etc/resolv.conf \
-  --volume var-run-calico,kind=host,source=/var/run/calico --mount volume=var-run-calico,target=/var/run/calico \
-  --volume var-lib-calico,kind=host,source=/var/lib/calico --mount volume=var-lib-calico,target=/var/lib/calico \
-  --volume modprobe,kind=host,source=/usr/sbin/modprobe --mount volume=modprobe,target=/usr/sbin/modprobe \
-  --volume lib-modules,kind=host,source=/lib/modules --mount volume=lib-modules,target=/lib/modules"
 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
 ExecStartPre=/usr/bin/mkdir -p /var/log/containers
 ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
@@ -27,29 +15,50 @@ ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
 #  https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731
 ExecStartPre=/sbin/sysctl -w net.ipv4.tcp_retries2=8
 ExecStartPre=/opt/bin/cfssl-new-cert
-ExecStartPre=-/usr/bin/rkt gc --grace-period=0s
-ExecStart=/usr/lib/coreos/kubelet-wrapper \
-  --kubeconfig=/var/lib/kubelet/kubeconfig \
-  --node-labels=role=${role} \
-  --container-runtime=docker \
-  --network-plugin=cni \
-  --cni-bin-dir=/opt/cni/bin \
-  --cni-conf-dir=/etc/cni/net.d \
-  --register-node=true \
-  --allow-privileged=true \
-  --pod-manifest-path=/etc/kubernetes/manifests \
-  --cluster-dns=${cluster_dns} \
-  --cluster_domain=cluster.local \
-  ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
-  --eviction-soft=memory.available<2Gi,nodefs.available<4Gi \
-  --eviction-soft-grace-period=memory.available=1m,nodefs.available=1m \
-  --eviction-max-pod-grace-period=30 \
-  --eviction-hard=memory.available<1Gi,nodefs.available<2Gi \
-  --serialize-image-pulls=false \
-  --v=0
-ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+ExecStart=/usr/bin/docker \
+  run \
+    --rm \
+    --net host \
+    --pid host \
+    --privileged \
+    --volume /dev:/dev:rw \
+    --volume /sys:/sys:ro \
+    --volume /var/run:/var/run:rw \
+    --volume /var/lib/cni/:/var/lib/cni:rw \
+    --volume /var/lib/docker/:/var/lib/docker:rw \
+    --volume /var/lib/kubelet/:/var/lib/kubelet:shared \
+    --volume /var/log:/var/log:shared \
+    --volume /etc/kubernetes:/etc/kubernetes:ro \
+    --volume /etc/cni/net.d:/etc/cni/net.d:rw \
+    --volume /etc/resolv.conf:/etc/resolv.conf:ro \
+    --volume /opt/cni/bin:/opt/cni/bin:rw \
+    --volume /var/run/calico:/var/run/calico:rw \
+    --volume /var/lib/calico:/var/lib/calico:rw \
+    --volume /usr/sbin/modprobe:/usr/sbin/modprobe:rw \
+    --volume /lib/modules:/lib/modules:rw \
+    --entrypoint /usr/local/bin/kubelet \
+  "${kubelet_image_url}:${kubelet_image_tag}" \
+    --kubeconfig=/var/lib/kubelet/kubeconfig \
+    --node-labels=role=${role} \
+    --container-runtime=docker \
+    --network-plugin=cni \
+    --cni-bin-dir=/opt/cni/bin \
+    --cni-conf-dir=/etc/cni/net.d \
+    --allow-privileged \
+    --pod-manifest-path=/etc/kubernetes/manifests \
+    --minimum-container-ttl-duration=6m0s \
+    --cluster-dns=${cluster_dns} \
+    --cluster_domain=cluster.local \
+    ${cloud_provider == "" ? "" : "--cloud-provider=${cloud_provider}"} \
+    --eviction-soft=memory.available<2Gi,nodefs.available<4Gi \
+    --eviction-soft-grace-period=memory.available=1m,nodefs.available=1m \
+    --eviction-max-pod-grace-period=30 \
+    --eviction-hard=memory.available<1Gi,nodefs.available<2Gi \
+    --serialize-image-pulls=false \
+    --lock-file=/var/run/lock/kubelet.lock \
+    --exit-on-lock-contention \
+    --v=0
 Restart=always
 RestartSec=10
-TimeoutStartSec=3m
 [Install]
 WantedBy=multi-user.target