From 04ff0600651d5824c5ac208cfcaea46dfdfaf2c4 Mon Sep 17 00:00:00 2001 From: Venkat Nagappan <45001120+justmeandopensource@users.noreply.github.com> Date: Sat, 2 Oct 2021 19:08:43 +0100 Subject: [PATCH] initial ha docs (#82) --- .../external-keepalived-haproxy/README.md | 220 ++++++++++++++++++ .../external-keepalived-haproxy/Vagrantfile | 101 ++++++++ .../external-keepalived-haproxy/bootstrap.sh | 11 + 3 files changed, 332 insertions(+) create mode 100644 kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/README.md create mode 100644 kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/Vagrantfile create mode 100644 kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/bootstrap.sh diff --git a/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/README.md b/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/README.md new file mode 100644 index 00000000000..f1c0365a4f5 --- /dev/null +++ b/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/README.md @@ -0,0 +1,220 @@ +# Set up a Highly Available Kubernetes Cluster using kubeadm +Follow this documentation to set up a highly available Kubernetes cluster using __Ubuntu 20.04 LTS__ with keepalived and haproxy + +This documentation guides you in setting up a cluster with three master nodes, one worker node and two load balancer node using HAProxy and Keepalived. + +## Vagrant Environment +|Role|FQDN|IP|OS|RAM|CPU| +|----|----|----|----|----|----| +|Load Balancer|loadbalancer1.example.com|172.16.16.51|Ubuntu 20.04|512M|1| +|Load Balancer|loadbalancer2.example.com|172.16.16.52|Ubuntu 20.04|512M|1| +|Master|kmaster1.example.com|172.16.16.101|Ubuntu 20.04|2G|2| +|Master|kmaster2.example.com|172.16.16.102|Ubuntu 20.04|2G|2| +|Master|kmaster3.example.com|172.16.16.103|Ubuntu 20.04|2G|2| +|Worker|kworker1.example.com|172.16.16.201|Ubuntu 20.04|2G|2| + +> * Password for the **root** account on all these virtual machines is **kubeadmin** +> * Perform all the commands as root user unless otherwise specified + +### Virtual IP managed by Keepalived on the load balancer nodes +|Virtual IP| +|----| +|172.16.16.100| + +## Pre-requisites +If you want to try this in a virtualized environment on your workstation +* Virtualbox installed +* Vagrant installed +* Host machine has atleast 12 cores +* Host machine has atleast 16G memory + +## Bring up all the virtual machines +``` +vagrant up +``` +If you are on Linux host and want to use KVM/Libvirt +``` +vagrant up --provider libvirt +``` + +## Set up load balancer nodes (loadbalancer1 & loadbalancer2) +##### Install Keepalived & Haproxy +``` +apt update && apt install -y keepalived haproxy +``` +##### configure keepalived +On both nodes create the health check script /etc/keepalived/check_apiserver.sh +``` +cat >> /etc/keepalived/check_apiserver.sh <&2 + exit 1 +} + +curl --silent --max-time 2 --insecure https://localhost:6443/ -o /dev/null || errorExit "Error GET https://localhost:6443/" +if ip addr | grep -q 172.16.16.100; then + curl --silent --max-time 2 --insecure https://172.16.16.100:6443/ -o /dev/null || errorExit "Error GET https://172.16.16.100:6443/" +fi +EOF + +chmod +x /etc/keepalived/check_apiserver.sh +``` +Create keepalived config /etc/keepalived/keepalived.conf +``` +cat >> /etc/keepalived/keepalived.conf <> /etc/haproxy/haproxy.cfg <> /etc/modules-load.d/containerd.conf <>/etc/sysctl.d/kubernetes.conf< /etc/containerd/config.toml + systemctl restart containerd + systemctl enable containerd +} +``` +##### Add apt repo for kubernetes +``` +{ + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main" +} +``` +##### Install Kubernetes components +``` +{ + apt update + apt install -y kubeadm=1.22.0-00 kubelet=1.22.0-00 kubectl=1.22.0-00 +} +``` +## Bootstrap the cluster +## On kmaster1 +##### Initialize Kubernetes Cluster +``` +kubeadm init --control-plane-endpoint="172.16.16.100:6443" --upload-certs --apiserver-advertise-address=172.16.16.101 --pod-network-cidr=192.168.0.0/16 +``` +Copy the commands to join other master nodes and worker nodes. +##### Deploy Calico network +``` +kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f https://docs.projectcalico.org/v3.18/manifests/calico.yaml +``` + +## Join other master nodes to the cluster +> Use the respective kubeadm join commands you copied from the output of kubeadm init command on the first master. + +> IMPORTANT: Don't forget the --apiserver-advertise-address option to the join command when you join the other master nodes. + +## Join worker nodes to the cluster +> Use the kubeadm join command you copied from the output of kubeadm init command on the first master + + +## Downloading kube config to your local machine +On your host machine +``` +mkdir ~/.kube +scp root@172.16.16.101:/etc/kubernetes/admin.conf ~/.kube/config +``` +Password for root account is kubeadmin (if you used my Vagrant setup) + +## Verifying the cluster +``` +kubectl cluster-info +kubectl get nodes +``` + +Have Fun!! diff --git a/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/Vagrantfile b/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/Vagrantfile new file mode 100644 index 00000000000..9075629ff9c --- /dev/null +++ b/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/Vagrantfile @@ -0,0 +1,101 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +ENV['VAGRANT_NO_PARALLEL'] = 'yes' + +Vagrant.configure(2) do |config| + + config.vm.provision "shell", path: "bootstrap.sh" + + # Load Balancer Nodes + LoadBalancerCount = 2 + + (1..LoadBalancerCount).each do |i| + + config.vm.define "loadbalancer#{i}" do |lb| + + lb.vm.box = "generic/ubuntu2004" + lb.vm.box_check_update = false + lb.vm.box_version = "3.3.0" + lb.vm.hostname = "loadbalancer#{i}.example.com" + + lb.vm.network "private_network", ip: "172.16.16.5#{i}" + + lb.vm.provider :virtualbox do |v| + v.name = "loadbalancer#{i}" + v.memory = 512 + v.cpus = 1 + end + + lb.vm.provider :libvirt do |v| + v.memory = 512 + v.cpus = 1 + end + + end + + end + + + # Kubernetes Master Nodes + MasterCount = 3 + + (1..MasterCount).each do |i| + + config.vm.define "kmaster#{i}" do |masternode| + + masternode.vm.box = "generic/ubuntu2004" + masternode.vm.box_check_update = false + masternode.vm.box_version = "3.3.0" + masternode.vm.hostname = "kmaster#{i}.example.com" + + masternode.vm.network "private_network", ip: "172.16.16.10#{i}" + + masternode.vm.provider :virtualbox do |v| + v.name = "kmaster#{i}" + v.memory = 2048 + v.cpus = 2 + end + + masternode.vm.provider :libvirt do |v| + v.nested = true + v.memory = 2048 + v.cpus = 2 + end + + end + + end + + + # Kubernetes Worker Nodes + WorkerCount = 1 + + (1..WorkerCount).each do |i| + + config.vm.define "kworker#{i}" do |workernode| + + workernode.vm.box = "generic/ubuntu2004" + workernode.vm.box_check_update = false + workernode.vm.box_version = "3.3.0" + workernode.vm.hostname = "kworker#{i}.example.com" + + workernode.vm.network "private_network", ip: "172.16.16.20#{i}" + + workernode.vm.provider :virtualbox do |v| + v.name = "kworker#{i}" + v.memory = 2048 + v.cpus = 2 + end + + workernode.vm.provider :libvirt do |v| + v.nested = true + v.memory = 2048 + v.cpus = 2 + end + + end + + end + +end diff --git a/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/bootstrap.sh b/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/bootstrap.sh new file mode 100644 index 00000000000..146b4ef0f6a --- /dev/null +++ b/kubeadm-ha-keepalived-haproxy/external-keepalived-haproxy/bootstrap.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Enable ssh password authentication +echo "[TASK 1] Enable ssh password authentication" +sed -i 's/^PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config +echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config +systemctl reload sshd + +# Set Root password +echo "[TASK 2] Set root password" +echo -e "kubeadmin\nkubeadmin" | passwd root >/dev/null 2>&1