Merge pull request #24 from invidian/updates

Add instructions how to run conformance tests and other improvements

.gitignore

@@ -10,3 +10,4 @@ bin
 kubeconfig
 .terraform.tfstate.lock.info
 .env
+*.tar.gz

Makefile

@@ -10,7 +10,7 @@ GOMOD=$(GOCMD) mod
 GOBUILD=CGO_ENABLED=$(CGO_ENABLED) $(GOCMD) build -v -buildmode=exe -ldflags $(LD_FLAGS)
 
 CC_TEST_REPORTER_ID=6e107e510c5479f40b0ce9166a254f3f1ee0bc547b3e48281bada1a5a32bb56d
-GOLANGCI_LINT_VERSION=v1.22.2
+GOLANGCI_LINT_VERSION=v1.23.1
 BIN_PATH=$$HOME/bin
 
 GO_PACKAGES=./...
@@ -34,13 +34,13 @@ TERRAFORM_BIN=$(TERRAFORM_ENV) /usr/bin/terraform
 # Default target when testing locally
 TEST_LOCAL=controlplane
 
-CONTROLLERS=$(shell grep CONTROLLERS .env | cut -d= f2 2>/dev/null || echo "1")
+CONTROLLERS=$(shell (grep CONTROLLERS .env 2>/dev/null || echo "1") | cut -d= -f2 2>/dev/null)
 
-WORKERS=$(shell grep WORKERS .env | cut -d= -f2 2>/dev/null || echo "0")
+WORKERS=$(shell (grep WORKERS .env 2>/dev/null || echo "2") | cut -d= -f2 2>/dev/null)
 
 NODES_CIDR="192.168.50.0/24"
 
-FLATCAR_CHANNEL=$(shell grep FLATCAR_CHANNEL .env | cut -d= -f2 2>/dev/null || echo "edge")
+FLATCAR_CHANNEL=$(shell (grep FLATCAR_CHANNEL .env 2>/dev/null || echo "stable") | cut -d= -f2 2>/dev/null)
 
 TERRAFORM_ENV=TF_VAR_flatcar_channel=$(FLATCAR_CHANNEL) TF_VAR_controllers_count=$(CONTROLLERS) TF_VAR_workers_count=$(WORKERS) TF_VAR_nodes_cidr=$(NODES_CIDR)
 
@@ -117,6 +117,18 @@ test-local-apply:
  cd cmd/terraform-provider-flexkube && go build -o ../../local-testing/terraform-provider-flexkube
  cd local-testing && $(TERRAFORM_BIN) init && $(TERRAFORM_BIN) apply -auto-approve
 
+.PHONY: test-conformance
+test-conformance:SHELL=/bin/bash
+test-conformance:
+ until kubectl get nodes >/dev/null; do sleep 1; done
+ sonobuoy run --mode=certified-conformance || true
+ until sonobuoy status | grep e2e | grep complete; do timeout --foreground 10m sonobuoy logs -f || true; sleep 1; done
+ sonobuoy results $$(sonobuoy retrieve)
+
+.PHONY: test-conformance-clean
+test-conformance-clean:
+ sonobuoy delete
+
 .PHONY: lint
 lint:
  golangci-lint run --enable-all --disable=$(DISABLED_LINTERS) --max-same-issues=0 --max-issues-per-linter=0 --build-tags integration $(GO_PACKAGES)
@@ -206,12 +218,10 @@ vagrant-integration-shell:
 .PHONY: vagrant-integration
 vagrant-integration: vagrant-up vagrant-rsync vagrant-integration-build vagrant-integration-run
 
-
 .PHONY: vagrant-build-bin
 vagrant-build-bin: vagrant-integration-build
  $(VAGRANTCMD) ssh -c "$(BUILD_CMD) make build-bin"
 
-
 .PHONY: vagrant-e2e-build
 vagrant-e2e-build:
  $(VAGRANTCMD) ssh -c "$(BUILD_CMD) make build-e2e"
@@ -235,3 +245,16 @@ vagrant-e2e-shell:
 
 .PHONY: vagrant-e2e
 vagrant-e2e: vagrant-e2e-run vagrant-e2e-destroy vagrant-destroy
+
+.PHONY: vagrant-conformance-run
+vagrant-conformance-run:
+ # Make sure static controlplane is shut down.
+ $(VAGRANTCMD) ssh -c "docker stop kube-apiserver kube-scheduler kube-controller-manager"
+ $(VAGRANTCMD) ssh -c "$(E2E_CMD) -c 'make test-conformance'"
+
+.PHONY: vagrant-conformance
+vagrant-conformance: vagrant-e2e-run vagrant-conformance-run vagrant-conformance-copy-results
+
+.PHONY: vagrant-conformance-copy-results
+vagrant-conformance-copy-results:
+ scp -P 2222 -i ~/.vagrant.d/insecure_private_key [email protected]:/home/core/libflexkube/*.tar.gz ./

README.md

@@ -49,7 +49,7 @@ Using this library has minimal target host (where containers will run) requireme
 - when deploying to remote hosts, SSH access with the user allowed to create containers (e.g. when using `Docker` as a container runtime,
  user must be part of `docker` group)
 
-`root` access on the target hosts is NOT required, as all configuration files are managed using temporary configuration containers.
+Direct `root` access (via SSH login or with e.g. `sudo`) on the target hosts is NOT required, as all configuration files are managed using temporary configuration containers.
 
 No Public DNS or any other public discovery service is required for getting cluster up and running either.
 
@@ -227,7 +227,6 @@ This can be done in 2 ways:
 ## Current known issues and limitations
 
 Currently, there are several things, which are either missing or broken. Here is the list of known problems:
-- network plug-ins are not configurable (currently `kubenet` is hardcoded)
 - gracefully replacing CA certificates (if private key does not change, it should work, but has not been tested)
 - adding/removing etcd members
 - no checkpointer for pods/apiserver. If static kube-apiserver container is stopped and node reboots, single node cluster will not come back.
@@ -242,8 +241,6 @@ And features, which are not yet implemented:
 - showing diff to the user (planning what will be done)
 - removal of config files, created data and containers
 - automatic shutdown/start of bootstrap control plane
-- taints and tolerations for control plane
-- role labels for kubelets
 
 ## Testing
 
@@ -302,6 +299,21 @@ If you just want to run E2E tests and clean everything up afterwards, run the fo
 make vagrant-e2e
 ```
 
+### Conformance tests
+
+To run conformance tests in the environment provided by `Vagrantfile`, run the following command:
+```
+make vagrant-conformance
+```
+
+The command will deploy E2E environment and then run conformance tests in there.
+
+The test should take a bit more than an hour to finish.
+
+By default, after scheduling the conformance tests, the command will start showing the logs of the tests. One can then use CTRL-C to stop showing the logs, as tests will be running in the background and the command is idempotent.
+
+Once tests are complete, the command should will the test results and archive file with the report will be copied into project's root directory, which can be then submitted to [k8s-conformance](https://github.com/cncf/k8s-conformance) repository.
+
 ### Local tests
 
 For testing standalone resources, e.g. just `etcd-cluster`, [local-testing](./local-testing) directory can be used, which will use the code from [e2e](./e2e) directory to create a cluster and then will dump all configuration and state files to separate directories, when tools from [cmd](./cmd) directory can be used directly. That allows to skip many sync steps, which speeds up the overall process, making development easier.
@@ -371,6 +383,12 @@ If you want to also remove all artifacts from the repository, like built binarie
 make clean
 ```
 
+## Helm charts
+
+All self-hosted control-plane deployments and CNI plugins are managed using [Helm](https://helm.sh/). All used charts are available via `https://flexkube.github.io/charts/` charts repository.
+
+The repository is hosted using GitHub Pages and and it's content can be found in this [charts](https://github.com/flexkube/charts) repository.
+
 ## Contributing
 
 All contributions to this project are welcome. If it does not satisfy your needs, feel free to raise an issue about it or implement the support yourself and create a pull request with the patch, so we can all benefit from it.

e2e/Dockerfile

@@ -1,6 +1,6 @@
 FROM hashicorp/terraform:light
 
-RUN apk add -U make bash bash-completion vim
+RUN apk add -U make bash bash-completion vim coreutils
 
 RUN wget https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz -O - | tar -xzvf - linux-amd64/helm && \
  mv linux-amd64/helm /usr/local/bin/
@@ -9,6 +9,12 @@ RUN wget -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-rel
  && chmod +x /usr/local/bin/kubectl \
  && echo -e 'source /usr/share/bash-completion/bash_completion\nsource <(kubectl completion bash)' >> $HOME/.bashrc
 
+RUN VERSION=0.17.1 OS=linux && \
+ wget "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_amd64.tar.gz" -O sonobuoy.tar.gz && \
+ tar -xzf sonobuoy.tar.gz -C /usr/local/bin && \
+ chmod +x /usr/local/bin/sonobuoy && \
+ rm sonobuoy.tar.gz
+
 RUN echo 'plugin_cache_dir = "/root/.terraform.d.host/plugin-cache"' >> /root/.terraformrc
 
 RUN helm repo add stable https://kubernetes-charts.storage.googleapis.com

e2e/main.tf

@@ -31,7 +31,7 @@ variable "pod_cidr" {
 }
 
 variable "network_plugin" {
- default = "cni"
+ default = "calico"
 }
 
 variable "node_ssh_port" {
@@ -236,6 +236,8 @@ EOF
  address = var.controllers_count > 1 ? "localhost" : cidrhost(var.nodes_cidr, 2)
  })
 
+ network_plugin = var.network_plugin == "kubenet" ? "kubenet" : "cni"
+
  kubelet_pool_config = templatefile("./templates/kubelet_config.yaml.tmpl", {
  kubelet_addresses = local.controller_ips
  bootstrap_kubeconfig = local.bootstrap_kubeconfig
@@ -245,7 +247,7 @@ EOF
  kubelet_pod_cidrs = local.controller_cidrs
  kubernetes_ca_certificate = module.kubernetes_pki.kubernetes_ca_cert
  kubelet_names = local.controller_names
- network_plugin = var.network_plugin
+ network_plugin = local.network_plugin
  labels = {}
  privileged_labels = {
  "node-role.kubernetes.io/master" = ""
@@ -266,7 +268,7 @@ EOF
  kubelet_pod_cidrs = local.worker_cidrs
  kubernetes_ca_certificate = module.kubernetes_pki.kubernetes_ca_cert
  kubelet_names = local.worker_names
- network_plugin = var.network_plugin
+ network_plugin = local.network_plugin
  labels = {}
  taints = {}
  privileged_labels = {}
@@ -354,6 +356,8 @@ resource "flexkube_helm_release" "kubelet-rubber-stamp" {
 }
 
 resource "flexkube_helm_release" "calico" {
+ count = var.network_plugin == "calico" ? 1 : 0
+
  kubeconfig = local.kubeconfig_admin
  namespace = "kube-system"
  chart = var.calico_helm_chart_source

pkg/kubelet/kubelet.go

@@ -233,8 +233,8 @@ func (k *kubelet) ToHostConfiguredContainer() (*container.HostConfiguredContaine
  Target: "/var/run/docker.sock",
  },
  {
- // For testing kubenet.
- // TODO do we need it?
+ // Required when using CNI plugin for networking, as kubelet will verify, that network configuration
+ // has been deployed there before creating pods.
  Source: "/etc/cni/net.d/",
  Target: "/etc/cni/net.d",
  },
@@ -247,23 +247,33 @@ func (k *kubelet) ToHostConfiguredContainer() (*container.HostConfiguredContaine
  Target: "/host/opt/cni/bin",
  },
  {
- // Required by kubelet when creating Docker containers. rslave borrowed from Rancher.
- // TODO add better explanation
- Source: "/var/lib/docker/",
- Target: "/var/lib/docker",
- Propagation: "rslave",
+ // Required by kubelet when creating Docker containers. This is required, when using Docker as container
+ // runtime, as cAdvisor, which is integrated into kubelet will try to identify image read-write layer for
+ // container when creating a handler for monitoring. This is needed to report disk usage inside the container.
+ //
+ // It is also needed, as kubelet creates a symlink from Docker container's log file to /var/log/pods.
+ Source: "/var/lib/docker/",
+ Target: "/var/lib/docker",
  },
  {
- // Required for kubelet when running Docker containers. Since kubelet mounts stuff there several times
- // mounts should be propagated, hence the "shared". "shared" borrowed from Rancher.
- // TODO add better explanation
+ // In there, kubelet persist generated certificates and information about pods. In case of a re-creation of
+ // kubelet containers, this information would get lost, so running pods would become orphans, which is not
+ // desired.
+ //
+ // Kubelet also mounts the pods mounts in there, so those directories must be shared with host (where actual
+ // Docker containers are created).
+ //
+ // "shared" propagation is needed, as those pods mounts should be visible for the kubelet as well, otherwise
+ // kubelet complains when trying to clean up pods volumes.
  Source: "/var/lib/kubelet/",
  Target: "/var/lib/kubelet",
  Propagation: "shared",
  },
  {
- // To persist CNI configuration managed by kubelet. Might be only required with 'kubenet' network plugin.
- // TODO check if this is needed. Maybe explain what is stored there.
+ // This is where kubelet stores information about the network configuration on the node when using 'kubenet'
+ // as network plugin, so it should be persisted.
+ //
+ // It is also used for caching network configuration for both 'kubenet' and CNI plugins.
  Source: "/var/lib/cni/",
  Target: "/var/lib/cni",
  },
@@ -273,7 +283,9 @@ func (k *kubelet) ToHostConfiguredContainer() (*container.HostConfiguredContaine
  Target: "/lib/modules",
  },
  {
- // Store pod logs on the host, so they are persistent and also can read by Loki.
+ // In this directory, kubelet creates symlinks to container log files, so this directory should be visible
+ // also for other containers. For example for centralised logging, as this is the location, where logging
+ // agent expect to find pods logs.
  Source: "/var/log/pods/",
  Target: "/var/log/pods",
  },
@@ -292,11 +304,10 @@ func (k *kubelet) ToHostConfiguredContainer() (*container.HostConfiguredContaine
  "--config=/etc/kubernetes/kubelet.yaml",
  // Specify kubeconfig file for kubelet. This enabled API server mode and
  // specifies when kubelet will write kubeconfig file after TLS bootstrapping.
- "--kubeconfig=/etc/kubernetes/kubeconfig",
+ "--kubeconfig=/var/lib/kubelet/kubeconfig",
  // kubeconfig with access token for TLS bootstrapping.
  "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig",
- // Use 'kubenet' network plugin, as it's the simplest one.
- // TODO allow to use different CNI plugins (just 'cni' to be precise)
+ // Set which network plugin to use.
  fmt.Sprintf("--network-plugin=%s", k.networkPlugin),
  // https://alexbrand.dev/post/why-is-my-kubelet-listening-on-a-random-port-a-closer-look-at-cri-and-the-docker-cri-shim/
  "--redirect-container-streaming=false",