eks-prow-build-cluster: GitOps proposal

infra/aws/terraform/prow-build-cluster/Makefile

@@ -15,46 +15,94 @@
 TF ?= terraform
 TF_ARGS ?=
 
+K ?= kubectl
+
 # TODO: figure out a workaround for: https://github.com/hashicorp/terraform-provider-kubernetes-alpha/issues/199#issuecomment-832614387
 DEPLOY_K8S_RESOURCES ?= true
 
 # Valid values are: canary, prod
 PROW_ENV ?= canary
 
+##@ Helpers:
+
+.PHONY: help
+help: ## Display this help.
+	@awk \
+		-v "col=${COLOR}" -v "nocol=${NOCOLOR}" \
+		' \
+			BEGIN { \
+				FS = ":.*##" ; \
+				printf "\nUsage:\n  make %s<target>%s\n", col, nocol \
+			} \
+			/^[a-zA-Z_-]+:.*?##/ { \
+				printf "  %s%-15s%s %s\n", col, $$1, nocol, $$2 \
+			} \
+			/^##@/ { \
+				printf "\n%s%s%s\n", col, substr($$0, 5), nocol \
+			} \
+		' $(MAKEFILE_LIST)
+
+##@ Terraform:
+
 .PHONY: init
-init:
+init: ## Initialize Terraform's state and download necessary providers.
 	$(TF) $@ \
 		-backend-config=./tfbackends/$(PROW_ENV).tfbackend
 
 .PHONY: plan
-plan:
+plan: ## Present plan for creating/updating Terraform resources.
 	$(TF) $@ $(TF_ARGS) \
 		-var-file=./terraform.tfvars \
 		-var-file=./terraform.$(PROW_ENV).tfvars \
 		-var="deploy_kubernetes_resources=$(DEPLOY_K8S_RESOURCES)"
 
 .PHONY: apply
-apply:
+apply: ## Create/Update Terraform resources.
 	$(TF) $@ $(TF_ARGS) \
 		-var-file=./terraform.tfvars \
 		-var-file=./terraform.$(PROW_ENV).tfvars \
 		-var="deploy_kubernetes_resources=$(DEPLOY_K8S_RESOURCES)"
 
 .PHONY: destroy
-destroy:
+destroy: ## Delete Terraform resources.
 	$(TF) $@ $(TF_ARGS) \
 		-var-file=./terraform.tfvars \
 		-var-file=./terraform.$(PROW_ENV).tfvars \
 		-var="deploy_kubernetes_resources=$(DEPLOY_K8S_RESOURCES)"
 
 .PHONY: fmt
-fmt:
+fmt: ## Format Terraform files.
 	$(TF) $@
 
 .PHONY: output
-output:
+output: ## Print Terraform output.
 	$(TF) $@
 
 .PHONY: clean
-clean:
+clean: ## Clean up Terraform cache and local state.
 	rm -rf ./.terraform
+
+##@ FluxCD:
+
+.PHONY: flux-install
+flux-install: ## Install FluxCD components inside flux-system namespace.
+	kubectl apply --server-side -f ./resources/flux-system/gotk-components.yaml
+
+.PHONY: flux-apply-sources
+flux-apply-sources: ## Apply FluxCD Sources located inside "./resources/flux-system" folder.
+	find ./resources/flux-system -type f -name "*-source-*" -exec ${K} apply -f {} \;
+
+.PHONY: flux-apply-kustomizations
+flux-apply-kustomizations: ## Apply FluxCD Kustomizations located inside "./resources/flux-system" folder.
+	find ./resources/flux-system -type f -name "ks-*" -exec ${K} apply -f {} \;
+
+.PHONY: flux-apply-helm-releases
+flux-apply-helm-releases: ## Apply HelmReleases located inside "./resources" folder.
+	find ./resources -type f -name "flux-hr-*" -exec ${K} apply -f {} \;
+
+PHONY: flux-apply-all
+flux-apply-all: flux-apply-sources flux-apply-kustomizations flux-apply-helm-releases ## Apply FluxCD Sources, Kustomizations and HelmReleases, all at once.
+
+.PHONY: flux-update
+flux-update: ## Update FluxCD manifests, generate Kustomizations and HelmReleases (requires flux cli).
+	./hack/flux-update.bash

infra/aws/terraform/prow-build-cluster/README.md

@@ -243,3 +243,63 @@ make destroy
 
 If you want to remove roles used for EKS creation go to `../iam/<aws_account_name>` and run `terraform destroy` command there.
 
+
+## GitOps
+
+To sync state from this Git repo into EKS cluster we use [FluxCD](https://fluxcd.io/).
+
+FluxCD has a dedicated [CLI tool](https://fluxcd.io/flux/installation/#install-the-flux-cli). We use it iniside our scripts to generate and monitor syncs.
+
+FluxCD resources stored inside the `./resources` directory and have been generated with use of `./hack/flux-update.bash`. The script prepares manifests for the [GitOps Tool Kit](https://fluxcd.io/flux/components/) and generates [Sources](https://fluxcd.io/flux/components/source/), [Kustomizations](https://fluxcd.io/flux/components/kustomize/kustomization/) and [Helm Releases](https://fluxcd.io/flux/components/helm/helmreleases/).
+
+The `flux-system` namespace contains all GitOps Tool Kit componenets as well as all Flux Sources and Kustomizations. Helm Releases ought to be deployed in the same namespaces as manifests they create. As a convention, Flux Helm Releases have to be prefixed with `flux-hr-`, Kustomizations with `ks-` and sources should contain `-source-` part. These are our internal conventions and are used for discovery process in our scripts:
+
+- Installing Flux:
+    ```bash
+    make flux-install
+    ```
+
+- Deploying Sources:
+    ```bash
+    make flux-apply-sources
+    ```
+
+- Deploying Kustomizations:
+    ```bash
+    make flux-apply-kustomizations
+    ```
+
+- Deploying Helm Releases:
+    ```bash
+    make flux-apply-helm
+    ```
+
+### Accessing Flux
+
+In order to access FluxCD, all you need is kubeconfig with broad cluster permissions and Flux CLI installed locally.
+
+### Adding New Kustomization
+
+1. Go to `./resources` directory and create a new folder containing your manifests.
+2. Open `./hack/flux-update.bash` in your editor of choice.
+3. Find `kustomizations` variable. It contains a list of kustomizations to generate. Each element of list corresonds to directory inside `./resources` folder.
+4. Extend the kustomization list with the name of folder you created in the first step.
+5. Regenerate kustomizations by running: `make flux-update`
+6. Commit your changes and wait for your resources to appear in the cluster. You can use `flux get ks <kustomization-name>` to monitor state.
+
+### Adding New Helm Release
+
+1. Open `./hack/flux-update.bash` in your editor of choice.
+2. Create a new helm source. If you already have helm source, you can move to the next step.
+    1. Find part of script responsible for createing `eks-charts` helm release.
+    2. Use the same pattern to create a new helm source.
+3. Create a new helm release.
+    1. Find part of script responsible for creating helm release for `node-termination-handler`.
+    2. Based on `node-termination-handler` example extend the script with your helm release.
+4. Regenerate helm-releases by running: `make flux-update`
+5. Apply new helm release by running: `make flux-apply-helm-releases`
+6. Check the status by executing: `flux get hr -n <namespace> <helm_release_name>` or `flux get hr -A` to list all helm releases inside the cluster.
+
+### Monitoring Flux
+
+__TODO__

infra/aws/terraform/prow-build-cluster/hack/flux-update.bash

@@ -0,0 +1,96 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+set -xeuo pipefail
+
+# TODO(pkprzekwas): point at k8s.io main
+github_org=pkprzekwas
+github_repo=k8s.io
+github_branch=eks-prow-build-cluster-gitops
+
+if ! command -v flux &> /dev/null
+then
+  echo "flux could not be found"
+  exit 2
+fi
+
+script_root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
+
+function boilerplate() {
+  cat ${script_root}/../../../../../hack/boilerplate/boilerplate.sh.txt | sed -e "s/\<YEAR\>/$(date +'%Y')/"
+  echo "# Code generated by make flux-update. DO NOT EDIT."
+}
+
+resources_dir=${script_root}/../resources
+
+# Generate all FluxCD resources.
+# gotk stands for GitOpsToolKit (https://fluxcd.io/flux/components/)
+boilerplate > ${resources_dir}/flux-system/gotk-components.yaml
+flux install --export >> ${resources_dir}/flux-system/gotk-components.yaml
+
+# sync_interval determines Flux Source Controller sync interval.
+# (https://github.com/fluxcd/source-controller)
+sync_interval=5m
+
+boilerplate > ${resources_dir}/flux-system/git-source-k8s.io.yaml
+flux create source git k8s-io \
+    --url=https://github.com/${github_org}/k8s.io \
+    --branch=${github_branch} \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/flux-system/git-source-k8s.io.yaml
+
+boilerplate > ${resources_dir}/flux-system/helm-source-eks-charts.yaml
+flux create source helm eks-charts \
+    --url=https://aws.github.io/eks-charts \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/flux-system/helm-source-eks-charts.yaml
+
+boilerplate > ${resources_dir}/kube-system/flux-hr-node-termination-handler.yaml
+flux create hr node-termination-handler \
+    --source=HelmRepository/eks-charts.flux-system \
+    --namespace=kube-system \
+    --chart=aws-node-termination-handler \
+    --chart-version=0.21.0 \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/kube-system/flux-hr-node-termination-handler.yaml
+
+# This list contains names of folders inside ./resources directory
+# that are used for generating FluxCD kustomizations.
+kustomizations=(
+    boskos
+    flux-system
+    kube-system
+    monitoring
+    node-problem-detector
+    rbac
+    test-pods
+    external-secrets
+)
+
+# Code below is used to figure out a relative path of
+# resources dir in relation to the root dir of git repo.
+pushd ${resources_dir} > /dev/null
+resources_git_repo_path=$(git rev-parse --show-prefix)
+popd > /dev/null
+
+for k in "${kustomizations[@]}"; do
+    boilerplate > ${resources_dir}/flux-system/ks-${k}.yaml
+    flux create kustomization ${k} \
+        --source=GitRepository/k8s-io.flux-system \
+        --path=${resources_git_repo_path}/${k} \
+        --interval=5m \
+        --prune \
+        --export >> ${resources_dir}/flux-system/ks-${k}.yaml
+done

infra/aws/terraform/prow-build-cluster/resources/external-secrets/namespace.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: external-secrets

infra/aws/terraform/prow-build-cluster/resources/flux-system/git-source-k8s.io.yaml

@@ -0,0 +1,26 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code generated by make flux-update. DO NOT EDIT.
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: k8s-io
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  ref:
+    branch: eks-prow-build-cluster-gitops
+  url: https://github.com/pkprzekwas/k8s.io