feat: create module for creating kubernetes cluster

civo-github/templates/gpu-cluster/infrastructure/workspace.yaml

@@ -19,3 +19,5 @@ spec:
       value: "<WORKLOAD_NODE_COUNT>"
     - key: node_type
       value: <WORKLOAD_NODE_TYPE>
+    - key: cluster_type
+      value: "talos"

civo-github/templates/workload-cluster/infrastructure/workspace.yaml

@@ -19,3 +19,5 @@ spec:
       value: "<WORKLOAD_NODE_COUNT>"
     - key: node_type
       value: <WORKLOAD_NODE_TYPE>
+    - key: cluster_type
+      value: "k3s"

civo-github/terraform/civo/modules/civo-cluster/main.tf

@@ -0,0 +1,222 @@
+resource "civo_network" "kubefirst" {
+  label = var.cluster_name
+}
+
+resource "civo_firewall" "kubefirst" {
+  name                 = var.cluster_name
+  network_id           = civo_network.kubefirst.id
+  create_default_rules = true
+}
+
+resource "civo_kubernetes_cluster" "kubefirst" {
+  name             = var.cluster_name
+  network_id       = civo_network.kubefirst.id
+  firewall_id      = civo_firewall.kubefirst.id
+  write_kubeconfig = true
+  cluster_type     = var.cluster_type
+  pools {
+    label      = var.cluster_name
+    size       = var.node_type
+    node_count = var.node_count
+    labels     = var.labels
+  }
+}
+
+resource "vault_generic_secret" "clusters" {
+  path = "secret/clusters/${var.cluster_name}"
+
+  data_json = jsonencode(
+    {
+      kubeconfig              = civo_kubernetes_cluster.kubefirst.kubeconfig
+      client_certificate      = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).users[0].user.client-certificate-data)
+      client_key              = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).users[0].user.client-key-data)
+      cluster_ca_certificate  = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).clusters[0].cluster.certificate-authority-data)
+      host                    = civo_kubernetes_cluster.kubefirst.api_endpoint
+      cluster_name            = var.cluster_name
+      argocd_manager_sa_token = kubernetes_secret_v1.argocd_manager.data.token
+    }
+  )
+}
+
+provider "kubernetes" {
+  host                   = civo_kubernetes_cluster.kubefirst.api_endpoint
+  client_certificate     = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).users[0].user.client-certificate-data)
+  client_key             = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).users[0].user.client-key-data)
+  cluster_ca_certificate = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).clusters[0].cluster.certificate-authority-data)
+}
+
+provider "helm" {
+  repository_config_path = "${path.module}/.helm/repositories.yaml"
+  repository_cache       = "${path.module}/.helm"
+  kubernetes {
+    host                   = civo_kubernetes_cluster.kubefirst.api_endpoint
+    client_certificate     = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).users[0].user.client-certificate-data)
+    client_key             = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).users[0].user.client-key-data)
+    cluster_ca_certificate = base64decode(yamldecode(civo_kubernetes_cluster.kubefirst.kubeconfig).clusters[0].cluster.certificate-authority-data)
+  }
+}
+
+resource "kubernetes_cluster_role_v1" "argocd_manager" {
+  metadata {
+    name = "argocd-manager-role"
+  }
+
+  rule {
+    api_groups = ["*"]
+    resources  = ["*"]
+    verbs      = ["*"]
+  }
+  rule {
+    non_resource_urls = ["*"]
+    verbs             = ["*"]
+  }
+}
+
+
+resource "kubernetes_cluster_role_binding_v1" "argocd_manager" {
+  metadata {
+    name = "argocd-manager-role-binding"
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role_v1.argocd_manager.metadata.0.name
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account_v1.argocd_manager.metadata.0.name
+    namespace = "kube-system"
+  }
+}
+
+resource "kubernetes_service_account_v1" "argocd_manager" {
+  metadata {
+    name      = "argocd-manager"
+    namespace = "kube-system"
+  }
+  secret {
+    name = "argocd-manager-token"
+  }
+}
+
+resource "kubernetes_secret_v1" "argocd_manager" {
+  metadata {
+    name      = "argocd-manager-token"
+    namespace = "kube-system"
+    annotations = {
+      "kubernetes.io/service-account.name" = "argocd-manager"
+    }
+  }
+  type       = "kubernetes.io/service-account-token"
+  depends_on = [kubernetes_service_account_v1.argocd_manager]
+}
+
+resource "kubernetes_namespace_v1" "external_dns" {
+  metadata {
+    name = "external-dns"
+  }
+}
+
+data "vault_generic_secret" "external_dns" {
+  path = "secret/external-dns"
+}
+
+resource "kubernetes_secret_v1" "external_dns" {
+  metadata {
+    name      = "external-dns-secrets"
+    namespace = kubernetes_namespace_v1.external_dns.metadata.0.name
+  }
+  data = {
+    token = data.vault_generic_secret.external_dns.data["token"]
+  }
+  type = "Opaque"
+}
+
+
+resource "kubernetes_namespace_v1" "external_secrets_operator" {
+  metadata {
+    name = "external-secrets-operator"
+  }
+}
+
+resource "kubernetes_namespace_v1" "environment" {
+  metadata {
+    name = var.cluster_name
+  }
+}
+
+data "vault_generic_secret" "docker_config" {
+  path = "secret/dockerconfigjson"
+}
+
+resource "kubernetes_secret_v1" "image_pull" {
+  metadata {
+    name      = "docker-config"
+    namespace = kubernetes_namespace_v1.environment.metadata.0.name
+  }
+
+  data = {
+    ".dockerconfigjson" = data.vault_generic_secret.docker_config.data["dockerconfig"]
+  }
+
+  type = "kubernetes.io/dockerconfigjson"
+}
+
+data "vault_generic_secret" "external_secrets_operator" {
+  path = "secret/atlantis"
+}
+
+resource "kubernetes_secret_v1" "external_secrets_operator_environment" {
+  metadata {
+    name      = "${var.cluster_name}-cluster-vault-bootstrap"
+    namespace = kubernetes_namespace_v1.environment.metadata.0.name
+  }
+  data = {
+    vault-token = data.vault_generic_secret.external_secrets_operator.data["VAULT_TOKEN"]
+  }
+  type = "Opaque"
+}
+
+resource "kubernetes_secret_v1" "external_secrets_operator" {
+  metadata {
+    name      = "${var.cluster_name}-cluster-vault-bootstrap"
+    namespace = kubernetes_namespace_v1.external_secrets_operator.metadata.0.name
+  }
+  data = {
+    vault-token = data.vault_generic_secret.external_secrets_operator.data["VAULT_TOKEN"]
+  }
+  type = "Opaque"
+}
+
+resource "kubernetes_service_account_v1" "external_secrets" {
+  metadata {
+    name      = "external-secrets"
+    namespace = kubernetes_namespace_v1.external_secrets_operator.metadata.0.name
+  }
+  secret {
+    name = "external-secrets-token"
+  }
+}
+
+resource "kubernetes_secret_v1" "external_secrets" {
+  metadata {
+    name      = "external-secrets-token"
+    namespace = kubernetes_namespace_v1.external_secrets_operator.metadata.0.name
+    annotations = {
+      "kubernetes.io/service-account.name" = "external-secrets"
+    }
+  }
+  type       = "kubernetes.io/service-account-token"
+  depends_on = [kubernetes_service_account_v1.external_secrets]
+}
+
+resource "kubernetes_config_map" "kubefirst_cm" {
+  metadata {
+    name      = "kubefirst-cm"
+    namespace = "kube-system"
+  }
+
+  data = {
+    mgmt_cluster_id = var.mgmt_cluster_id
+  }
+}

civo-github/terraform/civo/modules/civo-cluster/output.tf

@@ -0,0 +1,7 @@
+output "kubeconfig" {
+  value = civo_kubernetes_cluster.kubefirst.kubeconfig
+}
+
+output "api_endpoint" {
+  value = civo_kubernetes_cluster.kubefirst.api_endpoint
+}

civo-github/terraform/civo/modules/civo-cluster/provider.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    civo = {
+      source  = "civo/civo"
+      version = "~> 1.1.0"
+    }
+  }
+}

civo-github/terraform/civo/modules/civo-cluster/variables.tf

@@ -0,0 +1,32 @@
+variable "cluster_name" {
+  type = string
+}
+
+variable "cluster_region" {
+  type = string
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "node_count" {
+  type = number
+}
+
+variable "node_type" {
+  type = string
+}
+
+variable "cluster_type" {
+  type        = string
+  description = "type of cluster talos/k3s"
+}
+
+variable "labels" {
+  type = map(string)
+}
+
+variable "mgmt_cluster_id" {
+  type = string
+}