fix: allow specific AZs for EKS (#181)

* fix: allow specific AZs for EKS

This PR address the need of customizing the AZs used by the EKS cluster.
Related to
camunda/team-infrastructure-experience#439

modules/eks-cluster/README.md

@@ -51,6 +51,8 @@ module "eks_cluster" {
 |------|-------------|------|---------|:--------:|
 | <a name="input_access_entries"></a> [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster. | `any` | `{}` | no |
 | <a name="input_authentication_mode"></a> [authentication\_mode](#input\_authentication\_mode) | The authentication mode for the cluster. | `string` | `"API"` | no |
+| <a name="input_availability_zones"></a> [availability\_zones](#input\_availability\_zones) | A list of availability zone names in the region. By default, this is set to `null` and is not used; instead, `availability_zones_count` manages the number of availability zones. This value should not be updated directly. To make changes, please create a new resource. | `list(string)` | `null` | no |
+| <a name="input_availability_zones_count"></a> [availability\_zones\_count](#input\_availability\_zones\_count) | The count of availability zones to utilize within the specified AWS Region, where pairs of public and private subnets will be generated (minimum is `2`). Valid only when availability\_zones variable is not provided. | `number` | `3` | no |
 | <a name="input_cluster_node_ipv4_cidr"></a> [cluster\_node\_ipv4\_cidr](#input\_cluster\_node\_ipv4\_cidr) | The CIDR block for public and private subnets of loadbalancers and nodes. Between /28 and /16. | `string` | `"10.192.0.0/16"` | no |
 | <a name="input_cluster_service_ipv4_cidr"></a> [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. Between /24 and /12. | `string` | `"10.190.0.0/16"` | no |
 | <a name="input_cluster_tags"></a> [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no |
@@ -90,6 +92,7 @@ module "eks_cluster" {
 | <a name="output_private_subnet_ids"></a> [private\_subnet\_ids](#output\_private\_subnet\_ids) | Private subnet IDs |
 | <a name="output_private_vpc_cidr_blocks"></a> [private\_vpc\_cidr\_blocks](#output\_private\_vpc\_cidr\_blocks) | Private VPC CIDR blocks |
 | <a name="output_public_vpc_cidr_blocks"></a> [public\_vpc\_cidr\_blocks](#output\_public\_vpc\_cidr\_blocks) | Public VPC CIDR blocks |
+| <a name="output_vpc_azs"></a> [vpc\_azs](#output\_vpc\_azs) | VPC AZs of the cluster |
 | <a name="output_vpc_id"></a> [vpc\_id](#output\_vpc\_id) | VPC id of the cluster |
 | <a name="output_vpc_main_route_table_id"></a> [vpc\_main\_route\_table\_id](#output\_vpc\_main\_route\_table\_id) | The ID of the main route table associated with this VPC |
 <!-- END_TF_DOCS -->

modules/eks-cluster/outputs.tf

@@ -96,6 +96,11 @@ output "vpc_id" {
   value       = module.vpc.vpc_id
 }
 
+output "vpc_azs" {
+  description = "VPC AZs of the cluster"
+  value       = module.vpc.azs
+}
+
 output "private_vpc_cidr_blocks" {
   value       = module.vpc.private_subnets_cidr_blocks
   description = "Private VPC CIDR blocks"

modules/eks-cluster/variables.tf

@@ -105,3 +105,15 @@ variable "create_ebs_gp3_default_storage_class" {
   default     = true
   description = "Flag to determine if the kubernetes_storage_class should be created using EBS-CSI and set on GP3 by default. Set to 'false' to skip creating the storage class, useful for avoiding dependency issues during EKS cluster deletion."
 }
+
+variable "availability_zones_count" {
+  type        = number
+  description = "The count of availability zones to utilize within the specified AWS Region, where pairs of public and private subnets will be generated (minimum is `2`). Valid only when availability_zones variable is not provided."
+  default     = 3
+}
+
+variable "availability_zones" {
+  type        = list(string)
+  description = "A list of availability zone names in the region. By default, this is set to `null` and is not used; instead, `availability_zones_count` manages the number of availability zones. This value should not be updated directly. To make changes, please create a new resource."
+  default     = null
+}

modules/eks-cluster/vpc.tf

@@ -2,6 +2,23 @@ locals {
   vpc_name = "${var.name}-vpc"
 }
 
+locals {
+  # Generate the list of availability zones
+  azs = var.availability_zones != null ? var.availability_zones : [
+    for index in range(var.availability_zones_count) : "${var.region}${["a", "b", "c", "d", "e", "f"][index]}"
+  ]
+
+  # Private subnets for nodes
+  private_subnets = [
+    for index in range(length(local.azs)) : cidrsubnet(var.cluster_node_ipv4_cidr, 3, index)
+  ]
+
+  public_subnets = [
+    for index in range(length(local.azs)) : cidrsubnet(var.cluster_node_ipv4_cidr, 3, index + length(local.azs))
+  ]
+}
+
+
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "5.15.0"
@@ -11,16 +28,17 @@ module "vpc" {
   # AWS supports between /16 and 28
   cidr = var.cluster_node_ipv4_cidr
 
-  azs = ["${var.region}a", "${var.region}b", "${var.region}c"]
+  azs = local.azs
+
+  # Private subnets for nodes
+  private_subnets = local.private_subnets
+
+  # Public subnets for Load Balancers
+  public_subnets = local.public_subnets
 
-  # private subnets for nodes
-  private_subnets = [cidrsubnet(var.cluster_node_ipv4_cidr, 3, 0), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 1), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 2)]
   private_subnet_tags = {
     "kubernetes.io/role/internal-elb" = 1
   }
-
-  # public subnet for loadbalancers
-  public_subnets = [cidrsubnet(var.cluster_node_ipv4_cidr, 3, 3), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 4), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 5)]
   public_subnet_tags = {
     "kubernetes.io/role/elb" = 1
   }

test/src/custom_eks_opensearch_test.go

@@ -85,6 +85,8 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() {
 		"name":                  suite.clusterName,
 		"region":                suite.region,
 		"np_desired_node_count": suite.expectedNodes,
+		// we test the usage of a two zones (minimum)
+		"availability_zones_count": 2,
 	}
 
 	suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf)
@@ -126,6 +128,9 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() {
 	// idempotency test
 	terraform.InitAndApply(suite.T(), terraformOptions)
 
+	expectedVpcAZs := fmt.Sprintf("[%sa %sb]", suite.varTf["region"], suite.varTf["region"]) // must match availability_zones_count
+	suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs"))
+
 	sess, err := utils.GetAwsClient()
 	suite.Require().NoErrorf(err, "Failed to get aws client")
 
@@ -218,11 +223,13 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() {
 	}
 
 	varsConfigOpenSearch := map[string]interface{}{
-		"domain_name":             opensearchDomainName,
-		"subnet_ids":              result.Cluster.ResourcesVpcConfig.SubnetIds,
-		"cidr_blocks":             append(publicBlocks, privateBlocks...),
-		"vpc_id":                  *result.Cluster.ResourcesVpcConfig.VpcId,
-		"iam_roles_with_policies": iamRolesWithPolicies,
+		"domain_name":                            opensearchDomainName,
+		"subnet_ids":                             result.Cluster.ResourcesVpcConfig.SubnetIds,
+		"cidr_blocks":                            append(publicBlocks, privateBlocks...),
+		"vpc_id":                                 *result.Cluster.ResourcesVpcConfig.VpcId,
+		"iam_roles_with_policies":                iamRolesWithPolicies,
+		"zone_awareness_availability_zone_count": suite.varTf["availability_zones_count"], // must match VPC AZs of EKS
+		"instance_count":                         2,                                       // we must choose an even number of data nodes for a two Availability Zone deployment
 	}
 
 	tfModuleOpenSearch := "opensearch/"
@@ -270,7 +277,7 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() {
 
 	// Perform assertions on the OpenSearch domain configuration
 	suite.Assert().Equal(varsConfigOpenSearch["domain_name"].(string), *describeOpenSearchDomainOutput.DomainStatus.DomainName)
-	suite.Assert().Equal(int32(3), *describeOpenSearchDomainOutput.DomainStatus.ClusterConfig.InstanceCount)
+	suite.Assert().Equal(int32(2), *describeOpenSearchDomainOutput.DomainStatus.ClusterConfig.InstanceCount)
 	suite.Assert().Equal(types.OpenSearchPartitionInstanceType("t3.small.search"), describeOpenSearchDomainOutput.DomainStatus.ClusterConfig.InstanceType)
 	suite.Assert().Equal(varsConfigOpenSearch["vpc_id"].(string), *describeOpenSearchDomainOutput.DomainStatus.VPCOptions.VPCId)
 

test/src/custom_eks_rds_test.go

@@ -84,6 +84,8 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() {
 		"name":                  suite.clusterName,
 		"region":                suite.region,
 		"np_desired_node_count": suite.expectedNodes,
+		// we test the definition of specific AZs, also RDS requires exactly 3AZs
+		"availability_zones": []string{fmt.Sprintf("%sa", suite.region), fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)},
 	}
 
 	suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf)
@@ -125,6 +127,10 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() {
 	// idempotency test
 	terraform.InitAndApply(suite.T(), terraformOptions)
 
+	// basic tests after terraform apply
+	expectedVpcAZs := fmt.Sprintf("[%sa %sb %sc]", suite.varTf["region"], suite.varTf["region"], suite.varTf["region"])
+	suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs"))
+
 	sess, err := utils.GetAwsClient()
 	suite.Require().NoErrorf(err, "Failed to get aws client")
 
@@ -227,7 +233,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() {
 		"cluster_name":            auroraClusterName,
 		"subnet_ids":              result.Cluster.ResourcesVpcConfig.SubnetIds,
 		"vpc_id":                  *result.Cluster.ResourcesVpcConfig.VpcId,
-		"availability_zones":      []string{fmt.Sprintf("%sa", suite.region), fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)},
+		"availability_zones":      suite.varTf["availability_zones"], // we must match the zones of the EKS cluster
 		"cidr_blocks":             append(publicBlocks, privateBlocks...),
 		"iam_auth_enabled":        true,
 		"iam_roles_with_policies": iamRolesWithPolicies,

test/src/default_eks_test.go

@@ -141,6 +141,7 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti
 	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "ebs_cs_arn"))
 	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "external_dns_arn"))
 	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "vpc_id"))
+	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "vpc_azs"))
 	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_vpc_cidr_blocks"))
 	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_subnet_ids"))
 	suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "default_security_group_id"))
@@ -159,6 +160,9 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti
 	expectedPublicVpcCidrBlocks := "[10.192.96.0/19 10.192.128.0/19 10.192.160.0/19]"
 	suite.Assert().Equal(expectedPublicVpcCidrBlocks, terraform.Output(suite.T(), terraformOptions, "public_vpc_cidr_blocks"))
 
+	expectedVpcAZs := fmt.Sprintf("[%sa %sb %sc]", suite.varTf["region"], suite.varTf["region"], suite.varTf["region"])
+	suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs"))
+
 	sess, err := utils.GetAwsClient()
 	suite.Require().NoErrorf(err, "Failed to get aws client")
 

test/src/upgrade_eks_test.go

@@ -80,6 +80,8 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() {
 	suite.varTf = map[string]interface{}{
 		"name":   suite.clusterName,
 		"region": suite.region,
+		// we test the definition of specific AZs, 2 in this case
+		"availability_zones": []string{fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)},
 
 		"np_desired_node_count": suite.expectedNodes,
 
@@ -145,6 +147,10 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() {
 
 	suite.Assert().Equal(suite.kubeVersion, *result.Cluster.Version)
 
+	// test the custom AZs definition
+	expectedVpcAZs := fmt.Sprintf("[%sb %sc]", suite.varTf["region"], suite.varTf["region"])
+	suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs"))
+
 	utils.GenerateKubeConfigFromAWS(suite.T(), suite.region, suite.clusterName, utils.GetAwsProfile(), suite.kubeConfigPath)
 
 	// test suite: deploy a pod and check it is healthy
@@ -224,6 +230,10 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() {
 	suite.Assert().NoError(err)
 	suite.Assert().Equal(suite.varTf["kubernetes_version"], *result.Cluster.Version)
 
+	// test the custom AZs definition is not changed
+	expectedVpcAZs = fmt.Sprintf("[%sb %sc]", suite.varTf["region"], suite.varTf["region"])
+	suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs"))
+
 	// check everything works as expected
 	k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second)
 	// wait to ensure service available