diff --git a/modules/eks-cluster/README.md b/modules/eks-cluster/README.md index 77c34f4..24c62f0 100644 --- a/modules/eks-cluster/README.md +++ b/modules/eks-cluster/README.md @@ -51,6 +51,8 @@ module "eks_cluster" { |------|-------------|------|---------|:--------:| | [access\_entries](#input\_access\_entries) | Map of access entries to add to the cluster. | `any` | `{}` | no | | [authentication\_mode](#input\_authentication\_mode) | The authentication mode for the cluster. | `string` | `"API"` | no | +| [availability\_zones](#input\_availability\_zones) | A list of availability zone names in the region. By default, this is set to `null` and is not used; instead, `availability_zones_count` manages the number of availability zones. This value should not be updated directly. To make changes, please create a new resource. | `list(string)` | `null` | no | +| [availability\_zones\_count](#input\_availability\_zones\_count) | The count of availability zones to utilize within the specified AWS Region, where pairs of public and private subnets will be generated (minimum is `2`). Valid only when availability\_zones variable is not provided. | `number` | `3` | no | | [cluster\_node\_ipv4\_cidr](#input\_cluster\_node\_ipv4\_cidr) | The CIDR block for public and private subnets of loadbalancers and nodes. Between /28 and /16. | `string` | `"10.192.0.0/16"` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. Between /24 and /12. | `string` | `"10.190.0.0/16"` | no | | [cluster\_tags](#input\_cluster\_tags) | A map of additional tags to add to the cluster | `map(string)` | `{}` | no | @@ -90,6 +92,7 @@ module "eks_cluster" { | [private\_subnet\_ids](#output\_private\_subnet\_ids) | Private subnet IDs | | [private\_vpc\_cidr\_blocks](#output\_private\_vpc\_cidr\_blocks) | Private VPC CIDR blocks | | [public\_vpc\_cidr\_blocks](#output\_public\_vpc\_cidr\_blocks) | Public VPC CIDR blocks | +| [vpc\_azs](#output\_vpc\_azs) | VPC AZs of the cluster | | [vpc\_id](#output\_vpc\_id) | VPC id of the cluster | | [vpc\_main\_route\_table\_id](#output\_vpc\_main\_route\_table\_id) | The ID of the main route table associated with this VPC | diff --git a/modules/eks-cluster/outputs.tf b/modules/eks-cluster/outputs.tf index 8a7c217..a56bcfc 100644 --- a/modules/eks-cluster/outputs.tf +++ b/modules/eks-cluster/outputs.tf @@ -96,6 +96,11 @@ output "vpc_id" { value = module.vpc.vpc_id } +output "vpc_azs" { + description = "VPC AZs of the cluster" + value = module.vpc.azs +} + output "private_vpc_cidr_blocks" { value = module.vpc.private_subnets_cidr_blocks description = "Private VPC CIDR blocks" diff --git a/modules/eks-cluster/variables.tf b/modules/eks-cluster/variables.tf index 0587515..56434e6 100644 --- a/modules/eks-cluster/variables.tf +++ b/modules/eks-cluster/variables.tf @@ -105,3 +105,15 @@ variable "create_ebs_gp3_default_storage_class" { default = true description = "Flag to determine if the kubernetes_storage_class should be created using EBS-CSI and set on GP3 by default. Set to 'false' to skip creating the storage class, useful for avoiding dependency issues during EKS cluster deletion." } + +variable "availability_zones_count" { + type = number + description = "The count of availability zones to utilize within the specified AWS Region, where pairs of public and private subnets will be generated (minimum is `2`). Valid only when availability_zones variable is not provided." + default = 3 +} + +variable "availability_zones" { + type = list(string) + description = "A list of availability zone names in the region. By default, this is set to `null` and is not used; instead, `availability_zones_count` manages the number of availability zones. This value should not be updated directly. To make changes, please create a new resource." + default = null +} diff --git a/modules/eks-cluster/vpc.tf b/modules/eks-cluster/vpc.tf index 53aa3d4..48128b5 100644 --- a/modules/eks-cluster/vpc.tf +++ b/modules/eks-cluster/vpc.tf @@ -2,6 +2,23 @@ locals { vpc_name = "${var.name}-vpc" } +locals { + # Generate the list of availability zones + azs = var.availability_zones != null ? var.availability_zones : [ + for index in range(var.availability_zones_count) : "${var.region}${["a", "b", "c", "d", "e", "f"][index]}" + ] + + # Private subnets for nodes + private_subnets = [ + for index in range(length(local.azs)) : cidrsubnet(var.cluster_node_ipv4_cidr, 3, index) + ] + + public_subnets = [ + for index in range(length(local.azs)) : cidrsubnet(var.cluster_node_ipv4_cidr, 3, index + length(local.azs)) + ] +} + + module "vpc" { source = "terraform-aws-modules/vpc/aws" version = "5.15.0" @@ -11,16 +28,17 @@ module "vpc" { # AWS supports between /16 and 28 cidr = var.cluster_node_ipv4_cidr - azs = ["${var.region}a", "${var.region}b", "${var.region}c"] + azs = local.azs + + # Private subnets for nodes + private_subnets = local.private_subnets + + # Public subnets for Load Balancers + public_subnets = local.public_subnets - # private subnets for nodes - private_subnets = [cidrsubnet(var.cluster_node_ipv4_cidr, 3, 0), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 1), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 2)] private_subnet_tags = { "kubernetes.io/role/internal-elb" = 1 } - - # public subnet for loadbalancers - public_subnets = [cidrsubnet(var.cluster_node_ipv4_cidr, 3, 3), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 4), cidrsubnet(var.cluster_node_ipv4_cidr, 3, 5)] public_subnet_tags = { "kubernetes.io/role/elb" = 1 } diff --git a/test/src/custom_eks_opensearch_test.go b/test/src/custom_eks_opensearch_test.go index f27b637..f9bb655 100644 --- a/test/src/custom_eks_opensearch_test.go +++ b/test/src/custom_eks_opensearch_test.go @@ -85,6 +85,8 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() { "name": suite.clusterName, "region": suite.region, "np_desired_node_count": suite.expectedNodes, + // we test the usage of a two zones (minimum) + "availability_zones_count": 2, } suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) @@ -126,6 +128,9 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() { // idempotency test terraform.InitAndApply(suite.T(), terraformOptions) + expectedVpcAZs := fmt.Sprintf("[%sa %sb]", suite.varTf["region"], suite.varTf["region"]) // must match availability_zones_count + suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs")) + sess, err := utils.GetAwsClient() suite.Require().NoErrorf(err, "Failed to get aws client") @@ -218,11 +223,13 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() { } varsConfigOpenSearch := map[string]interface{}{ - "domain_name": opensearchDomainName, - "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, - "cidr_blocks": append(publicBlocks, privateBlocks...), - "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, - "iam_roles_with_policies": iamRolesWithPolicies, + "domain_name": opensearchDomainName, + "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, + "cidr_blocks": append(publicBlocks, privateBlocks...), + "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, + "iam_roles_with_policies": iamRolesWithPolicies, + "zone_awareness_availability_zone_count": suite.varTf["availability_zones_count"], // must match VPC AZs of EKS + "instance_count": 2, // we must choose an even number of data nodes for a two Availability Zone deployment } tfModuleOpenSearch := "opensearch/" @@ -270,7 +277,7 @@ func (suite *CustomEKSOpenSearchTestSuite) TestCustomEKSAndOpenSearch() { // Perform assertions on the OpenSearch domain configuration suite.Assert().Equal(varsConfigOpenSearch["domain_name"].(string), *describeOpenSearchDomainOutput.DomainStatus.DomainName) - suite.Assert().Equal(int32(3), *describeOpenSearchDomainOutput.DomainStatus.ClusterConfig.InstanceCount) + suite.Assert().Equal(int32(2), *describeOpenSearchDomainOutput.DomainStatus.ClusterConfig.InstanceCount) suite.Assert().Equal(types.OpenSearchPartitionInstanceType("t3.small.search"), describeOpenSearchDomainOutput.DomainStatus.ClusterConfig.InstanceType) suite.Assert().Equal(varsConfigOpenSearch["vpc_id"].(string), *describeOpenSearchDomainOutput.DomainStatus.VPCOptions.VPCId) diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index f23e1e7..dae22ba 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -84,6 +84,8 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { "name": suite.clusterName, "region": suite.region, "np_desired_node_count": suite.expectedNodes, + // we test the definition of specific AZs, also RDS requires exactly 3AZs + "availability_zones": []string{fmt.Sprintf("%sa", suite.region), fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)}, } suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) @@ -125,6 +127,10 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { // idempotency test terraform.InitAndApply(suite.T(), terraformOptions) + // basic tests after terraform apply + expectedVpcAZs := fmt.Sprintf("[%sa %sb %sc]", suite.varTf["region"], suite.varTf["region"], suite.varTf["region"]) + suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs")) + sess, err := utils.GetAwsClient() suite.Require().NoErrorf(err, "Failed to get aws client") @@ -227,7 +233,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { "cluster_name": auroraClusterName, "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, - "availability_zones": []string{fmt.Sprintf("%sa", suite.region), fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)}, + "availability_zones": suite.varTf["availability_zones"], // we must match the zones of the EKS cluster "cidr_blocks": append(publicBlocks, privateBlocks...), "iam_auth_enabled": true, "iam_roles_with_policies": iamRolesWithPolicies, diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index 9d2907c..8a74ae3 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -141,6 +141,7 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "ebs_cs_arn")) suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "external_dns_arn")) suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "vpc_id")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "vpc_azs")) suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_vpc_cidr_blocks")) suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_subnet_ids")) suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "default_security_group_id")) @@ -159,6 +160,9 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti expectedPublicVpcCidrBlocks := "[10.192.96.0/19 10.192.128.0/19 10.192.160.0/19]" suite.Assert().Equal(expectedPublicVpcCidrBlocks, terraform.Output(suite.T(), terraformOptions, "public_vpc_cidr_blocks")) + expectedVpcAZs := fmt.Sprintf("[%sa %sb %sc]", suite.varTf["region"], suite.varTf["region"], suite.varTf["region"]) + suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs")) + sess, err := utils.GetAwsClient() suite.Require().NoErrorf(err, "Failed to get aws client") diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 1675327..29c9d7c 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -80,6 +80,8 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { suite.varTf = map[string]interface{}{ "name": suite.clusterName, "region": suite.region, + // we test the definition of specific AZs, 2 in this case + "availability_zones": []string{fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)}, "np_desired_node_count": suite.expectedNodes, @@ -145,6 +147,10 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { suite.Assert().Equal(suite.kubeVersion, *result.Cluster.Version) + // test the custom AZs definition + expectedVpcAZs := fmt.Sprintf("[%sb %sc]", suite.varTf["region"], suite.varTf["region"]) + suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs")) + utils.GenerateKubeConfigFromAWS(suite.T(), suite.region, suite.clusterName, utils.GetAwsProfile(), suite.kubeConfigPath) // test suite: deploy a pod and check it is healthy @@ -224,6 +230,10 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { suite.Assert().NoError(err) suite.Assert().Equal(suite.varTf["kubernetes_version"], *result.Cluster.Version) + // test the custom AZs definition is not changed + expectedVpcAZs = fmt.Sprintf("[%sb %sc]", suite.varTf["region"], suite.varTf["region"]) + suite.Assert().Equal(expectedVpcAZs, terraform.Output(suite.T(), terraformOptions, "vpc_azs")) + // check everything works as expected k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) // wait to ensure service available