Merge pull request #5140 from giantswarm/support-maxhealthypercentage

✨ Support setting maxHealthyPercentage to configure ASG instance refresh

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml

@@ -969,6 +969,18 @@ spec:
  The default is to use the value for the health check grace period defined for the group.
  format: int64
  type: integer
+ maxHealthyPercentage:
+ description: |-
+ The amount of capacity as a percentage in ASG that can be in service and healthy, or pending,
+ to support your workload when replacing instances.
+ The value is expressed as a percentage of the desired capacity of the ASG. Value range is 100 to 200.
+ If you specify MaxHealthyPercentage , you must also specify MinHealthyPercentage , and the difference between
+ them cannot be greater than 100.
+ A larger range increases the number of instances that can be replaced at the same time.
+ format: int64
+ maximum: 200
+ minimum: 100
+ type: integer
  minHealthyPercentage:
  description: |-
  The amount of capacity as a percentage in ASG that must remain healthy

exp/api/v1beta1/conversion.go

@@ -42,8 +42,9 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
  if restored.Spec.SuspendProcesses != nil {
  dst.Spec.SuspendProcesses = restored.Spec.SuspendProcesses
  }
- if dst.Spec.RefreshPreferences != nil && restored.Spec.RefreshPreferences != nil {
+ if restored.Spec.RefreshPreferences != nil {
  dst.Spec.RefreshPreferences.Disable = restored.Spec.RefreshPreferences.Disable
+ dst.Spec.RefreshPreferences.MaxHealthyPercentage = restored.Spec.RefreshPreferences.MaxHealthyPercentage
  }
  if restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions != nil {
  dst.Spec.AWSLaunchTemplate.InstanceMetadataOptions = restored.Spec.AWSLaunchTemplate.InstanceMetadataOptions

exp/api/v1beta2/awsmachinepool_types.go

@@ -172,6 +172,17 @@ type RefreshPreferences struct {
  // during an instance refresh. The default is 90.
  // +optional
  MinHealthyPercentage *int64 `json:"minHealthyPercentage,omitempty"`
+
+ // The amount of capacity as a percentage in ASG that can be in service and healthy, or pending,
+ // to support your workload when replacing instances.
+ // The value is expressed as a percentage of the desired capacity of the ASG. Value range is 100 to 200.
+ // If you specify MaxHealthyPercentage , you must also specify MinHealthyPercentage , and the difference between
+ // them cannot be greater than 100.
+ // A larger range increases the number of instances that can be replaced at the same time.
+ // +optional
+ // +kubebuilder:validation:Minimum=100
+ // +kubebuilder:validation:Maximum=200
+ MaxHealthyPercentage *int64 `json:"maxHealthyPercentage,omitempty"`
 }
 
 // AWSMachinePoolStatus defines the observed state of AWSMachinePool.

exp/api/v1beta2/awsmachinepool_webhook.go

@@ -133,6 +133,7 @@ func (r *AWSMachinePool) validateAdditionalSecurityGroups() field.ErrorList {
  }
  return allErrs
 }
+
 func (r *AWSMachinePool) validateSpotInstances() field.ErrorList {
  var allErrs field.ErrorList
  if r.Spec.AWSLaunchTemplate.SpotMarketOptions != nil && r.Spec.MixedInstancesPolicy != nil {
@@ -141,6 +142,26 @@ func (r *AWSMachinePool) validateSpotInstances() field.ErrorList {
  return allErrs
 }
 
+func (r *AWSMachinePool) validateRefreshPreferences() field.ErrorList {
+ var allErrs field.ErrorList
+
+ if r.Spec.RefreshPreferences == nil {
+ return allErrs
+ }
+
+ if r.Spec.RefreshPreferences.MaxHealthyPercentage != nil && r.Spec.RefreshPreferences.MinHealthyPercentage == nil {
+ allErrs = append(allErrs, field.Forbidden(field.NewPath("spec.refreshPreferences.maxHealthyPercentage"), "If you specify spec.refreshPreferences.maxHealthyPercentage, you must also specify spec.refreshPreferences.minHealthyPercentage"))
+ }
+
+ if r.Spec.RefreshPreferences.MaxHealthyPercentage != nil && r.Spec.RefreshPreferences.MinHealthyPercentage != nil {
+ if *r.Spec.RefreshPreferences.MaxHealthyPercentage-*r.Spec.RefreshPreferences.MinHealthyPercentage > 100 {
+ allErrs = append(allErrs, field.Forbidden(field.NewPath("spec.refreshPreferences.maxHealthyPercentage"), "the difference between spec.refreshPreferences.maxHealthyPercentage and spec.refreshPreferences.minHealthyPercentage cannot be greater than 100"))
+ }
+ }
+
+ return allErrs
+}
+
 // ValidateCreate will do any extra validation when creating a AWSMachinePool.
 func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
  log.Info("AWSMachinePool validate create", "machine-pool", klog.KObj(r))
@@ -154,6 +175,7 @@ func (r *AWSMachinePool) ValidateCreate() (admission.Warnings, error) {
  allErrs = append(allErrs, r.validateSubnets()...)
  allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...)
  allErrs = append(allErrs, r.validateSpotInstances()...)
+ allErrs = append(allErrs, r.validateRefreshPreferences()...)
 
  if len(allErrs) == 0 {
  return nil, nil
@@ -175,6 +197,7 @@ func (r *AWSMachinePool) ValidateUpdate(_ runtime.Object) (admission.Warnings, e
  allErrs = append(allErrs, r.validateSubnets()...)
  allErrs = append(allErrs, r.validateAdditionalSecurityGroups()...)
  allErrs = append(allErrs, r.validateSpotInstances()...)
+ allErrs = append(allErrs, r.validateRefreshPreferences()...)
 
  if len(allErrs) == 0 {
  return nil, nil

exp/api/v1beta2/awsmachinepool_webhook_test.go

@@ -153,6 +153,27 @@ func TestAWSMachinePoolValidateCreate(t *testing.T) {
  },
  wantErr: true,
  },
+ {
+ name: "Should fail if MaxHealthyPercentage is set, but MinHealthyPercentage is not set",
+ pool: &AWSMachinePool{
+ Spec: AWSMachinePoolSpec{
+ RefreshPreferences: &RefreshPreferences{MaxHealthyPercentage: aws.Int64(100)},
+ },
+ },
+ wantErr: true,
+ },
+ {
+ name: "Should fail if the difference between MaxHealthyPercentage and MinHealthyPercentage is greater than 100",
+ pool: &AWSMachinePool{
+ Spec: AWSMachinePoolSpec{
+ RefreshPreferences: &RefreshPreferences{
+ MaxHealthyPercentage: aws.Int64(150),
+ MinHealthyPercentage: aws.Int64(25),
+ },
+ },
+ },
+ wantErr: true,
+ },
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
@@ -287,6 +308,27 @@ func TestAWSMachinePoolValidateUpdate(t *testing.T) {
  },
  wantErr: true,
  },
+ {
+ name: "Should fail if MaxHealthyPercentage is set, but MinHealthyPercentage is not set",
+ new: &AWSMachinePool{
+ Spec: AWSMachinePoolSpec{
+ RefreshPreferences: &RefreshPreferences{MaxHealthyPercentage: aws.Int64(100)},
+ },
+ },
+ wantErr: true,
+ },
+ {
+ name: "Should fail if the difference between MaxHealthyPercentage and MinHealthyPercentage is greater than 100",
+ new: &AWSMachinePool{
+ Spec: AWSMachinePoolSpec{
+ RefreshPreferences: &RefreshPreferences{
+ MaxHealthyPercentage: aws.Int64(150),
+ MinHealthyPercentage: aws.Int64(25),
+ },
+ },
+ },
+ wantErr: true,
+ },
  }
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {

pkg/cloud/services/autoscaling/autoscalinggroup.go

@@ -349,7 +349,7 @@ func (s *Service) CanStartASGInstanceRefresh(scope *scope.MachinePoolScope) (boo
 // StartASGInstanceRefresh will start an ASG instance with refresh.
 func (s *Service) StartASGInstanceRefresh(scope *scope.MachinePoolScope) error {
  strategy := ptr.To[string](autoscaling.RefreshStrategyRolling)
- var minHealthyPercentage, instanceWarmup *int64
+ var minHealthyPercentage, maxHealthyPercentage, instanceWarmup *int64
  if scope.AWSMachinePool.Spec.RefreshPreferences != nil {
  if scope.AWSMachinePool.Spec.RefreshPreferences.Strategy != nil {
  strategy = scope.AWSMachinePool.Spec.RefreshPreferences.Strategy
@@ -360,6 +360,9 @@ func (s *Service) StartASGInstanceRefresh(scope *scope.MachinePoolScope) error {
  if scope.AWSMachinePool.Spec.RefreshPreferences.MinHealthyPercentage != nil {
  minHealthyPercentage = scope.AWSMachinePool.Spec.RefreshPreferences.MinHealthyPercentage
  }
+ if scope.AWSMachinePool.Spec.RefreshPreferences.MaxHealthyPercentage != nil {
+ maxHealthyPercentage = scope.AWSMachinePool.Spec.RefreshPreferences.MaxHealthyPercentage
+ }
  }
 
  input := &autoscaling.StartInstanceRefreshInput{
@@ -368,6 +371,7 @@ func (s *Service) StartASGInstanceRefresh(scope *scope.MachinePoolScope) error {
  Preferences: &autoscaling.RefreshPreferences{
  InstanceWarmup: instanceWarmup,
  MinHealthyPercentage: minHealthyPercentage,
+ MaxHealthyPercentage: maxHealthyPercentage,
  },
  }
 

pkg/cloud/services/autoscaling/autoscalinggroup_test.go

@@ -1203,6 +1203,7 @@ func TestServiceStartASGInstanceRefresh(t *testing.T) {
  Preferences: &autoscaling.RefreshPreferences{
  InstanceWarmup: aws.Int64(100),
  MinHealthyPercentage: aws.Int64(80),
+ MaxHealthyPercentage: aws.Int64(100),
  },
  })).
  Return(nil, awserrors.NewNotFound("not found"))
@@ -1218,6 +1219,7 @@ func TestServiceStartASGInstanceRefresh(t *testing.T) {
  Preferences: &autoscaling.RefreshPreferences{
  InstanceWarmup: aws.Int64(100),
  MinHealthyPercentage: aws.Int64(80),
+ MaxHealthyPercentage: aws.Int64(100),
  },
  })).
  Return(&autoscaling.StartInstanceRefreshOutput{}, nil)
@@ -1312,6 +1314,7 @@ func getMachinePoolScope(client client.Client, clusterScope *scope.ClusterScope)
  Strategy: aws.String("Rolling"),
  InstanceWarmup: aws.Int64(100),
  MinHealthyPercentage: aws.Int64(80),
+ MaxHealthyPercentage: aws.Int64(100),
  },
  MixedInstancesPolicy: &expinfrav1.MixedInstancesPolicy{
  InstancesDistribution: &expinfrav1.InstancesDistribution{