Skip to content

Commit

Permalink
Fix for launch templates
Browse files Browse the repository at this point in the history 
- Move launch template functions to their own file
- Figure out which launch template needed to be used
- Minor changes to enable the launch template to reconcile in capa
  • Loading branch information
@mproffitt
mproffitt committed Nov 10, 2023
1 parent 5f6d843 commit 0481845
Show file tree
Hide file tree
Showing 7 changed files with 413 additions and 294 deletions.
170 changes: 38 additions & 132 deletions awsapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
asg "github.com/aws/aws-sdk-go-v2/service/autoscaling"
asgtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/aws/aws-sdk-go-v2/service/eks"
"github.com/aws/aws-sdk-go-v2/service/eks/types"
"github.com/crossplane/crossplane-runtime/pkg/errors"
Expand All @@ -18,7 +17,6 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
infrav2 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
expinfrav2 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
capiinfra "sigs.k8s.io/cluster-api/api/v1beta1"
)
Expand Down Expand Up @@ -46,18 +44,6 @@ func DescribeNodegroup(c context.Context, api EKSNodegroupsAPI, input *eks.Descr
return api.DescribeNodegroup(c, input)
}

// EC2API Describes the functions required to access data on the AWS EC2 api
type EC2API interface {
DescribeLaunchTemplateVersions(ctx context.Context,
params *ec2.DescribeLaunchTemplateVersionsInput,
optFns ...func(*ec2.Options)) (*ec2.DescribeLaunchTemplateVersionsOutput, error)
}

// Get the EC2 Launch template versions for a given launch template
func GetLaunchTemplate(c context.Context, api EC2API, input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) {
return api.DescribeLaunchTemplateVersions(c, input)
}

// AutoscalingAPI presents functions required for reading autoscaling groups from AWS
type AutoscalingAPI interface {
DescribeAutoScalingGroups(ctx context.Context,
Expand All @@ -70,13 +56,16 @@ func GetAutoScalingGroups(c context.Context, api AutoscalingAPI, input *asg.Desc
return api.DescribeAutoScalingGroups(c, input)
}

// CreateAWSNodegroupSpec will attempt to determine how the nodegroup is defined
// and map that back into objects for cluster-api and cluster-api-provider-aws
//
// This function will output both a MachinePool and an AWSManagedMachinepool object
func (f *Function) CreateAWSNodegroupSpec(ac *awsconfig) (err error) {
var (
res *eks.ListNodegroupsOutput
cfg aws.Config
)

// Set up the assume role clients
if cfg, err = xfnaws.Config(ac.region, ac.providerConfigRef); err != nil {
err = errors.Wrap(err, "failed to load aws config for assume role")
return
Expand All @@ -85,7 +74,6 @@ func (f *Function) CreateAWSNodegroupSpec(ac *awsconfig) (err error) {
eksclient := eks.NewFromConfig(cfg)
ec2client := ec2.NewFromConfig(cfg)
asgclient := asg.NewFromConfig(cfg)
// end setting up clients

clusterInput := &eks.ListNodegroupsInput{
ClusterName: ac.cluster,
Expand All @@ -103,17 +91,19 @@ func (f *Function) CreateAWSNodegroupSpec(ac *awsconfig) (err error) {
}
var group *eks.DescribeNodegroupOutput
if group, err = DescribeNodegroup(context.TODO(), eksclient, nodegroupInput); err != nil {
f.log.Debug(fmt.Sprintf("cannot describe nodegroup %s for cluster %s", nodegroup, *ac.cluster), "error was", err)
f.log.Debug("AWSAPI", "cannot describe nodegroup", nodegroup, "cluster", *ac.cluster, "error", err)
continue
}

var ng *expinfrav2.AWSManagedMachinePoolSpec
if ng, err = f.nodegroupToCapiObject(group.Nodegroup, ec2client, asgclient); err != nil {
f.log.Debug(fmt.Sprintf("cannot create nodegroup object for nodegroup %q in cluster %q", nodegroup, *ac.cluster), "error was", err)
f.log.Debug("AWSAPI", "cannot create nodegroup", nodegroup, "cluster", *ac.cluster, "error", err)
continue
}

var nodegroupName string = fmt.Sprintf("%s-%s", *ac.cluster, nodegroup)
ac.labels["giantswarm.io/machine-pool"] = nodegroup
var nodegroupName string = fmt.Sprintf("%s-awsmanagedmachinepool-%s", *ac.cluster, nodegroup)
f.log.Info("AWSAPI", "Creating nodegroup", nodegroupName)
var awsmmp expinfrav2.AWSManagedMachinePool = expinfrav2.AWSManagedMachinePool{
TypeMeta: metav1.TypeMeta{
Kind: "AWSManagedMachinePool",
Expand All @@ -135,13 +125,15 @@ func (f *Function) CreateAWSNodegroupSpec(ac *awsconfig) (err error) {
}

var dataSecretName string = ""
var machinepoolName string = fmt.Sprintf("%s-machinepool-%s", *ac.cluster, nodegroup)
f.log.Info("AWSAPI", "Creating machinepool", machinepoolName)
var machinepool *capiinfra.MachineDeployment = &capiinfra.MachineDeployment{
TypeMeta: metav1.TypeMeta{
Kind: "MachinePool",
APIVersion: "cluster.x-k8s.io/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodegroupName + "-mp",
Name: machinepoolName,
Namespace: *ac.namespace,
Labels: ac.labels,
Annotations: ac.annotations,
Expand All @@ -167,25 +159,25 @@ func (f *Function) CreateAWSNodegroupSpec(ac *awsconfig) (err error) {
}

var awsobject, mpobject *unstructured.Unstructured
if awsobject, err = composite.ToUnstructuredKubernetesObject(awsmmp, ac.composite.Spec.ClusterProviderConfigRef); err != nil {
f.log.Debug(fmt.Sprintf("failed to convert nodegroup %q to kubernetes object for cluster %q.", nodegroup, *ac.cluster), "error was", err)
if awsobject, err = composite.ToUnstructuredKubernetesObject(awsmmp, ac.composite.Spec.ClusterProviderConfigRef, ac.composite.Spec.ObjectDeletionPolicy); err != nil {
f.log.Debug("failed to convert nodegroup", nodegroupName, "cluster", *ac.cluster, "error", err, "object", awsmmp)
continue
}

f.log.Info("Adding nodegroup to required resources", "nodegroup", nodegroupName)
if err = ac.composed.AddDesired(nodegroupName, awsobject); err != nil {
f.log.Info(composedName, "add machinepool", errors.Wrap(err, "cannot add composed object "+nodegroupName))
f.log.Debug("failed to add nodegroup", nodegroupName, "cluster", *ac.cluster, "error", err, "object", awsobject)
continue
}

if mpobject, err = composite.ToUnstructuredKubernetesObject(machinepool, ac.composite.Spec.ClusterProviderConfigRef); err != nil {
f.log.Debug(fmt.Sprintf("failed to convert nodegroup %q to kubernetes object for cluster %q.", nodegroup, *ac.cluster), "error was", err)
if mpobject, err = composite.ToUnstructuredKubernetesObject(machinepool, ac.composite.Spec.ClusterProviderConfigRef, ac.composite.Spec.ObjectDeletionPolicy); err != nil {
f.log.Debug("failed to convert machinepool", machinepoolName, "cluster", *ac.cluster, "error", err, "object", machinepool)
continue
}

f.log.Info("Adding machinepool to required resources", "machinepool", nodegroupName)
if err = ac.composed.AddDesired(nodegroupName+"-mp", mpobject); err != nil {
f.log.Info(composedName, "add machinepool", errors.Wrap(err, "cannot add composed object "+nodegroupName))
if err = ac.composed.AddDesired(machinepoolName, mpobject); err != nil {
f.log.Debug("failed to add machinepool", machinepoolName, "cluster", *ac.cluster, "error", err, "object", mpobject)
continue
}
}
Expand All @@ -195,52 +187,44 @@ func (f *Function) CreateAWSNodegroupSpec(ac *awsconfig) (err error) {
// Pull all the information together to create a AWSManagedMachinePool object
func (f *Function) nodegroupToCapiObject(group *types.Nodegroup, ec2client *ec2.Client, asgclient *asg.Client) (pool *expinfrav2.AWSManagedMachinePoolSpec, err error) {
pool = &expinfrav2.AWSManagedMachinePoolSpec{}
pool.AdditionalTags = make(map[string]string)
for k, v := range group.Tags {
if !strings.HasPrefix(k, "aws:") {
pool.AdditionalTags[k] = v
}
}
pool.AMIType = (*expinfrav2.ManagedMachineAMIType)(&group.AmiType)

var (
name string
autoscaling *asgtypes.AutoScalingGroup
autoscalinglt *expinfrav2.AWSLaunchTemplate
asgName string
asg *asgtypes.AutoScalingGroup
asgLaunchTemplate *expinfrav2.AWSLaunchTemplate
)

if group.Resources != nil {
name = *group.Resources.AutoScalingGroups[0].Name
asgName = *group.Resources.AutoScalingGroups[0].Name
}

if autoscaling, autoscalinglt, err = getAutoscaling(name, asgclient, ec2client); err != nil {
if autoscaling == nil {
if asg, asgLaunchTemplate, err = getAutoscaling(asgName, asgclient, ec2client); err != nil {
if asg == nil {
return nil, err
}
}
pool.AMIType = (*expinfrav2.ManagedMachineAMIType)(&group.AmiType)
pool.AvailabilityZones = asg.AvailabilityZones

// pool.AMIVersion = group.Version

pool.AvailabilityZones = autoscaling.AvailabilityZones
if pool.AWSLaunchTemplate, err = getLaunchTemplate(group.LaunchTemplate, ec2client); err != nil {
f.log.Debug("AWSAPI", "AWSLaunchTemplate error", err)
}

if group.LaunchTemplate == nil {
if pool.AWSLaunchTemplate == nil {
pool.InstanceType = &group.InstanceTypes[0]
} else {
if pool.AWSLaunchTemplate, err = getLaunchTemplate(group.LaunchTemplate, ec2client); err != nil {
return pool, err
}
if pool.AWSLaunchTemplate.InstanceType == "" {
pool.AWSLaunchTemplate.InstanceType = group.InstanceTypes[0]
}

f.log.Debug("Autoscaling", "LaunchTemplate", pool.AWSLaunchTemplate)
f.log.Debug("Autoscaling", "autoscalinglt", autoscalinglt)
if autoscalinglt != nil {
f.log.Debug("Autoscaling", "AWSLaunchTemplate", pool.AWSLaunchTemplate)
f.log.Debug("Autoscaling", "asgLaunchTemplate", asgLaunchTemplate)
if asgLaunchTemplate != nil {
if pool.AWSLaunchTemplate.AMI.ID == nil {
pool.AWSLaunchTemplate.AMI.ID = autoscalinglt.AMI.ID
pool.AWSLaunchTemplate.AMI.ID = asgLaunchTemplate.AMI.ID
}

if pool.AWSLaunchTemplate.IamInstanceProfile == "" {
pool.AWSLaunchTemplate.IamInstanceProfile = autoscalinglt.IamInstanceProfile
pool.AWSLaunchTemplate.IamInstanceProfile = asgLaunchTemplate.IamInstanceProfile
}
}
}
Expand All @@ -256,7 +240,7 @@ func (f *Function) nodegroupToCapiObject(group *types.Nodegroup, ec2client *ec2.
pool.EKSNodegroupName = *group.NodegroupName
pool.Labels = group.Labels

for _, instance := range autoscaling.Instances {
for _, instance := range asg.Instances {
var pid string = fmt.Sprintf("aws:///%s/%s", *instance.AvailabilityZone, *instance.InstanceId)
pool.ProviderIDList = append(pool.ProviderIDList, pid)
}
Expand All @@ -268,8 +252,6 @@ func (f *Function) nodegroupToCapiObject(group *types.Nodegroup, ec2client *ec2.
}
}

// RoleAdditionalPolicies???

pool.RoleName = strings.Split(*group.NodeRole, "/")[1]

if group.ScalingConfig != nil {
Expand Down Expand Up @@ -321,7 +303,6 @@ func getAutoscaling(name string, client *asg.Client, ec2client *ec2.Client) (*as
}

var (
// I think we only need the first here
autoscaling asgtypes.AutoScalingGroup = res.AutoScalingGroups[0]
asglt *asgtypes.LaunchTemplateSpecification
lt types.LaunchTemplateSpecification
Expand All @@ -345,78 +326,3 @@ func getAutoscaling(name string, client *asg.Client, ec2client *ec2.Client) (*as

return &autoscaling, asgLaunchTemplate, err
}

func getLaunchTemplate(base *types.LaunchTemplateSpecification, client *ec2.Client) (*expinfrav2.AWSLaunchTemplate, error) {
var (
res *ec2.DescribeLaunchTemplateVersionsOutput
template expinfrav2.AWSLaunchTemplate
err error
)

input := ec2.DescribeLaunchTemplateVersionsInput{
LaunchTemplateId: base.Id,
Versions: []string{
*base.Version,
},
}

// OK... Which launch template should be used here?
// Both the nodegroup and the autoscaling group have a launch template
// and both seem to be different
if res, err = GetLaunchTemplate(context.TODO(), client, &input); err != nil {
return nil, err
}

if len(res.LaunchTemplateVersions) != 1 {
return nil, fmt.Errorf("wrong count for launch templates for template %s", *base.Name)
}

template.Name = *res.LaunchTemplateVersions[0].LaunchTemplateName
template.VersionNumber = res.LaunchTemplateVersions[0].VersionNumber

var data *ec2types.ResponseLaunchTemplateData = res.LaunchTemplateVersions[0].LaunchTemplateData
template.InstanceType = string(data.InstanceType)
template.SSHKeyName = data.KeyName

template.AMI = infrav2.AMIReference{
ID: data.ImageId,
}

if data.IamInstanceProfile != nil {
if data.IamInstanceProfile.Name != nil && !strings.HasPrefix(*data.IamInstanceProfile.Name, "eks-") {
template.IamInstanceProfile = *data.IamInstanceProfile.Name
} else if data.IamInstanceProfile.Arn != nil {
template.IamInstanceProfile = *data.IamInstanceProfile.Arn
}
}

if data.InstanceMarketOptions != nil && data.InstanceMarketOptions.SpotOptions != nil {
template.SpotMarketOptions = &infrav2.SpotMarketOptions{
MaxPrice: data.InstanceMarketOptions.SpotOptions.MaxPrice,
}
}

if len(data.BlockDeviceMappings) > 0 {
var (
device ec2types.LaunchTemplateBlockDeviceMapping = data.BlockDeviceMappings[0]
throughput int64 = int64(*device.Ebs.Throughput)
)
template.RootVolume = &infrav2.Volume{
DeviceName: *device.DeviceName,
Encrypted: device.Ebs.Encrypted,
IOPS: int64(*device.Ebs.Iops),
Size: int64(*device.Ebs.VolumeSize),
Throughput: &throughput,
Type: infrav2.VolumeType(device.Ebs.VolumeType),
}
}

for _, id := range data.SecurityGroupIds {
id := id
template.AdditionalSecurityGroups = append(template.AdditionalSecurityGroups, infrav2.AWSResourceReference{
ID: &id,
})
}

return &template, nil
}
2 changes: 1 addition & 1 deletion build.sh
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

VERSION=v0.0.4
VERSION=v0.0.27
go build . && {
rm package/*.xpkg
go generate ./...
Expand Down
2 changes: 2 additions & 0 deletions fn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ func (f *Function) RunFunction(_ context.Context, req *fnv1beta1.RunFunctionRequ
for k, v := range ac.composite.Spec.KubernetesAdditionalLabels {
ac.labels[k] = v
}
ac.labels["cluster.x-k8s.io/cluster-name"] = *ac.cluster
ac.labels["giantswarm.io/cluster"] = *ac.cluster

var provider string = ac.composite.Spec.CompositionSelector.MatchLabels.Provider
f.log.Info(provider)
Expand Down
Loading

0 comments on commit 0481845

Please sign in to comment.