diff --git a/api/v1beta1/packetcluster_types.go b/api/v1beta1/packetcluster_types.go index 43d554f4..edc152a2 100644 --- a/api/v1beta1/packetcluster_types.go +++ b/api/v1beta1/packetcluster_types.go @@ -35,6 +35,14 @@ const ( KUBEVIPID = "KUBE_VIP" ) +// AssignmentType describes the component responsible for allocating IP addresses to the machines. +type AssignmentType string + +const ( + AssignmentClusterAPI AssignmentType = "cluster-api" + AssignmentDHCP AssignmentType = "dhcp" +) + // VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer. type VIPManagerType string @@ -60,6 +68,24 @@ type PacketClusterSpec struct { // +kubebuilder:validation:Enum=CPEM;KUBE_VIP;EMLB // +kubebuilder:default:=CPEM VIPManager VIPManagerType `json:"vipManager"` + + // Networks is a list of network configurations for the PacketCluster + Networks []NetworkSpec `json:"networks,omitempty"` +} + +// NetworkSpec defines the network configuration for a PacketCluster. +type NetworkSpec struct { + // Name of the network, e.g. "storage VLAN", is optional + // +optional + Name string `json:"name,omitempty"` + // Description of the network, e.g. "Storage network", is optional + // +optional + Description string `json:"description,omitempty"` + // AddressRange for the cluster network for eg: VRF IP Ranges + Addresses []string `json:"addresses,omitempty"` + // Assignment is component responsible for allocating IP addresses to the machines, either cluster-api or dhcp + // +kubebuilder:validation:Enum=cluster-api;dhcp + Assignment AssignmentType `json:"assignment,omitempty"` } // PacketClusterStatus defines the observed state of PacketCluster. diff --git a/api/v1beta1/packetmachine_types.go b/api/v1beta1/packetmachine_types.go index f09ed8a2..17316b16 100644 --- a/api/v1beta1/packetmachine_types.go +++ b/api/v1beta1/packetmachine_types.go @@ -28,6 +28,9 @@ const ( // MachineFinalizer allows ReconcilePacketMachine to clean up Packet resources before // removing it from the apiserver. MachineFinalizer = "packetmachine.infrastructure.cluster.x-k8s.io" + // IPAddressClaimFinalizer allows the reconciler to prevent deletion of an + // IPAddressClaim that is in use. + IPAddressClaimFinalizer = "packetmachine.infrastructure.cluster.x-k8s.io/ip-claim-protection" ) const ( @@ -50,6 +53,32 @@ const ( WaitingForClusterInfrastructureReason = "WaitingForClusterInfrastructure" // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. WaitingForBootstrapDataReason = "WaitingForBootstrapData" + + Layer2NetworkConfigurationConditionSuccess = "Layer2NetworkConfigurationSuccess" + Layer2NetworkConfigurationConditionFailed = "Layer2NetworkConfigurationFailed" + +) + +const ( + // IPAddressClaimedCondition documents the status of claiming an IP address + // from an IPAM provider. + IPAddressClaimedCondition clusterv1.ConditionType = "IPAddressClaimed" + + // IPAddressClaimsBeingCreatedReason (Severity=Info) documents that claims for the + // IP addresses required by the PacketMachine are being created. + IPAddressClaimsBeingCreatedReason = "IPAddressClaimsBeingCreated" + + // WaitingForIPAddressReason (Severity=Info) documents that the PacketMachine is + // currently waiting for an IP address to be provisioned. + WaitingForIPAddressReason = "WaitingForIPAddress" + + // IPAddressInvalidReason (Severity=Error) documents that the IP address + // provided by the IPAM provider is not valid. + IPAddressInvalidReason = "IPAddressInvalid" + + // IPAddressClaimNotFoundReason (Severity=Error) documents that the IPAddressClaim + // cannot be found. + IPAddressClaimNotFoundReason = "IPAddressClaimNotFound" ) // PacketMachineSpec defines the desired state of PacketMachine. @@ -86,6 +115,48 @@ type PacketMachineSpec struct { // Tags is an optional set of tags to add to Packet resources managed by the Packet provider. // +optional Tags Tags `json:"tags,omitempty"` + + // NetworkPorts is an optional set of configurations for configuring layer2 seetings in a machine. + // +optional + NetworkPorts []*Port `json:"ports,omitempty"` +} + +// Port defines the Layer2(VLAN) Configuration that needs to be done on a port (eg: bond0). +type Port struct { + // name of the port e.g bond0,eth0 and eth1 for 2 NIC servers. + Name string `json:"name"` + // port bonded or not. + Bonded bool `json:"bonded,omitempty"` + // convert port to layer 2. is false by default on new devices. changes result in /ports/id/convert/layer-[2|3] API calls + Layer2 bool `json:"layer2,omitempty"` + // Network configurations for the port + Networks []Network `json:"networks"` +} + +// Network defines the network configuration for a port. +type Network struct { + // VLANs for EM API to find by vxlan, project, and metro match then attach to device. OS userdata template will also configure this VLAN on the bond device + VXLAN int `json:"vxlan,omitempty"` + // VLAN ID for the VLAN created on the EM Console + VLANID string `json:"vlanID,omitempty"` + // Netmask is the netmask for the network. + // eg: 255.255.255.248 + Netmask string `json:"netmask,omitempty"` + // AddressFromPool is a reference of IPAddressPool that should be assigned to IPAddressClaim. + // The machine's cloud-init metadata will be populated with IPAddresse fulfilled by an IPAM provider. + AddressFromPool corev1.TypedLocalObjectReference `json:"addressFromPool,omitempty"` + // AddressType is the type of address to assign to the machine. It can be either Internal or External. + // kubebuilder:validation:Enum=Internal;External + AddressType string `json:"addressType,omitempty"` + // List of Routes to be configured on the Packet Machine + // +optional + Routes []*RouteSpec `json:"routes,omitempty"` +} + +// RouteSpec defines the static route configuration for a PacketMachine. +type RouteSpec struct { + Destination string `json:"destination"` + Gateway string `json:"gateway"` } // PacketMachineStatus defines the observed state of PacketMachine. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d3a8b698..a4d7b5fd 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -27,12 +27,59 @@ import ( "sigs.k8s.io/cluster-api/errors" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Network) DeepCopyInto(out *Network) { + *out = *in + in.AddressFromPool.DeepCopyInto(&out.AddressFromPool) + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]*RouteSpec, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(RouteSpec) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. +func (in *Network) DeepCopy() *Network { + if in == nil { + return nil + } + out := new(Network) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { + *out = *in + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkSpec. +func (in *NetworkSpec) DeepCopy() *NetworkSpec { + if in == nil { + return nil + } + out := new(NetworkSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PacketCluster) DeepCopyInto(out *PacketCluster) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -90,6 +137,13 @@ func (in *PacketClusterList) DeepCopyObject() runtime.Object { func (in *PacketClusterSpec) DeepCopyInto(out *PacketClusterSpec) { *out = *in out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]NetworkSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PacketClusterSpec. @@ -201,6 +255,17 @@ func (in *PacketMachineSpec) DeepCopyInto(out *PacketMachineSpec) { *out = make(Tags, len(*in)) copy(*out, *in) } + if in.NetworkPorts != nil { + in, out := &in.NetworkPorts, &out.NetworkPorts + *out = make([]*Port, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Port) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PacketMachineSpec. @@ -345,6 +410,43 @@ func (in *PacketMachineTemplateSpec) DeepCopy() *PacketMachineTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Port) DeepCopyInto(out *Port) { + *out = *in + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]Network, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Port. +func (in *Port) DeepCopy() *Port { + if in == nil { + return nil + } + out := new(Port) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RouteSpec) DeepCopyInto(out *RouteSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec. +func (in *RouteSpec) DeepCopy() *RouteSpec { + if in == nil { + return nil + } + out := new(RouteSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Tags) DeepCopyInto(out *Tags) { { diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml index 96aec7f9..e251bb81 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml @@ -73,6 +73,35 @@ spec: metro: description: Metro represents the Packet metro for this cluster type: string + networks: + description: Networks is a list of network configurations for the + PacketCluster + items: + description: NetworkSpec defines the network configuration for a + PacketCluster. + properties: + addresses: + description: 'AddressRange for the cluster network for eg: VRF + IP Ranges' + items: + type: string + type: array + assignment: + description: Assignment is component responsible for allocating + IP addresses to the machines, either cluster-api or dhcp + enum: + - cluster-api + - dhcp + type: string + description: + description: Description of the network, e.g. "Storage network", + is optional + type: string + name: + description: Name of the network, e.g. "storage VLAN", is optional + type: string + type: object + type: array projectID: description: ProjectID represents the Packet Project where this cluster will be placed into diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachines.yaml index fb8b1115..c6bd78bc 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachines.yaml @@ -92,6 +92,94 @@ spec: type: string os: type: string + ports: + description: NetworkPorts is an optional set of configurations for + configuring layer2 seetings in a machine. + items: + description: 'Port defines the Layer2(VLAN) Configuration that needs + to be done on a port (eg: bond0).' + properties: + bonded: + description: port bonded or not. + type: boolean + layer2: + description: convert port to layer 2. is false by default on + new devices. changes result in /ports/id/convert/layer-[2|3] + API calls + type: boolean + name: + description: name of the port e.g bond0,eth0 and eth1 for 2 + NIC servers. + type: string + networks: + description: Network configurations for the port + items: + description: Network defines the network configuration for + a port. + properties: + addressFromPool: + description: |- + AddressFromPool is a reference of IPAddressPool that should be assigned to IPAddressClaim. + The machine's cloud-init metadata will be populated with IPAddresse fulfilled by an IPAM provider. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + addressType: + description: |- + AddressType is the type of address to assign to the machine. It can be either Internal or External. + kubebuilder:validation:Enum=Internal;External + type: string + netmask: + description: |- + Netmask is the netmask for the network. + eg: 255.255.255.248 + type: string + routes: + description: List of Routes to be configured on the Packet + Machine + items: + description: RouteSpec defines the static route configuration + for a PacketMachine. + properties: + destination: + type: string + gateway: + type: string + required: + - destination + - gateway + type: object + type: array + vlanID: + description: VLAN ID for the VLAN created on the EM Console + type: string + vxlan: + description: VLANs for EM API to find by vxlan, project, + and metro match then attach to device. OS userdata template + will also configure this VLAN on the bond device + type: integer + type: object + type: array + required: + - name + - networks + type: object + type: array providerID: description: ProviderID is the unique identifier as specified by the cloud provider. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachinetemplates.yaml index cf7f0d79..d9037e15 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_packetmachinetemplates.yaml @@ -80,6 +80,98 @@ spec: type: string os: type: string + ports: + description: NetworkPorts is an optional set of configurations + for configuring layer2 seetings in a machine. + items: + description: 'Port defines the Layer2(VLAN) Configuration + that needs to be done on a port (eg: bond0).' + properties: + bonded: + description: port bonded or not. + type: boolean + layer2: + description: convert port to layer 2. is false by default + on new devices. changes result in /ports/id/convert/layer-[2|3] + API calls + type: boolean + name: + description: name of the port e.g bond0,eth0 and eth1 + for 2 NIC servers. + type: string + networks: + description: Network configurations for the port + items: + description: Network defines the network configuration + for a port. + properties: + addressFromPool: + description: |- + AddressFromPool is a reference of IPAddressPool that should be assigned to IPAddressClaim. + The machine's cloud-init metadata will be populated with IPAddresse fulfilled by an IPAM provider. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + addressType: + description: |- + AddressType is the type of address to assign to the machine. It can be either Internal or External. + kubebuilder:validation:Enum=Internal;External + type: string + netmask: + description: |- + Netmask is the netmask for the network. + eg: 255.255.255.248 + type: string + routes: + description: List of Routes to be configured on + the Packet Machine + items: + description: RouteSpec defines the static route + configuration for a PacketMachine. + properties: + destination: + type: string + gateway: + type: string + required: + - destination + - gateway + type: object + type: array + vlanID: + description: VLAN ID for the VLAN created on the + EM Console + type: string + vxlan: + description: VLANs for EM API to find by vxlan, + project, and metro match then attach to device. + OS userdata template will also configure this + VLAN on the bond device + type: integer + type: object + type: array + required: + - name + - networks + type: object + type: array providerID: description: ProviderID is the unique identifier as specified by the cloud provider. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 88064780..eb1e7b06 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: quay.io/equinix-oss/cluster-api-provider-packet:dev + - image: docker.io/rahulsawra/cluster-api-provider-packet-amd64:dev name: manager diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml index cd7ae12c..74a0879c 100644 --- a/config/default/manager_pull_policy.yaml +++ b/config/default/manager_pull_policy.yaml @@ -8,4 +8,4 @@ spec: spec: containers: - name: manager - imagePullPolicy: IfNotPresent + imagePullPolicy: Always diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 84c65a52..3dcd1848 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -93,3 +93,22 @@ rules: - get - patch - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddresses + verbs: + - get + - list + - watch diff --git a/controllers/packetmachine_controller.go b/controllers/packetmachine_controller.go index 707fbb34..840dc163 100644 --- a/controllers/packetmachine_controller.go +++ b/controllers/packetmachine_controller.go @@ -20,17 +20,25 @@ import ( "context" "errors" "fmt" + "net" "net/http" + "slices" + "strconv" "strings" "time" + apitypes "k8s.io/apimachinery/pkg/types" + metal "github.com/equinix/equinix-sdk-go/services/metalv1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" capierrors "sigs.k8s.io/cluster-api/errors" - "sigs.k8s.io/cluster-api/util" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" + clusterutilv1 "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/predicates" @@ -38,15 +46,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1" "sigs.k8s.io/cluster-api-provider-packet/internal/emlb" + "sigs.k8s.io/cluster-api-provider-packet/internal/layer2" packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet" "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope" clog "sigs.k8s.io/cluster-api/util/log" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const ( @@ -97,7 +106,7 @@ func (r *PacketMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Fetch the Machine. - machine, err := util.GetOwnerMachine(ctx, r.Client, packetmachine.ObjectMeta) + machine, err := clusterutilv1.GetOwnerMachine(ctx, r.Client, packetmachine.ObjectMeta) if err != nil { log.Error(err, "Failed to get owner machine") return ctrl.Result{}, err @@ -111,7 +120,7 @@ func (r *PacketMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques ctx = ctrl.LoggerInto(ctx, log) // Fetch the Cluster. - cluster, err := util.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta) + cluster, err := clusterutilv1.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta) if err != nil { log.Info("Machine is missing cluster label or cluster does not exist") return ctrl.Result{}, err @@ -166,8 +175,8 @@ func (r *PacketMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques // Add finalizer first if not set to avoid the race condition between init and delete. // Note: Finalizers in general can only be added when the deletionTimestamp is not set. - if packetmachine.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(packetmachine, infrav1.MachineFinalizer) { - controllerutil.AddFinalizer(packetmachine, infrav1.MachineFinalizer) + if packetmachine.ObjectMeta.DeletionTimestamp.IsZero() && !ctrlutil.ContainsFinalizer(packetmachine, infrav1.MachineFinalizer) { + ctrlutil.AddFinalizer(packetmachine, infrav1.MachineFinalizer) return ctrl.Result{}, nil } @@ -182,7 +191,7 @@ func (r *PacketMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *PacketMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { log := ctrl.LoggerFrom(ctx) - clusterToPacketMachines, err := util.ClusterToTypedObjectsMapper(mgr.GetClient(), &infrav1.PacketMachineList{}, mgr.GetScheme()) + clusterToPacketMachines, err := clusterutilv1.ClusterToTypedObjectsMapper(mgr.GetClient(), &infrav1.PacketMachineList{}, mgr.GetScheme()) if err != nil { return fmt.Errorf("failed to create mapper for Cluster to PacketMachines: %w", err) } @@ -193,7 +202,7 @@ func (r *PacketMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log, r.WatchFilterValue)). Watches( &clusterv1.Machine{}, - handler.EnqueueRequestsFromMapFunc(util.MachineToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("PacketMachine"))), + handler.EnqueueRequestsFromMapFunc(clusterutilv1.MachineToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("PacketMachine"))), ). Watches( &infrav1.PacketCluster{}, @@ -205,6 +214,10 @@ func (r *PacketMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl builder.WithPredicates( predicates.ClusterUnpausedAndInfrastructureReady(log), ), + ). + Watches( + &ipamv1.IPAddressClaim{}, + handler.EnqueueRequestsFromMapFunc(ipAddressClaimToPacketMachine), ).Complete(r) if err != nil { return fmt.Errorf("failed setting up with a controller manager: %w", err) @@ -231,7 +244,7 @@ func (r *PacketMachineReconciler) PacketClusterToPacketMachines(ctx context.Cont return nil } - cluster, err := util.GetOwnerCluster(ctx, r.Client, c.ObjectMeta) + cluster, err := clusterutilv1.GetOwnerCluster(ctx, r.Client, c.ObjectMeta) switch { case apierrors.IsNotFound(err) || cluster == nil: log.Error(err, "owning cluster is not found, skipping mapping.") @@ -258,6 +271,29 @@ func (r *PacketMachineReconciler) PacketClusterToPacketMachines(ctx context.Cont return result } +func ipAddressClaimToPacketMachine(_ context.Context, a client.Object) []reconcile.Request { + ipAddressClaim, ok := a.(*ipamv1.IPAddressClaim) + if !ok { + return nil + } + + requests := []reconcile.Request{} + if clusterutilv1.HasOwner(ipAddressClaim.OwnerReferences, infrav1.GroupVersion.String(), []string{"PacketMachine"}) { + for _, ref := range ipAddressClaim.OwnerReferences { + if ref.Kind == "PacketMachine" { + requests = append(requests, reconcile.Request{ + NamespacedName: apitypes.NamespacedName{ + Name: ref.Name, + Namespace: ipAddressClaim.Namespace, + }, + }) + break + } + } + } + return requests +} + func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *scope.MachineScope) (ctrl.Result, error) { //nolint:gocyclo,maintidx log := ctrl.LoggerFrom(ctx, "machine", machineScope.Machine.Name, "cluster", machineScope.Cluster.Name) log.Info("Reconciling PacketMachine") @@ -289,6 +325,7 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s err error controlPlaneEndpoint *metal.IPReservation resp *http.Response + ipAddrCfg []packet.IPAddressCfg ) if deviceID != "" { @@ -341,10 +378,21 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s return ctrl.Result{}, patchErr } } + if len(machineScope.PacketMachine.Spec.NetworkPorts) > 0 { + if err := r.ReconcileIPAddresses(ctx, machineScope.PacketMachine); err != nil { + return ctrl.Result{}, err + } + } + + ipAddrCfg, err = getIPAddressCfg(ctx, r.Client, machineScope.PacketMachine) + if err != nil { + return ctrl.Result{}, err + } createDeviceReq := packet.CreateDeviceRequest{ MachineScope: machineScope, ExtraTags: packet.DefaultCreateTags(machineScope.Namespace(), machineScope.Machine.Name, machineScope.Cluster.Name), + IPAddresses: ipAddrCfg, } // when a node is a control plane node we need the elastic IP @@ -452,6 +500,44 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s } } + if machineScope.PacketMachine.Spec.NetworkPorts != nil { + eventsList, resp, err := r.PacketClient.EventsApi.FindDeviceEvents(ctx, *dev.Id).Execute() + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get device events: %w", err) + } + if resp.StatusCode != http.StatusOK { + return ctrl.Result{}, fmt.Errorf("failed to get device events: %w", err) + } + // check if the network configuration has been successful/failed by polling the /events endpoint + // if the network configuration has been successful, we can set the device to ready else we need to requeue + // we need to wait for either the network configuration to be successful or failed before we can proceed. + if len(eventsList.Events) > 0 { + if checkIfEventsContainNetworkConfigurationSuccess(eventsList) { + conditions.MarkTrue(machineScope.PacketMachine, infrav1.Layer2NetworkConfigurationConditionSuccess) + } else if checkIfEventsContainNetworkConfigurationFailure(eventsList) { + conditions.MarkTrue(machineScope.PacketMachine, infrav1.Layer2NetworkConfigurationConditionFailed) + return ctrl.Result{}, fmt.Errorf("failed to configure network on device") + } else { + // if the network configuration is still in progress, we need to requeue + // user data scripts might take some time to complete. + log.Info("waiting for layer2 network configurations to complete") + return ctrl.Result{RequeueAfter: time.Second * 10}, nil + } + } + } + + // once the network configuration has been successful, we can call the APIs to set the port configuration to layer2/bonded/bound VXLAN to port. + // reconstruct ipAddrCfg as earlier it was done when device was created first. During later reconciliations, this might be nil and we need to reconstruct it. + ipAddrCfg, err = getIPAddressCfg(ctx, r.Client, machineScope.PacketMachine) + if err != nil { + return ctrl.Result{}, err + } + if len(ipAddrCfg) > 0 { + if err := r.reconcilePortConfigurations(ctx, *dev.Id, ipAddrCfg); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set port configuration: %w", err) + } + } + machineScope.SetReady() conditions.MarkTrue(machineScope.PacketMachine, infrav1.DeviceReadyCondition) @@ -504,8 +590,7 @@ func (r *PacketMachineReconciler) reconcileDelete(ctx context.Context, machineSc if dev == nil { log.Info("Server not found by tags, nothing left to do") - controllerutil.RemoveFinalizer(packetmachine, infrav1.MachineFinalizer) - return nil + return r.deletePacketMachine(ctx, packetmachine) } device = dev @@ -519,15 +604,13 @@ func (r *PacketMachineReconciler) reconcileDelete(ctx context.Context, machineSc // When the server does not exist we do not have anything left to do. // Probably somebody manually deleted the server from the UI or via API. log.Info("Server not found by id, nothing left to do") - controllerutil.RemoveFinalizer(packetmachine, infrav1.MachineFinalizer) - return nil + return r.deletePacketMachine(ctx, packetmachine) } if resp.StatusCode == http.StatusForbidden { // When a server fails to provision it will return a 403 log.Info("Server appears to have failed provisioning, nothing left to do") - controllerutil.RemoveFinalizer(packetmachine, infrav1.MachineFinalizer) - return nil + return r.deletePacketMachine(ctx, packetmachine) } } @@ -539,7 +622,7 @@ func (r *PacketMachineReconciler) reconcileDelete(ctx context.Context, machineSc // We should never get there but this is a safety check if device == nil { - controllerutil.RemoveFinalizer(packetmachine, infrav1.MachineFinalizer) + _ = r.deletePacketMachine(ctx, packetmachine) return fmt.Errorf("%w: %s", errMissingDevice, packetmachine.Name) } @@ -559,6 +642,514 @@ func (r *PacketMachineReconciler) reconcileDelete(ctx context.Context, machineSc return fmt.Errorf("failed to delete the machine: %w", err) } - controllerutil.RemoveFinalizer(packetmachine, infrav1.MachineFinalizer) + return r.deletePacketMachine(ctx, packetmachine) +} + +// +kubebuilder:rbac:groups=ipam.cluster.x-k8s.io,resources=ipaddressclaims,verbs=get;create;patch;watch;list;update +// +kubebuilder:rbac:groups=ipam.cluster.x-k8s.io,resources=ipaddresses,verbs=get;list;watch + +// ReconcileIPAddresses reconciles ip addresses forpacket machine +func (r *PacketMachineReconciler) ReconcileIPAddresses(ctx context.Context, machine *infrav1.PacketMachine) error { + + log := ctrl.LoggerFrom(ctx) + + totalClaims, claimsCreated := 0, 0 + claimsFulfilled := 0 + + var ( + claims []conditions.Getter + errList []error + ) + + for portIdx, port := range machine.Spec.NetworkPorts { + for networkIdx, network := range port.Networks { + totalClaims++ + + ipAddrClaimName := packet.IPAddressClaimName(machine.Name, portIdx, networkIdx) + + ipAddrClaim := &ipamv1.IPAddressClaim{} + ipAddrClaimKey := client.ObjectKey{ + Namespace: machine.Namespace, + Name: ipAddrClaimName, + } + + log := log.WithValues("IPAddressClaim", klog.KRef(ipAddrClaimKey.Namespace, ipAddrClaimKey.Name)) + ctx := ctrl.LoggerInto(ctx, log) + + err := r.Client.Get(ctx, ipAddrClaimKey, ipAddrClaim) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get IPAddressClaim %s err: %v", klog.KRef(ipAddrClaimKey.Namespace, ipAddrClaimKey.Name), err) + } + + ipAddrClaim, created, err := createOrPatchIPAddressClaim(ctx, r.Client, machine, ipAddrClaimName, network.AddressFromPool) + if err != nil { + errList = append(errList, err) + continue + } + + if created { + claimsCreated++ + } + if ipAddrClaim.Status.AddressRef.Name != "" { + claimsFulfilled++ + } + + if conditions.Has(ipAddrClaim, clusterv1.ReadyCondition) { + claims = append(claims, ipAddrClaim) + } + } + } + + if len(errList) > 0 { + aggregatedErr := kerrors.NewAggregate(errList) + conditions.MarkFalse(machine, + infrav1.IPAddressClaimedCondition, + infrav1.IPAddressClaimNotFoundReason, + clusterv1.ConditionSeverityError, + aggregatedErr.Error()) + return aggregatedErr + } + + // Fallback logic to calculate the state of the IPAddressClaimed condition + switch { + case totalClaims == claimsFulfilled: + conditions.MarkTrue(machine, infrav1.IPAddressClaimedCondition) + case claimsFulfilled < totalClaims && claimsCreated > 0: + conditions.MarkFalse(machine, infrav1.IPAddressClaimedCondition, + infrav1.IPAddressClaimsBeingCreatedReason, clusterv1.ConditionSeverityInfo, + "%d/%d claims being created", claimsCreated, totalClaims) + case claimsFulfilled < totalClaims && claimsCreated == 0: + conditions.MarkFalse(machine, infrav1.IPAddressClaimedCondition, + infrav1.WaitingForIPAddressReason, clusterv1.ConditionSeverityInfo, + "%d/%d claims being processed", totalClaims-claimsFulfilled, totalClaims) + } + + return nil +} + +func createOrPatchIPAddressClaim(ctx context.Context, client client.Client, machine *infrav1.PacketMachine, name string, poolRef corev1.TypedLocalObjectReference) (*ipamv1.IPAddressClaim, bool, error) { + claim := &ipamv1.IPAddressClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: machine.Namespace, + }, + } + + mutateFn := func() (err error) { + claim.SetOwnerReferences(clusterutilv1.EnsureOwnerRef( + claim.OwnerReferences, + metav1.OwnerReference{ + APIVersion: infrav1.GroupVersion.String(), + Kind: "PacketMachine", + Name: machine.Name, + UID: machine.UID, + })) + + ctrlutil.AddFinalizer(claim, infrav1.IPAddressClaimFinalizer) + + if claim.Labels == nil { + claim.Labels = make(map[string]string) + } + claim.Labels[clusterv1.ClusterNameLabel] = machine.Labels[clusterv1.ClusterNameLabel] + + claim.Spec.PoolRef.APIGroup = poolRef.APIGroup + claim.Spec.PoolRef.Kind = poolRef.Kind + claim.Spec.PoolRef.Name = poolRef.Name + return nil + } + + log := ctrl.LoggerFrom(ctx) + + result, err := ctrlutil.CreateOrPatch(ctx, client, claim, mutateFn) + if err != nil { + return nil, false, fmt.Errorf("failed to CreateOrPatch IPAddressClaim, err: : %s", err) + } + switch result { + case ctrlutil.OperationResultCreated: + log.Info("Created IPAddressClaim") + return claim, true, nil + case ctrlutil.OperationResultUpdated: + log.Info("Updated IPAddressClaim") + case ctrlutil.OperationResultNone, ctrlutil.OperationResultUpdatedStatus, ctrlutil.OperationResultUpdatedStatusOnly: + log.V(3).Info("No change required for IPAddressClaim", "operationResult", result) + } + return claim, false, nil +} + +func getIPAddressCfg(ctx context.Context, client client.Client, machine *infrav1.PacketMachine) ([]packet.IPAddressCfg, error) { + log := ctrl.LoggerFrom(ctx) + + boundClaims, totalClaims := 0, 0 + ipaddrCfgs := []packet.IPAddressCfg{} + + for portIdx, port := range machine.Spec.NetworkPorts { + for networkIdx, network := range port.Networks { + totalClaims++ + + ipAddrClaimName := packet.IPAddressClaimName(machine.Name, portIdx, networkIdx) + + log := log.WithValues("IPAddressClaim", klog.KRef(machine.Namespace, ipAddrClaimName)) + + ctx := ctrl.LoggerInto(ctx, log) + + ipAddrClaim, err := getIPAddrClaim(ctx, client, ipAddrClaimName, machine.Namespace) + if err != nil { + if apierrors.IsNotFound(err) { + // it would be odd for this to occur, a findorcreate just happened in a previous step + continue + } + return nil, fmt.Errorf("failed to get IPAddressClaim %s, err: %v", klog.KRef(machine.Namespace, ipAddrClaimName), err) + } + + log.V(5).Info("Fetched IPAddressClaim") + ipAddrName := ipAddrClaim.Status.AddressRef.Name + if ipAddrName == "" { + log.V(5).Info("IPAddress not yet bound to IPAddressClaim") + continue + } + + ipAddr := &ipamv1.IPAddress{} + ipAddrKey := apitypes.NamespacedName{ + Namespace: machine.Namespace, + Name: ipAddrName, + } + if err := client.Get(ctx, ipAddrKey, ipAddr); err != nil { + // because the ref was set on the claim, it is expected this error should not occur + return nil, err + } + + routes := make([]layer2.RouteSpec, 0) + for _, route := range network.Routes { + routes = append(routes, layer2.RouteSpec{ + Destination: route.Destination, + Gateway: route.Gateway, + }) + } + + ipaddrCfgs = append(ipaddrCfgs, packet.IPAddressCfg{ + VXLAN: network.VXLAN, + Address: ipAddr.Spec.Address, + Netmask: net.IP(net.CIDRMask(ipAddr.Spec.Prefix, 32)).String(), + PortName: port.Name, + Layer2: port.Layer2, + Bonded: port.Bonded, + Routes: routes, + }) + boundClaims++ + } + } + + if boundClaims < totalClaims { + log.Info("Waiting for ip address claims to be bound", + "total claims", totalClaims, + "claims bound", boundClaims) + return nil, fmt.Errorf("waiting for IP address claims to be bound") + } + + return ipaddrCfgs, nil +} + +func checkIfEventsContainNetworkConfigurationSuccess(eventsList *metal.EventList) bool { + networkConfigurationSuccess := "network_configuration_success" + + for _, event := range eventsList.Events { + if event.Body == nil { + continue + } + + if *event.Body == networkConfigurationSuccess { + return true + } + } + return false +} + +func checkIfEventsContainNetworkConfigurationFailure(eventsList *metal.EventList) bool { + networkConfigurationFailure := "network_configuration_failed" + + for _, event := range eventsList.Events { + if event.Body == nil { + continue + } + if *event.Body == networkConfigurationFailure { + return true + } + } + return false +} + +// reconcilePortConfigurations manages port configurations for a given device +// It ensures that the ports are configured with the correct VXLAN, layer2, and bonding settings +func (r *PacketMachineReconciler) reconcilePortConfigurations(ctx context.Context, deviceID string, desiredConfigs []packet.IPAddressCfg) error { + // Fetch the device details + device, _, err := r.PacketClient.GetDevice(ctx, deviceID) + if err != nil { + return fmt.Errorf("failed to get device %s: %w", deviceID, err) + } + + // Collect all desired VXLANs + desiredVXLANs := make(map[string][]int32) + for _, config := range desiredConfigs { + // fetch the port ID + portID, err := getMetalPortID(config.PortName, device.NetworkPorts) + if portID == nil || err != nil { + return fmt.Errorf("failed to get port ID for %s: %w", config.PortName, err) + } + desiredVXLANs[*portID] = append(desiredVXLANs[*portID], int32(config.VXLAN)) + } + + if err := r.reconcileVXLAN(ctx, desiredVXLANs); err != nil { + return err + } + + for _, desiredConfig := range desiredConfigs { + if err := r.reconcilePortConfig(ctx, device.NetworkPorts, desiredConfig); err != nil { + return fmt.Errorf("failed to reconcile port %s: %w", desiredConfig.PortName, err) + } + } + + return nil +} + +// reconcilePortConfig handles the configuration for a single port +func (r *PacketMachineReconciler) reconcilePortConfig(ctx context.Context, networkPorts []metal.Port, desiredConfig packet.IPAddressCfg) error { + log := ctrl.LoggerFrom(ctx) + + portID, err := getMetalPortID(desiredConfig.PortName, networkPorts) + if err != nil { + return fmt.Errorf("failed to get port ID for %s: %w", desiredConfig.PortName, err) + } + + if err := r.reconcileLayer2AndBonding(ctx, *portID, desiredConfig); err != nil { + return err + } + + log.Info("Port configuration reconciled successfully", "port", desiredConfig.PortName) + return nil +} + +func (r *PacketMachineReconciler) reconcileVXLAN(ctx context.Context, desiredConfig map[string][]int32) error { + log := ctrl.LoggerFrom(ctx) + + for portID, vxlanList := range desiredConfig { + currentVXLANs, err := r.getCurrentVXLANAssignments(ctx, portID) + if err != nil { + return err + } + + for _, vxlan := range vxlanList { + vxlanStr := strconv.Itoa(int(vxlan)) + if !slices.Contains(currentVXLANs, vxlan) { + if err := r.assignVXLAN(ctx, portID, vxlanStr); err != nil { + return err + } + log.Info("VXLAN assigned successfully", "port", portID, "vxlan", vxlanStr) + } + } + + for _, vxlan := range currentVXLANs { + if !slices.Contains(vxlanList, vxlan) { + if err := r.unassignVXLAN(ctx, portID, vxlan); err != nil { + return err + } + log.Info("VXLAN unassigned successfully", "port", portID, "vxlan", vxlan) + } + } + } + return nil +} + +// reconcileLayer2AndBonding ensures the port is set to the correct layer2 and bonding configuration +func (r *PacketMachineReconciler) reconcileLayer2AndBonding(ctx context.Context, portID string, desiredConfig packet.IPAddressCfg) error { + port, _, err := r.PacketClient.PortsApi.FindPortById(ctx, portID).Execute() + if err != nil { + return fmt.Errorf("failed to get port %s: %w", portID, err) + } + if port == nil { + return fmt.Errorf("port %s not found", portID) + } + + if desiredConfig.Layer2 { + if err := r.setLayer2(ctx, portID, port, desiredConfig.PortName); err != nil { + return err + } + } + + if desiredConfig.Bonded { + if err := r.setBonding(ctx, portID, port, desiredConfig.PortName); err != nil { + return err + } + } + + return nil +} + +// getCurrentVXLANAssignments fetches the current VXLAN assignments for a port +func (r *PacketMachineReconciler) getCurrentVXLANAssignments(ctx context.Context, portID string) ([]int32, error) { + vlanAssignList, resp, err := r.PacketClient.PortsApi.FindPortVlanAssignments(ctx, portID).Execute() + if err != nil { + return nil, fmt.Errorf("failed to get port VLAN assignments: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get port VLAN assignments: unexpected status code %d", resp.StatusCode) + } + return getPortVXLANAssignments(vlanAssignList), nil +} + +// unassignVXLAN removes a VXLAN assignment from a port +func (r *PacketMachineReconciler) unassignVXLAN(ctx context.Context, portID string, vxlan int32) error { + vxlanStr := strconv.Itoa(int(vxlan)) + _, resp, err := r.PacketClient.PortsApi.UnassignPort(ctx, portID).PortAssignInput(metal.PortAssignInput{ + Vnid: &vxlanStr, + }).Execute() + if err != nil { + return fmt.Errorf("failed to unassign port from VXLAN %d: %w", vxlan, err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to unassign port from VXLAN %d: unexpected status code %d", vxlan, resp.StatusCode) + } + return nil +} + +// assignVXLAN assigns a VXLAN to a port +func (r *PacketMachineReconciler) assignVXLAN(ctx context.Context, portID, vxlanStr string) error { + _, resp, err := r.PacketClient.PortsApi.AssignPort(ctx, portID).PortAssignInput(metal.PortAssignInput{ + Vnid: &vxlanStr, + }).Execute() + if err != nil { + return fmt.Errorf("failed to assign port to VXLAN %s: %w", vxlanStr, err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to assign port to VXLAN %s: unexpected status code %d", vxlanStr, resp.StatusCode) + } + return nil +} + +// setLayer2 configures the port for layer2 if not already set +func (r *PacketMachineReconciler) setLayer2(ctx context.Context, portID string, port *metal.Port, portName string) error { + log := ctrl.LoggerFrom(ctx) + + if strings.Contains(string(*port.NetworkType), "layer2") { + log.Info("Port is already set to layer2", "port", portName) + return nil + } + + _, resp, err := r.PacketClient.PortsApi.ConvertLayer2(ctx, portID).Execute() + if err != nil { + return fmt.Errorf("failed to set port to layer2: %w", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to set port to layer2: unexpected status code %d", resp.StatusCode) + } + log.Info("Port set to layer2", "port", portName) + return nil +} + +// setBonding configures the port for bonding if not already set +func (r *PacketMachineReconciler) setBonding(ctx context.Context, portID string, port *metal.Port, portName string) error { + log := ctrl.LoggerFrom(ctx) + + if port.Data == nil { + return fmt.Errorf("failed to get port data to check bonded status") + } + if *port.Data.Bonded { + log.Info("Port is already bonded", "port", portName) + return nil + } + + _, resp, err := r.PacketClient.PortsApi.BondPort(ctx, portID).Execute() + if err != nil { + return fmt.Errorf("failed to set port to bonded: %w", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to set port to bonded: unexpected status code %d", resp.StatusCode) + } + log.Info("Port set to bonded", "port", portName) + return nil +} + +// getPortVXLANAssignments returns all current VXLAN assignments for a port +func getPortVXLANAssignments(vlanAssignList *metal.PortVlanAssignmentList) []int32 { + var assignments []int32 + for _, vlanAssign := range vlanAssignList.VlanAssignments { + if vlanAssign.Vlan != nil { + assignments = append(assignments, *vlanAssign.Vlan) + } + } + return assignments +} + +func getMetalPortID(portName string, networkPorts []metal.Port) (*string, error) { + for _, port := range networkPorts { + if *port.Name == portName { + return port.Id, nil + } + } + return nil, fmt.Errorf("failed to find port %s", portName) +} + +func getIPAddrClaim(ctx context.Context, client client.Client, ipAddrClaimName, namespace string) (*ipamv1.IPAddressClaim, error) { + log := ctrl.LoggerFrom(ctx) + + ipAddrClaim := &ipamv1.IPAddressClaim{} + ipAddrClaimKey := apitypes.NamespacedName{ + Namespace: namespace, + Name: ipAddrClaimName, + } + + log.V(5).Info("Fetching IPAddressClaim", "IPAddressClaim", klog.KRef(ipAddrClaimKey.Namespace, ipAddrClaimKey.Name)) + if err := client.Get(ctx, ipAddrClaimKey, ipAddrClaim); err != nil { + return nil, err + } + return ipAddrClaim, nil +} + +func (r *PacketMachineReconciler) deletePacketMachine(ctx context.Context, packetmachine *infrav1.PacketMachine) error { + + if packetmachine.Spec.NetworkPorts != nil { + if err := r.reconcileDeletePacketMachineIPAddressClaims(ctx, packetmachine); err != nil { + return err + } + } + + ctrlutil.RemoveFinalizer(packetmachine, infrav1.MachineFinalizer) + return nil +} + +// +kubebuilder:rbac:groups=ipam.cluster.x-k8s.io,resources=ipaddressclaims,verbs=get;create;patch;watch;list;update + +// reconcileDeletePacketMachineIPAddressClaims removes the finalizers from the IPAddressClaim objects of given packet machine +// so that they can be deleted as part of garbage collection +func (r *PacketMachineReconciler) reconcileDeletePacketMachineIPAddressClaims(ctx context.Context, packetMachine *infrav1.PacketMachine) error { + log := ctrl.LoggerFrom(ctx) + + for portIdx, port := range packetMachine.Spec.NetworkPorts { + for networkIdx := range port.Networks { + // check if claim exists + ipAddrClaim := &ipamv1.IPAddressClaim{} + ipAddrClaimName := packet.IPAddressClaimName(packetMachine.Name, portIdx, networkIdx) + ipAddrClaimKey := client.ObjectKey{ + Namespace: packetMachine.Namespace, + Name: ipAddrClaimName, + } + + if err := r.Client.Get(ctx, ipAddrClaimKey, ipAddrClaim); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return fmt.Errorf("failed to get IPAddressClaim %s to remove the finalizer,err: %v", ipAddrClaimName, err) + } + + if ctrlutil.RemoveFinalizer(ipAddrClaim, infrav1.IPAddressClaimFinalizer) { + log.Info(fmt.Sprintf("Removing finalizer %s", infrav1.IPAddressClaimFinalizer), "IPAddressClaim", klog.KObj(ipAddrClaim)) + + if err := r.Client.Update(ctx, ipAddrClaim); err != nil { + log.Error(err, "failed to update IPAddressClaim", "IPAddressClaim", klog.KObj(ipAddrClaim)) + return fmt.Errorf("failed to update IPAddressClaim %s, err: %v", klog.KObj(ipAddrClaim), err) + } + } + } + } + return nil } diff --git a/go.mod b/go.mod index deda4e6b..52b45951 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 golang.org/x/oauth2 v0.18.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.29.3 k8s.io/apimachinery v0.29.3 k8s.io/client-go v0.29.3 @@ -47,7 +48,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -71,13 +72,12 @@ require ( go.opentelemetry.io/otel/sdk v1.20.0 // indirect go.opentelemetry.io/otel/trace v1.20.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect - golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -87,7 +87,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.3 // indirect k8s.io/apiserver v0.29.3 // indirect k8s.io/cluster-bootstrap v0.29.3 // indirect diff --git a/go.sum b/go.sum index 61780eb5..8c39502c 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -236,8 +236,8 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -254,28 +254,28 @@ golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/layer2/layer2.go b/internal/layer2/layer2.go new file mode 100644 index 00000000..4ec0c1c4 --- /dev/null +++ b/internal/layer2/layer2.go @@ -0,0 +1,58 @@ +package layer2 + +import ( + "bytes" + "text/template" +) + +// PortNetwork represents a network configuration for a Layer2 network +type PortNetwork struct { + PortName string + Vxlan int + IPAddress string + Netmask string + Gateway string + Routes []RouteSpec +} + +// RouteSpec represents a static route. +type RouteSpec struct { + Destination string + Gateway string +} + +type Config struct { + // VLANs is a list of network configurations for the Layer2 + VLANs []PortNetwork +} + +// NewConfig returns a new Config object +func NewConfig() *Config { + return &Config{ + VLANs: make([]PortNetwork, 0), + } +} + +func (c *Config) AddPortNetwork(portName string, vxlan int, ipAddress string, netmask string, routes []RouteSpec) { + c.VLANs = append(c.VLANs, PortNetwork{ + PortName: portName, + Vxlan: vxlan, + IPAddress: ipAddress, + Netmask: netmask, + Routes: routes, + }) +} + +func (c *Config) GetTemplate() (string, error) { + tmpl, err := template.New("layer-2-user-data").Parse(configTemplate) + if err != nil { + return "", err + } + + // execute the template and save the output to a buffer + var output bytes.Buffer + if err := tmpl.Execute(&output, c); err != nil { + return "", err + } + return output.String(), nil +} diff --git a/internal/layer2/merge.go b/internal/layer2/merge.go new file mode 100644 index 00000000..222602ac --- /dev/null +++ b/internal/layer2/merge.go @@ -0,0 +1,152 @@ +package layer2 + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +const ( + cloudConfigHeader = "#cloud-config" + jinjaHeader = "## template: jinja" +) + +// CloudConfigMerger handles the merging of cloud configs +type CloudConfigMerger struct { +} + +// NewCloudConfigMerger creates a new instance of CloudConfigMerger +func NewCloudConfigMerger() *CloudConfigMerger { + return &CloudConfigMerger{} +} + +// configHeaders represents the headers found in a cloud-config file +type configHeaders struct { + hasJinja bool + hasCloudConfig bool +} + +// stripHeaders removes the template and cloud-config headers and returns the remaining content +func stripHeaders(data string) (string, configHeaders) { + headers := configHeaders{} + lines := strings.Split(strings.TrimSpace(data), "\n") + startIndex := 0 + + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + switch trimmedLine { + case jinjaHeader: + headers.hasJinja = true + startIndex = i + 1 + case cloudConfigHeader: + headers.hasCloudConfig = true + startIndex = i + 1 + default: + if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + return strings.Join(lines[startIndex:], "\n"), headers + } + } + } + return "", headers +} + +// deepMerge recursively merges two maps +func (m *CloudConfigMerger) deepMerge(base, overlay map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + + // Copy base map + for k, v := range base { + result[k] = v + } + + // Merge overlay + for k, v := range overlay { + if baseVal, exists := result[k]; exists { + // If both values are maps, merge them recursively + if baseMap, ok := baseVal.(map[string]interface{}); ok { + if overlayMap, ok := v.(map[string]interface{}); ok { + result[k] = m.deepMerge(baseMap, overlayMap) + continue + } + } + + // If either value is a slice or both values are different, create/extend a slice + baseSlice, baseIsSlice := baseVal.([]interface{}) + overlaySlice, overlayIsSlice := v.([]interface{}) + + if baseIsSlice && overlayIsSlice { + result[k] = append(baseSlice, overlaySlice...) + } else if baseIsSlice { + result[k] = append(baseSlice, v) + } else if overlayIsSlice { + result[k] = append([]interface{}{baseVal}, overlaySlice...) + } else { + result[k] = []interface{}{baseVal, v} + } + } else { + // Key doesn't exist in base, so add it + result[k] = v + } + } + + return result +} + +// buildHeader constructs the appropriate header based on the input configurations +func buildHeader(bootstrapHeaders, layer2Headers configHeaders) string { + var headers []string + + // If either input has the Jinja header, include it in the output + if bootstrapHeaders.hasJinja || layer2Headers.hasJinja { + headers = append(headers, jinjaHeader) + } + + // Always include the cloud-config header + headers = append(headers, cloudConfigHeader) + + return strings.Join(headers, "\n") +} + +// MergeConfigs combines bootstrap data with layer2 config +func (m *CloudConfigMerger) MergeConfigs(bootstrapData string, layer2UserData string) (string, error) { + // Strip headers and get header info + bootstrapStripped, bootstrapHeaders := stripHeaders(bootstrapData) + layer2Stripped, layer2Headers := stripHeaders(layer2UserData) + + // Validate that at least one input has the cloud-config header + if !bootstrapHeaders.hasCloudConfig && !layer2Headers.hasCloudConfig { + return "", fmt.Errorf("neither input contains #cloud-config header") + } + + var bootstrapConfig, layer2UserDataConfig map[string]interface{} + + if bootstrapStripped != "" { + if err := yaml.Unmarshal([]byte(bootstrapStripped), &bootstrapConfig); err != nil { + return "", fmt.Errorf("error parsing bootstrap YAML: %v", err) + } + } else { + bootstrapConfig = make(map[string]interface{}) + } + + if layer2Stripped != "" { + if err := yaml.Unmarshal([]byte(layer2Stripped), &layer2UserDataConfig); err != nil { + return "", fmt.Errorf("error parsing layer2 YAML: %v", err) + } + } else { + layer2UserDataConfig = make(map[string]interface{}) + } + + // Merge configurations + mergedConfig := m.deepMerge(layer2UserDataConfig, bootstrapConfig) + + // Convert merged config back to YAML + result, err := yaml.Marshal(mergedConfig) + if err != nil { + return "", fmt.Errorf("error marshaling merged config: %v", err) + } + + // Build appropriate headers and combine with content + header := buildHeader(bootstrapHeaders, layer2Headers) + return fmt.Sprintf("%s\n%s", header, string(result)), nil +} diff --git a/internal/layer2/merge_test.go b/internal/layer2/merge_test.go new file mode 100644 index 00000000..f30ffff3 --- /dev/null +++ b/internal/layer2/merge_test.go @@ -0,0 +1,264 @@ +package layer2 + +import ( + "testing" + "gopkg.in/yaml.v3" + "strings" + "reflect" +) + +func TestStripHeaders(t *testing.T) { + tests := []struct { + name string + input string + expected string + headers configHeaders + }{ + { + name: "Both headers present", + input: "## template: jinja\n#cloud-config\ncontent: here", + expected: "content: here", + headers: configHeaders{hasJinja: true, hasCloudConfig: true}, + }, + { + name: "Only cloud-config header", + input: "#cloud-config\ncontent: here", + expected: "content: here", + headers: configHeaders{hasJinja: false, hasCloudConfig: true}, + }, + { + name: "No headers", + input: "content: here", + expected: "content: here", + headers: configHeaders{hasJinja: false, hasCloudConfig: false}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + content, headers := stripHeaders(tt.input) + if content != tt.expected { + t.Errorf("Expected content %q, got %q", tt.expected, content) + } + if headers != tt.headers { + t.Errorf("Expected headers %v, got %v", tt.headers, headers) + } + }) + } +} + +func TestDeepMerge(t *testing.T) { + merger := NewCloudConfigMerger() + + tests := []struct { + name string + base map[string]interface{} + overlay map[string]interface{} + expected map[string]interface{} + }{ + { + name: "Merge simple maps with slice creation", + base: map[string]interface{}{ + "a": 1, + "b": 2, + }, + overlay: map[string]interface{}{ + "b": 3, + "c": 4, + }, + expected: map[string]interface{}{ + "a": 1, + "b": []interface{}{2, 3}, + "c": 4, + }, + }, + { + name: "Merge nested maps", + base: map[string]interface{}{ + "a": map[string]interface{}{ + "x": 1, + "y": 2, + }, + }, + overlay: map[string]interface{}{ + "a": map[string]interface{}{ + "y": 3, + "z": 4, + }, + }, + expected: map[string]interface{}{ + "a": map[string]interface{}{ + "x": 1, + "y": []interface{}{2, 3}, + "z": 4, + }, + }, + }, + { + name: "Merge existing slices", + base: map[string]interface{}{ + "a": []interface{}{1, 2}, + }, + overlay: map[string]interface{}{ + "a": []interface{}{3, 4}, + }, + expected: map[string]interface{}{ + "a": []interface{}{1, 2, 3, 4}, + }, + }, + { + name: "Merge slice with non-slice", + base: map[string]interface{}{ + "a": []interface{}{1, 2}, + }, + overlay: map[string]interface{}{ + "a": 3, + }, + expected: map[string]interface{}{ + "a": []interface{}{1, 2, 3}, + }, + }, + { + name: "two cloud configs", + base: map[string]interface{}{ + "write_files": []interface{}{ + map[string]interface{}{ + "path": "/etc/hostname", + "permissions": "0644", + "owner": "root", + "content": "node-{{ node_index }}", + }, + }, + }, + overlay: map[string]interface{}{ + "write_files": []interface{}{ + map[string]interface{}{ + "path": "/var/lib/cloud/instance/hostname", + "permissions": "0644", + "owner": "root", + "content": "node-{{ node_index }}", + }, + map[string]interface{}{ + "path": "/etc/hosts", + "permissions": "0644", + "owner": "root", + "content": "xyz", + }, + }, + }, + expected: map[string]interface{}{ + "write_files": []interface{}{ + map[string]interface{}{ + "path": "/etc/hostname", + "permissions": "0644", + "owner": "root", + "content": "node-{{ node_index }}", + }, + map[string]interface{}{ + "path": "/var/lib/cloud/instance/hostname", + "permissions": "0644", + "owner": "root", + "content": "node-{{ node_index }}", + }, + map[string]interface{}{ + "path": "/etc/hosts", + "permissions": "0644", + "owner": "root", + "content": "xyz", + }, + }, + }, + }} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := merger.deepMerge(tt.base, tt.overlay) + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestMergeConfigs(t *testing.T) { + merger := NewCloudConfigMerger() + + tests := []struct { + name string + bootstrap string + layer2 string + expectedYAML map[string]interface{} + expectedError bool + }{ + { + name: "Merge valid configs", + bootstrap: "#cloud-config\nbootstrap: data", + layer2: "#cloud-config\nlayer2: data", + expectedYAML: map[string]interface{}{ + "bootstrap": "data", + "layer2": "data", + }, + expectedError: false, + }, + { + name: "No cloud-config header", + bootstrap: "bootstrap: data", + layer2: "layer2: data", + expectedYAML: nil, + expectedError: true, + }, + { + name: "Merge with Jinja header", + bootstrap: "## template: jinja\n#cloud-config\nbootstrap: data", + layer2: "#cloud-config\nlayer2: data", + expectedYAML: map[string]interface{}{ + "bootstrap": "data", + "layer2": "data", + }, + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := merger.MergeConfigs(tt.bootstrap, tt.layer2) + + if tt.expectedError { + if err == nil { + t.Errorf("Expected an error, but got none") + } + return + } + + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + + // Parse the result YAML + var resultYAML map[string]interface{} + err = yaml.Unmarshal([]byte(result), &resultYAML) + if err != nil { + t.Errorf("Error parsing result YAML: %v", err) + return + } + + // Compare the parsed YAML + if !reflect.DeepEqual(resultYAML, tt.expectedYAML) { + t.Errorf("Expected YAML %v, got %v", tt.expectedYAML, resultYAML) + } + + // Check for correct headers + lines := strings.Split(result, "\n") + if tt.bootstrap == "## template: jinja\n#cloud-config\nbootstrap: data" { + if lines[0] != "## template: jinja" || lines[1] != "#cloud-config" { + t.Errorf("Expected Jinja and cloud-config headers, got %v", lines[:2]) + } + } else { + if lines[0] != "#cloud-config" { + t.Errorf("Expected cloud-config header, got %v", lines[0]) + } + } + }) + } +} \ No newline at end of file diff --git a/internal/layer2/template.go b/internal/layer2/template.go new file mode 100644 index 00000000..610cd923 --- /dev/null +++ b/internal/layer2/template.go @@ -0,0 +1,183 @@ +package layer2 + +const configTemplate = ` +#cloud-config + +package_update: true +package_upgrade: true +packages: + - jq + - vlan + +write_files: + - path: /var/lib/capi_network_settings/final_configuration.sh + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + + echo "Running final configuration commands" + apt-get update -qq + apt-get install -y -qq jq vlan + + modprobe 8021q + echo "8021q" >> /etc/modules + + # Generate the network configuration and append it to /etc/network/interfaces for each VLAN-tagged sub-interface. + cat <> /etc/network/interfaces +{{ range .VLANs }} + auto {{ .PortName }}.{{ .Vxlan }} + iface {{ .PortName }}.{{ .Vxlan }} inet static + pre-up sleep 5 + address {{ .IPAddress }} + netmask {{ .Netmask }} + {{- if .Gateway }} + gateway {{ .Gateway }} + {{- end }} + vlan-raw-device {{ .PortName }} + {{- range .Routes }} + up ip route add {{ .Destination }} via {{ .Gateway }} + {{- end }} +{{ end }} + EOL + + echo "VLAN configuration appended to /etc/network/interfaces." + + # Function to send user state events + url="$(curl -sf https://metadata.platformequinix.com/metadata | jq -r .user_state_url)" + + # Function to send user state events + send_user_state_event() { + local state="$1" + local code="$2" + local message="$3" + local data + local max_retries=3 + local retry_count=0 + + data=$(jq -n --arg state "$state" --arg code "$code" --arg message "$message" \ + '{state: $state, code: ($code | tonumber), message: $message}') + + while [ $retry_count -lt $max_retries ]; do + # Make the POST request and capture the HTTP status code + http_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$data" "$url") + + echo "HTTP Status Code: $http_code" + + if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then + echo "User state event sent successfully on attempt $((retry_count + 1))" + return 0 + else + echo "Warning: Received non-success status code: $http_code" + fi + + retry_count=$((retry_count + 1)) + if [ $retry_count -lt $max_retries ]; then + echo "Retrying in 5 seconds..." + sleep 5 + fi + done + + echo "Error: Failed to send user state event after $max_retries attempts" + return 1 + } + + + send_user_state_event running 1000 "network_configuration_started" + + # Restart networking to apply VLAN configurations + echo "Restarting networking service..." + systemctl restart networking + + # Wait for interfaces to be fully up + echo "Waiting for interfaces to be up..." + sleep 5 + + # Verify network configuration + verification_failed=false +{{ range .VLANs }} + if ip addr show {{ .PortName }}.{{ .Vxlan }} | grep -q {{ .IPAddress }}; then + echo "Configuration for VLAN {{ .Vxlan }} on {{ .PortName }} with IP {{ .IPAddress }} successful" + else + echo "Configuration for VLAN {{ .Vxlan }} on {{ .PortName }} with IP {{ .IPAddress }} failed" >&2 + verification_failed=true + fi + + # Verify static routes +{{ range .Routes }} + if ip route | grep -q "{{ .Destination }} via {{ .Gateway }}"; then + echo "Static route to {{ .Destination }} via {{ .Gateway }} added successfully" + else + echo "Failed to add static route to {{ .Destination }} via {{ .Gateway }}" >&2 + verification_failed=true + fi +{{ end }} +{{ end }} + + if [ "$verification_failed" = true ]; then + send_user_state_event failed 1002 "network_configuration_failed" + exit 1 + else + send_user_state_event succeeded 1001 "network_configuration_success" + fi + + - path: /var/lib/capi_network_settings/initial_configuration.sh + permissions: '0755' + content: | + #!/bin/bash + set -eu + + # Fetch metadata from Equinix Metal + metadata=$(curl https://metadata.platformequinix.com/metadata) + + # Extract MAC addresses for eth0 and eth1 + mac_eth0=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth0") | .mac') + mac_eth1=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth1") | .mac') + + # Check if MAC addresses were successfully extracted + if [ -z "$mac_eth0" ] || [ -z "$mac_eth1" ]; then + echo "Error: Failed to extract MAC addresses" >&2 + exit 1 + fi + + # Display extracted MAC addresses + echo "Extracted MAC addresses - eth0: $mac_eth0, eth1: $mac_eth1" + + # Function to find network interface by MAC address + find_interface_by_mac() { + local mac="$1" + for iface in $(ls /sys/class/net/); do + iface_mac=$(ethtool -P "$iface" 2>/dev/null | awk '{print $NF}') + if [ "$iface_mac" == "$mac" ]; then + echo "$iface" + return + fi + done + echo "Interface not found for MAC $mac" >&2 + return 1 + } + + # Find interfaces for eth0 and eth1 MAC addresses + iface_eth0=$(find_interface_by_mac "$mac_eth0") + iface_eth1=$(find_interface_by_mac "$mac_eth1") + + # Check and replace eth0 in /var/lib/capi_network_settings/final_configuration.sh + if grep -q "eth0" /var/lib/capi_network_settings/final_configuration.sh; then + sed -i "s/eth0/${iface_eth0}/g" /var/lib/capi_network_settings/final_configuration.sh + echo "Replaced eth0 with ${iface_eth0} in /var/lib/capi_network_settings/final_configuration.sh" + else + echo "No occurrences of eth0 found in /var/lib/capi_network_settings/final_configuration.sh. No changes made." + fi + + # Check and replace eth1 in /var/lib/capi_network_settings/final_configuration.sh + if grep -q "eth1" /var/lib/capi_network_settings/final_configuration.sh; then + sed -i "s/eth1/${iface_eth1}/g" /var/lib/capi_network_settings/final_configuration.sh + echo "Replaced eth1 with ${iface_eth1} in /var/lib/capi_network_settings/final_configuration.sh" + else + echo "No occurrences of eth1 found in /var/lib/capi_network_settings/final_configuration.sh. No changes made." + fi + +runcmd: + - /var/lib/capi_network_settings/initial_configuration.sh + - /var/lib/capi_network_settings/final_configuration.sh +` \ No newline at end of file diff --git a/main.go b/main.go index 3f7116a2..13ed9ea5 100644 --- a/main.go +++ b/main.go @@ -52,6 +52,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1" "sigs.k8s.io/cluster-api-provider-packet/controllers" packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" // +kubebuilder:scaffold:imports ) @@ -66,6 +67,7 @@ func init() { _ = infrav1.AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) _ = bootstrapv1.AddToScheme(scheme) + _ = ipamv1.AddToScheme(scheme) } var ( diff --git a/pkg/cloud/packet/client.go b/pkg/cloud/packet/client.go index 8e20bb3f..b0067b39 100644 --- a/pkg/cloud/packet/client.go +++ b/pkg/cloud/packet/client.go @@ -33,6 +33,7 @@ import ( "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-packet/internal/layer2" "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope" "sigs.k8s.io/cluster-api-provider-packet/version" ) @@ -106,6 +107,17 @@ type CreateDeviceRequest struct { ControlPlaneEndpoint string CPEMLBConfig string EMLBID string + IPAddresses []IPAddressCfg +} + +type IPAddressCfg struct { + Netmask string + VXLAN int + Address string + PortName string + Layer2 bool + Bonded bool + Routes []layer2.RouteSpec } // NewDevice creates a new device. @@ -125,6 +137,7 @@ func (p *Client) NewDevice(ctx context.Context, req CreateDeviceRequest) (*metal } stringWriter := &strings.Builder{} + userData := string(userDataRaw) userDataValues := map[string]interface{}{ "kubernetesVersion": ptr.Deref(req.MachineScope.Machine.Spec.Version, ""), @@ -164,8 +177,27 @@ func (p *Client) NewDevice(ctx context.Context, req CreateDeviceRequest) (*metal return nil, fmt.Errorf("error executing userdata template: %w", err) } - userData = stringWriter.String() + // Todo: move this to a separate function + var layer2UserData string + // check if layer2 is enabled and add the layer2 user data + if packetMachineSpec.NetworkPorts != nil { + layer2Config := layer2.NewConfig() + for _, ipAddr := range req.IPAddresses { + layer2Config.AddPortNetwork(ipAddr.PortName, ipAddr.VXLAN, ipAddr.Address, ipAddr.Netmask, ipAddr.Routes) + } + + layer2UserData, err = layer2Config.GetTemplate() + if err != nil { + return nil, fmt.Errorf("error generating layer2 user data: %w", err) + } + userData, err = layer2.NewCloudConfigMerger().MergeConfigs(stringWriter.String(), layer2UserData) + if err != nil { + return nil, fmt.Errorf("error combining user data: %w", err) + } + } else { + userData = stringWriter.String() + } // If Metro or Facility are specified at the Machine level, we ignore the // values set at the Cluster level facility := packetClusterSpec.Facility diff --git a/pkg/cloud/packet/util.go b/pkg/cloud/packet/util.go index a5d895d6..5f454413 100644 --- a/pkg/cloud/packet/util.go +++ b/pkg/cloud/packet/util.go @@ -71,3 +71,7 @@ func DefaultCreateTags(namespace, name, clusterName string) []string { GenerateNamespaceTag(namespace), } } + +func IPAddressClaimName(machineName string, portIndex, networkIndex int) string { + return fmt.Sprintf("%s-port-%d-network-%d", machineName, portIndex, networkIndex) +} diff --git a/templates/cluster-template-crs-cni.yaml b/templates/cluster-template-crs-cni.yaml index ae6336b4..c3c19594 100644 --- a/templates/cluster-template-crs-cni.yaml +++ b/templates/cluster-template-crs-cni.yaml @@ -70,6 +70,7 @@ spec: sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd + - "" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster diff --git a/templates/cluster-template-development.yaml b/templates/cluster-template-development.yaml index d419b715..0933e8db 100644 --- a/templates/cluster-template-development.yaml +++ b/templates/cluster-template-development.yaml @@ -51,6 +51,7 @@ spec: sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd + - "" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster diff --git a/templates/cluster-template-emlb-crs-cni.yaml b/templates/cluster-template-emlb-crs-cni.yaml index 238d8907..3a7413c9 100644 --- a/templates/cluster-template-emlb-crs-cni.yaml +++ b/templates/cluster-template-emlb-crs-cni.yaml @@ -70,6 +70,7 @@ spec: sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd + - "" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster diff --git a/templates/cluster-template-emlb.yaml b/templates/cluster-template-emlb.yaml index 2e948990..9aedceb6 100644 --- a/templates/cluster-template-emlb.yaml +++ b/templates/cluster-template-emlb.yaml @@ -51,6 +51,7 @@ spec: sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd + - "" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index d419b715..28eda575 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -51,6 +51,11 @@ spec: sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml systemctl restart containerd + - | + + + + --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -241,3 +246,6 @@ spec: sshKeys: - ${SSH_KEY} tags: [] + + +