Skip to content

Commit 7863504

Browse files
committed
feat(operator): fetch cluster name from providers
Signed-off-by: Rishabh Soni <[email protected]> Add support for EKS Signed-off-by: Rishabh Soni <[email protected]> Remove AKS reference Signed-off-by: Rishabh Soni <[email protected]> Make provider hostname and endpoint configurable Signed-off-by: Rishabh Soni <[email protected]> Make provider hostname and endpoint configurable Signed-off-by: Rishabh Soni <[email protected]>
1 parent 3dee887 commit 7863504

File tree

6 files changed

+174
-4
lines changed

6 files changed

+174
-4
lines changed

KubeArmor/core/kubeUpdate.go

100644100755
+2
Original file line numberDiff line numberDiff line change
@@ -2761,6 +2761,7 @@ func (dm *KubeArmorDaemon) WatchConfigMap() cache.InformerSynced {
27612761
if cm, ok := obj.(*corev1.ConfigMap); ok && cm.Namespace == cmNS {
27622762
cfg.GlobalCfg.HostVisibility = cm.Data[cfg.ConfigHostVisibility]
27632763
cfg.GlobalCfg.Visibility = cm.Data[cfg.ConfigVisibility]
2764+
cfg.GlobalCfg.Cluster = cm.Data[cfg.ConfigCluster]
27642765
if _, ok := cm.Data[cfg.ConfigDefaultPostureLogs]; ok {
27652766
cfg.GlobalCfg.DefaultPostureLogs = (cm.Data[cfg.ConfigDefaultPostureLogs] == "true")
27662767
}
@@ -2806,6 +2807,7 @@ func (dm *KubeArmorDaemon) WatchConfigMap() cache.InformerSynced {
28062807
if cm, ok := new.(*corev1.ConfigMap); ok && cm.Namespace == cmNS {
28072808
cfg.GlobalCfg.HostVisibility = cm.Data[cfg.ConfigHostVisibility]
28082809
cfg.GlobalCfg.Visibility = cm.Data[cfg.ConfigVisibility]
2810+
cfg.GlobalCfg.Cluster = cm.Data[cfg.ConfigCluster]
28092811
if _, ok := cm.Data[cfg.ConfigDefaultPostureLogs]; ok {
28102812
cfg.GlobalCfg.DefaultPostureLogs = (cm.Data[cfg.ConfigDefaultPostureLogs] == "true")
28112813
}

deployments/helm/KubeArmorOperator/templates/deployment.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ spec:
3131
{{- if or (eq $tag "latest") (and (hasPrefix "v" $tag) (semverCompare "^1.4.0" $tag)) }}
3232
# initDeploy flag is only supported from v1.4.0
3333
args:
34-
- --initDeploy={{.Values.kubearmorOperator.initDeploy }}
34+
{{- if .Values.kubearmorOperator.args -}}
35+
{{- toYaml .Values.kubearmorOperator.args | trim | nindent 8 }}
36+
{{- end }}
3537
{{- end }}
3638

3739
serviceAccountName: {{ .Values.kubearmorOperator.name }}

deployments/helm/KubeArmorOperator/values.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ kubearmorOperator:
3636
repository: kubearmor/kubearmor-operator
3737
tag: ""
3838
imagePullPolicy: IfNotPresent
39-
initDeploy: true
39+
args:
40+
- "--initDeploy=true"
4041

4142
kubearmorConfig:
4243
defaultCapabilitiesPosture: audit

pkg/KubeArmorOperator/cmd/operator/main.go

100644100755
+4-1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ var Opv1Client *opv1client.Clientset
3232
var Secv1Client *secv1client.Clientset
3333
var InitDeploy bool
3434
var LogLevel string
35+
var ProviderHostname, ProviderEndpoint string
3536

3637
// Cmd represents the base command when called without any subcommands
3738
var Cmd = &cobra.Command{
@@ -55,7 +56,7 @@ var Cmd = &cobra.Command{
5556
return nil
5657
},
5758
Run: func(cmd *cobra.Command, args []string) {
58-
nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, Secv1Client, PathPrefix, DeploymentName, InitDeploy)
59+
nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, Secv1Client, PathPrefix, DeploymentName, ProviderHostname, ProviderEndpoint, InitDeploy)
5960
go nodeWatcher.WatchConfigCrd()
6061
nodeWatcher.WatchNodes()
6162

@@ -81,6 +82,8 @@ func init() {
8182
Cmd.PersistentFlags().StringVar(&LsmOrder, "lsm", "bpf,apparmor,selinux", "lsm preference order to use")
8283
Cmd.PersistentFlags().StringVar(&PathPrefix, "pathprefix", "/rootfs/", "path prefix for runtime search")
8384
Cmd.PersistentFlags().StringVar(&DeploymentName, "deploymentName", "kubearmor-operator", "operator deployment name")
85+
Cmd.PersistentFlags().StringVar(&ProviderHostname, "providerHostname", "", "IMDS URL hostname for retrieving cluster name")
86+
Cmd.PersistentFlags().StringVar(&ProviderEndpoint, "providerEndpoint", "", "IMDS URL endpoint for retrieving cluster name")
8487
// TODO:- set initDeploy to false by default once this change is added to stable
8588
Cmd.PersistentFlags().BoolVar(&InitDeploy, "initDeploy", true, "Init container deployment")
8689
Cmd.PersistentFlags().StringVar(&LogLevel, "loglevel", "info", "log level, e.g., debug, info, warn, error")

pkg/KubeArmorOperator/internal/controller/cluster.go

100644100755
+5-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ var deployment_uuid types.UID
4545
var deployment_name string = "kubearmor-operator"
4646
var PathPrefix string
4747
var initDeploy bool
48+
var ProviderHostname, ProviderEndpoint string
4849

4950
type ClusterWatcher struct {
5051
Nodes []Node
@@ -68,7 +69,7 @@ type Node struct {
6869
Seccomp string
6970
}
7071

71-
func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, secv1Client *secv1client.Clientset, pathPrefix, deploy_name string, initdeploy bool) *ClusterWatcher {
72+
func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, secv1Client *secv1client.Clientset, pathPrefix, deploy_name, providerHostname, providerEndpoint string, initdeploy bool) *ClusterWatcher {
7273
if informer == nil {
7374
informer = informers.NewSharedInformerFactory(client, 0)
7475
}
@@ -85,6 +86,9 @@ func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, ext
8586
PathPrefix = pathPrefix
8687
deployment_name = deploy_name
8788
initDeploy = initdeploy
89+
ProviderHostname = providerHostname
90+
ProviderEndpoint = providerEndpoint
91+
8892
return &ClusterWatcher{
8993
Nodes: []Node{},
9094
Daemonsets: make(map[string]int),

pkg/KubeArmorOperator/internal/controller/resources.go

100644100755
+158
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ import (
77
"bytes"
88
"context"
99
"fmt"
10+
"io"
11+
"net/http"
12+
"regexp"
1013
"strconv"
1114
"strings"
1215
"time"
@@ -470,6 +473,160 @@ func (clusterWatcher *ClusterWatcher) deployControllerDeployment(deployment *app
470473
return nil
471474
}
472475

476+
func (clusterWatcher *ClusterWatcher) getProvider(providerHostname, providerEndpoint string) (string, string, string) {
477+
nodes, err := clusterWatcher.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
478+
if err != nil {
479+
clusterWatcher.Log.Warnf("Error listing nodes: %s\n", err.Error())
480+
}
481+
482+
for _, node := range nodes.Items {
483+
for key, label := range node.Labels {
484+
if strings.Contains(key, "gke") || strings.Contains(label, "gke") {
485+
if providerHostname != "" && providerEndpoint == "" {
486+
providerEndpoint = "/computeMetadata/v1/instance/attributes/cluster-name"
487+
} else if providerHostname == "" && providerEndpoint != "" {
488+
providerHostname = "http://metadata.google.internal"
489+
} else if providerHostname == "" && providerEndpoint == "" {
490+
providerHostname = "http://metadata.google.internal"
491+
providerEndpoint = "/computeMetadata/v1/instance/attributes/cluster-name"
492+
}
493+
return "gke", providerHostname, providerEndpoint
494+
} else if strings.Contains(key, "eks") || strings.Contains(label, "eks") {
495+
if providerHostname != "" && providerEndpoint == "" {
496+
providerEndpoint = "/latest/user-data"
497+
} else if providerHostname == "" && providerEndpoint != "" {
498+
providerHostname = "http://169.254.169.254"
499+
} else if providerHostname == "" && providerEndpoint == "" {
500+
providerHostname = "http://169.254.169.254"
501+
providerEndpoint = "/latest/user-data"
502+
}
503+
return "eks", providerHostname, providerEndpoint
504+
}
505+
}
506+
}
507+
return "default", "", ""
508+
}
509+
510+
func (clusterWatcher *ClusterWatcher) fetchClusterNameFromGKE(providerHostname, providerEndpoint string) (string, error) {
511+
url := providerHostname + providerEndpoint
512+
req, err := http.NewRequest("GET", url, nil)
513+
if err != nil {
514+
clusterWatcher.Log.Warnf("failed to create request: %w, check provider host name and endpoint", err)
515+
return "", err
516+
}
517+
518+
// Set the required header
519+
req.Header.Set("Metadata-Flavor", "Google")
520+
521+
// Create an HTTP client and make the request
522+
client := &http.Client{}
523+
resp, err := client.Do(req)
524+
if err != nil {
525+
clusterWatcher.Log.Warnf("error making request: %w, check provider host name and endpoint", err)
526+
return "", err
527+
}
528+
defer resp.Body.Close()
529+
530+
// Check for a successful response
531+
if resp.StatusCode != http.StatusOK {
532+
clusterWatcher.Log.Warnf("failed to fetch from metadata, status code: %d", resp.StatusCode)
533+
return "", err
534+
}
535+
536+
// Read the response body
537+
body, err := io.ReadAll(resp.Body)
538+
if err != nil {
539+
clusterWatcher.Log.Warnf("error reading response body: %w", err)
540+
return "", err
541+
}
542+
543+
return string(body), nil
544+
}
545+
546+
func (clusterWatcher *ClusterWatcher) fetchClusterNameFromAWS(providerHostname, providerEndpoint string) (string, error) {
547+
var token []byte
548+
client := &http.Client{Timeout: 2 * time.Second}
549+
req, err := http.NewRequest("PUT", providerHostname+"/latest/api/token", nil)
550+
if err != nil {
551+
clusterWatcher.Log.Warnf("failed to create request for fetching token: %w, check provider host name", err)
552+
return "", err
553+
}
554+
req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
555+
556+
resp, err := client.Do(req)
557+
if err != nil {
558+
clusterWatcher.Log.Warnf("error making request: %w", err)
559+
return "", err
560+
}
561+
defer resp.Body.Close()
562+
563+
if resp.StatusCode == http.StatusOK {
564+
token, err = io.ReadAll(resp.Body)
565+
if err != nil {
566+
clusterWatcher.Log.Warnf("failed to read token: %d", err)
567+
return "", err
568+
}
569+
}
570+
571+
// Fetch the EKS cluster name from user data
572+
url := providerHostname + providerEndpoint
573+
req, err = http.NewRequest("GET", url, nil)
574+
client = &http.Client{Timeout: 2 * time.Second}
575+
if err != nil {
576+
clusterWatcher.Log.Warnf("failed to create request for fetching metadata: %w, check provider host name and endpoint", err)
577+
return "", err
578+
}
579+
req.Header.Set("X-aws-ec2-metadata-token", string(token))
580+
581+
resp, err = client.Do(req)
582+
if err != nil {
583+
clusterWatcher.Log.Warnf("error making request: %w", err)
584+
return "", err
585+
}
586+
defer resp.Body.Close()
587+
588+
if resp.StatusCode != http.StatusOK {
589+
clusterWatcher.Log.Warnf("failed to fetch from metadata, status code: %d", resp.StatusCode)
590+
return "", err
591+
}
592+
593+
body, err := io.ReadAll(resp.Body)
594+
if err != nil {
595+
clusterWatcher.Log.Warnf("failed to read metadata: %d", err)
596+
return "", err
597+
}
598+
599+
// Extract EKS cluster name
600+
re := regexp.MustCompile(`/etc/eks/bootstrap\.sh (\S+)`)
601+
match := re.FindStringSubmatch(string(body))
602+
if len(match) > 0 {
603+
return match[1], nil
604+
}
605+
606+
return "", err
607+
}
608+
609+
func (clusterWatcher *ClusterWatcher) GetClusterName(providerHostname, providerEndpoint string) string {
610+
provider, pHostname, pEndpoint := clusterWatcher.getProvider(ProviderHostname, providerEndpoint)
611+
if provider == "gke" {
612+
clusterWatcher.Log.Infof("Provider is GKE")
613+
if clusterName, err := clusterWatcher.fetchClusterNameFromGKE(pHostname, pEndpoint); err != nil {
614+
clusterWatcher.Log.Warnf("Cannot fetch cluster name for GKE %s", err.Error())
615+
} else {
616+
return clusterName
617+
}
618+
} else if provider == "eks" {
619+
clusterWatcher.Log.Infof("Provider is EKS")
620+
if clusterName, err := clusterWatcher.fetchClusterNameFromAWS(pHostname, pEndpoint); err != nil {
621+
clusterWatcher.Log.Warnf("Cannot fetch cluster name for EKS %s", err.Error())
622+
} else {
623+
return clusterName
624+
}
625+
}
626+
627+
return "default"
628+
}
629+
473630
func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
474631
var caCert, tlsCrt, tlsKey *bytes.Buffer
475632
var kGenErr, err, installErr error
@@ -653,6 +810,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
653810
// kubearmor configmap
654811
configmap := addOwnership(deployments.GetKubearmorConfigMap(common.Namespace, deployments.KubeArmorConfigMapName)).(*corev1.ConfigMap)
655812
configmap.Data = common.ConfigMapData
813+
configmap.Data["cluster"] = clusterWatcher.GetClusterName(ProviderHostname, ProviderEndpoint)
656814

657815
for {
658816
caCert, tlsCrt, tlsKey, kGenErr = common.GeneratePki(common.Namespace, deployments.KubeArmorControllerWebhookServiceName)

0 commit comments

Comments
 (0)