feat(core): add support for kata and confidential containers

KubeArmor/config/config.go

@@ -54,6 +54,7 @@ type KubearmorConfig struct {
 	DefaultPostureLogs bool     // Enable/Disable Default Posture logs for AppArmor LSM
 	InitTimeout        string   // Timeout for main thread init stages
 
+	UseOCIHooks bool // Use OCI hooks for container visibility instead of CRI socket
 	StateAgent bool // enable KubeArmor state agent
 
 	AlertThrottling   bool // Enable/Disable Alert Throttling
@@ -96,6 +97,7 @@ const (
 	ConfigCoverageTest                   string = "coverageTest"
 	ConfigK8sEnv                         string = "k8s"
 	ConfigDebug                          string = "debug"
+	UseOCIHooks 						 string = "useOCIHooks"
 	ConfigUntrackedNs                    string = "untrackedNs"
 	LsmOrder                             string = "lsm"
 	BPFFsPath                            string = "bpfFsPath"
@@ -156,6 +158,8 @@ func readCmdLineParams() {
 
 	stateAgent := flag.Bool(ConfigStateAgent, false, "enabling KubeArmor State Agent client")
 
+	useOCIHooks := flag.Bool(UseOCIHooks, false, "Use OCI hooks to get new containers instead of using container runtime socket")
+
 	alertThrottling := flag.Bool(ConfigAlertThrottling, true, "enabling Alert Throttling")
 
 	maxAlertPerSec := flag.Int(ConfigMaxAlertPerSec, 10, "Maximum alerts allowed per second")
@@ -220,6 +224,8 @@ func readCmdLineParams() {
 
 	viper.SetDefault(ConfigStateAgent, *stateAgent)
 
+	viper.SetDefault(UseOCIHooks, *useOCIHooks)
+
 	viper.SetDefault(ConfigAlertThrottling, *alertThrottling)
 
 	viper.SetDefault(ConfigMaxAlertPerSec, *maxAlertPerSec)
@@ -324,6 +330,8 @@ func LoadConfig() error {
 
 	GlobalCfg.StateAgent = viper.GetBool(ConfigStateAgent)
 
+	GlobalCfg.UseOCIHooks = viper.GetBool(UseOCIHooks)
+
 	GlobalCfg.AlertThrottling = viper.GetBool(ConfigAlertThrottling)
 	GlobalCfg.MaxAlertPerSec = viper.GetInt(ConfigMaxAlertPerSec)
 	GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec)

KubeArmor/core/hookHandler.go

@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2022 Authors of KubeArmor
+
+package core
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/fsnotify/fsnotify"
+	kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
+	cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
+	"github.com/kubearmor/KubeArmor/KubeArmor/state"
+	tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
+)
+
+func (dm *KubeArmorDaemon) HandleFile(file string) {
+	f, err := os.Open(file)
+	if err != nil {
+		log.Println("Error opening file:", err)
+	}
+
+	decoder := json.NewDecoder(f)
+	for {
+		var containerData tp.Container
+
+		err = decoder.Decode(&containerData)
+		if err != nil {
+			dm.Logger.Warnf("Reached EOF")
+		}
+		dm.handleContainerCreate(containerData)
+		if err == io.EOF {
+			// End of file reached
+			break
+		}
+	}
+	f.Close()
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatal("Error creating new watcher:", err)
+	}
+	defer w.Close()
+
+	err = w.Add(file)
+	if err != nil {
+		log.Fatal("Error adding file to watcher:", err)
+	}
+
+	for {
+		select {
+		case err, ok := <-w.Errors:
+			if !ok {
+				dm.Logger.Warnf("Returning 1")
+				return
+			}
+			log.Println("Watcher error:", err)
+
+		case e, ok := <-w.Events:
+			if !ok {
+				dm.Logger.Warnf("Returning 2")
+				return
+			}
+
+			if e.Op&fsnotify.Write == fsnotify.Write {
+				dm.Logger.Warnf("Detected changes in output.json")
+				f, err := os.Open(file)
+				if err != nil {
+					log.Println("Error opening file:", err)
+					continue
+				}
+				defer f.Close()
+
+				decoder := json.NewDecoder(f)
+				for {
+					var containerData tp.Container
+
+					err = decoder.Decode(&containerData)
+					if err != nil {
+						dm.Logger.Warnf("Reached EOF")
+					}
+					dm.handleContainerCreate(containerData)
+					if err == io.EOF {
+						// End of file reached
+						break
+					}
+				}
+			}
+		}
+	}
+}
+
+func (dm *KubeArmorDaemon) handleContainerCreate(container tp.Container) {
+	// endpoint := types.EndPoint{}
+
+	// dm.ContainersLock.Lock()
+	// defer dm.ContainersLock.Unlock()
+	// if _, ok := dm.Containers[container.ContainerID]; !ok {
+	// 	dm.Containers[container.ContainerID] = container
+	// } else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 {
+	// 	c := dm.Containers[container.ContainerID]
+	// 	c.MntNS = container.MntNS
+	// 	c.PidNS = container.PidNS
+	// 	c.AppArmorProfile = container.AppArmorProfile
+	// 	dm.Containers[c.ContainerID] = c
+	// 	dm.EndPointsLock.Lock()
+	// 	for idx, endPoint := range dm.EndPoints {
+	// 		if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
+
+	// 			// update apparmor profiles
+	// 			if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) {
+	// 				dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
+	// 			}
+
+	// 			if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
+	// 				dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
+	// 			}
+
+	// 			endpoint = dm.EndPoints[idx]
+
+	// 			break
+	// 		}
+	// 	}
+	// 	dm.EndPointsLock.Unlock()
+	// }
+
+	// if len(dm.OwnerInfo) > 0 {
+	// 	container.Owner = dm.OwnerInfo[container.EndPointName]
+	// }
+
+	// if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
+	// 	dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
+	// 	dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
+
+	// 	if len(endpoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endpoint yet
+	// 		dm.Logger.UpdateSecurityPolicies("ADDED", endpoint)
+	// 		if dm.RuntimeEnforcer != nil && endpoint.PolicyEnabled == types.KubeArmorPolicyEnabled {
+	// 			// enforce security policies
+	// 			dm.RuntimeEnforcer.UpdateSecurityPolicies(endpoint)
+	// 		}
+	// 	}
+	// }
+
+	// if container.ContainerID == "" {
+	// 	return false
+	// }
+
+	endPoint := tp.EndPoint{}
+
+	dm.ContainersLock.Lock()
+	defer dm.ContainersLock.Unlock()
+	if _, ok := dm.Containers[container.ContainerID]; !ok {
+		dm.Containers[container.ContainerID] = container
+
+		// create/update endpoint in non-k8s mode
+		if !dm.K8sEnabled {
+			// for policy matching
+			container.NamespaceName = "container_namespace"
+			labels := []string{}
+			labels = append(labels, "namespaceName="+container.NamespaceName)
+			labels = append(labels, "kubearmor.io/container.name="+container.ContainerName)
+			container.Labels = strings.Join(labels, ",")
+
+
+			containerLabels, containerIdentities := kl.GetLabelsFromString(container.Labels)
+			dm.EndPointsLock.Lock()
+
+			endPoint.EndPointName = container.ContainerName
+			endPoint.NamespaceName = container.NamespaceName
+			endPoint.Containers = []string{container.ContainerID}
+			endPoint.Labels = containerLabels
+			endPoint.Identities = containerIdentities
+			endPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
+			endPoint.ProcessVisibilityEnabled = true
+			endPoint.FileVisibilityEnabled = true
+			endPoint.NetworkVisibilityEnabled = true
+			endPoint.CapabilitiesVisibilityEnabled = true
+
+			endPoint.AppArmorProfiles = []string{"kubearmor_" + container.ContainerName}
+
+			globalDefaultPosture := tp.DefaultPosture{
+				FileAction:         cfg.GlobalCfg.DefaultFilePosture,
+				NetworkAction:      cfg.GlobalCfg.DefaultNetworkPosture,
+				CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
+			}
+			endPoint.DefaultPosture = globalDefaultPosture
+
+			dm.SecurityPoliciesLock.RLock()
+			for _, secPol := range dm.SecurityPolicies {
+				if kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) {
+					endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
+				}
+			}
+			dm.SecurityPoliciesLock.RUnlock()
+
+			dm.EndPoints = append(dm.EndPoints, endPoint)
+			dm.EndPointsLock.Unlock()
+		}
+	} else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 {
+		c := dm.Containers[container.ContainerID]
+		c.MntNS = container.MntNS
+		c.PidNS = container.PidNS
+		c.AppArmorProfile = container.AppArmorProfile
+		dm.Containers[c.ContainerID] = c
+
+		dm.EndPointsLock.Lock()
+		for idx, endpoint := range dm.EndPoints {
+			if endpoint.NamespaceName == container.NamespaceName && endpoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
+
+				// update apparmor profiles
+				if !kl.ContainsElement(endpoint.AppArmorProfiles, container.AppArmorProfile) {
+					dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
+				}
+
+				if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
+					dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
+				}
+
+				endPoint = dm.EndPoints[idx]
+
+				break
+			}
+		}
+		dm.EndPointsLock.Unlock()
+	}
+
+	if len(dm.OwnerInfo) > 0 {
+		container.Owner = dm.OwnerInfo[container.EndPointName]
+	}
+
+	if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
+		// for throttling
+		dm.SystemMonitor.Logger.ContainerNsKey[container.ContainerID] = kl.OuterKey{
+			MntNs: container.MntNS,
+			PidNs: container.PidNS,
+		}
+
+		// update NsMap
+		dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
+		dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
+
+		if len(endPoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endPoint yet
+			dm.Logger.UpdateSecurityPolicies("ADDED", endPoint)
+			if dm.RuntimeEnforcer != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
+				// enforce security policies
+				dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
+			}
+		}
+	}
+
+	if cfg.GlobalCfg.StateAgent {
+		container.Status = "running"
+		go dm.StateAgent.PushContainerEvent(container, state.EventAdded)
+	}
+
+	dm.Logger.Printf("Detected a container (added/%.12s/pidns=%d/mntns=%d)", container.ContainerID, container.PidNS, container.MntNS)
+}

KubeArmor/core/kubeArmor.go

@@ -569,8 +569,10 @@ func KubeArmor() {
 
 		dm.SetContainerNSVisibility()
 
-		// Check if cri socket set, if not then auto detect
-		if cfg.GlobalCfg.CRISocket == "" {
+		if cfg.GlobalCfg.UseOCIHooks {
+			go dm.HandleFile("/opt/output.json")
+		} else if cfg.GlobalCfg.CRISocket == "" {
+			// Check if cri socket set, if not then auto detect
 			if kl.GetCRISocket("") == "" {
 				dm.Logger.Warnf("Error while looking for CRI socket file")
 				enableContainerPolicy = false
@@ -593,15 +595,19 @@ func KubeArmor() {
 			go dm.MonitorCrioEvents()
 		} else {
 			dm.Logger.Warnf("Failed to monitor containers: %s is not a supported CRI socket.", cfg.GlobalCfg.CRISocket)
-			enableContainerPolicy = false
+			if !cfg.GlobalCfg.UseOCIHooks {
+				enableContainerPolicy = false
+			}
 		}
 
 		dm.Logger.Printf("Using %s for monitoring containers", cfg.GlobalCfg.CRISocket)
 	}
 
 	if dm.K8sEnabled && cfg.GlobalCfg.Policy {
-		// check if the CRI socket set while executing kubearmor exists
-		if cfg.GlobalCfg.CRISocket != "" {
+		if cfg.GlobalCfg.UseOCIHooks {
+			go dm.HandleFile("/opt/output.json")
+			} else if cfg.GlobalCfg.CRISocket != "" { // check if the CRI socket set while executing kubearmor exists
+			// check if the CRI socket set while executing kubearmor exists
 			trimmedSocket := strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://")
 			if _, err := os.Stat(trimmedSocket); err != nil {
 				dm.Logger.Warnf("Error while looking for CRI socket file: %s", err.Error())

KubeArmor/go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/containerd/containerd v1.7.13
 	github.com/containerd/typeurl/v2 v2.1.1
 	github.com/docker/docker v25.0.5+incompatible
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0
 	github.com/kubearmor/KubeArmor/pkg/KubeArmorController v0.0.0-20240110164432-c2c1b121cd94
@@ -64,7 +65,6 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect