Merge pull request #112 from openebs/disbale-rdma-device

fix: disable rdma device and restart csi node pod

Author: rohan2794

common/e2e_config/e2e_config.go

@@ -42,7 +42,7 @@ type ProductSpec struct {
 	CpuCount                          string            `yaml:"cpuCount" env-default:"2"`
 	CrdGroupName                      string            `yaml:"crdGroupName" env-default:"openebs.io"`
 	CrdPoolsResourceName              string            `yaml:"crdPoolsResourceName" env-default:"mayastorpools"`
-	CsiDaemonsetName                  string            `yaml:"csiDaemonsetName" env-default:"mayastor-csi"`
+	CsiDaemonsetName                  string            `yaml:"csiDaemonsetName" env-default:"mayastor-csi-node"`
 	CsiNodeServiceAppLabel            string            `yaml:"csiNodeServiceAppLabel" env-default:"csi-node"`
 	CsiNodeServiceDaemonset           string            `yaml:"csiNodeServiceDaemonset" env-default:"mayastor-csi-node"`
 	CsiNodeContainerName              string            `yaml:"csiNodeContainerName" env-default:"csi-node"`
@@ -241,7 +241,8 @@ type E2EConfig struct {
 
 	IOEngineNvmeTimeout int `yaml:"ioEngineNvmeTimeout" env-default:"0"`
 
-	// Individual Test parameters
+	// disable rdma device over list of interface
+	DisabledRdmaDevices []string `yaml:"disabledRdmaDevices"`
 }
 
 var once sync.Once

common/k8stest/util_rdma.go

@@ -181,12 +181,11 @@ func RemoveRdmaDeviceOnNode(node string) error {
 	return nil
 }
 
-func DisableRdmaOnNode(node string) error {
+func DisableRdmaOnNode(node string, networkInterface string) error {
 	logf.Log.Info("Disable rdma from IO engine node", "name", node)
 
-	iface := e2e_config.GetConfig().NetworkInterface
 	// get dev link port wrt to interface
-	rdmaDevName, err := GetDevLinkName(node, iface)
+	rdmaDevName, err := GetDevLinkName(node, networkInterface)
 	if err != nil {
 		return err
 	}
@@ -212,7 +211,9 @@ func DisableRdmaOnNode(node string) error {
 		}
 	}
 
-	return nil
+	// Restart csi node pod on the node
+	return RestartCsiNodePodOnNode(node, 240, 120)
+
 }
 
 func RemoveRdmaDeviceOnAllWorkerNodes() error {
@@ -223,7 +224,7 @@ func RemoveRdmaDeviceOnAllWorkerNodes() error {
 	logf.Log.Info("Remove rdma from IO engine node")
 	for _, node := range workerNodes.Items {
 		logf.Log.Info("IO engine", "Node", node.Name)
-		err := DisableRdmaOnNode(node.Name)
+		err := DisableRdmaOnNode(node.Name, e2e_config.GetConfig().NetworkInterface)
 		if err != nil {
 			return err
 		}
@@ -239,20 +240,19 @@ func EnableRdmaDeviceOnAllWorkerNodes() error {
 	logf.Log.Info("Enable rdma from IO engine node")
 	for _, node := range workerNodes.Items {
 		logf.Log.Info("IO engine", "Node", node.Name)
-		err := EnableRdmaOnNode(node.Name)
+		err := EnableRdmaOnNode(node.Name, e2e_config.GetConfig().NetworkInterface)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func EnableRdmaOnNode(node string) error {
+func EnableRdmaOnNode(node string, networkInterface string) error {
 	logf.Log.Info("Enable rdma on IO engine node", "name", node)
-	// get interface name
-	iface := e2e_config.GetConfig().NetworkInterface
+
 	// get dev link port wrt to interface
-	rdmaDevPortName, err := GetDevLinkName(node, iface)
+	rdmaDevPortName, err := GetDevLinkName(node, networkInterface)
 	if err != nil {
 		return err
 	}
@@ -317,3 +317,55 @@ func GetDevLinkName(node, iface string) (string, error) {
 	}
 	return "", nil
 }
+
+// DisableConfiguredDisabledRdmaDevicesOnAllMayastorNodes disable rdma devices which are configured in platform config
+// on all mayastor nodes. If no rdma devices are configured then it will return without doing anything.
+// In some cases, multiple rdma devices are configured in cluster, so it should be disabled all the devices on nodes
+func DisableConfiguredDisabledRdmaDevicesOnAllMayastorNodes() error {
+	ifaceList := e2e_config.GetConfig().DisabledRdmaDevices
+	if len(ifaceList) == 0 {
+		logf.Log.Info("No rdma devices configured which needs to be disabled")
+		return nil
+	}
+	workerNodes, err := ListIOEngineNodes()
+	if err != nil {
+		return err
+	}
+	for _, node := range workerNodes.Items {
+		logf.Log.Info("Disable rdma device", "Node", node.Name)
+		for _, iface := range ifaceList {
+			err := DisableRdmaOnNode(node.Name, iface)
+			if err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}
+
+// RestoreConfiguredDisabledRdmaDevicesOnAllMayastorNodes enable rdma devices which are configured in platform config
+// on all mayastor nodes. If no rdma devices are configured then it will return without doing anything.
+// In some cases, multiple rdma devices are configured in cluster were disabled, so it should be enabled all the devices on nodes
+func RestoreConfiguredDisabledRdmaDevicesOnAllMayastorNodes() error {
+	ifaceList := e2e_config.GetConfig().DisabledRdmaDevices
+	if len(ifaceList) == 0 {
+		logf.Log.Info("No rdma devices configured which needs to be restored")
+		return nil
+	}
+	workerNodes, err := ListIOEngineNodes()
+	if err != nil {
+		return err
+	}
+	for _, node := range workerNodes.Items {
+		logf.Log.Info("Enable rdma device", "Node", node.Name)
+		for _, iface := range ifaceList {
+			err := EnableRdmaOnNode(node.Name, iface)
+			if err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}

common/k8stest/util_testpods.go

@@ -1218,3 +1218,69 @@ func GetMayastorPodNameonNodeByPrefix(prefix string, nodeName string) (string, e
 	}
 	return "", fmt.Errorf("failed to get mayastor pod with prefix %s on node %s", prefix, nodeName)
 }
+
+// GetCsiNodePodNameOnNodeByPrefix return csi node pod on a node with a given prefix
+func GetPodNameOnNodeByPrefix(podPrefix, nodeName, namespace string) (string, error) {
+	podApi := gTestEnv.KubeInt.CoreV1().Pods
+	pods, err := podApi(namespace).List(context.TODO(), metaV1.ListOptions{})
+	if err != nil {
+		return "", fmt.Errorf("failed to list pod in %s namespace, error: %v", common.NSMayastor(), err)
+	}
+	for _, pod := range pods.Items {
+		if strings.HasPrefix(pod.Name, podPrefix) {
+			if pod.Spec.NodeName == nodeName {
+				return pod.Name, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("failed to get mayastor pod with prefix %s on node %s", podPrefix, nodeName)
+}
+
+func restartCsiNodePodsOnNode(nodeName string) error {
+	podApi := gTestEnv.KubeInt.CoreV1().Pods
+
+	podName, err := GetPodNameOnNodeByPrefix(e2e_config.GetConfig().Product.CsiDaemonsetName, nodeName, common.NSMayastor())
+
+	if err == nil {
+		logf.Log.Info("Restarting", "pod", podName)
+		time.Sleep(1 * time.Second)
+
+		delErr := podApi(common.NSMayastor()).Delete(context.TODO(), podName, metaV1.DeleteOptions{})
+		if delErr != nil {
+			logf.Log.Info("Failed to delete", "pod", podName, "error", delErr)
+			err = delErr
+		} else {
+			logf.Log.Info("Deleted", "pod", podName)
+		}
+	}
+	return err
+}
+
+// RestartCsiNodePodOnNode restart csi node pod scheduled on a given node and wait for mayastor pods and pool readiness
+func RestartCsiNodePodOnNode(nodeName string, readyTOSecs int, poolsTOSecs int) error {
+	ready := false
+
+	err := restartCsiNodePodsOnNode(nodeName)
+	if err != nil {
+		logf.Log.Info("Warning: RestartMayastorPodsOnNode failed", "error", err)
+	}
+
+	ready, err = MayastorReady(10, readyTOSecs)
+	if err != nil {
+		return fmt.Errorf("failure waiting for mayastor to be ready %v", err)
+	}
+	if !ready {
+		return fmt.Errorf("mayastor is not ready after deleting all pods")
+	}
+
+	const sleepTime = 10
+	for ix := 0; ix < (poolsTOSecs+sleepTime-1)/sleepTime; ix++ {
+		time.Sleep(sleepTime * time.Second)
+		err = custom_resources.CheckAllMsPoolsAreOnline()
+		if err == nil {
+			break
+		}
+	}
+
+	return err
+}

configurations/maasci_config.yaml

@@ -6,4 +6,6 @@ configName: maasci
 grpcMandated: true
 deferredAssert: true
 beforeEachCheckAndRestart: true
-networkInterface: ens1f0
+networkInterface: ens1f0
+disabledRdmaDevices:
+  - "ens1f1"