feat(rdma): verify volume connection protocol by checking volume live path in nvme sub system connection

Signed-off-by: rohan2794 <[email protected]>

Author: rohan2794

common/k8stest/util_rdma.go

@@ -12,7 +12,11 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
-const RdmaDeviceName = "rxe0"
+const (
+	RdmaDeviceName = "rxe0"
+	TcpProtocol    = "tcp"
+	RdmaProtocol   = "rdma"
+)
 
 type RdmaDeviceNetworkInterface struct {
 	IfIndex       int    `json:"ifindex"`
@@ -369,3 +373,27 @@ func RestoreConfiguredDisabledRdmaDevicesOnAllMayastorNodes() error {
 	}
 	return nil
 }
+
+// IsVolumeConnectedOverSpecifiedProtocol return true if volume is connected over specified protocol
+// by checking nvme list subsystem live path protocol for specified volume
+func IsVolumeConnectedOverSpecifiedProtocol(volUuid, initiatorNode, protocol string) (bool, error) {
+	// get node IP
+	nodeIp, err := GetNodeIPAddress(initiatorNode)
+	if err != nil {
+		return false, fmt.Errorf("failed to get node IP for node %s, error: %v", initiatorNode, err)
+	} else if *nodeIp == "" {
+		return false, fmt.Errorf("node IP for node %s not found", initiatorNode)
+	}
+	logf.Log.Info("Node IP", "node", initiatorNode, "IP", *nodeIp)
+
+	//volume should get connected over tcp
+	logf.Log.Info("verify volume connection", "expected protocol", protocol)
+	nvmeListSubsysProtocol, err := GetNvmeListSubsystemVolumeEntryLivePathProtocol(*nodeIp, volUuid)
+	if err != nil {
+		return false, fmt.Errorf("failed to get nvme list subsystem live path protocol for volume %s on node %s, error: %v", volUuid, initiatorNode, err)
+	} else if nvmeListSubsysProtocol == "" {
+		return false, fmt.Errorf("nvme list subsystem live path protocol for volume %s on node %s not found", volUuid, initiatorNode)
+	}
+	logf.Log.Info("Volume connection protocol", "volume", volUuid, "protocol", nvmeListSubsysProtocol)
+	return true, nil
+}

common/k8stest/util_replica.go

@@ -1037,3 +1037,60 @@ func CheckReplicaComparisonPossible(volUuid string, volName string, volType comm
 
 	return err
 }
+
+// GetNvmeListSubSys returns the list of nvme subsystems on the initiator node
+func GetNvmeListSubSys(initiatorNodeIP string) (map[string][]nvmeListSubsystemEntry, error) {
+	subSysList := make(map[string][]nvmeListSubsystemEntry)
+	output, err := agent.NvmeListSubSys(initiatorNodeIP)
+	if output == "" || err != nil {
+		logf.Log.Info("nvme list-subsys failed", "output", output, "err", err)
+		return nil, err
+	} else {
+		output = trimForJson(output)
+		if err = json.Unmarshal([]byte(output), &subSysList); err != nil {
+			logf.Log.Info("Failed to unmarshal target", "error", err)
+			return nil, err
+		}
+	}
+	return subSysList, nil
+}
+
+// GetNvmeListSubsystemVolumeEntryLivePath returns the live path of the volume entry in the subsystem
+func GetNvmeListSubsystemVolumeEntryLivePath(initiatorNodeIP string, volumeUuid string) (map[string]string, error) {
+	subSysPath := make(map[string]string)
+	subSysList, err := GetNvmeListSubSys(initiatorNodeIP)
+	if err != nil {
+		logf.Log.Info("nvme list-subsys failed", "err", err)
+		return subSysPath, err
+	}
+	for _, subSys := range subSysList["Subsystems"] {
+		if strings.Contains(subSys.NQN, volumeUuid) {
+			for _, path := range subSys.Paths {
+				if path["State"] == "live" {
+					subSysPath = path
+					break
+				}
+			}
+		}
+	}
+	return subSysPath, nil
+}
+
+// GetNvmeListSubsystemVolumeEntryLivePathProtocol returns the transport protocol of the volume entry in the subsystem
+func GetNvmeListSubsystemVolumeEntryLivePathProtocol(initiatorNodeIP string, volumeUuid string) (string, error) {
+	subSysLivePath, err := GetNvmeListSubsystemVolumeEntryLivePath(initiatorNodeIP, volumeUuid)
+	if err != nil {
+		logf.Log.Info("failed to get live nvme sub sys path",
+			"initiatorNodeIP", initiatorNodeIP,
+			"volumeUuid", volumeUuid,
+			"err", err)
+		return "", err
+	}
+	logf.Log.Info("Volume sub system live path",
+		"volumeUuid", volumeUuid,
+		"subSysLivePath", subSysLivePath)
+	if subSysLivePath["Transport"] == "" {
+		return "", fmt.Errorf("failed to get live nvme sub sys path transport for volume %s with initiatorNodeIP %s", volumeUuid, initiatorNodeIP)
+	}
+	return subSysLivePath["Transport"], nil
+}