From 47cace4818bb8a27cfd2209d103fdb027c38a99b Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 11 Oct 2022 03:24:01 +0300 Subject: [PATCH] fix packet interdomain testing Signed-off-by: denis-tingaikin --- entry_point_test.go | 5 +- packet/packet.yaml | 22 +++--- scripts/packet/create-kubernetes-cluster.sh | 6 -- scripts/packet/k8s/start-master.sh | 2 +- scripts/packet/sriov/config-SRIOV.sh | 58 -------------- scripts/packet/sriov/enable-SRIOV.sh | 6 -- scripts/packet/sriov/enable-VFIO.sh | 21 ----- scripts/packet/sriov/setup-SRIOV.sh | 85 --------------------- 8 files changed, 15 insertions(+), 190 deletions(-) delete mode 100755 scripts/packet/sriov/config-SRIOV.sh delete mode 100755 scripts/packet/sriov/enable-SRIOV.sh delete mode 100755 scripts/packet/sriov/enable-VFIO.sh delete mode 100755 scripts/packet/sriov/setup-SRIOV.sh diff --git a/entry_point_test.go b/entry_point_test.go index 7eec0e7..c8c00a6 100644 --- a/entry_point_test.go +++ b/entry_point_test.go @@ -1,5 +1,7 @@ // Copyright (c) 2021 Doc.ai and/or its affiliates. // +// Copyright (c) 2022 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,10 +23,11 @@ import ( "os" "testing" - "github.com/networkservicemesh/integration-tests/suites/floating_interdomain" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // This is required for GKE authentication + + "github.com/networkservicemesh/integration-tests/suites/floating_interdomain" ) func TestFloatingInterdomainBasicSuite(t *testing.T) { diff --git a/packet/packet.yaml b/packet/packet.yaml index b9926ec..6454ea1 100644 --- a/packet/packet.yaml +++ b/packet/packet.yaml @@ -19,26 +19,24 @@ providers: - KUBERNETES_VERSION packet: ssh-key: sshkey.pub - hardware-devices: + devices: - name: "interdomain-Master" host-name: "SR-IOV-Master-${CLUSTER_NAME}" os: "ubuntu_20_04" billing-cycle: "hourly" - port-vlans: - eth3: 1044 + plan: "n3.xlarge.x86" - name: "interdomain-Worker" host-name: "SR-IOV-Worker-${CLUSTER_NAME}" os: "ubuntu_20_04" billing-cycle: "hourly" - port-vlans: - eth3: 1044 - hardware-reservations: - - 2cf78481-53b0-46c8-a084-6e9815acdb0b - - 2361d3c2-f694-4fa7-a683-a9f69e2abe7c - - 70b51e39-2e65-4a64-8191-dd28386c0b80 - - 4f27da36-a911-45a5-ab81-0e789348e3fe - - 7741d4a8-8555-492b-8d73-63c2f0715d16 - - d173dbde-8510-454e-b0c2-c120c397cdf5 + plan: "n3.xlarge.x86" + facilities: + - baremetal + - backend_transfer + - layer_2 + - global_ipv4 + - ibx + preferred-facility: da11 scripts: install: "echo all deps installed" setup: | diff --git a/scripts/packet/create-kubernetes-cluster.sh b/scripts/packet/create-kubernetes-cluster.sh index 6e850f6..478d303 100755 --- a/scripts/packet/create-kubernetes-cluster.sh +++ b/scripts/packet/create-kubernetes-cluster.sh @@ -21,12 +21,6 @@ function wait_pids() { return 0 } -# Setup SR-IOV -pids="" -/bin/bash scripts/packet/sriov/setup-SRIOV.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" & -pids+=" $!" -wait_pids "${pids}" "SR-IOV config failed" || exit 1 - # Create k8s scripts directory on nodes ssh ${SSH_OPTS} root@${master_ip} mkdir k8s ssh ${SSH_OPTS} root@${worker_ip} mkdir k8s diff --git a/scripts/packet/k8s/start-master.sh b/scripts/packet/k8s/start-master.sh index 7d4628c..1e9f665 100755 --- a/scripts/packet/k8s/start-master.sh +++ b/scripts/packet/k8s/start-master.sh @@ -12,7 +12,7 @@ mkdir -p "$HOME"/.kube sudo cp -f /etc/kubernetes/admin.conf "$HOME"/.kube/config sudo chown "$(id -u):$(id -g)" "$HOME"/.kube/config -kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')&env.IPALLOC_RANGE=192.168.0.0/16" +kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml kubectl taint nodes --all node-role.kubernetes.io/master- diff --git a/scripts/packet/sriov/config-SRIOV.sh b/scripts/packet/sriov/config-SRIOV.sh deleted file mode 100755 index fe61fce..0000000 --- a/scripts/packet/sriov/config-SRIOV.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2064,SC2129 - -CONFIG_DIRECTORY="/var/lib/networkservicemesh" -CONFIG_FILE="${CONFIG_DIRECTORY}/sriov.config" - -function softlink_target() { - softlink="$1" - - raw_target="$(stat -c %N "${softlink}")" - test $? -eq 0 || return 1 - - target=$(echo "${raw_target}" | sed -E "s/(.*\/)(.*)'/\2/g") - test $? -eq 0 || return 2 - - echo "${target}" - return 0 -} - -function config_link() { - device="/sys/class/net/$1/device" - IFS=","; read -ra domains <<< "$2"; unset IFS - - pci_addr="$(softlink_target "${device}")" - test $? -eq 0 || return 1 - - pf_kernel_driver="$(softlink_target "${device}/driver")" - test $? -eq 0 || return 1 - - trap "echo 0 > '${device}/sriov_numvfs'" err exit - echo 1 > "${device}/sriov_numvfs" || return 1 - vf_kernel_driver="$(softlink_target "${device}/virtfn0/driver")" - test $? -eq 0 || return 1 - - echo " ${pci_addr}:" >> "${CONFIG_FILE}" - echo " pfKernelDriver: ${pf_kernel_driver}" >> "${CONFIG_FILE}" - echo " vfKernelDriver: ${vf_kernel_driver}" >> "${CONFIG_FILE}" - echo " capabilities:" >> "${CONFIG_FILE}" - echo " - intel" >> "${CONFIG_FILE}" - echo " - 10G" >> "${CONFIG_FILE}" - echo " serviceDomains:" >> "${CONFIG_FILE}" - for domain in "${domains[@]}"; do - echo " - ${domain}" >> "${CONFIG_FILE}" - done - - return 0 -} - -mkdir -p "${CONFIG_DIRECTORY}" - -echo "---" > "${CONFIG_FILE}" -echo "physicalFunctions:" >> "${CONFIG_FILE}" - -for link_domains in "$@"; do - IFS="="; read -ra args <<< "${link_domains}"; unset IFS - config_link "${args[0]}" "${args[1]}" - test $? -eq 0 || exit 1 -done diff --git a/scripts/packet/sriov/enable-SRIOV.sh b/scripts/packet/sriov/enable-SRIOV.sh deleted file mode 100755 index 012c097..0000000 --- a/scripts/packet/sriov/enable-SRIOV.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -sed -Ei "s/(GRUB_CMDLINE_LINUX=.*)'/\1 intel_iommu=on'/" /etc/default/grub -grub-mkconfig -o /boot/grub/grub.cfg - -nohup bash -c "sleep 5; reboot" >/dev/null 2>&1 & diff --git a/scripts/packet/sriov/enable-VFIO.sh b/scripts/packet/sriov/enable-VFIO.sh deleted file mode 100755 index 0e4de22..0000000 --- a/scripts/packet/sriov/enable-VFIO.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2002,SC2064 - -device="/sys/class/net/$1/device" - -# modprobe VFIO driver -VFIO_DIR="/sys/bus/pci/drivers/vfio-pci" -ls -l "${VFIO_DIR}" || modprobe vfio-pci || exit 1 - -# Don't forget to remove VFs for the link -trap "echo 0 >'${device}/sriov_numvfs'" err exit - -# Add 1 VF for the link -echo 1 > "${device}/sriov_numvfs" || exit 2 - -# Get VF pci id -pci_id=$(cat "${device}/virtfn0/uevent" | grep "PCI_ID" | sed -E "s/PCI_ID=(.*):(.*)/\1 \2/g") -test $? -eq 0 || exit 3 - -# Enable VFIO driver for the VF -echo "${pci_id}" > "${VFIO_DIR}/new_id" || exit 4 diff --git a/scripts/packet/sriov/setup-SRIOV.sh b/scripts/packet/sriov/setup-SRIOV.sh deleted file mode 100755 index bf9ac1e..0000000 --- a/scripts/packet/sriov/setup-SRIOV.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -x -# shellcheck disable=SC2086 - -master_ip="$1" -worker_ip="$2" -SSH_OPTS="$3" - -function wait_pids() { - pids="$1" - message="$2" - for pid in ${pids}; do - echo "waiting for PID ${pid}" - wait ${pid} - code=$? - if test $code -ne 0; then - echo "${message}: process exited with code $code, aborting..." - return 1 - fi - done - return 0 -} - -SRIOV_DIR=$(dirname "$0") - -# Create SR-IOV scripts directory on nodes -ssh ${SSH_OPTS} root@${master_ip} mkdir sriov -ssh ${SSH_OPTS} root@${worker_ip} mkdir sriov - -# Enable SR-IOV and wait for the servers to reboot -scp ${SSH_OPTS} ${SRIOV_DIR}/enable-SRIOV.sh root@${master_ip}:sriov/enable-SRIOV.sh || exit 1 -scp ${SSH_OPTS} ${SRIOV_DIR}/enable-SRIOV.sh root@${worker_ip}:sriov/enable-SRIOV.sh || exit 2 - -pids="" -ssh ${SSH_OPTS} root@${master_ip} ./sriov/enable-SRIOV.sh & -pids+=" $!" -ssh ${SSH_OPTS} root@${worker_ip} ./sriov/enable-SRIOV.sh & -pids+=" $!" -wait_pids "${pids}" "SR-IOV setup failed" || exit 3 - -sleep 5 - -for ip in ${master_ip} ${worker_ip}; do - success_attempts=0 - # ~15 minutes to start - for i in {1..60}; do - if [[ ${i} == 60 ]]; then - echo "timeout waiting for the ${ip} to start, aborting..." - exit 4 - fi - - if ssh ${SSH_OPTS} -o ConnectTimeout=1 -o BatchMode=yes root@${ip} true; then - ((success_attempts++)) - else - success_attempts=0 - fi - - if [[ ${success_attempts} == 3 ]]; then - break - fi - - sleep 15 - done -done - -# Create SR-IOV config -scp ${SSH_OPTS} ${SRIOV_DIR}/config-SRIOV.sh root@${master_ip}:sriov/config-SRIOV.sh || exit 5 -scp ${SSH_OPTS} ${SRIOV_DIR}/config-SRIOV.sh root@${worker_ip}:sriov/config-SRIOV.sh || exit 6 - -pids="" -ssh ${SSH_OPTS} root@${master_ip} ./sriov/config-SRIOV.sh eno4=worker.domain & -pids+=" $!" -ssh ${SSH_OPTS} root@${worker_ip} ./sriov/config-SRIOV.sh eno4=master.domain & -pids+=" $!" -wait_pids "${pids}" "SR-IOV config failed" || exit 7 - -# Enable VFIO driver -scp ${SSH_OPTS} ${SRIOV_DIR}/enable-VFIO.sh root@${master_ip}:sriov/enable-VFIO.sh || exit 8 -scp ${SSH_OPTS} ${SRIOV_DIR}/enable-VFIO.sh root@${worker_ip}:sriov/enable-VFIO.sh || exit 9 - -pids="" -ssh ${SSH_OPTS} root@${master_ip} ./sriov/enable-VFIO.sh eno4 & -pids+=" $!" -ssh ${SSH_OPTS} root@${worker_ip} ./sriov/enable-VFIO.sh eno4 & -pids+=" $!" -wait_pids "${pids}" "VFIO enabling failed" || exit 10