From 7a90b0c5f268606ab77a441543581f7bca1b5750 Mon Sep 17 00:00:00 2001 From: Huy Mai Date: Thu, 20 Jun 2024 08:12:23 +0000 Subject: [PATCH] Add new Go module to deploy VM for tests Signed-off-by: Huy Mai --- .github/workflows/e2e-test.yml | 3 +- clean_bmcs.sh | 27 ++ hack/ci-e2e.sh | 6 +- hack/clean-e2e.sh | 4 + test/createVM/main.go | 407 +++++++++++++++++++ test/e2e/bmc.go | 13 +- test/e2e/e2e_suite_test.go | 3 +- test/go.mod | 2 + test/go.sum | 4 + tools/bmh_test/clean_local_bmh_test_setup.sh | 6 +- 10 files changed, 464 insertions(+), 11 deletions(-) create mode 100755 clean_bmcs.sh create mode 100644 test/createVM/main.go diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 31878ee4f7..cd6ae7e2df 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -31,7 +31,7 @@ jobs: - name: Install libvirt run: | sudo apt-get update - sudo apt-get install -y libvirt-daemon-system qemu-kvm virt-manager + sudo apt-get install -y libvirt-daemon-system qemu-kvm virt-manager libvirt-dev - name: Run BMO e2e Tests env: @@ -45,6 +45,7 @@ jobs: sudo -s -u $USER --preserve-env bash ${{ github.workspace }}/hack/ci-e2e.sh - name: Upload artifacts + if: {{ !cancelled() }} uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: artifacts-${{ inputs.bmc-protocol }}.tar.gz diff --git a/clean_bmcs.sh b/clean_bmcs.sh new file mode 100755 index 0000000000..c3ea8ffcd0 --- /dev/null +++ b/clean_bmcs.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# This script reads BMC information in a config file and prepare VMs +# whose info match those config +# +set -ex + +REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")") +cd "${REPO_ROOT}" + +virsh pool-destroy default || true +virsh pool-delete default || true +virsh pool-undefine default || true + +CONFIG_FILE=$1 +NETWORK=${2:-"baremetal-e2e"} + +readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${CONFIG_FILE}") + +for bmc in "${BMCS[@]}"; do + bootMacAddress=$(echo "${bmc}" | jq -r '.bootMacAddress') + ipAddress=$(echo "${bmc}" | jq -r '.ipAddress') + virsh -c qemu:///system net-update "${NETWORK}" delete ip-dhcp-host "" --live --config +done +"${REPO_ROOT}/tools/bmh_test/clean_local_bmh_test_setup.sh" "^bmo-e2e-" +rm -rf /tmp/bmo-e2e-* + diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 55da95ff50..26ff2ffe31 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -105,7 +105,10 @@ else exit 1 fi -"${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e +# "${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e +pushd "${REPO_ROOT}/test/createVM" || exit 1 +go run main.go --yaml-source-file "${E2E_BMCS_CONF_FILE}" +popd # Image server variables CIRROS_VERSION="0.6.2" @@ -118,6 +121,7 @@ mkdir -p "${IMAGE_DIR}" ## Download disk images wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}" wget --quiet -P "${IMAGE_DIR}/" https://fastly-cdn.system-rescue.org/releases/11.00/systemrescue-11.00-amd64.iso +wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso ## Start the image server docker run --name image-server-e2e -d \ diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh index 1a36c5f55c..b739186a40 100755 --- a/hack/clean-e2e.sh +++ b/hack/clean-e2e.sh @@ -13,3 +13,7 @@ docker rm -f sushy-tools rm -rf "${REPO_ROOT}/test/e2e/_artifacts" rm -rf "${REPO_ROOT}"/artifacts-* rm -rf "${REPO_ROOT}/test/e2e/images" + +# Clear network +virsh -c qemu:///system net-destroy baremetal-e2e +virsh -c qemu:///system net-undefine baremetal-e2e diff --git a/test/createVM/main.go b/test/createVM/main.go new file mode 100644 index 0000000000..4029cf1144 --- /dev/null +++ b/test/createVM/main.go @@ -0,0 +1,407 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "text/template" + + "github.com/dypflying/go-qcow2lib/qcow2" + "github.com/libvirt/libvirt-go" + bmoe2e "github.com/metal3-io/baremetal-operator/test/e2e" +) + +// CreateVolumePool creates a volume pool with specified name if a pool with +// that name does not exist yet +func CreateVolumePool(poolName string) error { + // Connect to libvirt daemon + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + fmt.Println("Failed to connect to qemu:///system") + return err + } + defer conn.Close() + + _, err = conn.LookupStoragePoolByName(poolName) + + if err == nil { + fmt.Println("Pool already exists") + return nil + } + + xmlConfigDef := ` + + {{ .PoolName }} + + {{ .PoolPath }} + + 0755 + -1 + -1 + + +` + xmlTpl, err := template.New("xml").Parse(xmlConfigDef) + + if err != nil { + fmt.Println("Failed to read pool XML file") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + poolPath := "/tmp/pool" + + data := struct { + PoolName string + PoolPath string + }{ + PoolName: poolName, + PoolPath: poolPath, + } + + var buf bytes.Buffer + + err = xmlTpl.Execute(&buf, data) + + if err != nil { + return err + } + + // Create the volume pool + pool, err := conn.StoragePoolCreateXML(buf.String(), 0) + + if err != nil { + fmt.Println("Failed to create volume pool") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + defer pool.Free() + + // Set the storage pool to autostart + // err = pool.SetAutostart(true) + // if err != nil { + // fmt.Println("Failed to set the storage pool to autostart") + // fmt.Printf("Error occurred: %v\n", err) + // return err + // } + + fmt.Println("Volume pool created successfully") + return nil +} + +func CreateLibvirtVM(hostName, networkName, macAddress string) error { + opts := make(map[string]any) + opts[qcow2.OPT_SIZE] = 3 * (1 << 30) // qcow2 file's size is 3g + opts[qcow2.OPT_FMT] = "qcow2" // qcow2 format + opts[qcow2.OPT_SUBCLUSTER] = true // enable sub-cluster + + err := qcow2.Blk_Create("/tmp/"+hostName+".qcow2", opts) + + if err != nil { + fmt.Println("Failed to create qcow2 file") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + if err = CreateVolumePool("default"); err != nil { + return err + } + + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + fmt.Println("Failed to connect to qemu:///system") + return err + } + defer conn.Close() + + domainDef := ` + + {{ .HostName }} + Virtualized BareMetalHost + + + + + + + + + + + + + + + + + + + + 4194304 + 2 + + hvm + + + + + + + + + + + + + + + + + + /usr/bin/qemu-system-x86_64 + + + + +
+ + +
+ + + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + +
+ + +
+ + + + + + + + + + + + + + + + + + +
+ + + /dev/urandom +
+ + + ` + + xmlTpl, err := template.New("xml").Parse(domainDef) + + if err != nil { + fmt.Println("Failed to init the template") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + data := struct { + HostName string + Network string + MacAddress string + }{ + HostName: hostName, + Network: networkName, + MacAddress: macAddress, + } + + var buf bytes.Buffer + + err = xmlTpl.Execute(&buf, data) + + if err != nil { + return err + } + + fmt.Println(buf.String()) + + dom, err := conn.DomainDefineXML(buf.String()) + + if err != nil { + fmt.Println("Failed to define domain") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + if err := dom.Create(); err != nil { + fmt.Println("Failed to create domain") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + fmt.Println("Domain created successfully") + return nil +} + +func CreateLibvirtBMC(macAddress, hostName, ipAddress, networkName string) error { + var err error + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + return err + } + defer conn.Close() + + network, err := conn.LookupNetworkByName(networkName) + if err != nil { + return err + } + + xmlTpl, err := template.New("xml").Parse("") + + if err != nil { + return err + } + + data := struct { + MacAddress string + HostName string + IPAddress string + }{ + MacAddress: macAddress, + HostName: hostName, + IPAddress: ipAddress, + } + + var buf bytes.Buffer + + err = xmlTpl.Execute(&buf, data) + + if err != nil { + fmt.Printf("Error occurred: %v\n", err) + return err + } + + if err = network.Update( + libvirt.NETWORK_UPDATE_COMMAND_ADD_LAST, + libvirt.NETWORK_SECTION_IP_DHCP_HOST, + -1, + buf.String(), + libvirt.NETWORK_UPDATE_AFFECT_LIVE|libvirt.NETWORK_UPDATE_AFFECT_CONFIG, + ); err != nil { + fmt.Printf("Error occurred: %v\n", err) + return err + } + if err = CreateLibvirtVM(hostName, networkName, macAddress); err != nil { + fmt.Printf("Error occurred: %v\n", err) + return err + } + return nil +} + +func main() { + var vmName = flag.String( + "vm-name", "VM-1", "The name of the VM to create") + var networkName = flag.String( + "network-name", "baremetal-e2e", "The name of the network that the new VM should be attached to") + var macAddress = flag.String( + "mac-address", "00:60:2f:31:81:01", "Mac address of the VM on the network") + var ipAddress = flag.String( + "ip-address", "192.168.222.122", "IP address of the VM on the network") + var configFile = flag.String( + "yaml-source-file", "", "yaml file where BMCS are defined. If this is set, ignore all other options") + flag.Parse() + var err error + if *configFile == "" { + if err = CreateLibvirtBMC(*macAddress, *vmName, *ipAddress, *networkName); err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } + } else { + bmcs, err := bmoe2e.LoadBMCConfig(*configFile) + if err != nil { + os.Exit(1) + } + for _, bmc := range *bmcs { + if err = CreateLibvirtBMC(bmc.BootMacAddress, bmc.HostName, bmc.IPAddress, "baremetal-e2e"); err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } + } + } +} diff --git a/test/e2e/bmc.go b/test/e2e/bmc.go index 6bea16b5a4..6d67ad350c 100644 --- a/test/e2e/bmc.go +++ b/test/e2e/bmc.go @@ -3,7 +3,6 @@ package e2e import ( "os" - . "github.com/onsi/gomega" "gopkg.in/yaml.v2" ) @@ -28,10 +27,14 @@ type BMC struct { SSHPort string `yaml:"sshPort,omitempty"` } -func LoadBMCConfig(configPath string) *[]BMC { +func LoadBMCConfig(configPath string) (*[]BMC, error) { configData, err := os.ReadFile(configPath) //#nosec - Expect(err).ToNot(HaveOccurred(), "Failed to read the bmcs config file") var bmcs []BMC - Expect(yaml.Unmarshal(configData, &bmcs)).To(Succeed()) - return &bmcs + if err != nil { + return nil, err + } + if err := yaml.Unmarshal(configData, &bmcs); err != nil { + return nil, err + } + return &bmcs, nil } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index eb9d31ce3d..a4edf7a73a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -170,7 +170,8 @@ var _ = SynchronizedBeforeSuite(func() []byte { err := metal3api.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred()) e2eConfig = LoadE2EConfig(configPath) - bmcs = LoadBMCConfig(bmcConfigPath) + bmcs, err := LoadBMCConfig(bmcConfigPath) + Expect(err).ToNot(HaveOccurred(), "Failed to read the bmcs config file") bmc = (*bmcs)[GinkgoParallelProcess()-1] clusterProxy = framework.NewClusterProxy("bmo-e2e", kubeconfigPath, scheme) }) diff --git a/test/go.mod b/test/go.mod index d6c5f876b8..efe450db4d 100644 --- a/test/go.mod +++ b/test/go.mod @@ -4,6 +4,8 @@ go 1.22 require ( github.com/cert-manager/cert-manager v1.10.0 + github.com/dypflying/go-qcow2lib v1.0.0 + github.com/libvirt/libvirt-go v7.4.0+incompatible github.com/metal3-io/baremetal-operator/apis v0.5.1 github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.5.1 github.com/onsi/ginkgo/v2 v2.17.1 diff --git a/test/go.sum b/test/go.sum index 159e7d2cd3..b43c4c15d4 100644 --- a/test/go.sum +++ b/test/go.sum @@ -70,6 +70,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= +github.com/dypflying/go-qcow2lib v1.0.0 h1:TYWEEwrBj0e7WM+t89VbbIgp3fxdEonrdv/CcJ+cqFo= +github.com/dypflying/go-qcow2lib v1.0.0/go.mod h1:pQW9aFBFRaz24xpODWDjI7j2gR16RUsV+dG1E0YgBfI= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -183,6 +185,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libvirt/libvirt-go v7.4.0+incompatible h1:crnSLkwPqCdXtg6jib/FxBG/hweAc/3Wxth1AehCXL4= +github.com/libvirt/libvirt-go v7.4.0+incompatible/go.mod h1:34zsnB4iGeOv7Byj6qotuW8Ya4v4Tr43ttjz/F0wjLE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= diff --git a/tools/bmh_test/clean_local_bmh_test_setup.sh b/tools/bmh_test/clean_local_bmh_test_setup.sh index 641d0083e4..6ae0c3b049 100755 --- a/tools/bmh_test/clean_local_bmh_test_setup.sh +++ b/tools/bmh_test/clean_local_bmh_test_setup.sh @@ -18,8 +18,8 @@ else fi # Clear vbmc -docker rm -f vbmc +# docker rm -f vbmc # Clear network -virsh -c qemu:///system net-destroy baremetal-e2e -virsh -c qemu:///system net-undefine baremetal-e2e +# virsh -c qemu:///system net-destroy baremetal-e2e +# virsh -c qemu:///system net-undefine baremetal-e2e