From 398d95915c03ba1bfded7c6238f3149c1937dba6 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 20 Aug 2024 15:21:14 -0300 Subject: [PATCH 1/7] versions: bump version of kcli In order to use kcli to create a k8s cluster with configured with cri-o, it will be needed to use a version newer than 07/02/2024 which containers the https://github.com/karmab/kcli/commit/77cf2cb88bfd58394180941bd477db6abe09ee29 fix. So picking the latest version available at the time of this commit. Signed-off-by: Wainer dos Santos Moschetta --- src/cloud-api-adaptor/versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloud-api-adaptor/versions.yaml b/src/cloud-api-adaptor/versions.yaml index 81f917f10..b8cf07827 100644 --- a/src/cloud-api-adaptor/versions.yaml +++ b/src/cloud-api-adaptor/versions.yaml @@ -24,7 +24,7 @@ tools: rust: 1.75.0 protoc: 3.15.0 packer: v1.9.4 - kcli: 99.0.202407031308 + kcli: 99.0.202408152044 iptables-wrapper: v0.0.0-20240819165702-06cad2ec6cb5 oras: 1.2.0 # Referenced Git repositories From c179d8c5d42e5c243503b542c5ad42b1a79bff81 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 20 Aug 2024 15:31:31 -0300 Subject: [PATCH 2/7] libvirt/kcli_cluster.sh: add support to CRI-O By exporting the CONTAINER_RUNTIME=crio variable, kcli will create a k8s cluster configured CRI-O: $ export CONTAINER_RUNTIME=crio $ ./src/cloud-api-adaptor/libvirt/kcli_cluster.sh create Fixes #1981 Signed-off-by: Wainer dos Santos Moschetta --- src/cloud-api-adaptor/libvirt/kcli_cluster.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cloud-api-adaptor/libvirt/kcli_cluster.sh b/src/cloud-api-adaptor/libvirt/kcli_cluster.sh index 8242bf796..1c5656f3e 100755 --- a/src/cloud-api-adaptor/libvirt/kcli_cluster.sh +++ b/src/cloud-api-adaptor/libvirt/kcli_cluster.sh @@ -15,6 +15,7 @@ CLUSTER_NAME="${CLUSTER_NAME:-peer-pods}" CLUSTER_IMAGE="${CLUSTER_IMAGE:-ubuntu2204}" CLUSTER_VERSION="${CLUSTER_VERSION:-1.30.0}" CLUSTER_WORKERS="${CLUSTER_WORKERS:-1}" +CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}" # Either "containerd" or "crio" LIBVIRT_NETWORK="${LIBVIRT_NETWORK:-default}" LIBVIRT_POOL="${LIBVIRT_POOL:-default}" @@ -53,7 +54,8 @@ create () { -P sdn=flannel \ -P nfs=false \ -P disk_size=$CLUSTER_DISK_SIZE \ - -P version=$CLUSTER_VERSION" + -P version=$CLUSTER_VERSION \ + -P engine=$CONTAINER_RUNTIME" # The autolabeller and multus images do not support s390x arch yet # disable them for s390x cluster if [[ ${TARGET_ARCH} == "s390x" ]]; then @@ -120,7 +122,8 @@ usage () { CLUSTER_VERSION (default "${CLUSTER_VERSION}") LIBVIRT_NETWORK (default "${LIBVIRT_NETWORK}") LIBVIRT_POOL (default "${LIBVIRT_POOL}") - CLUSTER_WORKERS (default "${CLUSTER_WORKERS}"). + CLUSTER_WORKERS (default "${CLUSTER_WORKERS}") + CONTAINER_RUNTIME (default "${CONTAINER_RUNTIME}") delete Delete the cluster. Specify the cluster name with CLUSTER_NAME (default "${CLUSTER_NAME}"). EOF From dc89125c717de1fcd232e0b39df00eff83943336 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 27 Sep 2024 15:49:42 -0300 Subject: [PATCH 3/7] test/e2e: new container_runtime property for libvirt Commit a0247aef461c4 introduced a new parameter (CONTAINER_RUNTIME) for docker provider, allowing users to specify the container runtime used. Some tests will take decisions based on that property, for example, whether nydus snapshotter messages should be inspected or not. Likewise, this added the handler for that property for libvirt, so allowing to test with cri-o too. Fixes #1981 Signed-off-by: Wainer dos Santos Moschetta --- src/cloud-api-adaptor/test/e2e/README.md | 1 + .../provisioner/libvirt/provision_common.go | 57 ++++++++++--------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/README.md b/src/cloud-api-adaptor/test/e2e/README.md index 49ed2e627..3bfeaf294 100644 --- a/src/cloud-api-adaptor/test/e2e/README.md +++ b/src/cloud-api-adaptor/test/e2e/README.md @@ -150,6 +150,7 @@ Use the properties on the table below for Libvirt: |Property|Description|Default| |---|---|---| +|container_runtime|Test cluster configured container runtime. Either **containerd** or **crio** |containerd| |libvirt_network|Libvirt Network|"default"| |libvirt_storage|Libvirt storage pool|"default"| |libvirt_vol_name|Volume name|"podvm-base.qcow2"| diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 226ec2c3c..0de8c9796 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -21,16 +21,17 @@ import ( // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt. type LibvirtProvisioner struct { - conn *libvirt.Connect // Libvirt connection - network string // Network name - ssh_key_file string // SSH key file used to connect to Libvirt - storage string // Storage pool name - uri string // Libvirt URI - wd string // libvirt's directory path on this repository - volumeName string // Podvm volume name - clusterName string // Cluster name - kbs_image string // KBS Service OCI Image URL - kbs_image_tag string // KBS Service OCI Image Tag + conn *libvirt.Connect // Libvirt connection + containerRuntime string // Name of the container runtime + network string // Network name + ssh_key_file string // SSH key file used to connect to Libvirt + storage string // Storage pool name + uri string // Libvirt URI + wd string // libvirt's directory path on this repository + volumeName string // Podvm volume name + clusterName string // Cluster name + kbs_image string // KBS Service OCI Image URL + kbs_image_tag string // KBS Service OCI Image Tag } // LibvirtInstallOverlay implements the InstallOverlay interface @@ -94,16 +95,17 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e // TODO: Check network and storage are not nil? return &LibvirtProvisioner{ - conn: conn, - network: network, - ssh_key_file: ssh_key_file, - storage: storage, - uri: uri, - wd: wd, - volumeName: vol_name, - clusterName: clusterName, - kbs_image: kbs_image, - kbs_image_tag: kbs_image_tag, + conn: conn, + containerRuntime: properties["container_runtime"], + network: network, + ssh_key_file: ssh_key_file, + storage: storage, + uri: uri, + wd: wd, + volumeName: vol_name, + clusterName: clusterName, + kbs_image: kbs_image, + kbs_image_tag: kbs_image_tag, }, nil } @@ -203,13 +205,14 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { return map[string]string{ - "network": l.network, - "podvm_volume": l.volumeName, - "ssh_key_file": l.ssh_key_file, - "storage": l.storage, - "uri": l.uri, - "KBS_IMAGE": l.kbs_image, - "KBS_IMAGE_TAG": l.kbs_image_tag, + "CONTAINER_RUNTIME": l.containerRuntime, + "network": l.network, + "podvm_volume": l.volumeName, + "ssh_key_file": l.ssh_key_file, + "storage": l.storage, + "uri": l.uri, + "KBS_IMAGE": l.kbs_image, + "KBS_IMAGE_TAG": l.kbs_image_tag, } } From 8a12b1a52cdeac96c0b55ca91e69628b1cda7aad Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 27 Sep 2024 16:54:12 -0300 Subject: [PATCH 4/7] workflows/libvirt: add e2e job to test with CRI-O Added a new container_runtime matrix column to generate one job for each runtime: containerd and crio. Fixes #1981 Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/e2e_libvirt.yaml | 7 +++++++ .github/workflows/e2e_run_all.yaml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index 31b4b6d51..790437770 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -23,6 +23,11 @@ on: description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. required: false type: string + container_runtime: + default: 'containerd' + description: Name of the container runtime. Either containerd or crio. + required: false + type: string env: CLOUD_PROVIDER: libvirt @@ -86,6 +91,7 @@ jobs: run: | ./libvirt/config_libvirt.sh echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties + echo "container_runtime=\"${{ inputs.container_runtime }}\"" >> libvirt.properties # For debugging cat libvirt.properties @@ -117,6 +123,7 @@ jobs: REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }} run: | export CLOUD_PROVIDER=libvirt + export CONTAINER_RUNTIME="${{ inputs.container_runtime }}" export DEPLOY_KBS=true export TEST_PROVISION="yes" export TEST_TEARDOWN="no" diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index 6bdb036b2..c111c5a2e 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -159,6 +159,9 @@ jobs: strategy: fail-fast: false matrix: + container_runtime: + - containerd + - crio os: - ubuntu provider: @@ -168,6 +171,7 @@ jobs: uses: ./.github/workflows/e2e_libvirt.yaml with: caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev + container_runtime: ${{ matrix.container_runtime }} podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }} install_directory_artifact: install_directory git_ref: ${{ inputs.git_ref }} From 0931173696066958768d01eeaaac17101980870d Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 27 Sep 2024 17:09:21 -0300 Subject: [PATCH 5/7] workflows/libvirt: allow the e2e crio job to fail Let's keep it running for a while on CI, once it's stable we can remove the continue-on-error. Fixes #1981 Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/e2e_libvirt.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index 790437770..234d8f744 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -40,6 +40,8 @@ defaults: jobs: test: runs-on: az-ubuntu-2204 + # TODO: remove this when the crio job gets stable + continue-on-error: ${{ inputs.container_runtime == 'crio' && true || false }} steps: - name: Checkout Code uses: actions/checkout@v3 From 1687ba8393da2ca1cc20d206d2e5ee7a29036145 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Thu, 3 Oct 2024 19:02:50 -0300 Subject: [PATCH 6/7] test/e2e: adapt DoTestRestrictivePolicyBlocksExec for CRI-O The DoTestRestrictivePolicyBlocksExec test for CRI-O will have the "error executing command in container" error message instead of "failed to exec in container". So adjusted the expected strings on the error message to consider the output of CRI-O too. Signed-off-by: Wainer dos Santos Moschetta --- src/cloud-api-adaptor/test/e2e/common_suite.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index c575c8103..fdb969b05 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -632,8 +632,10 @@ func DoTestRestrictivePolicyBlocksExec(t *testing.T, e env.Environment, assert C Command: []string{"ls"}, ContainerName: pod.Spec.Containers[0].Name, TestErrorFn: func(err error) bool { - if strings.Contains(err.Error(), "failed to exec in container") && strings.Contains(err.Error(), "ExecProcessRequest is blocked by policy") { - t.Logf("Exec process was blocked %s", err.Error()) + if (strings.Contains(err.Error(), "failed to exec in container") || // containerd + strings.Contains(err.Error(), "error executing command in container")) && // cri-o + strings.Contains(err.Error(), "ExecProcessRequest is blocked by policy") { + t.Logf("Exec process was blocked: %s", err.Error()) return true } else { t.Errorf("Exec process was allowed: %s", err.Error()) From 711ee01f1e37811f3f3e6d3033b73996a22cf33f Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 8 Oct 2024 18:12:09 -0300 Subject: [PATCH 7/7] test/e2e: skip libvirt tests for CRI-O These test are already skipped on CI, also disabled them when running locally because they fail. Related-to: #2100 Signed-off-by: Wainer dos Santos Moschetta --- src/cloud-api-adaptor/test/e2e/libvirt_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 0c6e17f31..9c99187f1 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -30,6 +30,9 @@ func TestLibvirtCreatePodWithSecret(t *testing.T) { func TestLibvirtCreatePeerPodContainerWithExternalIPAccess(t *testing.T) { SkipTestOnCI(t) + if isTestOnCrio() { + t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)") + } assert := LibvirtAssert{} DoTestCreatePeerPodContainerWithExternalIPAccess(t, testEnv, assert) @@ -101,6 +104,9 @@ func TestLibvirtDeletePod(t *testing.T) { func TestLibvirtPodToServiceCommunication(t *testing.T) { // This test is causing issues on CI with instability, so skip until we can resolve this. SkipTestOnCI(t) + if isTestOnCrio() { + t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)") + } assert := LibvirtAssert{} DoTestPodToServiceCommunication(t, testEnv, assert) } @@ -108,6 +114,9 @@ func TestLibvirtPodToServiceCommunication(t *testing.T) { func TestLibvirtPodsMTLSCommunication(t *testing.T) { // This test is causing issues on CI with instability, so skip until we can resolve this. SkipTestOnCI(t) + if isTestOnCrio() { + t.Skip("Fails with CRI-O (confidential-containers/cloud-api-adaptor#2100)") + } assert := LibvirtAssert{} DoTestPodsMTLSCommunication(t, testEnv, assert) }