[integration-k8s-kind#325] Calico CI

.cloudtest_calico.yaml

@@ -0,0 +1,31 @@
+---
+version: 1.0
+root: "./.tests/cloud_test_calico/"
+timeout: 7200  # 2 hour total total timeout
+shuffle-enabled: true
+statistics:
+  enabled: true
+  interval: 60  # 60 seconds for statistics
+import:
+  - cloudtest/packet.yaml
+  - cloudtest/tests.yaml
+
+retest:  # Allow to do test re-run if some kind of failures are detected, line CNI network plugin errors.
+  count: 1  # Allow 5 times to do restart
+  warmup-time: 15  # Put 15 seconds warmup for cluster instance to be used again.
+  allowed-retests: 2  # If cluster instance have few attempts with retest requests one after another, we need to restart cluster.
+  pattern:
+    - "NetworkPlugin cni failed to set up pod"    # Error in AWS dur to leak of IPs or not ability to assign them.
+    - "etcdserver: request timed out"  # Error in any could, reason unknown.
+    - "unable to establish connection to VPP (VPP API socket file /run/vpp/api.sock does not exist)"  # a VPP is not started, it will be re-started in general, but will cause test fail.
+    # Sometimes (rarely) docker registry is unavailable for a moment
+    - "Error response from daemon: Get https://.*docker.io/.*: dial tcp: lookup registry"
+    - "Error response from daemon: Get https://.*docker.io/.*: net/http: request canceled while waiting for connection"
+    - "Failed create pod sandbox"
+reporting:
+  junit-report: "results/junit.xml"
+health-check:
+  - message: "Branch is not up to date"
+    interval: 60  # 1 minute
+    run: |
+      echo "Health check!"

.github/workflows/ci.yaml

@@ -66,8 +66,12 @@ jobs:
           git diff --name-only --exit-code go.sum || ( echo "Run go tidy" && false )
 
   packet:
-    name: packet
+    name: packet (Calico ${{ matrix.calico }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        calico: ["off", "on"]
     steps:
       - name: Set up /bin permissions   # 1. Set up /bin permissions
         run: |
@@ -87,31 +91,42 @@ jobs:
         with:
           repository: networkservicemesh/deployments-k8s
           path: networkservicemesh/deployments-k8s
-      - name: Checkout files            # 5. Checkout files
+      - name: Compute suffix            # 5. Compute suffix for cloudtest input and output paths
+        id: suffix
+        run: |
+          if [[ "${CALICO}" == "on" ]]; then
+            echo ::set-output name=val::_calico
+          fi
+        env:
+          CALICO: ${{ matrix.calico }}
+      - name: Checkout files            # 6. Checkout files
         uses: actions/checkout@v2
         with:
           path: ${{ github.repository }}
-      - name: Run tests with cloudtest  # 6. Run tests with cloudtest
+      - name: Run tests with cloudtest  # 7. Run tests with cloudtest
         working-directory: ${{ github.repository }}
         run: |
-          cloudtest
+          cloudtest --config=.cloudtest${suffix}.yaml
         env:
           PACKET_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}
           PACKET_PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18
           KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }}
-      - name: Publish test report       # 7. Publish test report
+          CALICO: ${{ matrix.calico }}
+          suffix: ${{ steps.suffix.outputs.val }}
+      - name: Publish test report       # 8. Publish test report
         uses: mikepenz/[email protected]
         if: ${{ always() }}
         with:
-          report_paths: "**/cloud_test/results/junit.xml"
+          report_paths: "**/cloud_test${{ steps.suffix.outputs.val }}/results/junit.xml"
           suite_regex: "Test*"
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Upload logs               # 8. Upload logs
+          check_name: "JUnit Test Report (Calico ${{ matrix.calico }})"
+      - name: Upload logs               # 9. Upload logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: logs-${{ github.run_number }}
-          path: ${{ github.repository }}/.tests/cloud_test/
+          path: ${{ github.repository }}/.tests/
 
   packet-cleanup:
     name: packet cleanup

cloudtest/packet.yaml

@@ -25,13 +25,15 @@ providers:
           os: "ubuntu_20_04"
           billing-cycle: "hourly"
           port-vlans:
-            eth3: 1044
+            eth1: 3000  # calico VLAN
+            eth3: 1044  # SR-IOV VLAN
         - name: "Worker"
           host-name: "SR-IOV-Worker-${CLUSTER_NAME}"
           os: "ubuntu_20_04"
           billing-cycle: "hourly"
           port-vlans:
-            eth3: 1044
+            eth1: 3000  # calico VLAN
+            eth3: 1044  # SR-IOV VLAN
       hardware-reservations:
         - 2cf78481-53b0-46c8-a084-6e9815acdb0b
         - 2361d3c2-f694-4fa7-a683-a9f69e2abe7c

main_test.go

@@ -17,17 +17,26 @@
 package main_test
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
 
+	"github.com/networkservicemesh/integration-tests/suites/basic"
 	"github.com/networkservicemesh/integration-tests/suites/heal"
 	"github.com/networkservicemesh/integration-tests/suites/memory"
 	"github.com/networkservicemesh/integration-tests/suites/multiforwarder"
 	"github.com/networkservicemesh/integration-tests/suites/sriov"
 )
 
+func isCalico() bool {
+	return os.Getenv("CALICO") == "on"
+}
+
 func TestMemory(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(memory.Suite))
 }
 
@@ -36,9 +45,22 @@ func TestSRIOV(t *testing.T) {
 }
 
 func TestMultiForwarder(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(multiforwarder.Suite))
 }
 
 func TestHeal(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(heal.Suite))
 }
+
+func TestBasic(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
+	suite.Run(t, new(basic.Suite))
+}

scripts/calico/deploy-calico.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+function on_error() {
+  kubectl describe pods --all-namespaces
+  exit 1
+}
+trap 'on_error' ERR
+
+kubectl apply -k scripts/calico
+
+kubectl -n calico-vpp-dataplane rollout status daemonset/calico-vpp-node --timeout=5m
+kubectl -n kube-system rollout status deployment/calico-kube-controllers --timeout=5m

scripts/calico/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v0.17.0-calicov3.20.2/yaml/generated/calico-vpp-nohuge.yaml
+
+patchesStrategicMerge:
+  - patch.yaml

scripts/calico/patch.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: calico-vpp-config
+  namespace: calico-vpp-dataplane
+data:
+  vpp_dataplane_interface: eno2

scripts/calico/setup-interfaces.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+ip="$1"
+
+ip addr add "${ip}" dev eno2
+ip link set up dev eno2

scripts/calico/setup-node-ip.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+ip="$1"
+
+sed -Ei "s/(.*)\"/\1 --node-ip=${ip}\"/g" /var/lib/kubelet/kubeadm-flags.env
+systemctl restart kubelet