Skip to content

Commit afae298

Browse files
denis-tingaikindenis-tingaikin
authored
Merge pull request #224 from networkservicemesh/calico_a
Add Calico
2 parents 4fb47ca + 38199a4 commit afae298

22 files changed

+363
-115
lines changed

‎.cloudtest_calico.yaml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
version: 1.0
3+
root: "./.tests/cloud_test_calico/"
4+
timeout: 7200 # 2 hour total total timeout
5+
shuffle-enabled: true
6+
statistics:
7+
enabled: true
8+
interval: 60 # 60 seconds for statistics
9+
import:
10+
- cloudtest/packet.yaml
11+
- cloudtest/tests.yaml
12+
13+
retest: # Allow to do test re-run if some kind of failures are detected, line CNI network plugin errors.
14+
count: 1 # Allow 5 times to do restart
15+
warmup-time: 15 # Put 15 seconds warmup for cluster instance to be used again.
16+
allowed-retests: 2 # If cluster instance have few attempts with retest requests one after another, we need to restart cluster.
17+
pattern:
18+
- "NetworkPlugin cni failed to set up pod" # Error in AWS dur to leak of IPs or not ability to assign them.
19+
- "etcdserver: request timed out" # Error in any could, reason unknown.
20+
- "unable to establish connection to VPP (VPP API socket file /run/vpp/api.sock does not exist)" # a VPP is not started, it will be re-started in general, but will cause test fail.
21+
# Sometimes (rarely) docker registry is unavailable for a moment
22+
- "Error response from daemon: Get https://.*docker.io/.*: dial tcp: lookup registry"
23+
- "Error response from daemon: Get https://.*docker.io/.*: net/http: request canceled while waiting for connection"
24+
- "Failed create pod sandbox"
25+
reporting:
26+
junit-report: "results/junit.xml"
27+
health-check:
28+
- message: "Branch is not up to date"
29+
interval: 60 # 1 minute
30+
run: |
31+
echo "Health check!"

‎.github/workflows/ci.yaml

+23-8
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,12 @@ jobs:
6666
git diff --name-only --exit-code go.sum || ( echo "Run go tidy" && false )
6767
6868
packet:
69-
name: packet
69+
name: packet (CNI ${{ matrix.CNI }})
7070
runs-on: ubuntu-latest
71+
strategy:
72+
fail-fast: false
73+
matrix:
74+
CNI: ["default", "calico"]
7175
steps:
7276
- name: Set up /bin permissions # 1. Set up /bin permissions
7377
run: |
@@ -87,31 +91,42 @@ jobs:
8791
with:
8892
repository: networkservicemesh/deployments-k8s
8993
path: networkservicemesh/deployments-k8s
90-
- name: Checkout files # 5. Checkout files
94+
- name: Compute suffix # 5. Compute suffix for cloudtest input and output paths
95+
id: suffix
96+
run: |
97+
if [[ "${CNI}" == "calico" ]]; then
98+
echo ::set-output name=val::_calico
99+
fi
100+
env:
101+
CNI: ${{ matrix.CNI }}
102+
- name: Checkout files # 6. Checkout files
91103
uses: actions/checkout@v2
92104
with:
93105
path: ${{ github.repository }}
94-
- name: Run tests with cloudtest # 6. Run tests with cloudtest
106+
- name: Run tests with cloudtest # 7. Run tests with cloudtest
95107
working-directory: ${{ github.repository }}
96108
run: |
97-
cloudtest
109+
cloudtest --config=.cloudtest${suffix}.yaml
98110
env:
99111
PACKET_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}
100112
PACKET_PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18
101113
KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }}
102-
- name: Publish test report # 7. Publish test report
114+
CNI: ${{ matrix.CNI }}
115+
suffix: ${{ steps.suffix.outputs.val }}
116+
- name: Publish test report # 8. Publish test report
103117
uses: mikepenz/[email protected]
104118
if: ${{ always() }}
105119
with:
106-
report_paths: "**/cloud_test/results/junit.xml"
120+
report_paths: "**/cloud_test${{ steps.suffix.outputs.val }}/results/junit.xml"
107121
suite_regex: "Test*"
108122
github_token: ${{ secrets.GITHUB_TOKEN }}
109-
- name: Upload logs # 8. Upload logs
123+
check_name: "JUnit Test Report (CNI ${{ matrix.CNI }})"
124+
- name: Upload logs # 9. Upload logs
110125
uses: actions/upload-artifact@v2
111126
if: ${{ always() }}
112127
with:
113128
name: logs-${{ github.run_number }}
114-
path: ${{ github.repository }}/.tests/cloud_test/
129+
path: ${{ github.repository }}/.tests/
115130

116131
packet-cleanup:
117132
name: packet cleanup

‎cloudtest/packet.yaml

+4-2
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ providers:
2525
os: "ubuntu_20_04"
2626
billing-cycle: "hourly"
2727
port-vlans:
28-
eth3: 1044
28+
eth1: 3000 # calico VLAN
29+
eth3: 1044 # SR-IOV VLAN
2930
- name: "Worker"
3031
host-name: "SR-IOV-Worker-${CLUSTER_NAME}"
3132
os: "ubuntu_20_04"
3233
billing-cycle: "hourly"
3334
port-vlans:
34-
eth3: 1044
35+
eth1: 3000 # calico VLAN
36+
eth3: 1044 # SR-IOV VLAN
3537
hardware-reservations:
3638
- 2cf78481-53b0-46c8-a084-6e9815acdb0b
3739
- 2361d3c2-f694-4fa7-a683-a9f69e2abe7c

‎scripts/calico/deploy-calico.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
function on_error() {
4+
kubectl describe pods --all-namespaces
5+
exit 1
6+
}
7+
trap 'on_error' ERR
8+
9+
kubectl apply -f https://projectcalico.docs.tigera.io/v3.22/manifests/tigera-operator.yaml
10+
kubectl apply -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/calico/installation-default.yaml
11+
kubectl apply -k scripts/calico
12+
13+
kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=10m

‎scripts/calico/kustomization.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
5+
resources:
6+
- https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/generated/calico-vpp-nohuge.yaml
7+
8+
patchesStrategicMerge:
9+
- patch.yaml

‎scripts/calico/patch.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: calico-vpp-config
6+
namespace: calico-vpp-dataplane
7+
data:
8+
vpp_dataplane_interface: eno2

‎scripts/calico/setup-interfaces.sh

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
cdr2mask ()
6+
{
7+
# Number of args to shift, 255..255, first non-255 byte, zeroes
8+
set -- $(( 5 - ("$1" / 8) )) 255 255 255 255 $(( (255 << (8 - ("$1" % 8))) & 255 )) 0 0 0
9+
if [[ "$1" -gt 1 ]]
10+
then
11+
shift "$1"
12+
else
13+
shift
14+
fi
15+
echo "${1-0}"."${2-0}"."${3-0}"."${4-0}"
16+
}
17+
18+
iface="$1"
19+
ip="$2"
20+
cidr="$3"
21+
mask=$(cdr2mask "$3")
22+
23+
# Unbond interface and set IP address
24+
cd /etc/network/
25+
awk -v pattern="iface $1 inet" -v ip="$2" -v mask="$mask" '
26+
$0 ~ pattern {
27+
printf "%s static\n",pattern;
28+
printf " address %s\n",ip;
29+
printf " netmask %s\n",mask;
30+
getline;
31+
while ($0 != "") {
32+
if ($1=="bond-master") {
33+
next;
34+
break
35+
};
36+
print;
37+
getline
38+
}
39+
} 1
40+
' interfaces > interfaces.tmp && mv interfaces.tmp interfaces
41+
cd
42+
ifenslave -d bond0 "${iface}"
43+
ip addr change "${ip}/${cidr}" dev "${iface}"
44+
ip link set up dev "${iface}"

‎scripts/calico/setup-node-ip.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
ip="$1"
6+
7+
sed -Ei "s/(.*)\"/\1 --node-ip=${ip}\"/g" /var/lib/kubelet/kubeadm-flags.env
8+
systemctl restart kubelet

‎scripts/create-kubernetes-cluster.sh

+109-48
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,143 @@
11
#!/bin/bash -x
2-
# shellcheck disable=SC2086
2+
# shellcheck disable=SC2086,SC2029
33

44
master_ip=$1
55
worker_ip=$2
66
sshkey=$3
77

8-
SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i ${sshkey}"
9-
10-
function wait_pids() {
11-
pids="$1"
12-
message="$2"
13-
for pid in ${pids}; do
14-
echo "waiting for PID ${pid}"
15-
wait ${pid}
16-
code=$?
17-
if test $code -ne 0; then
18-
echo "${message}: process exited with code $code, aborting..." && return 1
19-
fi
20-
done
21-
return 0
22-
}
23-
24-
# Setup SR-IOV
8+
SSH_CONFIG="ssh_config"
9+
SSH_OPTS="-F ${SSH_CONFIG} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i ${sshkey}"
10+
11+
if [[ "$CNI" == "calico" ]]; then # calico
12+
# Use a new 10.0.0.${base_ip}/30 subnet to prevent IP addresses collisions
13+
# ${base_ip} should be <= 248, because 10.0.0.252/30 subnet is reserved for manual testing
14+
base_ip=$(( GITHUB_RUN_NUMBER % 63 * 4 ))
15+
16+
CALICO_MASTER_IP="10.0.0.$(( base_ip + 1 ))"
17+
CALICO_WORKER_IP="10.0.0.$(( base_ip + 2 ))"
18+
CALICO_CIDR_PREFIX="30"
19+
CALICO_INTERFACE="eno2"
20+
fi
21+
22+
ENVS="KUBERNETES_VERSION CNI"
23+
24+
# wait_pids pid_1 ... pid_n
25+
source scripts/include/wait-pids.sh
26+
# wait_start ip_1 ... ip_n
27+
source scripts/include/wait-start.sh
28+
29+
# 0. Setup SendEnv on the local side.
30+
cp /etc/ssh/ssh_config ${SSH_CONFIG} || exit 1
31+
echo "Host *
32+
SendEnv ${ENVS}" >> ${SSH_CONFIG} || exit 2
33+
34+
wait_start ${master_ip} ${worker_ip} || exit 3
35+
36+
# 1. Setup AcceptEnv on the servers sides and wait for sshd to restart.
37+
scp ${SSH_OPTS} scripts/setup-sshd.sh root@${master_ip}:setup-sshd.sh || exit 11
38+
scp ${SSH_OPTS} scripts/setup-sshd.sh root@${worker_ip}:setup-sshd.sh || exit 12
39+
40+
pids=""
41+
ssh ${SSH_OPTS} root@${master_ip} ./setup-sshd.sh "${ENVS}" &
42+
pids+=" $!"
43+
ssh ${SSH_OPTS} root@${worker_ip} ./setup-sshd.sh "${ENVS}" &
44+
pids+=" $!"
45+
wait_pids "${pids}" "sshd config failed" || exit 13
46+
47+
wait_start ${master_ip} ${worker_ip} || exit 14
48+
49+
## 2. Setup SR-IOV.
2550
pids=""
2651
/bin/bash scripts/sriov/setup-SRIOV.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" &
2752
pids+=" $!"
28-
wait_pids "${pids}" "SR-IOV config failed" || exit 1
53+
wait_pids "${pids}" "SR-IOV config failed" || exit 21
54+
55+
if [[ "$CNI" == "calico" ]]; then # calico
56+
# 3. Create Calico scripts directory on nodes.
57+
ssh ${SSH_OPTS} root@${master_ip} mkdir -p calico || exit 31
58+
ssh ${SSH_OPTS} root@${worker_ip} mkdir -p calico || exit 32
59+
60+
# 4. Setup Calico interfaces.
61+
scp ${SSH_OPTS} scripts/calico/setup-interfaces.sh root@${master_ip}:calico/setup-interfaces.sh || exit 41
62+
scp ${SSH_OPTS} scripts/calico/setup-interfaces.sh root@${worker_ip}:calico/setup-interfaces.sh || exit 42
2963

30-
# Create k8s scripts directory on nodes
31-
ssh ${SSH_OPTS} root@${master_ip} mkdir k8s
32-
ssh ${SSH_OPTS} root@${worker_ip} mkdir k8s
64+
pids=""
65+
ssh ${SSH_OPTS} root@${master_ip} ./calico/setup-interfaces.sh "${CALICO_INTERFACE}" "${CALICO_MASTER_IP}" "${CALICO_CIDR_PREFIX}" &
66+
pids+=" $!"
67+
ssh ${SSH_OPTS} root@${worker_ip} ./calico/setup-interfaces.sh "${CALICO_INTERFACE}" "${CALICO_WORKER_IP}" "${CALICO_CIDR_PREFIX}" &
68+
pids+=" $!"
69+
wait_pids "${pids}" "setup Calico interfaces failed" || exit 43
70+
fi
3371

34-
# Setup docker ulimit
35-
scp ${SSH_OPTS} scripts/k8s/docker-ulimit.sh root@${master_ip}:k8s/docker-ulimit.sh || exit 2
36-
scp ${SSH_OPTS} scripts/k8s/docker-ulimit.sh root@${worker_ip}:k8s/docker-ulimit.sh || exit 3
72+
# 5. Create k8s scripts directory on nodes.
73+
ssh ${SSH_OPTS} root@${master_ip} mkdir -p k8s || exit 51
74+
ssh ${SSH_OPTS} root@${worker_ip} mkdir -p k8s || exit 52
75+
76+
# 6. Config docker.
77+
scp ${SSH_OPTS} scripts/k8s/config-docker.sh root@${master_ip}:k8s/config-docker.sh || exit 61
78+
scp ${SSH_OPTS} scripts/k8s/config-docker.sh root@${worker_ip}:k8s/config-docker.sh || exit 62
3779

3880
pids=""
39-
ssh ${SSH_OPTS} root@${master_ip} ./k8s/docker-ulimit.sh &
81+
ssh ${SSH_OPTS} root@${master_ip} ./k8s/config-docker.sh &
4082
pids+=" $!"
41-
ssh ${SSH_OPTS} root@${worker_ip} ./k8s/docker-ulimit.sh &
83+
ssh ${SSH_OPTS} root@${worker_ip} ./k8s/config-docker.sh &
4284
pids+=" $!"
43-
wait_pids "${pids}" "kubernetes install failed" || exit 4
85+
wait_pids "${pids}" "docker config failed" || exit 63
4486

45-
# Install kubeadm, kubelet and kubectl
46-
scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${master_ip}:k8s/install-kubernetes.sh || exit 5
47-
scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${worker_ip}:k8s/install-kubernetes.sh || exit 6
87+
# 7. Install kubeadm, kubelet and kubectl.
88+
scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${master_ip}:k8s/install-kubernetes.sh || exit 71
89+
scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${worker_ip}:k8s/install-kubernetes.sh || exit 72
4890

4991
pids=""
50-
ssh ${SSH_OPTS} root@${master_ip} ./k8s/install-kubernetes.sh ${KUBERNETES_VERSION} &
92+
ssh ${SSH_OPTS} root@${master_ip} ./k8s/install-kubernetes.sh &
5193
pids+=" $!"
52-
ssh ${SSH_OPTS} root@${worker_ip} ./k8s/install-kubernetes.sh ${KUBERNETES_VERSION} &
94+
ssh ${SSH_OPTS} root@${worker_ip} ./k8s/install-kubernetes.sh &
5395
pids+=" $!"
54-
wait_pids "${pids}" "kubernetes install failed" || exit 7
96+
wait_pids "${pids}" "kubernetes install failed" || exit 73
5597

56-
# master: start kubernetes and create join script
57-
# worker: download kubernetes images
58-
scp ${SSH_OPTS} scripts/k8s/start-master.sh root@${master_ip}:k8s/start-master.sh || exit 8
59-
scp ${SSH_OPTS} scripts/k8s/download-worker-images.sh root@${worker_ip}:k8s/download-worker-images.sh || exit 9
98+
# 8.
99+
# master: start kubernetes and create join script.
100+
# worker: download kubernetes images.
101+
scp ${SSH_OPTS} scripts/k8s/start-master.sh root@${master_ip}:k8s/start-master.sh || exit 81
102+
scp ${SSH_OPTS} scripts/k8s/download-worker-images.sh root@${worker_ip}:k8s/download-worker-images.sh || exit 82
60103

61104
pids=""
62-
ssh ${SSH_OPTS} root@${master_ip} ./k8s/start-master.sh ${KUBERNETES_VERSION} &
105+
ssh ${SSH_OPTS} root@${master_ip} ./k8s/start-master.sh ${master_ip} ${CALICO_MASTER_IP} &
63106
pids+=" $!"
64107
ssh ${SSH_OPTS} root@${worker_ip} ./k8s/download-worker-images.sh &
65108
pids+=" $!"
66-
wait_pids "${pids}" "node setup failed" || exit 10
109+
wait_pids "${pids}" "nodes setup failed" || exit 83
67110

68-
# Download worker join script
111+
# 9. Download, upload and run worker join script.
69112
mkdir -p /tmp/${master_ip}
70-
scp ${SSH_OPTS} root@${master_ip}:k8s/join-cluster.sh /tmp/${master_ip}/join-cluster.sh || exit 11
71-
chmod +x /tmp/${master_ip}/join-cluster.sh || exit 12
113+
scp ${SSH_OPTS} root@${master_ip}:k8s/join-cluster.sh /tmp/${master_ip}/join-cluster.sh || exit 91
114+
chmod +x /tmp/${master_ip}/join-cluster.sh || exit 92
72115

73-
# Upload and run worker join script
74-
scp ${SSH_OPTS} /tmp/${master_ip}/join-cluster.sh root@${worker_ip}:k8s/join-cluster.sh || exit 13
116+
scp ${SSH_OPTS} /tmp/${master_ip}/join-cluster.sh root@${worker_ip}:k8s/join-cluster.sh || exit 93
75117

76118
pids=""
77119
ssh ${SSH_OPTS} root@${worker_ip} ./k8s/join-cluster.sh &
78120
pids+=" $!"
79-
wait_pids "${pids}" "worker join failed" || exit 14
121+
wait_pids "${pids}" "worker join failed" || exit 94
122+
123+
# 10. Save KUBECONFIG to file.
124+
scp ${SSH_OPTS} root@${master_ip}:.kube/config ${KUBECONFIG} || exit 101
125+
126+
if [[ "$CNI" == "calico" ]]; then # calico
127+
# 11. Setup cluster nodes IPs.
128+
scp ${SSH_OPTS} scripts/calico/setup-node-ip.sh root@${master_ip}:calico/setup-node-ip.sh || exit 111
129+
scp ${SSH_OPTS} scripts/calico/setup-node-ip.sh root@${worker_ip}:calico/setup-node-ip.sh || exit 112
130+
131+
pids=""
132+
ssh ${SSH_OPTS} root@${master_ip} ./calico/setup-node-ip.sh "${CALICO_MASTER_IP}" &
133+
pids+=" $!"
134+
ssh ${SSH_OPTS} root@${worker_ip} ./calico/setup-node-ip.sh "${CALICO_WORKER_IP}" &
135+
pids+=" $!"
136+
wait_pids "${pids}" "nodes IPs setup failed" || exit 113
137+
138+
# 12. Deploy Calico CNI.
139+
/bin/bash scripts/calico/deploy-calico.sh || exit 121
140+
fi
80141

81-
echo "Save KUBECONFIG to file"
82-
scp ${SSH_OPTS} root@${master_ip}:.kube/config ${KUBECONFIG} || exit 15
142+
# Get pods
143+
kubectl get pods --all-namespaces

0 commit comments

Comments
 (0)