-
Notifications
You must be signed in to change notification settings - Fork 544
193 lines (175 loc) · 7.16 KB
/
e2e-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
name: e2e
on:
schedule:
- cron: '30 5,17 * * *' # run this every day at 5:30 and 17:30 UTC (00:30 and 12:30 ET)
push:
branches:
- master
pull_request:
workflow_dispatch:
merge_group:
jobs:
# Build the OLM image and save it as an artifact
build:
runs-on: ubuntu-latest
outputs:
sha: ${{ steps.vars.outputs.sha }}
steps:
# checkout code and setup go
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: "go.mod"
# build binaries and image for e2e test (includes experimental features)
- name: Build controller image
run: make e2e-build
- name: Save image
run: docker save quay.io/operator-framework/olm:local -o olm-image.tar
- name: Upload Docker image as artifact
uses: actions/upload-artifact@v4
with:
name: olm-image.tar
path: olm-image.tar
# Run e2e tests in parallel jobs
# Take olm image from the previous stage
e2e:
needs: build
strategy:
fail-fast: false
matrix:
parallel-id: [0, 1, 2, 3]
runs-on: ubuntu-latest
env:
# absolute path to test artifacts directory
ARTIFACT_DIR: ${{ github.workspace }}/artifacts
E2E_TEST_CHUNK: ${{ matrix.parallel-id }}
E2E_NODES: 2
E2E_KUBECONFIG_ROOT: ${{ github.workspace }}/kubeconfigs
steps:
# checkout code and setup go
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: "go.mod"
# load the olm image
- name: Load OLM Docker image
uses: actions/download-artifact@v4
with:
name: olm-image.tar
path: .
- run: docker load < olm-image.tar
# set e2e environment variables
# Set ginkgo output and parallelism
- run: echo "GINKGO_OPTS=-output-dir ${ARTIFACT_DIR} -junit-report junit_e2e.xml -nodes ${E2E_NODES}" >> $GITHUB_ENV
# Setting -kubeconfig-root tells the e2e test suite to look for kubeconfigs
# in <kubeconfig-root>/kubeconfig-<node-number>
# This is used to run tests in parallel on multiple clusters as the current e2e
# test suite does not support running tests in parallel on a single cluster
- run: echo "E2E_OPTS=-kubeconfig-root=${E2E_KUBECONFIG_ROOT}" >> $GITHUB_ENV
# run e2e tests
# create artifacts directory
- run: mkdir -p ${ARTIFACT_DIR}
# deploy test clusters
- name: Deploy test cluster(s)
# create kubeconfig root and store the kubeconfig for each cluster within it as you create the clusters
# Add kind and helm options to specify kubeconfig location
# Deploy the new cluster and helm install olm for testing
run: |
mkdir -p ${E2E_KUBECONFIG_ROOT}
for i in $(seq 1 ${E2E_NODES}); do
KIND_CLUSTER_NAME="kind-olmv0-${i}" \
KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
make kind-create deploy;
done
# run non-flakes if matrix-id is not 'flakes'
- name: Run e2e tests
# calculate the number of chunks as the number of parallel jobs
# use the split tool to split the test suite into chunks and run the chunk corresponding to the matrix-id
# focus on those tests and skip tests marked as FLAKE
run: |
E2E_TEST_NUM_CHUNKS=$(( ${{ strategy.job-total }})) \
GINKGO_OPTS="${GINKGO_OPTS} -focus '$(go run ./test/e2e/split/... -chunks $E2E_TEST_NUM_CHUNKS -print-chunk $E2E_TEST_CHUNK ./test/e2e)' -skip '\[FLAKE\]'" \
make e2e;
# archive test results
- name: Archive Test Artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: e2e-test-output-${{ (github.event.pull_request.head.sha || github.sha) }}-${{ github.run_id }}-${{ matrix.parallel-id }}
path: ${{ env.ARTIFACT_DIR }}/*
# TODO: create job to combine test artifacts using code in https://github.com/operator-framework/operator-lifecycle-manager/pull/1476
e2e-tests:
if: ${{ always() }}
runs-on: ubuntu-latest
needs: e2e
steps:
- run: |
echo "Matrix result: ${{ needs.e2e.result }}"
- name: check individual matrix results
if: ${{ needs.e2e.result == 'failure' }}
run: |
echo 'Failure: at least one e2e matrix job has failed'
exit 1
# Run e2e tests in parallel jobs
# Take olm image from the previous stage
e2e-flakes:
needs: build
strategy:
fail-fast: false
runs-on: ubuntu-latest
env:
# absolute path to test artifacts directory
ARTIFACT_DIR: ${{ github.workspace }}/artifacts
E2E_NODES: 1
E2E_KUBECONFIG_ROOT: ${{ github.workspace }}/kubeconfigs
steps:
# checkout code and setup go
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: "go.mod"
# load the olm image
- name: Load OLM Docker image
uses: actions/download-artifact@v4
with:
name: olm-image.tar
path: .
- run: docker load < olm-image.tar
# set e2e environment variables
# Set ginkgo output and parallelism
- run: echo "GINKGO_OPTS=-output-dir ${ARTIFACT_DIR} -junit-report junit_e2e.xml -nodes ${E2E_NODES}" >> $GITHUB_ENV
# Setting -kubeconfig-root tells the e2e test suite to look for kubeconfigs
# in <kubeconfig-root>/kubeconfig-<node-number>
# This is used to run tests in parallel on multiple clusters as the current e2e
# test suite does not support running tests in parallel on a single cluster
- run: echo "E2E_OPTS=-kubeconfig-root=${E2E_KUBECONFIG_ROOT}" >> $GITHUB_ENV
# run e2e tests
# create artifacts directory
- run: mkdir -p ${ARTIFACT_DIR}
# deploy test clusters
- name: Deploy test cluster(s)
# create kubeconfig root and store the kubeconfig for each cluster within it as you create the clusters
# Add kind and helm options to specify kubeconfig location
# Deploy the new cluster and helm install olm for testing
run: |
mkdir -p ${E2E_KUBECONFIG_ROOT}
for i in $(seq 1 ${E2E_NODES}); do
KIND_CLUSTER_NAME="kind-olmv0-${i}" \
KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
make kind-create deploy;
done
# run e2e tests for flakes if matrix-id is 'flakes'
- name: Run flaky e2e tests
# focus on tests marked as FLAKE
run: |
GINKGO_OPTS="${GINKGO_OPTS} -focus '\[FLAKE\]'" make e2e
# archive test results
- name: Archive Test Artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: e2e-test-output-${{ (github.event.pull_request.head.sha || github.sha) }}-${{ github.run_id }}-flakes
path: ${{ env.ARTIFACT_DIR }}/*
# TODO: create job to combine test artifacts using code in https://github.com/operator-framework/operator-lifecycle-manager/pull/1476