-
Notifications
You must be signed in to change notification settings - Fork 2.1k
301 lines (258 loc) · 12.9 KB
/
upgrade_downgrade_test_backups_manual.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
name: Backups - Manual - Upgrade Downgrade Testing
on:
push:
pull_request:
concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual')
cancel-in-progress: true
permissions: read-all
jobs:
# This job usually execute in ± 20 minutes
upgrade_downgrade_test_manual:
timeout-minutes: 40
name: Run Upgrade Downgrade Test - Backups - Manual
runs-on: gh-hosted-runners-16cores-1-24.04
steps:
- name: Skip CI
run: |
if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
echo "skipping CI due to the 'Skip CI' label"
exit 1
fi
- name: Check if workflow needs to be skipped
id: skip-workflow
run: |
skip='false'
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
skip='true'
fi
echo Skip ${skip}
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
# Checkout to this build's commit
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
fetch-depth: 0
- name: Set output with latest release branch
id: output-previous-release-ref
if: steps.skip-workflow.outputs.skip-workflow == 'false'
run: |
previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
echo $previous_release_ref
echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1
id: changes
with:
token: ''
filters: |
end_to_end:
- 'test/config.json'
- 'go/**'
- 'go/**/*.go'
- 'test.go'
- 'Makefile'
- 'build.env'
- 'go.sum'
- 'go.mod'
- 'proto/*.proto'
- 'tools/**'
- 'config/**'
- 'bootstrap.sh'
- '.github/workflows/upgrade_downgrade_test_backups_manual.yml'
- 'examples/**'
- name: Set up Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
go-version: 1.22.10
- name: Set up python
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
- name: Tune the OS
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
- name: Get base dependencies
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
sudo DEBIAN_FRONTEND="noninteractive" apt-get update
# Uninstall any previously installed MySQL first
sudo systemctl stop apparmor
sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
sudo apt-get -y autoremove
sudo apt-get -y autoclean
sudo deluser mysql
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
# Install MySQL 8.0
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
sudo apt-get update
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
# Install everything else we need, and configure
sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep
sudo service mysql stop
sudo service etcd stop
sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
# install JUnit report formatter
go install github.com/vitessio/go-junit-report@HEAD
wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get install -y gnupg2
sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
sudo percona-release enable-only tools
sudo apt-get update
sudo apt-get install -y percona-xtrabackup-80
# Checkout to the last release of Vitess
- name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }})
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
- name: Get dependencies for the last release
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
go mod download
- name: Building last release's binaries
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env
NOVTADMINBUILD=1 make build
mkdir -p /tmp/vitess-build-other/
cp -R bin /tmp/vitess-build-other/
rm -Rf bin/*
# Checkout to this build's commit
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Get dependencies for this commit
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
go mod download
- name: Run make minimaltools
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
make minimaltools
- name: Building the binaries for this commit
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env
NOVTADMINBUILD=1 make build
mkdir -p /tmp/vitess-build-current/
cp -R bin /tmp/vitess-build-current/
# We create a sharded Vitess cluster following the local example.
# We also insert a few rows in our three tables.
- name: Create the example Vitess cluster with all components using version N
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 10
run: |
source build.env ; cd examples/backups
./start_cluster.sh
# Taking a backup
- name: Take a backup of all the shards
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/backups
./take_backups.sh
# We insert more data in every table after the backup.
# When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now.
# The initial number of rows for each table is:
# - customer: 5
# - product: 2
# - corder: 5
# We shall see the same number of rows after restoring the backup.
- name: Insert more data after the backup
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples; source ./common/env.sh
echo "insert into customer(email) values('[email protected]');" | mysql
echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql
# Stop all the tablets and remove their data
- name: Stop tablets
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 10
run: |
source build.env ; cd examples/backups
./stop_tablets.sh
# We downgrade: we use the version N-1 of vttablet
- name: Downgrade - Swap binaries, use VTTablet N-1
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env
rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
vttablet --version
# Starting the tablets again, they will automatically start restoring the last backup.
- name: Start new tablets and restore
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 10
run: |
source build.env ; cd examples/backups
./restart_tablets.sh
# give enough time to the tablets to restore the backup
sleep 90
# Count the number of rows in each table to make sure the restoration is successful.
- name: Assert the number of rows in every table
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples ; source ./common/env.sh
echo "select count(sku) from product;" | mysql 2>&1| grep 2
echo "select count(email) from customer;" | mysql 2>&1| grep 5
echo "select count(sku) from corder;" | mysql 2>&1| grep 5
# We insert one more row in every table.
- name: Insert more rows in the tables
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples ; source ./common/env.sh
echo "insert into customer(email) values('[email protected]');" | mysql
echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql
# Taking a second backup of the cluster.
- name: Take a second backup of all the shards
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 10
run: |
source build.env ; cd examples/backups
./take_backups.sh
# We upgrade: we swap binaries and use the version N of the tablet.
- name: Upgrade - Swap binaries, use VTTablet N
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env
rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
vttablet --version
# Starting the tablets again and restoring the previous backup.
- name: Start new tablets and restore
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 10
run: |
source build.env ; cd examples/backups
./upgrade_cluster.sh
# We count the number of rows in every table to check that the restore step was successful.
- name: Assert the number of rows in every table
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples ; source ./common/env.sh
echo "select count(sku) from product;" | mysql 2>&1| grep 3
echo "select count(email) from customer;" | mysql 2>&1| grep 6
echo "select count(sku) from corder;" | mysql 2>&1| grep 6
- name: Stop the Vitess cluster
if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local
./401_teardown.sh || true