Skip to content

Commit 962ad81

Browse files
ballista01ballista01
committed
feat(testing): Enhance E2E suite with scale and promotion tests
Signed-off-by: Wenxue Zhao <[email protected]>
1 parent 7a73c27 commit 962ad81

File tree

2 files changed

+260
-4
lines changed

2 files changed

+260
-4
lines changed

‎test/e2e/e2e_test.go

Lines changed: 174 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
"sigs.k8s.io/e2e-framework/pkg/envconf"
3434
"sigs.k8s.io/e2e-framework/pkg/features"
3535

36-
ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1"
36+
ec_v1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1"
3737
)
3838

3939
var etcdVersion = os.Getenv("ETCD_VERSION")
@@ -49,7 +49,7 @@ func TestInvalidClusterSize(t *testing.T) {
4949
"with-negative-members": -1,
5050
}
5151

52-
etcdClusterSpec := &ecv1alpha1.EtcdCluster{
52+
etcdClusterSpec := &ec_v1alpha1.EtcdCluster{
5353
TypeMeta: metav1.TypeMeta{
5454
APIVersion: "operator.etcd.io/v1alpha1",
5555
Kind: "EtcdCluster",
@@ -58,7 +58,7 @@ func TestInvalidClusterSize(t *testing.T) {
5858
Name: etcdClusterName,
5959
Namespace: namespace,
6060
},
61-
Spec: ecv1alpha1.EtcdClusterSpec{
61+
Spec: ec_v1alpha1.EtcdClusterSpec{
6262
Size: 0,
6363
Version: etcdVersion,
6464
},
@@ -82,7 +82,7 @@ func TestInvalidClusterSize(t *testing.T) {
8282
feature.Assess(fmt.Sprintf("etcdCluster %s should not be created when etcdCluster.Spec.Size is %d", name, size),
8383
func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
8484

85-
var etcdCluster ecv1alpha1.EtcdCluster
85+
var etcdCluster ec_v1alpha1.EtcdCluster
8686
err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster)
8787
if !errors.IsNotFound(err) {
8888
t.Fatalf("found unexpected etcdCluster %s with size %d. Got: %v", name, size, err)
@@ -125,3 +125,173 @@ func TestClusterHealthy(t *testing.T) {
125125
// 'testEnv' is the env.Environment you set up in TestMain
126126
_ = testEnv.Test(t, feature.Feature())
127127
}
128+
129+
func TestScaleDownFrom3To1(t *testing.T) {
130+
feature := features.New("scale-down")
131+
etcdClusterName := "etcd-scale-down"
132+
133+
feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
134+
createEtcdCluster(ctx, t, c, etcdClusterName, 3)
135+
waitForStsReady(ctx, t, c, etcdClusterName, 3)
136+
return ctx
137+
})
138+
139+
feature.Assess("scale down to 1", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
140+
var etcdCluster ec_v1alpha1.EtcdCluster
141+
if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster); err != nil {
142+
t.Fatalf("Failed to get EtcdCluster: %v", err)
143+
}
144+
145+
etcdCluster.Spec.Size = 1
146+
if err := c.Client().Resources().Update(ctx, &etcdCluster); err != nil {
147+
t.Fatalf("Failed to update EtcdCluster: %v", err)
148+
}
149+
150+
waitForStsReady(ctx, t, c, etcdClusterName, 1)
151+
return ctx
152+
})
153+
154+
feature.Assess("verify member list", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
155+
podName := fmt.Sprintf("%s-0", etcdClusterName)
156+
stdout, stderr, err := execInPod(t, c, podName, namespace, []string{"etcdctl", "member", "list"})
157+
if err != nil {
158+
t.Fatalf("Failed to exec in pod: %v, stderr: %s", err, stderr)
159+
}
160+
161+
if len(strings.Split(strings.TrimSpace(stdout), "\n")) != 1 {
162+
t.Errorf("Expected to find 1 member in member list, but got: %s", stdout)
163+
}
164+
return ctx
165+
})
166+
167+
feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
168+
var etcdCluster ec_v1alpha1.EtcdCluster
169+
if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster); err == nil {
170+
if err := c.Client().Resources().Delete(ctx, &etcdCluster); err != nil {
171+
t.Logf("Failed to delete EtcdCluster: %v", err)
172+
}
173+
}
174+
return ctx
175+
})
176+
177+
_ = testEnv.Test(t, feature.Feature())
178+
}
179+
180+
func TestScaleUpFrom1To3(t *testing.T) {
181+
feature := features.New("scale-up")
182+
etcdClusterName := "etcd-scale-up"
183+
184+
feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
185+
createEtcdCluster(ctx, t, c, etcdClusterName, 1)
186+
waitForStsReady(ctx, t, c, etcdClusterName, 1)
187+
return ctx
188+
})
189+
190+
feature.Assess("scale up to 3", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
191+
var etcdCluster ec_v1alpha1.EtcdCluster
192+
if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster); err != nil {
193+
t.Fatalf("Failed to get EtcdCluster: %v", err)
194+
}
195+
196+
etcdCluster.Spec.Size = 3
197+
if err := c.Client().Resources().Update(ctx, &etcdCluster); err != nil {
198+
t.Fatalf("Failed to update EtcdCluster: %v", err)
199+
}
200+
201+
waitForStsReady(ctx, t, c, etcdClusterName, 3)
202+
return ctx
203+
})
204+
205+
feature.Assess("verify member list", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
206+
podName := fmt.Sprintf("%s-0", etcdClusterName)
207+
stdout, stderr, err := execInPod(t, c, podName, namespace, []string{"etcdctl", "member", "list"})
208+
if err != nil {
209+
t.Fatalf("Failed to exec in pod: %v, stderr: %s", err, stderr)
210+
}
211+
212+
if len(strings.Split(strings.TrimSpace(stdout), "\n")) != 3 {
213+
t.Errorf("Expected to find 3 members in member list, but got: %s", stdout)
214+
}
215+
return ctx
216+
})
217+
218+
feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
219+
var etcdCluster ec_v1alpha1.EtcdCluster
220+
if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster); err == nil {
221+
if err := c.Client().Resources().Delete(ctx, &etcdCluster); err != nil {
222+
t.Logf("Failed to delete EtcdCluster: %v", err)
223+
}
224+
}
225+
return ctx
226+
})
227+
228+
_ = testEnv.Test(t, feature.Feature())
229+
}
230+
231+
func TestPromoteReadyLearner(t *testing.T) {
232+
feature := features.New("promote-learner")
233+
etcdClusterName := "etcd-promote-learner"
234+
235+
feature.Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
236+
createEtcdCluster(ctx, t, c, etcdClusterName, 2)
237+
waitForStsReady(ctx, t, c, etcdClusterName, 2)
238+
// Manually add a third member as a learner
239+
podName := fmt.Sprintf("%s-0", etcdClusterName)
240+
_, stderr, err := execInPod(t, c, podName, namespace, []string{"etcdctl", "member", "add", "etcd-promote-learner-2", "--peer-urls=http://etcd-promote-learner-2.etcd-promote-learner.etcd-operator-system.svc.cluster.local:2380", "--learner"})
241+
if err != nil {
242+
t.Fatalf("Failed to add learner: %v, stderr: %s", err, stderr)
243+
}
244+
return ctx
245+
})
246+
247+
feature.Assess("promote learner", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
248+
var etcdCluster ec_v1alpha1.EtcdCluster
249+
if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster); err != nil {
250+
t.Fatalf("Failed to get EtcdCluster: %v", err)
251+
}
252+
253+
etcdCluster.Spec.Size = 3
254+
if err := c.Client().Resources().Update(ctx, &etcdCluster); err != nil {
255+
t.Fatalf("Failed to update EtcdCluster: %v", err)
256+
}
257+
258+
// Wait for the learner to be promoted.
259+
wait.For(func(ctx context.Context) (done bool, err error) {
260+
podName := fmt.Sprintf("%s-0", etcdClusterName)
261+
command := []string{"etcdctl", "member", "list", "-w", "table"}
262+
stdout, _, err := execInPod(t, c, podName, namespace, command)
263+
if err != nil {
264+
return false, nil
265+
}
266+
return !strings.Contains(stdout, "true"), nil
267+
}, wait.WithTimeout(2*time.Minute), wait.WithInterval(5*time.Second))
268+
269+
return ctx
270+
})
271+
272+
feature.Assess("verify member list", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
273+
podName := fmt.Sprintf("%s-0", etcdClusterName)
274+
command := []string{"etcdctl", "member", "list", "-w", "table"}
275+
stdout, stderr, err := execInPod(t, c, podName, namespace, command)
276+
if err != nil {
277+
t.Fatalf("Failed to exec in pod: %v, stderr: %s", err, stderr)
278+
}
279+
280+
if strings.Contains(stdout, "true") {
281+
t.Errorf("Expected all members not to be learners, but found a learner. Member list:\n%s", stdout)
282+
}
283+
return ctx
284+
})
285+
286+
feature.Teardown(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
287+
var etcdCluster ec_v1alpha1.EtcdCluster
288+
if err := c.Client().Resources().Get(ctx, etcdClusterName, namespace, &etcdCluster); err == nil {
289+
if err := c.Client().Resources().Delete(ctx, &etcdCluster); err != nil {
290+
t.Logf("Failed to delete EtcdCluster: %v", err)
291+
}
292+
}
293+
return ctx
294+
})
295+
296+
_ = testEnv.Test(t, feature.Feature())
297+
}

‎test/e2e/helpers.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package e2e
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"fmt"
7+
"testing"
8+
"time"
9+
10+
ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1"
11+
appsv1 "k8s.io/api/apps/v1"
12+
corev1 "k8s.io/api/core/v1"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"sigs.k8s.io/e2e-framework/klient/k8s"
15+
"sigs.k8s.io/e2e-framework/klient/wait"
16+
"sigs.k8s.io/e2e-framework/klient/wait/conditions"
17+
"sigs.k8s.io/e2e-framework/pkg/envconf"
18+
)
19+
20+
func createEtcdCluster(ctx context.Context, t *testing.T, c *envconf.Config, name string, size int) *ecv1alpha1.EtcdCluster {
21+
t.Helper()
22+
etcdCluster := &ecv1alpha1.EtcdCluster{
23+
ObjectMeta: metav1.ObjectMeta{
24+
Name: name,
25+
Namespace: namespace,
26+
},
27+
Spec: ecv1alpha1.EtcdClusterSpec{
28+
Size: size,
29+
Version: etcdVersion,
30+
},
31+
}
32+
if err := c.Client().Resources().Create(ctx, etcdCluster); err != nil {
33+
t.Fatalf("Failed to create EtcdCluster: %v", err)
34+
}
35+
return etcdCluster
36+
}
37+
38+
func waitForStsReady(ctx context.Context, t *testing.T, c *envconf.Config, name string, expectedReplicas int) {
39+
t.Helper()
40+
var sts appsv1.StatefulSet
41+
err := wait.For(func(ctx context.Context) (done bool, err error) {
42+
if err := c.Client().Resources().Get(ctx, name, namespace, &sts); err != nil {
43+
return false, err
44+
}
45+
if sts.Status.ReadyReplicas == int32(expectedReplicas) {
46+
return true, nil
47+
}
48+
return false, nil
49+
}, wait.WithTimeout(3*time.Minute), wait.WithInterval(10*time.Second))
50+
if err != nil {
51+
t.Fatalf("StatefulSet %s failed to have %d ready replicas: %v", name, expectedReplicas, err)
52+
}
53+
54+
// Additional wait for the StatefulSet to be fully ready
55+
if err := wait.For(
56+
conditions.New(c.Client().Resources()).ResourceScaled(&sts, func(object k8s.Object) int32 {
57+
return sts.Status.ReadyReplicas
58+
}, int32(expectedReplicas)),
59+
wait.WithTimeout(3*time.Minute),
60+
wait.WithInterval(10*time.Second),
61+
); err != nil {
62+
t.Fatalf("unable to create sts with size %d: %s", int32(expectedReplicas), err)
63+
}
64+
}
65+
66+
func execInPod(t *testing.T, cfg *envconf.Config, podName string, namespace string, command []string) (string, string, error) {
67+
t.Helper()
68+
var stdout, stderr bytes.Buffer
69+
client := cfg.Client()
70+
71+
// Find the pod
72+
var pod corev1.Pod
73+
if err := client.Resources().Get(context.Background(), podName, namespace, &pod); err != nil {
74+
return "", "", fmt.Errorf("failed to get pod %s/%s: %w", namespace, podName, err)
75+
}
76+
77+
// Find the container
78+
if len(pod.Spec.Containers) == 0 {
79+
return "", "", fmt.Errorf("no containers in pod %s/%s", namespace, podName)
80+
}
81+
containerName := pod.Spec.Containers[0].Name
82+
83+
// Exec command
84+
err := client.Resources().ExecInPod(context.Background(), namespace, podName, containerName, command, &stdout, &stderr)
85+
return stdout.String(), stderr.String(), err
86+
}

0 commit comments

Comments
 (0)