fix: verify flaky tests (#9050)

* fix: flaky verify tests
GoogleContainerTools · Aug 29, 2023 · a66f3fa · a66f3fa
1 parent f2cacc3
commit a66f3fa
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 22 deletions.
diff --git a/integration/exec_k8s_actions_test.go b/integration/exec_k8s_actions_test.go
@@ -73,14 +73,15 @@ func TestExec_K8SActions(t *testing.T) {
 
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
-			MarkIntegrationTest(t.T, NeedsGcp)
+			MarkIntegrationTest(t.T, CanRunWithoutGcp)
 			args := []string{test.action}
 
 			if test.envFile != "" {
 				args = append(args, "--env-file", test.envFile)
 			}
+			ns, _ := SetupNamespace(t.T)
 
-			out, err := skaffold.Exec(args...).InDir("testdata/custom-actions-k8s").RunWithCombinedOutput(t.T)
+			out, err := skaffold.Exec(args...).InNs(ns.Name).InDir("testdata/custom-actions-k8s").RunWithCombinedOutput(t.T)
 			t.CheckError(test.shouldErr, err)
 			logs := string(out)
 
@@ -123,17 +124,17 @@ func TestExec_K8SActionWithLocalArtifact(t *testing.T) {
 
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
-			MarkIntegrationTest(t.T, NeedsGcp)
+			MarkIntegrationTest(t.T, CanRunWithoutGcp)
 			dir := "testdata/custom-actions-k8s"
 			args := []string{test.action}
-
+			ns, _ := SetupNamespace(t.T)
 			if test.shouldBuild {
 				tmpfile := testutil.TempFile(t.T, "", []byte{})
-				skaffold.Build("--file-output", tmpfile, "--tag", uuid.New().String(), "--check-cluster-node-platforms=true").InDir(dir).RunOrFail(t.T)
+				skaffold.Build("--file-output", tmpfile, "--tag", uuid.New().String(), "--check-cluster-node-platforms=true").InNs(ns.Name).InDir(dir).RunOrFail(t.T)
 				args = append(args, "--build-artifacts", tmpfile)
 			}
 
-			out, err := skaffold.Exec(args...).InDir(dir).RunWithCombinedOutput(t.T)
+			out, err := skaffold.Exec(args...).InNs(ns.Name).InDir(dir).RunWithCombinedOutput(t.T)
 			t.CheckError(test.shouldErr, err)
 
 			for _, expectedMsg := range test.expectedMsgs {
@@ -193,14 +194,15 @@ func TestExec_K8SActionsEvents(t *testing.T) {
 
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
-			MarkIntegrationTest(t.T, NeedsGcp)
+			MarkIntegrationTest(t.T, CanRunWithoutGcp)
+			ns, _ := SetupNamespace(t.T)
 			rpcAddr := randomPort()
 			tmp := t.TempDir()
 			logFile := filepath.Join(tmp, uuid.New().String()+"logs.json")
 
 			args := []string{test.action, "--rpc-port", rpcAddr, "--event-log-file", logFile}
 
-			_, err := skaffold.Exec(args...).InDir("testdata/custom-actions-k8s").RunWithCombinedOutput(t.T)
+			_, err := skaffold.Exec(args...).InNs(ns.Name).InDir("testdata/custom-actions-k8s").RunWithCombinedOutput(t.T)
 
 			t.CheckError(test.shouldErr, err)
 

diff --git a/integration/testdata/verify-succeed-k8s/skaffold.yaml b/integration/testdata/verify-succeed-k8s/skaffold.yaml
@@ -19,9 +19,9 @@ verify:
       image: docker.io/hello-world:latest
     executionMode:
       kubernetesCluster: {}
-  - name: alpine-1
+  - name: verify-succeed-k8s-1
     container:
-      name: alpine-1
+      name: verify-succeed-k8s-1
       image: alpine:3.15.4
       command: ["/bin/sh"]
       args: ["-c", "echo $FOO; sleep 10; echo bye"]
@@ -39,11 +39,11 @@ verify:
 profiles:
   - name: no-duplicated-logs
     verify:
-      - name: alpine-1
+      - name: no-duplicated-logs-1
         executionMode:
           kubernetesCluster: {}
         container:
-          name: alpine-1
+          name: no-duplicated-logs-1
           image: alpine:3.15.4
           command: ["/bin/sh"]
           args: ["-c", "echo alpine-1; sleep 1; echo bye alpine-1"]
@@ -91,11 +91,11 @@ profiles:
 
   - name: local-built-artifact
     verify:
-      - name: alpine-1
+      - name: local-built-artifact-1
         executionMode:
           kubernetesCluster: {}
         container:
-          name: alpine-1
+          name: local-built-artifact-1
           image: alpine:3.15.4
           command: ["/bin/sh"]
           args: ["-c", "echo alpine-1; sleep 2; echo bye alpine-1"]

diff --git a/integration/verify_test.go b/integration/verify_test.go
@@ -109,14 +109,15 @@ func TestKubernetesJobVerifyPassingTestsWithEnvVar(t *testing.T) {
 
 	rpcPort := randomPort()
 	// `--default-repo=` is used to cancel the default repo that is set by default.
+	ns, _ := SetupNamespace(t)
 	out, err := skaffold.Verify("--default-repo=", "--rpc-port", rpcPort,
-		"--event-log-file", logFile, "--env-file", "verify.env").InDir("testdata/verify-succeed-k8s").RunWithCombinedOutput(t)
+		"--event-log-file", logFile, "--env-file", "verify.env").InNs(ns.Name).InDir("testdata/verify-succeed-k8s").RunWithCombinedOutput(t)
 	logs := string(out)
 
 	testutil.CheckError(t, false, err)
 	testutil.CheckContains(t, "Hello from Docker!", logs)
 	testutil.CheckContains(t, "foo-var", logs)
-	testutil.CheckContains(t, "alpine-1", logs)
+	testutil.CheckContains(t, "verify-succeed-k8s-1", logs)
 	testutil.CheckContains(t, "alpine-2", logs)
 
 	// verify logs are in the event output as well
@@ -138,8 +139,9 @@ func TestKubernetesJobVerifyOneTestFailsWithEnvVar(t *testing.T) {
 
 	rpcPort := randomPort()
 	// `--default-repo=` is used to cancel the default repo that is set by default.
+	ns, _ := SetupNamespace(t)
 	out, err := skaffold.Verify("--default-repo=", "--rpc-port", rpcPort,
-		"--event-log-file", logFile, "--env-file", "verify.env").InDir("testdata/verify-fail-k8s").RunWithCombinedOutput(t)
+		"--event-log-file", logFile, "--env-file", "verify.env").InNs(ns.Name).InDir("testdata/verify-fail-k8s").RunWithCombinedOutput(t)
 	logs := string(out)
 
 	testutil.CheckError(t, true, err)
@@ -217,8 +219,8 @@ func TestNoDuplicateLogsK8SJobs(t *testing.T) {
 			dir:         "testdata/verify-succeed-k8s",
 			profile:     "no-duplicated-logs",
 			expectedUniqueLogs: []string{
-				"[alpine-1] alpine-1",
-				"[alpine-1] bye alpine-1",
+				"[no-duplicated-logs-1] alpine-1",
+				"[no-duplicated-logs-1] bye alpine-1",
 			},
 		},
 		{
@@ -237,10 +239,11 @@ func TestNoDuplicateLogsK8SJobs(t *testing.T) {
 
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
-			MarkIntegrationTest(t.T, NeedsGcp)
+			MarkIntegrationTest(t.T, CanRunWithoutGcp)
 
 			args := []string{"-p", test.profile}
-			out, err := skaffold.Verify(args...).InDir(test.dir).RunWithCombinedOutput(t.T)
+			ns, _ := SetupNamespace(t.T)
+			out, err := skaffold.Verify(args...).InNs(ns.Name).InDir(test.dir).RunWithCombinedOutput(t.T)
 
 			t.CheckError(test.shouldErr, err)
 
@@ -339,7 +342,8 @@ func TestTimeoutK8s(t *testing.T) {
 			MarkIntegrationTest(t.T, CanRunWithoutGcp)
 
 			args := []string{"-p", test.profile}
-			out, err := skaffold.Verify(args...).InDir(test.dir).RunWithCombinedOutput(t.T)
+			ns, _ := SetupNamespace(t.T)
+			out, err := skaffold.Verify(args...).InNs(ns.Name).InDir(test.dir).RunWithCombinedOutput(t.T)
 			logs := string(out)
 
 			t.CheckError(test.shouldErr, err)