Fix last flaky terratest (k8gb-io#1374)

* Fix last flaky terratest

* Do not fail immediately on 503
* Fixes k8gb-io#1345

Signed-off-by: Yury Tsarev <[email protected]>

* Remove concurrency limits from the jobs

* Remove part of the config introduced by k8gb-io#1123 as the
  terratest suite is stable now

Signed-off-by: Yury Tsarev <[email protected]>

* Change way to check for 503 and also sleep for 1 second in case of
detection

Signed-off-by: Yury Tsarev <[email protected]>

* Add more verbose logging to 503 edge case

Signed-off-by: Yury Tsarev <[email protected]>

---------

Signed-off-by: Yury Tsarev <[email protected]>

Author: ytsarev

.github/workflows/terratest-more-clusters.yaml

@@ -8,8 +8,6 @@ on:
 permissions:
   contents: read
 
-concurrency: terratest-${{ github.head_ref || github.run_id }}
-
 jobs:
   terratest-n-clusters:
     runs-on: ubuntu-22.04

.github/workflows/terratest.yaml

@@ -21,8 +21,6 @@ on:
 permissions:
   contents: read
 
-concurrency: terratest-${{ github.head_ref || github.run_id }}
-
 jobs:
   skip-check:
     runs-on: ubuntu-latest

.github/workflows/upgrade-testing.yaml

@@ -21,8 +21,6 @@ on:
 permissions:
   contents: read
 
-concurrency: terratest-${{ github.head_ref || github.run_id }}
-
 jobs:
   skip-check:
     runs-on: ubuntu-latest

terratest/utils/extensions.go

@@ -21,14 +21,15 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/gruntwork-io/terratest/modules/retry"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/gruntwork-io/terratest/modules/retry"
+
 	"github.com/AbsaOSS/gopkg/dns"
 	gopkgstr "github.com/AbsaOSS/gopkg/string"
 	"github.com/gruntwork-io/terratest/modules/helm"
@@ -432,8 +433,13 @@ func (i *Instance) HitTestApp() (result *TestAppResult) {
 	result = new(TestAppResult)
 	coreDNSIP := i.GetCoreDNSIP()
 	command := []string{"sh", "-c", fmt.Sprintf("wget -qO - %s", i.w.state.gslb.host)}
-	for t := 0; t < 3; t++ {
+	for t := 0; t < 60; t++ {
 		result.Body, err = RunBusyBoxCommand(i.w.t, i.w.k8sOptions, coreDNSIP, command)
+		if strings.Contains(result.Body, "503") {
+			i.w.t.Log("podinfo returns 503, trying again....")
+			time.Sleep(time.Second * 1)
+			continue
+		}
 		require.NoError(i.w.t, err, "busybox", command, result.Body)
 		if strings.HasPrefix(result.Body, "{") {
 			break