NW-3975 | Added retry for watch pods (#13)

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.1-alpine3.14 AS builder
+FROM golang:1.17.2-alpine3.14 AS builder
 RUN mkdir /build
 WORKDIR /build
 COPY . /build/

cmd/kubexit/main.go

@@ -20,6 +20,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/util/retry"
 )
 
 func main() {
@@ -176,11 +177,21 @@ func waitForBirthDeps(birthDeps []string, namespace, podName string, timeout tim
 	defer stopPodWatcher()
 
 	log.Info("Watching pod updates...")
-	err := kubernetes.WatchPod(ctx, namespace, podName,
-		onReadyOfAll(birthDeps, stopPodWatcher),
+	err := retry.OnError(
+		retry.DefaultBackoff,
+		isRetryableError,
+		func() error {
+			watchErr := kubernetes.WatchPod(ctx, namespace, podName,
+				onReadyOfAll(birthDeps, stopPodWatcher),
+			)
+			if watchErr != nil {
+				return fmt.Errorf("failed to watch pod: %v", watchErr)
+			}
+			return nil
+		},
 	)
 	if err != nil {
-		return fmt.Errorf("failed to watch pod: %v", err)
+		return fmt.Errorf("retry watching pods failed: %v", err)
 	}
 
 	// Block until all birth deps are ready
@@ -197,6 +208,10 @@ func waitForBirthDeps(birthDeps []string, namespace, podName string, timeout tim
 	return nil
 }
 
+func isRetryableError(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded)
+}
+
 // withCancelOnSignal calls cancel when one of the specified signals is recieved.
 func withCancelOnSignal(ctx context.Context, signals ...os.Signal) context.Context {
 	ctx, cancel := context.WithCancel(ctx)

pkg/supervisor/supervisor.go

@@ -32,7 +32,7 @@ func New(name string, args ...string) *Supervisor {
 	cmd.Stderr = os.Stderr
 	cmd.Env = os.Environ()
 	return &Supervisor{
-		cmd: cmd,
+		cmd:      cmd,
 		shutdown: false,
 	}
 }