Skip to content

Commit

Permalink
Check status of all the core pods for microshift
Browse files Browse the repository at this point in the history
In past we observed having kube api access doesn't mean all the required
service pods are running and cluster is working as expected. This PR
adds a list of core namespace for microshift preset and make sure all
the pods in that namespace is running before letting user to know to
consume the cluster.
  • Loading branch information
praveenkumar committed Feb 6, 2024
1 parent f708329 commit 217216c
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 1 deletion.
29 changes: 29 additions & 0 deletions pkg/crc/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -510,3 +510,32 @@ func DeleteMCOLeaderLease(ctx context.Context, ocConfig oc.Config) error {
_, _, err := ocConfig.RunOcCommand("delete", "-A", "lease", "--all")
return err
}

func CheckCorePodsRunning(ctx context.Context, ocConfig oc.Config) error {
if err := WaitForOpenshiftResource(ctx, ocConfig, "pod"); err != nil {
return err
}
coreNameSpaces := []string{"kube-system", "openshift-dns", "openshift-ingress", "openshift-ovn-kubernetes", "openshift-service-ca"}
waitForPods := func() error {
for _, namespace := range coreNameSpaces {
if !podRunningForNamespace(ocConfig, namespace) {
logging.Debugf("Pods in %s namespace are not running", namespace)
return &errors.RetriableError{Err: fmt.Errorf("pods in %s namespace are not running", namespace)}
}
}
return nil
}
return errors.Retry(ctx, 2*time.Minute, waitForPods, 2*time.Second)
}

func podRunningForNamespace(ocConfig oc.Config, namespace string) bool {
stdout, stderr, err := ocConfig.WithFailFast().RunOcCommand("get", "pods", "-n", namespace, "--field-selector=status.phase!=Running")
if err != nil {
logging.Debugf("Failed to get pods in %s namespace, stderr: %s", namespace, stderr)
return false
}
if len(stdout) != 0 {
return false
}
return true
}
5 changes: 4 additions & 1 deletion pkg/crc/machine/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -1064,7 +1064,10 @@ func startMicroshift(ctx context.Context, sshRunner *crcssh.Runner, ocConfig oc.
return err
}

return cluster.WaitForAPIServer(ctx, ocConfig)
if err := cluster.WaitForAPIServer(ctx, ocConfig); err != nil {
return err
}
return cluster.CheckCorePodsRunning(ctx, ocConfig)
}

func ensurePullSecretPresentInVM(sshRunner *crcssh.Runner, pullSec cluster.PullSecretLoader) error {
Expand Down

0 comments on commit 217216c

Please sign in to comment.