Added a polling method to wait for Ceph daemons

Signed-off-by: utkarsh bhatt <[email protected]>

.github/workflows/build_and_test.yaml

@@ -125,17 +125,20 @@ jobs:
           lxc exec cephadm -- sh -c "~/cephadm_helper.sh deploy_cephadm $reg_addr:5000/canonical/ceph:latest"
           sleep 2m
 
+      - name: Check health
+        run: |
+          # Check mon/mgr count is 1 each
+          lxc exec cephadm -- sh -c "~/cephadm_helper.sh poll_obj_count mon 1"
+          lxc exec cephadm -- sh -c "~/cephadm_helper.sh poll_obj_count mgr 1"
+
       - name: deploy 3 OSDs
         run: |
           lxc exec cephadm -- sh -c "ceph orch apply osd --all-available-devices" 
           sleep 2m
           lxc exec cephadm --sh -c "ceph -s"
 
       - name: check healthy
-        run: ls
-
-          reg_addr=$(./test/scripts/cephadm_helper.sh get_ip)
-          sudo python test/deploy.py --osd-num 3 --ceph-version reef image "$reg_addr:5000/canonical/ceph:latest"
+        run: lxc exec cephadm -- sh -c "~/cephadm_helper.sh poll_obj_count osd 3"
 
   RookTest:
     needs: build-rock

test/scripts/cephadm_helper.sh

@@ -86,6 +86,30 @@ function test_num_objs() {
     fi
 }
 
+function poll_obj_count() {
+  local what=${1:?missing}
+  local count=${2:?missing}
+  local timeout=${3:?missing}
+
+  echo "Polling for $what to reach $count under $timeout"
+  i=0
+  for i in $(seq 1 10); do
+    num_objs=$( get_num_objs $what )
+    if [ $num_objs == $count ]; then
+      echo "$what reached $count in ${i}th iteration."
+      break
+    else 
+      echo "."
+      sleep 30
+    fi
+  done
+
+  if [ $i -eq 10 ]; then
+    echo "Timeout waiting for $what, only reached $( get_num_objs $what )"
+    exit -1
+  fi
+}
+
 FUNCTION="$1"
 shift
 $FUNCTION "$@"