Enable unattented recovery from zero replicas

10conf-d.yml

@@ -93,7 +93,7 @@ data:
 
     if [ $HOST_ID -eq 0 ]; then
       echo "This is the 1st statefulset pod. Checking if the statefulset is down ..."
-      getent hosts mariadb
+      getent hosts mariadb-ready
       [ $? -eq 2 ] && {
         # https://github.com/docker-library/mariadb/commit/f76084f0f9dc13f29cce48c727440eb79b4e92fa#diff-b0fa4b30392406b32de6b8ffe36e290dR80
         if [ ! -d "$DATADIR/mysql" ]; then
@@ -112,11 +112,16 @@ data:
           #echo "    It may not be safe to bootstrap the cluster from this node. It was not the last one to leave the cluster and may not contain all the updates. To force cluster bootstrap with this node, edit the grastate.dat file manually and set safe_to_bootstrap to 1 ."
           echo "Or to try a regular start (for example after recovery + manual intervention), run:"
           echo "  $SUGGEST_EXEC_COMMAND touch /tmp/confirm-resume"
-          echo "Waiting for response ..."
+          if [ ! -z "$AUTO_RECOVERY_MODE" ]; then
+            echo "The AUTO_RECOVERY_MODE env was set to $AUTO_RECOVERY_MODE, will trigger that choice"
+            touch /tmp/$AUTO_RECOVERY_MODE
+          else
+            echo "Waiting for response ..."
+          fi
           while [ ! -f /tmp/confirm-resume ]; do
-            sleep 1
             if [ "$AUTO_NEW_CLUSTER" = "true" ]; then
               echo "The AUTO_NEW_CLUSTER env was set to $AUTO_NEW_CLUSTER, will proceed without confirmation"
+              echo "NOTE this env is deprecated, use AUTO_RECOVERY_MODE instead"
               wsrepNewCluster
               touch /tmp/confirm-resume
             elif [ -f /tmp/confirm-new-cluster ]; then
@@ -133,11 +138,19 @@ data:
               wsrepRecover
               touch /tmp/confirm-resume
             fi
+            sleep 1
           done
           rm /tmp/confirm-*
           set -x
         fi
       }
+    else
+      getent hosts mariadb-ready
+      [ $? -eq 2 ] && {
+        echo "This is NOT the 1st statefulset pod. Must not go up as primary."
+        echo "Found no ready pods. Will exit to trigger a crash loop back off."
+        exit 1
+      }
     fi
 
     # https://github.com/docker-library/mariadb/blob/master/10.2/docker-entrypoint.sh#L62

21mariadb-ready-service.yml

@@ -0,0 +1,14 @@
+# Used with `getent hosts` to check how many pods that are ready
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mariadb-ready
+  namespace: mysql
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "false"
+spec:
+  publishNotReadyAddresses: false
+  clusterIP: None
+  selector:
+    app: mariadb

50mariadb.yml

@@ -9,7 +9,7 @@ spec:
       app: mariadb
   serviceName: "mariadb"
   replicas: 3
-  podManagementPolicy: OrderedReady
+  podManagementPolicy: Parallel
   template:
     metadata:
       labels:
@@ -37,8 +37,8 @@ spec:
               fieldPath: metadata.namespace
         - name: DATADIR
           value: /data/db
-        - name: AUTO_NEW_CLUSTER
-          value: "false"
+        - name: AUTO_RECOVERY_MODE
+          value: confirm-force-bootstrap
         - name: WSREP_CLUSTER_ADDRESS
           value: "gcomm://mariadb-0.mariadb,mariadb-1.mariadb,mariadb-2.mariadb"
         workingDir: /etc/mysql/conf.d-configmap
@@ -51,10 +51,10 @@ spec:
           mountPath: /etc/mysql/conf.d
         - name: initdb
           mountPath: /docker-entrypoint-initdb.d
-        image: mariadb:10.2.31@sha256:1e1193ba3708a73ee28c2ab56c5c6527b39ac58510d19252f020c0903aa5d86a
+        image: mariadb:10.2.32-bionic@sha256:7f9ef5c4dfd39efccbd80e6add152c672e6c63486874d83ce3f60c5a6ba33b47
       containers:
       - name: mariadb
-        image: mariadb:10.2.31@sha256:1e1193ba3708a73ee28c2ab56c5c6527b39ac58510d19252f020c0903aa5d86a
+        image: mariadb:10.2.32-bionic@sha256:7f9ef5c4dfd39efccbd80e6add152c672e6c63486874d83ce3f60c5a6ba33b47
         ports:
         - containerPort: 3306
           name: mysql

kustomization.yaml

@@ -1,5 +1,6 @@
 resources:
 - 10conf-d.yml
 - 20mariadb-service.yml
+- 21mariadb-ready-service.yml
 - 30mysql-service.yml
 - 50mariadb.yml