Fixing broken mysql-router configuration

A customer faced an issue when they face full disk.
Mysql router configuration file was broken.
As this charm doesn't handle whole configuration file,
charm wrote just part of them based on template.
So re-bootstrapping is needed when this symptom happens.

To do that, I put code to check configuration file size,
and if it is under 1000bytes, run bootstrap.

func-test-pr: openstack-charmers/zaza-openstack-tests#1196
Closes-Bug: #2004088
Change-Id: I02863c6afa0b4d95d3dbf550339a1860fe5ea8c1

src/lib/charm/openstack/mysql_router.py

@@ -479,7 +479,25 @@ def custom_assess_status_check(self):
 
         return None, None
 
-    def bootstrap_mysqlrouter(self):
+    def validate_configuration(self):
+        """Validate Configuration
+
+        Check if mysql router configuration file is less than 1024 bytes.
+        If so, then the configuration file is probably damaged, and then
+        re-run the `bootstrap_mysqlrouter()` function with True to force
+        it.
+        """
+
+        if os.path.exists(self.mysqlrouter_conf):
+            conf_size = os.path.getsize(self.mysqlrouter_conf)
+            if conf_size <= 1024:
+                self.bootstrap_mysqlrouter(True)
+        else:
+            ch_core.hookenv.log(
+                "mysql router configuration file is not exist yet.",
+                "WARNING")
+
+    def bootstrap_mysqlrouter(self, force=False):
         """Bootstrap MySQL Router.
 
         Execute the mysqlrouter bootstrap command. MySQL Router bootstraps into
@@ -494,7 +512,7 @@ def bootstrap_mysqlrouter(self):
         :rtype: None
         """
 
-        if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED):
+        if not force and reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED):
             ch_core.hookenv.log(
                 "Bootstrap mysqlrouter is being called after we set the "
                 "bootstrapped flag: {}. This may require manual intervention,"
@@ -522,8 +540,19 @@ def bootstrap_mysqlrouter(self):
 
         # If we have attempted to bootstrap before but unsuccessfully,
         # use the force option to avoid LP Bug#1919560
-        if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED):
+        is_bootstrap_attempted = reactive.flags.is_flag_set(
+            MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)
+        if is_bootstrap_attempted or force:
             cmd.append("--force")
+            # clear configuration before force bootstrap
+            # because if there are some regex string, it fails
+            try:
+                with open(self.mysqlrouter_conf, "wt") as f:
+                    f.write("[DEFAULT]\n")
+            except Exception:
+                ch_core.hookenv.log(
+                    "ignored, because the bootstrap will overwrite the file.")
+                pass
 
         # Set and attempt the bootstrap
         reactive.flags.set_flag(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)

src/reactive/mysql_router_handlers.py

@@ -11,7 +11,6 @@
 charm.use_defaults(
     'charm.installed',
     'config.changed',
-    'update-status',
     'upgrade-charm')
 
 
@@ -99,6 +98,7 @@ def proxy_shared_db_responses(shared_db, db_router):
     :type db_router_interface: MySQLRouterRequires object
     """
     with charm.provide_charm_instance() as instance:
+        instance.validate_configuration()
         instance.config_changed()
         instance.proxy_db_and_user_responses(db_router, shared_db)
         instance.assess_status()
@@ -112,3 +112,10 @@ def stop_charm():
     with charm.provide_charm_instance() as instance:
         instance.stop_mysqlrouter()
         instance.config_cleanup()
+
+
+@reactive.hook('update-status')
+def update_status():
+    with charm.provide_charm_instance() as instance:
+        instance.validate_configuration()
+        instance.assess_status()

unit_tests/test_lib_charm_openstack_mysql_router.py

@@ -484,6 +484,108 @@ def test_bootstrap_mysqlrouter(self):
         self.clear_flag.assert_called_once_with(
             mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)
 
+    def test_bootstrap_mysqlrouter_force(self):
+        _json_addr = '"10.10.10.60"'
+        _json_pass = '"clusterpass"'
+        _pass = json.loads(_json_pass)
+        _addr = json.loads(_json_addr)
+        _user = "mysql"
+        _port = "3006"
+        self.patch_object(mysql_router.reactive.flags, "is_flag_set")
+        self.endpoint_from_flag.return_value = self.db_router
+        self.db_router.password.return_value = _json_pass
+        self.db_router.db_host.return_value = _json_addr
+        self.is_flag_set.return_value = False
+
+        mrc = mysql_router.MySQLRouterCharm()
+        mrc.options.system_user = _user
+        mrc.options.base_port = _port
+
+        _relations = ["relid"]
+
+        self.patch_object(mysql_router.ch_core.hookenv, "relation_ids")
+        self.relation_ids.return_value = _relations
+
+        _related_units = ["relunits"]
+
+        self.patch_object(mysql_router.ch_core.hookenv, "related_units")
+        self.related_units.return_value = _related_units
+
+        _config_data = {
+            "mysqlrouter_password": json.dumps(_pass),
+            "db_host": json.dumps(_addr),
+        }
+
+        self.patch_object(mysql_router.ch_core.hookenv, "relation_get")
+        self.relation_get.return_value = _config_data
+
+        self.cmp_pkgrevno.return_value = 1
+        self.is_flag_set.side_effect = [False, True]
+        self.subprocess.check_output.side_effect = None
+        mrc.bootstrap_mysqlrouter(True)
+        self.subprocess.check_output.assert_called_once_with(
+            [mrc.mysqlrouter_bin, "--user", _user, "--name", mrc.name,
+             "--bootstrap", "{}:{}@{}"
+             .format(mrc.db_router_user, _pass, _addr),
+             "--directory", mrc.mysqlrouter_working_dir,
+             "--conf-use-sockets",
+             "--conf-bind-address", mrc.shared_db_address,
+             "--report-host", mrc.db_router_address,
+             "--conf-base-port", _port,
+             "--disable-rest", "--force"],
+            stderr=self.stdout)
+        self.set_flag.assert_has_calls([
+            mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED),
+            mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAPPED)])
+        self.clear_flag.assert_called_once_with(
+            mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)
+
+    def test_validate_configuration_file_exists_and_small_size(self):
+        self.patch_object(mysql_router.os.path, "exists",
+                          return_value=True)
+        self.patch_object(mysql_router.os.path, "getsize",
+                          return_value=500)
+        self.patch_object(mysql_router.ch_core.hookenv, "log")
+        self.patch_object(mysql_router.MySQLRouterCharm,
+                          'bootstrap_mysqlrouter')
+
+        mrc = mysql_router.MySQLRouterCharm()
+        mrc.validate_configuration()
+
+        self.bootstrap_mysqlrouter.assert_called_once_with(True)
+        self.log.assert_not_called()
+
+    def test_validate_configuration_file_exists_and_large_size(self):
+        self.patch_object(mysql_router.os.path, "exists",
+                          return_value=True)
+        self.patch_object(mysql_router.os.path, "getsize",
+                          return_value=1500)
+        self.patch_object(mysql_router.ch_core.hookenv, "log")
+        self.patch_object(mysql_router.MySQLRouterCharm,
+                          'bootstrap_mysqlrouter')
+
+        mrc = mysql_router.MySQLRouterCharm()
+        mrc.validate_configuration()
+
+        self.bootstrap_mysqlrouter.assert_not_called()
+        self.log.assert_not_called()
+
+    def test_validate_configuration_file_not_exists(self):
+        self.patch_object(mysql_router.os.path, "exists",
+                          return_value=False)
+        self.patch_object(mysql_router.ch_core.hookenv, "log")
+        self.patch_object(mysql_router.MySQLRouterCharm,
+                          'bootstrap_mysqlrouter')
+
+        mrc = mysql_router.MySQLRouterCharm()
+        mrc.validate_configuration()
+
+        self.bootstrap_mysqlrouter.assert_not_called()
+        self.log.assert_called_once_with(
+            "mysql router configuration file is not exist yet.",
+            "WARNING"
+        )
+
     def test_start_mysqlrouter(self):
         self.patch_object(mysql_router.ch_core.host, "service_start")
         _name = "keystone-mysql-router"

unit_tests/test_mysql_router_handlers.py

@@ -25,7 +25,6 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
     def test_hooks(self):
         defaults = [
             "config.changed",
-            "update-status",
             "upgrade-charm",
             "charm.installed",
         ]
@@ -56,7 +55,10 @@ def test_hooks(self):
             "hook": {
                 "stop_charm": (
                     "stop",
-                )
+                ),
+                "update_status": (
+                    "update-status",
+                ),
             }
         }
         # test that the hooks were registered via the