From 080b63013a2b742d5b0b4c1148e644a9cb6ce326 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 22 Mar 2023 13:02:38 +0000 Subject: [PATCH 1/7] Add test to rotate mysql service user password This new test verifies that keystone can have its password rotated and then still operate afterwards. It verifies that the on-disk password is changed in the keystone application and that the user list can be performed. (cherry picked from commit e16406116b3e9ae8ec7dc31443e6451b62a3f489) --- zaza/openstack/charm_tests/mysql/tests.py | 96 +++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index eb379ffc5..c78f505c6 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -14,6 +14,7 @@ """MySQL/Percona Cluster Testing.""" +import configparser import json import logging import os @@ -567,6 +568,101 @@ def test_911_restart_after_group_replication_cache_set(self): ) +class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): + """Mysql-innodb-cluster charm tests. + + Note: The restart on changed and pause/resume tests also validate the + changing of the R/W primary. On each mysqld shutodown a new R/W primary is + elected automatically by MySQL. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql-innodb-cluster tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + + def test_rotate_keystone_service_user_password(self): + """Verify action used to rotate a service user (keystone) password.""" + KEYSTONE_APP = 'keystone' + KEYSTONE_PASSWD_KEY = "mysql-keystone.passwd" + KEYSTONE_CONF_FILE = '/etc/keystone/keystone.conf' + + def _get_password_from_keystone_leader(): + conf = zaza.model.file_contents( + 'keystone/leader', KEYSTONE_CONF_FILE) + config = configparser.ConfigParser() + config.read_string(conf) + connection_info = config['database']['connection'] + match = re.match(r"^mysql.*keystone:(.+)@", connection_info) + if match: + return match[1] + self.fail("Couldn't find mysql password in {}" + .format(connection_info)) + + # only do the test if keystone is in the model + applications = zaza.model.sync_deployed(self.model_name) + if KEYSTONE_APP not in applications: + self.skipTest( + '{} is not deployed, so not doing password rotation' + .format(KEYSTONE_APP)) + + # get the users via the 'list-service-usernames' action. + logging.info( + "Getting usernames from mysql that can have password rotated.") + action = zaza.model.run_action_on_leader( + self.application, + 'list-service-usernames', + action_params={} + ) + usernames = action.data['results']['usernames'] + self.assertIn('keystone', usernames) + logging.info("... usernames: %s", ', '.join(usernames)) + + # grab the password for keystone from the leader / to verify the change + old_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + old_keystone_passwd_conf = _get_password_from_keystone_leader() + + # verify that keystone is working. + admin_keystone_session = ( + openstack_utils.get_overcloud_keystone_session()) + keystone_client = openstack_utils.get_keystone_session_client( + admin_keystone_session) + keystone_client.users.list() + + # now rotate the password for keystone + # run the action to rotate the password. + logging.info("Rotating password for keystone in mysql.") + zaza.model.run_action_on_leader( + self.application_name, + 'rotate-service-user-password', + action_params={'service-user': 'keystone'}, + ) + + # let everything settle. + logging.info("Waiting for model to settle.") + zaza.model.block_until_all_units_idle() + + # verify that the password has changed. + new_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + new_keystone_passwd_conf = _get_password_from_keystone_leader() + self.assertNotEqual(old_keystone_passwd_on_mysql, + new_keystone_passwd_on_mysql) + self.assertNotEqual(old_keystone_passwd_conf, + new_keystone_passwd_conf) + self.assertEqual(new_keystone_passwd_on_mysql, + new_keystone_passwd_conf) + + # finally, verify that keystone is still working. + admin_keystone_session = ( + openstack_utils.get_overcloud_keystone_session()) + keystone_client = openstack_utils.get_keystone_session_client( + admin_keystone_session) + keystone_client.users.list() + + class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): """Percona Cluster cold start tests.""" From 61c6e6b673b4bf7cfd8d95418351ccab2ee23485 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 24 Mar 2023 12:23:28 +0000 Subject: [PATCH 2/7] Add wait_for_agent_status in 'settle' code This is to guarantee that the block check didn't happen prior to the action being started. (cherry picked from commit 32f2a052b1fe44793946f451c44eb9e76cd3e524) --- zaza/openstack/charm_tests/mysql/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index c78f505c6..80393386d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -642,6 +642,7 @@ def _get_password_from_keystone_leader(): # let everything settle. logging.info("Waiting for model to settle.") + zaza.model.wait_for_agent_status() zaza.model.block_until_all_units_idle() # verify that the password has changed. From a2f9456541a88e8e71139e453ac44cb3083dc208 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 29 Mar 2023 12:24:35 +0100 Subject: [PATCH 3/7] Fix handling of usernames in password rotation mysql test This patch ensures that the usernames are parsed correctly from the action to ensure that the usernames are handled correctly. (cherry picked from commit 140805d321305d047deaad43a725e4c9bec40dff) --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 80393386d..e7ab00306 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -615,7 +615,7 @@ def _get_password_from_keystone_leader(): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'] + usernames = action.data['results']['usernames'].split(',') self.assertIn('keystone', usernames) logging.info("... usernames: %s", ', '.join(usernames)) From aed8191a6c70ce83e2ba9800ed728709ba8caf36 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 5 Apr 2023 19:12:54 +0100 Subject: [PATCH 4/7] Add a tenacity retry to the password change checks (cherry picked from commit a61653d031382899506ce48f1c2123936a001ec1) --- zaza/openstack/charm_tests/mysql/tests.py | 31 ++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index e7ab00306..2ed7f0fd2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -646,15 +646,28 @@ def _get_password_from_keystone_leader(): zaza.model.block_until_all_units_idle() # verify that the password has changed. - new_keystone_passwd_on_mysql = juju_utils.leader_get( - self.application_name, KEYSTONE_PASSWD_KEY).strip() - new_keystone_passwd_conf = _get_password_from_keystone_leader() - self.assertNotEqual(old_keystone_passwd_on_mysql, - new_keystone_passwd_on_mysql) - self.assertNotEqual(old_keystone_passwd_conf, - new_keystone_passwd_conf) - self.assertEqual(new_keystone_passwd_on_mysql, - new_keystone_passwd_conf) + # Due to the async-ness of the whole model and when the various hooks + # will fire between mysql-innodb-cluster, the mysql-router and + # keystone, so we retry a reasonable time to wait for everything to + # propagate through. + for attempt in tenacity.Retrying( + reraise=True, + wait=tenacity.wait_fixed(30), + stop=tenacity.stop_after_attempt(20), # wait for max 10m + ): + with attempt: + new_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + new_keystone_passwd_conf = _get_password_from_keystone_leader() + self.assertNotEqual(old_keystone_passwd_on_mysql, + new_keystone_passwd_on_mysql) + self.assertNotEqual(old_keystone_passwd_conf, + new_keystone_passwd_conf) + self.assertEqual(new_keystone_passwd_on_mysql, + new_keystone_passwd_conf) + + # really wait for keystone to finish it's thing + zaza.model.block_until_all_units_idle() # finally, verify that keystone is still working. admin_keystone_session = ( From 4bd80a45334dade737056a56e8ed1e53ff3ff885 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 4 May 2023 09:31:40 +0200 Subject: [PATCH 5/7] Use json.loads() to read results of password rotation usernames (cherry picked from commit ce2b26a6e02322bb1a9433d77ee72f63e13b72ac) --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 2ed7f0fd2..c5e3f1821 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -615,9 +615,9 @@ def _get_password_from_keystone_leader(): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'].split(',') - self.assertIn('keystone', usernames) + usernames = json.loads(action.data['results']['usernames']) logging.info("... usernames: %s", ', '.join(usernames)) + self.assertIn('keystone', usernames) # grab the password for keystone from the leader / to verify the change old_keystone_passwd_on_mysql = juju_utils.leader_get( From e86ca5ffdc4f86cff59bf180873b1515b9e12476 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 5 May 2023 09:07:14 +0200 Subject: [PATCH 6/7] Switch from json.loads to yaml.safe_load for decoded usernames In the rotate password test, the results from the action return a list of strings that are deliminted by single quotes. This isn't compatible with json.loads(), but yaml.safe_load() is able to load the string as an array of strings. (cherry picked from commit 0fe8e9d666dc79c99217ea86f74e7251474e6c52) --- zaza/openstack/charm_tests/mysql/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index c5e3f1821..830985b9d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -21,6 +21,7 @@ import re import tempfile import tenacity +import yaml import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model @@ -615,7 +616,7 @@ def _get_password_from_keystone_leader(): 'list-service-usernames', action_params={} ) - usernames = json.loads(action.data['results']['usernames']) + usernames = yaml.safe_load(action.data['results']['usernames']) logging.info("... usernames: %s", ', '.join(usernames)) self.assertIn('keystone', usernames) From 5dcbcfd809aaa97ed68f8db1eb389464e5ebfe11 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 19 May 2023 12:36:28 +0100 Subject: [PATCH 7/7] Make username list on log look normal (cherry picked from commit f0a6e802cced47ba3933c5fc6812f6ab1452d619) --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 830985b9d..ce9898d84 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,7 +617,7 @@ def _get_password_from_keystone_leader(): action_params={} ) usernames = yaml.safe_load(action.data['results']['usernames']) - logging.info("... usernames: %s", ', '.join(usernames)) + logging.info("... usernames: %s", usernames) self.assertIn('keystone', usernames) # grab the password for keystone from the leader / to verify the change