From 23d3657560295a867c6f853e4b2a3c6b28047a10 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 22 Mar 2023 13:02:38 +0000 Subject: [PATCH 1/7] Add test to rotate mysql service user password This new test verifies that keystone can have its password rotated and then still operate afterwards. It verifies that the on-disk password is changed in the keystone application and that the user list can be performed. (cherry picked from commit e16406116b3e9ae8ec7dc31443e6451b62a3f489) --- zaza/openstack/charm_tests/mysql/tests.py | 96 +++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index f040a874b..a7d4c59dd 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -14,6 +14,7 @@ """MySQL/Percona Cluster Testing.""" +import configparser import json import logging import os @@ -549,6 +550,101 @@ def test_120_set_cluster_option(self): logging.info("Passed set cluster option action test.") +class MySQLInnoDBClusterRotatePasswordTests(MySQLCommonTests): + """Mysql-innodb-cluster charm tests. + + Note: The restart on changed and pause/resume tests also validate the + changing of the R/W primary. On each mysqld shutodown a new R/W primary is + elected automatically by MySQL. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql-innodb-cluster tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + + def test_rotate_keystone_service_user_password(self): + """Verify action used to rotate a service user (keystone) password.""" + KEYSTONE_APP = 'keystone' + KEYSTONE_PASSWD_KEY = "mysql-keystone.passwd" + KEYSTONE_CONF_FILE = '/etc/keystone/keystone.conf' + + def _get_password_from_keystone_leader(): + conf = zaza.model.file_contents( + 'keystone/leader', KEYSTONE_CONF_FILE) + config = configparser.ConfigParser() + config.read_string(conf) + connection_info = config['database']['connection'] + match = re.match(r"^mysql.*keystone:(.+)@", connection_info) + if match: + return match[1] + self.fail("Couldn't find mysql password in {}" + .format(connection_info)) + + # only do the test if keystone is in the model + applications = zaza.model.sync_deployed(self.model_name) + if KEYSTONE_APP not in applications: + self.skipTest( + '{} is not deployed, so not doing password rotation' + .format(KEYSTONE_APP)) + + # get the users via the 'list-service-usernames' action. + logging.info( + "Getting usernames from mysql that can have password rotated.") + action = zaza.model.run_action_on_leader( + self.application, + 'list-service-usernames', + action_params={} + ) + usernames = action.data['results']['usernames'] + self.assertIn('keystone', usernames) + logging.info("... usernames: %s", ', '.join(usernames)) + + # grab the password for keystone from the leader / to verify the change + old_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + old_keystone_passwd_conf = _get_password_from_keystone_leader() + + # verify that keystone is working. + admin_keystone_session = ( + openstack_utils.get_overcloud_keystone_session()) + keystone_client = openstack_utils.get_keystone_session_client( + admin_keystone_session) + keystone_client.users.list() + + # now rotate the password for keystone + # run the action to rotate the password. + logging.info("Rotating password for keystone in mysql.") + zaza.model.run_action_on_leader( + self.application_name, + 'rotate-service-user-password', + action_params={'service-user': 'keystone'}, + ) + + # let everything settle. + logging.info("Waiting for model to settle.") + zaza.model.block_until_all_units_idle() + + # verify that the password has changed. + new_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + new_keystone_passwd_conf = _get_password_from_keystone_leader() + self.assertNotEqual(old_keystone_passwd_on_mysql, + new_keystone_passwd_on_mysql) + self.assertNotEqual(old_keystone_passwd_conf, + new_keystone_passwd_conf) + self.assertEqual(new_keystone_passwd_on_mysql, + new_keystone_passwd_conf) + + # finally, verify that keystone is still working. + admin_keystone_session = ( + openstack_utils.get_overcloud_keystone_session()) + keystone_client = openstack_utils.get_keystone_session_client( + admin_keystone_session) + keystone_client.users.list() + + class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): """Percona Cluster cold start tests.""" From 74d4bf59732e2f967872d979bf4dedd24ae344f2 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 24 Mar 2023 12:23:28 +0000 Subject: [PATCH 2/7] Add wait_for_agent_status in 'settle' code This is to guarantee that the block check didn't happen prior to the action being started. (cherry picked from commit 32f2a052b1fe44793946f451c44eb9e76cd3e524) --- zaza/openstack/charm_tests/mysql/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index a7d4c59dd..8203f3e0a 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -624,6 +624,7 @@ def _get_password_from_keystone_leader(): # let everything settle. logging.info("Waiting for model to settle.") + zaza.model.wait_for_agent_status() zaza.model.block_until_all_units_idle() # verify that the password has changed. From f1c5604b753657f2cb6206915188826a23355f37 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 29 Mar 2023 12:24:35 +0100 Subject: [PATCH 3/7] Fix handling of usernames in password rotation mysql test This patch ensures that the usernames are parsed correctly from the action to ensure that the usernames are handled correctly. (cherry picked from commit 140805d321305d047deaad43a725e4c9bec40dff) --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 8203f3e0a..e4216a48a 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -597,7 +597,7 @@ def _get_password_from_keystone_leader(): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'] + usernames = action.data['results']['usernames'].split(',') self.assertIn('keystone', usernames) logging.info("... usernames: %s", ', '.join(usernames)) From c819a5e3f2a53767170343ee1afdccf195a8ef85 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 5 Apr 2023 19:12:54 +0100 Subject: [PATCH 4/7] Add a tenacity retry to the password change checks (cherry picked from commit a61653d031382899506ce48f1c2123936a001ec1) --- zaza/openstack/charm_tests/mysql/tests.py | 31 ++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index e4216a48a..1abc1d479 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -628,15 +628,28 @@ def _get_password_from_keystone_leader(): zaza.model.block_until_all_units_idle() # verify that the password has changed. - new_keystone_passwd_on_mysql = juju_utils.leader_get( - self.application_name, KEYSTONE_PASSWD_KEY).strip() - new_keystone_passwd_conf = _get_password_from_keystone_leader() - self.assertNotEqual(old_keystone_passwd_on_mysql, - new_keystone_passwd_on_mysql) - self.assertNotEqual(old_keystone_passwd_conf, - new_keystone_passwd_conf) - self.assertEqual(new_keystone_passwd_on_mysql, - new_keystone_passwd_conf) + # Due to the async-ness of the whole model and when the various hooks + # will fire between mysql-innodb-cluster, the mysql-router and + # keystone, so we retry a reasonable time to wait for everything to + # propagate through. + for attempt in tenacity.Retrying( + reraise=True, + wait=tenacity.wait_fixed(30), + stop=tenacity.stop_after_attempt(20), # wait for max 10m + ): + with attempt: + new_keystone_passwd_on_mysql = juju_utils.leader_get( + self.application_name, KEYSTONE_PASSWD_KEY).strip() + new_keystone_passwd_conf = _get_password_from_keystone_leader() + self.assertNotEqual(old_keystone_passwd_on_mysql, + new_keystone_passwd_on_mysql) + self.assertNotEqual(old_keystone_passwd_conf, + new_keystone_passwd_conf) + self.assertEqual(new_keystone_passwd_on_mysql, + new_keystone_passwd_conf) + + # really wait for keystone to finish it's thing + zaza.model.block_until_all_units_idle() # finally, verify that keystone is still working. admin_keystone_session = ( From c2526b2591b1f8a59490eb315c6f572f069ef049 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 4 May 2023 09:31:40 +0200 Subject: [PATCH 5/7] Use json.loads() to read results of password rotation usernames (cherry picked from commit ce2b26a6e02322bb1a9433d77ee72f63e13b72ac) --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 1abc1d479..4a2d33b13 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -597,9 +597,9 @@ def _get_password_from_keystone_leader(): 'list-service-usernames', action_params={} ) - usernames = action.data['results']['usernames'].split(',') - self.assertIn('keystone', usernames) + usernames = json.loads(action.data['results']['usernames']) logging.info("... usernames: %s", ', '.join(usernames)) + self.assertIn('keystone', usernames) # grab the password for keystone from the leader / to verify the change old_keystone_passwd_on_mysql = juju_utils.leader_get( From 68d15ada4b20b17bbe564126e165d9dacc01d71c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 5 May 2023 09:07:14 +0200 Subject: [PATCH 6/7] Switch from json.loads to yaml.safe_load for decoded usernames In the rotate password test, the results from the action return a list of strings that are deliminted by single quotes. This isn't compatible with json.loads(), but yaml.safe_load() is able to load the string as an array of strings. (cherry picked from commit 0fe8e9d666dc79c99217ea86f74e7251474e6c52) --- zaza/openstack/charm_tests/mysql/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 4a2d33b13..c6e24a5bf 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -21,6 +21,7 @@ import re import tempfile import tenacity +import yaml import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model @@ -597,7 +598,7 @@ def _get_password_from_keystone_leader(): 'list-service-usernames', action_params={} ) - usernames = json.loads(action.data['results']['usernames']) + usernames = yaml.safe_load(action.data['results']['usernames']) logging.info("... usernames: %s", ', '.join(usernames)) self.assertIn('keystone', usernames) From 3b8fce25c80903f21f9d35b1d5f7b66779e55cdf Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 19 May 2023 12:36:28 +0100 Subject: [PATCH 7/7] Make username list on log look normal (cherry picked from commit f0a6e802cced47ba3933c5fc6812f6ab1452d619) --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index c6e24a5bf..d6900d3a1 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -599,7 +599,7 @@ def _get_password_from_keystone_leader(): action_params={} ) usernames = yaml.safe_load(action.data['results']['usernames']) - logging.info("... usernames: %s", ', '.join(usernames)) + logging.info("... usernames: %s", usernames) self.assertIn('keystone', usernames) # grab the password for keystone from the leader / to verify the change