From 56408f442f1043447f130d64e431d22f87543e8f Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 23 Aug 2023 17:18:33 -0400 Subject: [PATCH 1/9] Add Watcher client support Add get_watcher_session_client() helper function to build a watcherclient.v1.Client instance authenticated with a keystone session that uses admin credentials by default. (cherry picked from commit fd824768b5414733b7c78e185b5d1074c829aa96) (cherry picked from commit 03875e1b6edf818668c018effacf8f7ca7e73bf7) (cherry picked from commit 32308a19d02e471e2986499f937539b2b42ec18a) --- requirements.txt | 1 + setup.py | 1 + zaza/openstack/utilities/openstack.py | 10 ++++++++++ 3 files changed, 12 insertions(+) diff --git a/requirements.txt b/requirements.txt index 938807430..4d49fcce6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,6 +49,7 @@ python-neutronclient python-novaclient python-octaviaclient python-swiftclient +python-watcherclient tenacity paramiko diff --git a/setup.py b/setup.py index 8fe59bfae..5cc8e9bf9 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ 'python-ceilometerclient', 'python-cinderclient<6.0.0', 'python-swiftclient<3.9.0', + 'python-watcherclient', 'zaza@git+https://github.com/openstack-charmers/zaza.git@stable/yoga#egg=zaza', ] diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index fc9e8d83e..52dd505bc 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -62,6 +62,7 @@ v3, v2, ) +from watcherclient import client as watcher_client import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env import zaza.utilities.juju as juju_utils @@ -465,6 +466,15 @@ def get_manila_session_client(session, version='2'): return manilaclient.Client(session=session, client_version=version) +def get_watcher_session_client(session): + """Return Watcher client authenticated by keystone session. + + :param session: Keystone session object + :returns: Authenticated watcher client + """ + return watcher_client.get_client(session=session, api_version='1') + + def get_keystone_scope(model_name=None): """Return Keystone scope based on OpenStack release of the overcloud. From 44fdfeacedfd26bcdeb8ed005423d700240cf526 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 23 Aug 2023 17:23:39 -0400 Subject: [PATCH 2/9] Use None as get_nova_session_client() version default This allows callers to pass None and let get_nova_session_client() to use a sane default API, specifically this allows intermediate users (e.g. launch_guest() ) to proxy values passed by the caller. (cherry picked from commit 3283ed47baef6d313dfaed6deaea39ffa0b87d37) (cherry picked from commit dd72af4fa302abb0e193baa179271811f75db403) (cherry picked from commit 3d17c1e5c9f01a36241a0fd3272bb601f3f6cf18) --- zaza/openstack/utilities/openstack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 52dd505bc..836d01565 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -329,7 +329,7 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session, version=2): +def get_nova_session_client(session, version=None): """Return novaclient authenticated by keystone session. :param session: Keystone session object @@ -339,6 +339,8 @@ def get_nova_session_client(session, version=2): :returns: Authenticated novaclient :rtype: novaclient.Client object """ + if not version: + version = 2 return novaclient_client.Client(version, session=session) From 865f55b90c15af1efedffa2568b009bbe56cf3c0 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 24 Aug 2023 15:54:56 -0400 Subject: [PATCH 3/9] launch_instance: expose nova api version and target host This change exposes two new parameters in the launch_instance() function: - nova_api_version: Set the microversion the novaclient should use. - host: Request to launch the instance on a specific hypervisor host. (cherry picked from commit be6ba2a02b35db84dc01654047d9c6188d03229d) (cherry picked from commit 41d4e42b924a2339202235659441a240ce33bfde) (cherry picked from commit e0cf425e877c84a80734b348f85c06129fad638d) --- zaza/openstack/configure/guest.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index d5034b36d..844f72acf 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -53,7 +53,10 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, private_network_name=None, image_name=None, flavor_name=None, external_network_name=None, meta=None, - userdata=None, attach_to_external_network=False): + userdata=None, attach_to_external_network=False, + keystone_session=None, perform_connectivity_check=True, + host=None, nova_api_version=None + ): """Launch an instance. :param instance_key: Key to collect associated config data with. @@ -79,11 +82,24 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, :param attach_to_external_network: Attach instance directly to external network. :type attach_to_external_network: bool + :param keystone_session: Keystone session to use. + :type keystone_session: Optional[keystoneauth1.session.Session] + :param perform_connectivity_check: Whether to perform a connectivity check. + :type perform_connectivity_check: bool + :param host: Requested host to create servers + :type host: str + :param nova_api_version: Nova API version to use + :type nova_api_version: str :returns: the created instance :rtype: novaclient.Server """ - keystone_session = openstack_utils.get_overcloud_keystone_session() - nova_client = openstack_utils.get_nova_session_client(keystone_session) + if not keystone_session: + keystone_session = openstack_utils.get_overcloud_keystone_session() + + nova_client = openstack_utils.get_nova_session_client( + keystone_session, + version=nova_api_version, + ) neutron_client = openstack_utils.get_neutron_session_client( keystone_session) @@ -131,7 +147,9 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, key_name=nova_utils.KEYPAIR_NAME, meta=meta, nics=nics, - userdata=userdata) + userdata=userdata, + host=host, + ) # Test Instance is ready. logging.info('Checking instance is active') @@ -142,7 +160,10 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, # NOTE(lourot): in some models this may sometimes take more than 15 # minutes. See lp:1945991 wait_iteration_max_time=120, - stop_after_attempt=16) + stop_after_attempt=16, + stop_status='ERROR', + msg='instance', + ) logging.info('Checking cloud init is complete') openstack_utils.cloud_init_complete( From 333951b58364fe8fdea8e818824f0f0047378492 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 24 Aug 2023 15:58:01 -0400 Subject: [PATCH 4/9] resource_reaches_status: add new parameter stop_status The stop_status parameter allows callers to ask stop retrying based on a list of statuses that are known to be final (and error) states, this saves time failing earlier. Usage example for fail early when an instance reaches to ERROR status: openstack_utils.resource_reaches_status(self.nova_client.servers, instance_uuid, resource_attribute='state', expected_status='ACTIVE', stop_status='ERROR') (cherry picked from commit 11c3f80d5e061f7a8e51fda0f657ee9da8be961f) (cherry picked from commit b7c38edc51a9d5bd9b6a55b5f3c02cadc360297e) (cherry picked from commit cfc0fb0d61ae88807b00077127906849ad9ac3fc) --- zaza/openstack/utilities/exceptions.py | 6 ++++++ zaza/openstack/utilities/openstack.py | 24 ++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index c638c2dd3..dce306ca7 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -220,3 +220,9 @@ class LoadBalancerUnrecoverableError(Exception): """The LoadBalancer has reached to an unrecoverable error state.""" pass + + +class StatusError(Exception): + """The resource status is in error state.""" + + pass diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 836d01565..3ef47983d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2324,7 +2324,8 @@ def download_image(image_url, target_file): def _resource_reaches_status(resource, resource_id, expected_status='available', msg='resource', - resource_attribute='status'): + resource_attribute='status', + stop_status=None): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -2342,11 +2343,20 @@ def _resource_reaches_status(resource, resource_id, :type msg: str :param resource_attribute: Resource attribute to check against :type resource_attribute: str + :param stop_status: Stop retrying when this status is reached + :type stop_status: str :raises: AssertionError + :raises: StatusError """ resource_status = getattr(resource.get(resource_id), resource_attribute) logging.info("{}: resource {} in {} state, waiting for {}".format( msg, resource_id, resource_status, expected_status)) + if stop_status: + if isinstance(stop_status, list) and resource_status in stop_status: + raise exceptions.StatusError(resource_status, expected_status) + elif isinstance(stop_status, str) and resource_status == stop_status: + raise exceptions.StatusError(resource_status, expected_status) + assert resource_status == expected_status @@ -2358,6 +2368,7 @@ def resource_reaches_status(resource, wait_exponential_multiplier=1, wait_iteration_max_time=60, stop_after_attempt=8, + stop_status=None, ): """Wait for an openstack resources status to reach an expected status. @@ -2382,23 +2393,28 @@ def resource_reaches_status(resource, :param wait_iteration_max_time: Wait a max of wait_iteration_max_time between retries. :type wait_iteration_max_time: int - :param stop_after_attempt: Stop after stop_after_attempt retires. + :param stop_after_attempt: Stop after stop_after_attempt retries :type stop_after_attempt: int :raises: AssertionError + :raises: StatusError """ retryer = tenacity.Retrying( wait=tenacity.wait_exponential( multiplier=wait_exponential_multiplier, max=wait_iteration_max_time), reraise=True, - stop=tenacity.stop_after_attempt(stop_after_attempt)) + stop=tenacity.stop_after_attempt(stop_after_attempt), + retry=tenacity.retry_if_exception_type(AssertionError), + ) retryer( _resource_reaches_status, resource, resource_id, expected_status, msg, - resource_attribute) + resource_attribute, + stop_status, + ) def _resource_removed(resource, resource_id, msg="resource"): From 0db3aec98c7de937f865187117a462e7ac4cde72 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 24 Aug 2023 16:02:53 -0400 Subject: [PATCH 5/9] Log available attributes on AttributeError This simplifies when developing new tests and the programmer is not too familiar with the different attributes an object has. (cherry picked from commit 55b3145865b2dd5ce6ffd80dbc92c2a1fada775b) (cherry picked from commit 42e940416133b138d6f32722cace05691184c08a) (cherry picked from commit b129d5774eebdc3b4f701be5a59dcd630c9555f0) --- zaza/openstack/utilities/openstack.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3ef47983d..08a5a6e69 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2348,7 +2348,13 @@ def _resource_reaches_status(resource, resource_id, :raises: AssertionError :raises: StatusError """ - resource_status = getattr(resource.get(resource_id), resource_attribute) + try: + res_object = resource.get(resource_id) + resource_status = getattr(res_object, resource_attribute) + except AttributeError: + logging.error('attributes available: %s' % str(dir(res_object))) + raise + logging.info("{}: resource {} in {} state, waiting for {}".format( msg, resource_id, resource_status, expected_status)) if stop_status: From b121804a53081cb689fd3f722c7c6e1b117da2bd Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 24 Aug 2023 16:04:19 -0400 Subject: [PATCH 6/9] Add watcher test. This test launches 1 instance per hypervisor, and then launches a new audit to optimize the use of hypervisors and consolidate the instances in a single hypervisor, but also disabling the nova-compute service to avoid new instances get allocated. (cherry picked from commit 6787c84267e9dddc62a696c6b4ce7fb785b623b4) (cherry picked from commit 35741f1ee7d292c0b95a4271d50261e267e0d4b7) (cherry picked from commit 8d600ffa3b799a2b03c0754e8f07664e9d841046) --- .../openstack/charm_tests/watcher/__init__.py | 15 ++ zaza/openstack/charm_tests/watcher/tests.py | 135 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 zaza/openstack/charm_tests/watcher/__init__.py create mode 100644 zaza/openstack/charm_tests/watcher/tests.py diff --git a/zaza/openstack/charm_tests/watcher/__init__.py b/zaza/openstack/charm_tests/watcher/__init__.py new file mode 100644 index 000000000..d92201549 --- /dev/null +++ b/zaza/openstack/charm_tests/watcher/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing Watcher.""" diff --git a/zaza/openstack/charm_tests/watcher/tests.py b/zaza/openstack/charm_tests/watcher/tests.py new file mode 100644 index 000000000..209eb4b16 --- /dev/null +++ b/zaza/openstack/charm_tests/watcher/tests.py @@ -0,0 +1,135 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Encapsulate Cinder testing.""" + +import logging + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.configure.guest as guest +import watcherclient.common.apiclient.exceptions as watcherclient_exceptions + +logger = logging.getLogger(__name__) + + +class WatcherTests(test_utils.OpenStackBaseTest): + """Encapsulate Watcher tests.""" + + AUDIT_TEMPLATE_NAME = 'zaza-at1' + AUDIT_TEMPLATE_GOAL = 'server_consolidation' + AUDIT_TEMPLATE_STRATEGY = 'vm_workload_consolidation' + AUDIT_TYPE = 'ONESHOT' + + BLOCK_SECS = 600 + + @classmethod + def setUpClass(cls): + """Configure Watcher tests class.""" + super().setUpClass() + cls.watcher_client = openstack_utils.get_watcher_session_client( + cls.keystone_session, + ) + + def test_server_consolidation(self): + """Test server consolidation policy.""" + try: + at = self.watcher_client.audit_template.get( + self.AUDIT_TEMPLATE_NAME + ) + logger.info('Re-using audit template: %s (%s)', at.name, at.uuid) + except watcherclient_exceptions.NotFound: + at = self.watcher_client.audit_template.create( + name=self.AUDIT_TEMPLATE_NAME, + goal=self.AUDIT_TEMPLATE_GOAL, + strategy=self.AUDIT_TEMPLATE_STRATEGY, + ) + logger.info('Audit template created: %s (%s)', at.name, at.uuid) + + hypervisors_before = { + 'enabled': [], + 'disabled': [], + } + for i, hypervisor in enumerate(self.nova_client.hypervisors.list()): + hypervisors_before[hypervisor.status].append( + hypervisor.hypervisor_hostname + ) + # There is a need to have instances running to allow Watcher not + # fail when calling gnocchi for cpu_util metric measures. + logger.info('Launching instance on hypervisor %s', + hypervisor.hypervisor_hostname) + guest.launch_instance( + 'cirros', + vm_name='zaza-watcher-%s' % i, + perform_connectivity_check=False, + host=hypervisor.hypervisor_hostname, + nova_api_version='2.74', + ) + + audit = self.watcher_client.audit.create( + audit_template_uuid=at.uuid, + audit_type=self.AUDIT_TYPE, + parameters={'period': 600, 'granularity': 300}, + ) + logger.info('Audit created: %s', audit.uuid) + + openstack_utils.resource_reaches_status(self.watcher_client.audit, + audit.uuid, + msg='audit', + resource_attribute='state', + expected_status='SUCCEEDED', + wait_iteration_max_time=180, + stop_after_attempt=30, + stop_status='FAILED') + action_plans = self.watcher_client.action_plan.list(audit=audit.uuid) + assert len(action_plans) == 1 + action_plan = action_plans[0] + actions = self.watcher_client.action.list(action_plan=action_plan.uuid) + + for action in actions: + logger.info('Action %s: %s %s', + action.uuid, action.state, action.action_type) + self.assertEqual(action.state, 'PENDING', + 'Action %s state %s != PENDING' % (action.uuid, + action.state)) + + self.watcher_client.action_plan.start(action_plan.uuid) + + openstack_utils.resource_reaches_status( + self.watcher_client.action_plan, + action_plan.uuid, + resource_attribute='state', + expected_status='SUCCEEDED', + wait_iteration_max_time=180, + stop_after_attempt=30, + ) + # get fresh list of action objects + actions = self.watcher_client.action.list(action_plan=action_plan.uuid) + for action in actions: + logger.info('Action %s: %s %s', + action.uuid, action.state, action.action_type) + self.assertEqual( + action.state, 'SUCCEEDED', + 'Action %s state %s != SUCCEEDED' % (action.uuid, + action.state), + ) + + hypervisors_after = { + 'enabled': [], + 'disabled': [], + } + for i, hypervisor in enumerate(self.nova_client.hypervisors.list()): + hypervisors_after[hypervisor.status].append( + hypervisor.hypervisor_hostname + ) + self.assertNotEqual(hypervisors_before, hypervisors_after) From c140ad290b5306bc243dbdff7c8c549af14063ea Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 24 Aug 2023 16:24:11 -0400 Subject: [PATCH 7/9] Add 'watcher' to tempest list of services (cherry picked from commit 1d42b643f465b2f6209365b578851da8f80063c4) (cherry picked from commit 9771dd29af2988a42153f697e29b2847a0a39e64) (cherry picked from commit 88f07b1e07b422a42d563c183ae76101f8a846cb) --- zaza/openstack/charm_tests/tempest/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/utils.py b/zaza/openstack/charm_tests/tempest/utils.py index 830600ec2..e640dcd76 100644 --- a/zaza/openstack/charm_tests/tempest/utils.py +++ b/zaza/openstack/charm_tests/tempest/utils.py @@ -39,7 +39,7 @@ TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon', 'ironic', 'neutron', 'nova', 'octavia', 'sahara', 'swift', - 'trove', 'zaqar'] + 'trove', 'watcher', 'zaqar'] def render_tempest_config_keystone_v2(): From a877c7e1f20d6ef001f5457fae167a50ff4664d9 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 25 Aug 2023 12:44:30 -0400 Subject: [PATCH 8/9] Fix version and nova_api_version data types declared in docstring (cherry picked from commit 449164284f297303d67ba61dc6451863b646d027) (cherry picked from commit b2a8f5a9e319be99b89adbcdba3928df7e61ec23) (cherry picked from commit 72457ba450f76867308a9f40a3bcf73af1afa963) --- zaza/openstack/configure/guest.py | 2 +- zaza/openstack/utilities/openstack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index 844f72acf..11babd766 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -89,7 +89,7 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, :param host: Requested host to create servers :type host: str :param nova_api_version: Nova API version to use - :type nova_api_version: str + :type nova_api_version: str | None :returns: the created instance :rtype: novaclient.Server """ diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 08a5a6e69..70810fd90 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -335,7 +335,7 @@ def get_nova_session_client(session, version=None): :param session: Keystone session object :type session: keystoneauth1.session.Session object :param version: Version of client to request. - :type version: float + :type version: float | str | None :returns: Authenticated novaclient :rtype: novaclient.Client object """ From 6139c29b09c02ec24c7ffa5631b6a8e811eb561a Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Mon, 4 Sep 2023 15:49:31 -0300 Subject: [PATCH 9/9] Retry check to avoid live migration unstability. (cherry picked from commit b087c0c2bda4fa19337b6d4ea2bedd445799d69d) (cherry picked from commit e093e1b1fa9d5c8b5bc3f233963b6dd9c28399cf) (cherry picked from commit 851ac245c094f7f6eccdcc64b4e4d221601312fc) --- zaza/openstack/charm_tests/watcher/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zaza/openstack/charm_tests/watcher/tests.py b/zaza/openstack/charm_tests/watcher/tests.py index 209eb4b16..b43c11b4e 100644 --- a/zaza/openstack/charm_tests/watcher/tests.py +++ b/zaza/openstack/charm_tests/watcher/tests.py @@ -14,6 +14,7 @@ """Encapsulate Cinder testing.""" import logging +import tenacity import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -43,6 +44,16 @@ def setUpClass(cls): def test_server_consolidation(self): """Test server consolidation policy.""" + for i, attempt in enumerate(tenacity.Retrying( + wait=tenacity.wait_fixed(2), + retry=tenacity.retry_if_exception_type(AssertionError), + reraise=True, + stop=tenacity.stop_after_attempt(4))): + with attempt: + logger.info('Attempt number %d', i + 1) + self._check_server_consolidation() + + def _check_server_consolidation(self): try: at = self.watcher_client.audit_template.get( self.AUDIT_TEMPLATE_NAME