diff --git a/requirements.txt b/requirements.txt index 938807430..4d49fcce6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,6 +49,7 @@ python-neutronclient python-novaclient python-octaviaclient python-swiftclient +python-watcherclient tenacity paramiko diff --git a/setup.py b/setup.py index 8fe59bfae..5cc8e9bf9 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ 'python-ceilometerclient', 'python-cinderclient<6.0.0', 'python-swiftclient<3.9.0', + 'python-watcherclient', 'zaza@git+https://github.com/openstack-charmers/zaza.git@stable/yoga#egg=zaza', ] diff --git a/zaza/openstack/charm_tests/tempest/utils.py b/zaza/openstack/charm_tests/tempest/utils.py index 830600ec2..e640dcd76 100644 --- a/zaza/openstack/charm_tests/tempest/utils.py +++ b/zaza/openstack/charm_tests/tempest/utils.py @@ -39,7 +39,7 @@ TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon', 'ironic', 'neutron', 'nova', 'octavia', 'sahara', 'swift', - 'trove', 'zaqar'] + 'trove', 'watcher', 'zaqar'] def render_tempest_config_keystone_v2(): diff --git a/zaza/openstack/charm_tests/watcher/__init__.py b/zaza/openstack/charm_tests/watcher/__init__.py new file mode 100644 index 000000000..d92201549 --- /dev/null +++ b/zaza/openstack/charm_tests/watcher/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing Watcher.""" diff --git a/zaza/openstack/charm_tests/watcher/tests.py b/zaza/openstack/charm_tests/watcher/tests.py new file mode 100644 index 000000000..b43c11b4e --- /dev/null +++ b/zaza/openstack/charm_tests/watcher/tests.py @@ -0,0 +1,146 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Encapsulate Cinder testing.""" + +import logging +import tenacity + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.configure.guest as guest +import watcherclient.common.apiclient.exceptions as watcherclient_exceptions + +logger = logging.getLogger(__name__) + + +class WatcherTests(test_utils.OpenStackBaseTest): + """Encapsulate Watcher tests.""" + + AUDIT_TEMPLATE_NAME = 'zaza-at1' + AUDIT_TEMPLATE_GOAL = 'server_consolidation' + AUDIT_TEMPLATE_STRATEGY = 'vm_workload_consolidation' + AUDIT_TYPE = 'ONESHOT' + + BLOCK_SECS = 600 + + @classmethod + def setUpClass(cls): + """Configure Watcher tests class.""" + super().setUpClass() + cls.watcher_client = openstack_utils.get_watcher_session_client( + cls.keystone_session, + ) + + def test_server_consolidation(self): + """Test server consolidation policy.""" + for i, attempt in enumerate(tenacity.Retrying( + wait=tenacity.wait_fixed(2), + retry=tenacity.retry_if_exception_type(AssertionError), + reraise=True, + stop=tenacity.stop_after_attempt(4))): + with attempt: + logger.info('Attempt number %d', i + 1) + self._check_server_consolidation() + + def _check_server_consolidation(self): + try: + at = self.watcher_client.audit_template.get( + self.AUDIT_TEMPLATE_NAME + ) + logger.info('Re-using audit template: %s (%s)', at.name, at.uuid) + except watcherclient_exceptions.NotFound: + at = self.watcher_client.audit_template.create( + name=self.AUDIT_TEMPLATE_NAME, + goal=self.AUDIT_TEMPLATE_GOAL, + strategy=self.AUDIT_TEMPLATE_STRATEGY, + ) + logger.info('Audit template created: %s (%s)', at.name, at.uuid) + + hypervisors_before = { + 'enabled': [], + 'disabled': [], + } + for i, hypervisor in enumerate(self.nova_client.hypervisors.list()): + hypervisors_before[hypervisor.status].append( + hypervisor.hypervisor_hostname + ) + # There is a need to have instances running to allow Watcher not + # fail when calling gnocchi for cpu_util metric measures. + logger.info('Launching instance on hypervisor %s', + hypervisor.hypervisor_hostname) + guest.launch_instance( + 'cirros', + vm_name='zaza-watcher-%s' % i, + perform_connectivity_check=False, + host=hypervisor.hypervisor_hostname, + nova_api_version='2.74', + ) + + audit = self.watcher_client.audit.create( + audit_template_uuid=at.uuid, + audit_type=self.AUDIT_TYPE, + parameters={'period': 600, 'granularity': 300}, + ) + logger.info('Audit created: %s', audit.uuid) + + openstack_utils.resource_reaches_status(self.watcher_client.audit, + audit.uuid, + msg='audit', + resource_attribute='state', + expected_status='SUCCEEDED', + wait_iteration_max_time=180, + stop_after_attempt=30, + stop_status='FAILED') + action_plans = self.watcher_client.action_plan.list(audit=audit.uuid) + assert len(action_plans) == 1 + action_plan = action_plans[0] + actions = self.watcher_client.action.list(action_plan=action_plan.uuid) + + for action in actions: + logger.info('Action %s: %s %s', + action.uuid, action.state, action.action_type) + self.assertEqual(action.state, 'PENDING', + 'Action %s state %s != PENDING' % (action.uuid, + action.state)) + + self.watcher_client.action_plan.start(action_plan.uuid) + + openstack_utils.resource_reaches_status( + self.watcher_client.action_plan, + action_plan.uuid, + resource_attribute='state', + expected_status='SUCCEEDED', + wait_iteration_max_time=180, + stop_after_attempt=30, + ) + # get fresh list of action objects + actions = self.watcher_client.action.list(action_plan=action_plan.uuid) + for action in actions: + logger.info('Action %s: %s %s', + action.uuid, action.state, action.action_type) + self.assertEqual( + action.state, 'SUCCEEDED', + 'Action %s state %s != SUCCEEDED' % (action.uuid, + action.state), + ) + + hypervisors_after = { + 'enabled': [], + 'disabled': [], + } + for i, hypervisor in enumerate(self.nova_client.hypervisors.list()): + hypervisors_after[hypervisor.status].append( + hypervisor.hypervisor_hostname + ) + self.assertNotEqual(hypervisors_before, hypervisors_after) diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index d5034b36d..11babd766 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -53,7 +53,10 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, private_network_name=None, image_name=None, flavor_name=None, external_network_name=None, meta=None, - userdata=None, attach_to_external_network=False): + userdata=None, attach_to_external_network=False, + keystone_session=None, perform_connectivity_check=True, + host=None, nova_api_version=None + ): """Launch an instance. :param instance_key: Key to collect associated config data with. @@ -79,11 +82,24 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, :param attach_to_external_network: Attach instance directly to external network. :type attach_to_external_network: bool + :param keystone_session: Keystone session to use. + :type keystone_session: Optional[keystoneauth1.session.Session] + :param perform_connectivity_check: Whether to perform a connectivity check. + :type perform_connectivity_check: bool + :param host: Requested host to create servers + :type host: str + :param nova_api_version: Nova API version to use + :type nova_api_version: str | None :returns: the created instance :rtype: novaclient.Server """ - keystone_session = openstack_utils.get_overcloud_keystone_session() - nova_client = openstack_utils.get_nova_session_client(keystone_session) + if not keystone_session: + keystone_session = openstack_utils.get_overcloud_keystone_session() + + nova_client = openstack_utils.get_nova_session_client( + keystone_session, + version=nova_api_version, + ) neutron_client = openstack_utils.get_neutron_session_client( keystone_session) @@ -131,7 +147,9 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, key_name=nova_utils.KEYPAIR_NAME, meta=meta, nics=nics, - userdata=userdata) + userdata=userdata, + host=host, + ) # Test Instance is ready. logging.info('Checking instance is active') @@ -142,7 +160,10 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, # NOTE(lourot): in some models this may sometimes take more than 15 # minutes. See lp:1945991 wait_iteration_max_time=120, - stop_after_attempt=16) + stop_after_attempt=16, + stop_status='ERROR', + msg='instance', + ) logging.info('Checking cloud init is complete') openstack_utils.cloud_init_complete( diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index c638c2dd3..dce306ca7 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -220,3 +220,9 @@ class LoadBalancerUnrecoverableError(Exception): """The LoadBalancer has reached to an unrecoverable error state.""" pass + + +class StatusError(Exception): + """The resource status is in error state.""" + + pass diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index fc9e8d83e..70810fd90 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -62,6 +62,7 @@ v3, v2, ) +from watcherclient import client as watcher_client import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env import zaza.utilities.juju as juju_utils @@ -328,16 +329,18 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session, version=2): +def get_nova_session_client(session, version=None): """Return novaclient authenticated by keystone session. :param session: Keystone session object :type session: keystoneauth1.session.Session object :param version: Version of client to request. - :type version: float + :type version: float | str | None :returns: Authenticated novaclient :rtype: novaclient.Client object """ + if not version: + version = 2 return novaclient_client.Client(version, session=session) @@ -465,6 +468,15 @@ def get_manila_session_client(session, version='2'): return manilaclient.Client(session=session, client_version=version) +def get_watcher_session_client(session): + """Return Watcher client authenticated by keystone session. + + :param session: Keystone session object + :returns: Authenticated watcher client + """ + return watcher_client.get_client(session=session, api_version='1') + + def get_keystone_scope(model_name=None): """Return Keystone scope based on OpenStack release of the overcloud. @@ -2312,7 +2324,8 @@ def download_image(image_url, target_file): def _resource_reaches_status(resource, resource_id, expected_status='available', msg='resource', - resource_attribute='status'): + resource_attribute='status', + stop_status=None): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -2330,11 +2343,26 @@ def _resource_reaches_status(resource, resource_id, :type msg: str :param resource_attribute: Resource attribute to check against :type resource_attribute: str + :param stop_status: Stop retrying when this status is reached + :type stop_status: str :raises: AssertionError + :raises: StatusError """ - resource_status = getattr(resource.get(resource_id), resource_attribute) + try: + res_object = resource.get(resource_id) + resource_status = getattr(res_object, resource_attribute) + except AttributeError: + logging.error('attributes available: %s' % str(dir(res_object))) + raise + logging.info("{}: resource {} in {} state, waiting for {}".format( msg, resource_id, resource_status, expected_status)) + if stop_status: + if isinstance(stop_status, list) and resource_status in stop_status: + raise exceptions.StatusError(resource_status, expected_status) + elif isinstance(stop_status, str) and resource_status == stop_status: + raise exceptions.StatusError(resource_status, expected_status) + assert resource_status == expected_status @@ -2346,6 +2374,7 @@ def resource_reaches_status(resource, wait_exponential_multiplier=1, wait_iteration_max_time=60, stop_after_attempt=8, + stop_status=None, ): """Wait for an openstack resources status to reach an expected status. @@ -2370,23 +2399,28 @@ def resource_reaches_status(resource, :param wait_iteration_max_time: Wait a max of wait_iteration_max_time between retries. :type wait_iteration_max_time: int - :param stop_after_attempt: Stop after stop_after_attempt retires. + :param stop_after_attempt: Stop after stop_after_attempt retries :type stop_after_attempt: int :raises: AssertionError + :raises: StatusError """ retryer = tenacity.Retrying( wait=tenacity.wait_exponential( multiplier=wait_exponential_multiplier, max=wait_iteration_max_time), reraise=True, - stop=tenacity.stop_after_attempt(stop_after_attempt)) + stop=tenacity.stop_after_attempt(stop_after_attempt), + retry=tenacity.retry_if_exception_type(AssertionError), + ) retryer( _resource_reaches_status, resource, resource_id, expected_status, msg, - resource_attribute) + resource_attribute, + stop_status, + ) def _resource_removed(resource, resource_id, msg="resource"):