Watcher yoga

requirements.txt

@@ -49,6 +49,7 @@ python-neutronclient
 python-novaclient
 python-octaviaclient
 python-swiftclient
+python-watcherclient
 tenacity
 paramiko
 

setup.py

@@ -65,6 +65,7 @@
     'python-ceilometerclient',
     'python-cinderclient<6.0.0',
     'python-swiftclient<3.9.0',
+    'python-watcherclient',
     'zaza@git+https://github.com/openstack-charmers/zaza.git@stable/yoga#egg=zaza',
 ]
 

zaza/openstack/charm_tests/tempest/utils.py

@@ -39,7 +39,7 @@
 TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest'
 TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon',
                     'ironic', 'neutron', 'nova', 'octavia', 'sahara', 'swift',
-                    'trove', 'zaqar']
+                    'trove', 'watcher', 'zaqar']
 
 
 def render_tempest_config_keystone_v2():

zaza/openstack/charm_tests/watcher/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing Watcher."""

zaza/openstack/charm_tests/watcher/tests.py

@@ -0,0 +1,146 @@
+# Copyright 2023 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Encapsulate Cinder testing."""
+
+import logging
+import tenacity
+
+import zaza.openstack.charm_tests.test_utils as test_utils
+import zaza.openstack.utilities.openstack as openstack_utils
+import zaza.openstack.configure.guest as guest
+import watcherclient.common.apiclient.exceptions as watcherclient_exceptions
+
+logger = logging.getLogger(__name__)
+
+
+class WatcherTests(test_utils.OpenStackBaseTest):
+    """Encapsulate Watcher tests."""
+
+    AUDIT_TEMPLATE_NAME = 'zaza-at1'
+    AUDIT_TEMPLATE_GOAL = 'server_consolidation'
+    AUDIT_TEMPLATE_STRATEGY = 'vm_workload_consolidation'
+    AUDIT_TYPE = 'ONESHOT'
+
+    BLOCK_SECS = 600
+
+    @classmethod
+    def setUpClass(cls):
+        """Configure Watcher tests class."""
+        super().setUpClass()
+        cls.watcher_client = openstack_utils.get_watcher_session_client(
+            cls.keystone_session,
+        )
+
+    def test_server_consolidation(self):
+        """Test server consolidation policy."""
+        for i, attempt in enumerate(tenacity.Retrying(
+                wait=tenacity.wait_fixed(2),
+                retry=tenacity.retry_if_exception_type(AssertionError),
+                reraise=True,
+                stop=tenacity.stop_after_attempt(4))):
+            with attempt:
+                logger.info('Attempt number %d', i + 1)
+                self._check_server_consolidation()
+
+    def _check_server_consolidation(self):
+        try:
+            at = self.watcher_client.audit_template.get(
+                self.AUDIT_TEMPLATE_NAME
+            )
+            logger.info('Re-using audit template: %s (%s)', at.name, at.uuid)
+        except watcherclient_exceptions.NotFound:
+            at = self.watcher_client.audit_template.create(
+                name=self.AUDIT_TEMPLATE_NAME,
+                goal=self.AUDIT_TEMPLATE_GOAL,
+                strategy=self.AUDIT_TEMPLATE_STRATEGY,
+            )
+            logger.info('Audit template created: %s (%s)', at.name, at.uuid)
+
+        hypervisors_before = {
+            'enabled': [],
+            'disabled': [],
+        }
+        for i, hypervisor in enumerate(self.nova_client.hypervisors.list()):
+            hypervisors_before[hypervisor.status].append(
+                hypervisor.hypervisor_hostname
+            )
+            # There is a need to have instances running to allow Watcher not
+            # fail when calling gnocchi for cpu_util metric measures.
+            logger.info('Launching instance on hypervisor %s',
+                        hypervisor.hypervisor_hostname)
+            guest.launch_instance(
+                'cirros',
+                vm_name='zaza-watcher-%s' % i,
+                perform_connectivity_check=False,
+                host=hypervisor.hypervisor_hostname,
+                nova_api_version='2.74',
+            )
+
+        audit = self.watcher_client.audit.create(
+            audit_template_uuid=at.uuid,
+            audit_type=self.AUDIT_TYPE,
+            parameters={'period': 600, 'granularity': 300},
+        )
+        logger.info('Audit created: %s', audit.uuid)
+
+        openstack_utils.resource_reaches_status(self.watcher_client.audit,
+                                                audit.uuid,
+                                                msg='audit',
+                                                resource_attribute='state',
+                                                expected_status='SUCCEEDED',
+                                                wait_iteration_max_time=180,
+                                                stop_after_attempt=30,
+                                                stop_status='FAILED')
+        action_plans = self.watcher_client.action_plan.list(audit=audit.uuid)
+        assert len(action_plans) == 1
+        action_plan = action_plans[0]
+        actions = self.watcher_client.action.list(action_plan=action_plan.uuid)
+
+        for action in actions:
+            logger.info('Action %s: %s %s',
+                        action.uuid, action.state, action.action_type)
+            self.assertEqual(action.state, 'PENDING',
+                             'Action %s state %s != PENDING' % (action.uuid,
+                                                                action.state))
+
+        self.watcher_client.action_plan.start(action_plan.uuid)
+
+        openstack_utils.resource_reaches_status(
+            self.watcher_client.action_plan,
+            action_plan.uuid,
+            resource_attribute='state',
+            expected_status='SUCCEEDED',
+            wait_iteration_max_time=180,
+            stop_after_attempt=30,
+        )
+        # get fresh list of action objects
+        actions = self.watcher_client.action.list(action_plan=action_plan.uuid)
+        for action in actions:
+            logger.info('Action %s: %s %s',
+                        action.uuid, action.state, action.action_type)
+            self.assertEqual(
+                action.state, 'SUCCEEDED',
+                'Action %s state %s != SUCCEEDED' % (action.uuid,
+                                                     action.state),
+            )
+
+        hypervisors_after = {
+            'enabled': [],
+            'disabled': [],
+        }
+        for i, hypervisor in enumerate(self.nova_client.hypervisors.list()):
+            hypervisors_after[hypervisor.status].append(
+                hypervisor.hypervisor_hostname
+            )
+        self.assertNotEqual(hypervisors_before, hypervisors_after)

zaza/openstack/configure/guest.py

@@ -53,7 +53,10 @@
 def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
                     private_network_name=None, image_name=None,
                     flavor_name=None, external_network_name=None, meta=None,
-                    userdata=None, attach_to_external_network=False):
+                    userdata=None, attach_to_external_network=False,
+                    keystone_session=None, perform_connectivity_check=True,
+                    host=None, nova_api_version=None
+                    ):
     """Launch an instance.
 
     :param instance_key: Key to collect associated config data with.
@@ -79,11 +82,24 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
     :param attach_to_external_network: Attach instance directly to external
                                        network.
     :type attach_to_external_network: bool
+    :param keystone_session: Keystone session to use.
+    :type keystone_session: Optional[keystoneauth1.session.Session]
+    :param perform_connectivity_check: Whether to perform a connectivity check.
+    :type perform_connectivity_check: bool
+    :param host: Requested host to create servers
+    :type host: str
+    :param nova_api_version: Nova API version to use
+    :type nova_api_version: str | None
     :returns: the created instance
     :rtype: novaclient.Server
     """
-    keystone_session = openstack_utils.get_overcloud_keystone_session()
-    nova_client = openstack_utils.get_nova_session_client(keystone_session)
+    if not keystone_session:
+        keystone_session = openstack_utils.get_overcloud_keystone_session()
+
+    nova_client = openstack_utils.get_nova_session_client(
+        keystone_session,
+        version=nova_api_version,
+    )
     neutron_client = openstack_utils.get_neutron_session_client(
         keystone_session)
 
@@ -131,7 +147,9 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
         key_name=nova_utils.KEYPAIR_NAME,
         meta=meta,
         nics=nics,
-        userdata=userdata)
+        userdata=userdata,
+        host=host,
+    )
 
     # Test Instance is ready.
     logging.info('Checking instance is active')
@@ -142,7 +160,10 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
         # NOTE(lourot): in some models this may sometimes take more than 15
         # minutes. See lp:1945991
         wait_iteration_max_time=120,
-        stop_after_attempt=16)
+        stop_after_attempt=16,
+        stop_status='ERROR',
+        msg='instance',
+    )
 
     logging.info('Checking cloud init is complete')
     openstack_utils.cloud_init_complete(

zaza/openstack/utilities/exceptions.py

@@ -220,3 +220,9 @@ class LoadBalancerUnrecoverableError(Exception):
     """The LoadBalancer has reached to an unrecoverable error state."""
 
     pass
+
+
+class StatusError(Exception):
+    """The resource status is in error state."""
+
+    pass

zaza/openstack/utilities/openstack.py

@@ -62,6 +62,7 @@
     v3,
     v2,
 )
+from watcherclient import client as watcher_client
 import zaza.openstack.utilities.cert as cert
 import zaza.utilities.deployment_env as deployment_env
 import zaza.utilities.juju as juju_utils
@@ -328,16 +329,18 @@ def get_designate_session_client(**kwargs):
                            **kwargs)
 
 
-def get_nova_session_client(session, version=2):
+def get_nova_session_client(session, version=None):
     """Return novaclient authenticated by keystone session.
 
     :param session: Keystone session object
     :type session: keystoneauth1.session.Session object
     :param version: Version of client to request.
-    :type version: float
+    :type version: float | str | None
     :returns: Authenticated novaclient
     :rtype: novaclient.Client object
     """
+    if not version:
+        version = 2
     return novaclient_client.Client(version, session=session)
 
 
@@ -465,6 +468,15 @@ def get_manila_session_client(session, version='2'):
     return manilaclient.Client(session=session, client_version=version)
 
 
+def get_watcher_session_client(session):
+    """Return Watcher client authenticated by keystone session.
+
+    :param session: Keystone session object
+    :returns: Authenticated watcher client
+    """
+    return watcher_client.get_client(session=session, api_version='1')
+
+
 def get_keystone_scope(model_name=None):
     """Return Keystone scope based on OpenStack release of the overcloud.
 
@@ -2312,7 +2324,8 @@ def download_image(image_url, target_file):
 def _resource_reaches_status(resource, resource_id,
                              expected_status='available',
                              msg='resource',
-                             resource_attribute='status'):
+                             resource_attribute='status',
+                             stop_status=None):
     """Wait for an openstack resources status to reach an expected status.
 
        Wait for an openstack resources status to reach an expected status
@@ -2330,11 +2343,26 @@ def _resource_reaches_status(resource, resource_id,
     :type msg: str
     :param resource_attribute: Resource attribute to check against
     :type resource_attribute: str
+    :param stop_status: Stop retrying when this status is reached
+    :type stop_status: str
     :raises: AssertionError
+    :raises: StatusError
     """
-    resource_status = getattr(resource.get(resource_id), resource_attribute)
+    try:
+        res_object = resource.get(resource_id)
+        resource_status = getattr(res_object, resource_attribute)
+    except AttributeError:
+        logging.error('attributes available: %s' % str(dir(res_object)))
+        raise
+
     logging.info("{}: resource {} in {} state, waiting for {}".format(
         msg, resource_id, resource_status, expected_status))
+    if stop_status:
+        if isinstance(stop_status, list) and resource_status in stop_status:
+            raise exceptions.StatusError(resource_status, expected_status)
+        elif isinstance(stop_status, str) and resource_status == stop_status:
+            raise exceptions.StatusError(resource_status, expected_status)
+
     assert resource_status == expected_status
 
 
@@ -2346,6 +2374,7 @@ def resource_reaches_status(resource,
                             wait_exponential_multiplier=1,
                             wait_iteration_max_time=60,
                             stop_after_attempt=8,
+                            stop_status=None,
                             ):
     """Wait for an openstack resources status to reach an expected status.
 
@@ -2370,23 +2399,28 @@ def resource_reaches_status(resource,
     :param wait_iteration_max_time: Wait a max of wait_iteration_max_time
                                     between retries.
     :type wait_iteration_max_time: int
-    :param stop_after_attempt: Stop after stop_after_attempt retires.
+    :param stop_after_attempt: Stop after stop_after_attempt retries
     :type stop_after_attempt: int
     :raises: AssertionError
+    :raises: StatusError
     """
     retryer = tenacity.Retrying(
         wait=tenacity.wait_exponential(
             multiplier=wait_exponential_multiplier,
             max=wait_iteration_max_time),
         reraise=True,
-        stop=tenacity.stop_after_attempt(stop_after_attempt))
+        stop=tenacity.stop_after_attempt(stop_after_attempt),
+        retry=tenacity.retry_if_exception_type(AssertionError),
+    )
     retryer(
         _resource_reaches_status,
         resource,
         resource_id,
         expected_status,
         msg,
-        resource_attribute)
+        resource_attribute,
+        stop_status,
+    )
 
 
 def _resource_removed(resource, resource_id, msg="resource"):