INTAKERPC-141: Add neutron ovs agent check. (#391)

* INTAKERPC-141:  Add neutron ovs agent check.

* Add Maas tasks

* Adjust neutron_plugin_type condition to task

* Add condition for installing linuxbridge agent check

* Further changes condition of installing linuxbridge agent check

* TURTLES-354: Fixing pep8 issue, add aio test conditionally installing
OVS agent

* Adjust indent

* Fixing bash gate.

playbooks/files/rax-maas/plugins/README.md

@@ -271,6 +271,20 @@ Hostname or IP address of service to test
     metric neutron-linuxbridge-agent_e86cee43-47bf-42cd-872f-a623e458e549_on_host_big3.localdomain uint32 1
     ...
 
+***
+#### neutron_ovs_agent_check.py
+
+##### Description:
+polls the neutron agent to ensure openvswtich agent is up and running
+
+##### Example Output:
+
+    metric agents_found uint32 1
+    metric container_success uint32 1
+    metric ovsdb-server_process_status uint32 1
+    metric ovs-vswitchd_process_status uint32 1
+    metric neutron-openvswitch-agent_process_status uint32 1
+
 ***
 #### neutron_metadata_local_check.py
 

playbooks/files/rax-maas/plugins/neutron_ovs_agent_check.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import re
+
+import lxc
+
+from maas_common import metric_bool
+from maas_common import print_output
+from maas_common import status_err
+from process_check_container import get_processes
+
+
+def check(args):
+
+    containers = lxc.list_containers()
+    neutron_agent_container = None
+    for container in containers:
+        if 'neutron_agents' in container:
+            neutron_agent_container = container
+            metric_bool('agents_found', True, m_name='maas_neutron')
+            break
+    else:
+        metric_bool('agents_found', False, m_name='maas_neutron')
+        status_err('no running neutron agents containers found',
+                   m_name='maas_neutron')
+        return
+
+    # Get the neutron_agent_container's init PID.
+    try:
+        c = lxc.Container(neutron_agent_container)
+        # If the container wasn't found, exit now.
+        if c.init_pid == -1:
+            metric_bool('container_success', False,
+                        m_name='maas_neutron_agent_container')
+            status_err(
+                'Could not find PID for container {}'.format(
+                    neutron_agent_container
+                ),
+                m_name='maas_neutron_agent_container'
+            )
+    except (Exception, SystemError) as e:
+        metric_bool('container_success', False,
+                    m_name='maas_neutron_agent_container')
+        status_err(
+            'Container lookup failed on "{}". ERROR: "{}"'.format(
+                neutron_agent_container,
+                e
+            ),
+            m_name='maas_neutron_agent_container'
+        )
+    else:
+        metric_bool('container_success', True,
+                    m_name='maas_neutron_agent_container')
+
+    # Get the processes within the neutron agent container.
+    procs = get_processes(parent_pid=c.init_pid)
+
+    # Make a list of command lines from each PID. There's a chance that one or
+    # more PIDs may have exited already and this causes a NoSuchProcess
+    # exception.
+    cmdlines = []
+    for proc in procs:
+        try:
+            # In psutil 1.2.1, cmdline is an attribute, but in 5.x, it's now
+            # a callable method.
+            cmdline_check = getattr(proc, "cmdline", None)
+            if callable(cmdline_check):
+                cmdlines.append(map(os.path.basename, proc.cmdline()))
+            else:
+                cmdlines.append(map(os.path.basename, proc.cmdline))
+        except Exception as e:
+            status_err('Error while retrieving process %s' %
+                       proc.cmdline,
+                       m_name='maas_neutron')
+            pass
+
+    # Loop through the process names provided on the command line to see
+    # if ovsdb-server,  ovs-vswitchd, and neutron-openvswitch exist on
+    # the system or in a container.
+    # suppress some character which throw MaaS off
+    # ovsdb-server and ovs-vswitchd are not directly in the command line
+    # parsing so we use condition
+    # `process_name in x or process_name in x[0]`
+    process_names = ['ovsdb-server', 'ovs-vswitchd',
+                     'neutron-openvswitch-agent']
+    pattern = re.compile('[^-\w]+')
+    for process_name in process_names:
+        matches = [x for x in cmdlines
+                   if process_name in x or process_name in x[0]
+                   ]
+
+        metric_bool('%s_process_status' % pattern.sub('', process_name),
+                    len(matches) > 0)
+
+
+def main(args):
+    check(args)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Check neutron OVS'
+                                                 'agents running')
+    parser.add_argument('--telegraf-output',
+                        action='store_true',
+                        default=False,
+                        help='Set the output format to telegraf')
+    args = parser.parse_args()
+    with print_output(print_telegraf=args.telegraf_output):
+        main(args)

playbooks/maas-openstack-neutron.yml

@@ -161,6 +161,31 @@
         group: "root"
         mode: "0644"
       delegate_to: "{{ physical_host | default(ansible_host) }}"
+      when: neutron_plugin_type is not defined or (neutron_plugin_type | search("linuxbridge"))  
+      notify:
+        - Restart rax-maas
+
+  handlers:
+    - include: handlers/main.yml
+  vars_files:
+    - vars/main.yml
+    - vars/maas-openstack.yml
+  tags:
+    - maas-openstack-neutron
+
+- name: Install checks for openstack neutron OVS agent
+  hosts: neutron_openvswitch_agent
+  gather_facts: false
+  tasks:
+    - name: Install neutron OVS agent checks
+      template:
+        src: "templates/rax-maas/neutron_ovs_agent_check.yaml.j2"
+        dest: "/etc/rackspace-monitoring-agent.conf.d/neutron_ovs_agent_check--{{ inventory_hostname }}.yaml"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      delegate_to: "{{ physical_host | default(ansible_host) }}"
+      when: neutron_plugin_type | search("ovs")  
       notify:
         - Restart rax-maas
 
@@ -195,7 +220,7 @@
   tags:
     - maas-openstack-neutron
 
-- name: Install checks for openstack neutron metadata agent
+- name: Install checks for openstack neutron metering agent
   hosts: neutron_metering_agent
   gather_facts: false
   tasks:

playbooks/templates/rax-maas/neutron_ovs_agent_check.yaml.j2

@@ -0,0 +1,20 @@
+{% set label = "neutron_ovs_agent_check" %}
+{% set check_name = label+'--'+inventory_hostname %}
+type        : agent.plugin
+label       : "{{ check_name }}"
+period      : "{{ maas_check_period_override[label] | default(maas_check_period) }}"
+timeout     : "{{ maas_check_timeout_override[label] | default(maas_check_timeout) }}"
+disabled    : "{{ (check_name | match(maas_excluded_checks_regex)) | ternary('true', 'false') }}"
+details     :
+    file    : run_plugin_in_venv.sh
+    args    : ["{{ maas_plugin_dir }}/neutron_ovs_agent_check.py"]
+alarms      :
+    neutron_ovs_agent_status :
+        label                   : neutron_ovs_agent_status--{{ inventory_hostname }}
+        notification_plan_id    : "{{ maas_notification_plan_override[label] | default(maas_notification_plan) }}"
+        disabled                : {{ (('neutron_ovs_agent_status--'+inventory_hostname) | match(maas_excluded_alarms_regex)) | ternary('true', 'false') }}
+        criteria                : |
+            :set consecutiveCount={{ maas_alarm_local_consecutive_count }}
+            if (metric["neutron-openvswitch-agent_process_status"] != 1) {
+                return new AlarmStatus(CRITICAL, "neutron-ovs-agent down");
+            }

tests/aio-create.sh

@@ -84,6 +84,25 @@ function enable_ironic {
       - name: ironic.yml.aio" | tee -a /etc/openstack_deploy/user_ironic.yml
 }
 
+function install_ovs {
+  if [ ! -z ${use_ovs+x} ]; then
+    sed -i 's/neutron_linuxbridge_agent/neutron_openvswitch_agent/' /opt/openstack-ansible/etc/openstack_deploy/openstack_user_config.yml.aio
+    echo '
+openstack_host_specific_kernel_modules:
+  - name: "openvswitch"
+  - pattern: "CONFIG_OPENVSWITCH"
+  - group: "network_hosts"
+
+neutron_plugin_type: ml2.ovs
+neutron_ml2_drivers_type: "vlan"
+neutron_provider_networks:
+  - network_flat_networks: "*"
+  - network_types: "vlan"
+  - network_vlan_ranges: "physnet1:1:1"
+  - network_mappings: "physnet1:br-provider"' | tee -a /etc/openstack_deploy/user_variables.yml
+  fi
+}
+
 ## Main ----------------------------------------------------------------------
 
 echo "Gate test starting
@@ -166,6 +185,9 @@ pushd /opt/openstack-ansible
     enable_ironic
   fi
 
+  # Install ovs agent if applicable
+  install_ovs
+
   # Disbale tempest on newer releases
   if [[ -f "tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2" ]]; then
     sed -i 's|^tempest_install.*|tempest_install: no|g' tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2