Merge pull request #559 from tonytan4ever/TURTLES-1425

MaaS Queens fixes

playbooks/common-tasks/maas_get_maasrc.yml

@@ -0,0 +1,36 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# maasrc distribution
+# (This playbook could be removed when upstream decides to include
+# OS_IMAGE_API_VERSION and OS_VOLUME_API_VERSION in openrc
+#
+- name: Retrieve maasrc file
+  slurp:
+    src: /root/maasrc
+  register: maasrc_data
+  delegate_to: "{{ groups['utility_all'][0] }}"
+
+- name: Register a fact for the maasrc
+  set_fact:
+     maasrc_fact: "{{ maasrc_data.content }}"
+
+- name: Distribute maasrc
+  copy:
+    dest: "/root/maasrc"
+    content: "{{ maasrc_fact | b64decode }}"
+    mode: "0640"
+  delegate_to: "{{ physical_host | default(ansible_host) }}"

playbooks/files/rax-maas/plugins/cinder_api_local_check.py

@@ -16,11 +16,11 @@
 
 import argparse
 import collections
-
 import ipaddr
 # Technically maas_common isn't third-party but our own thing but hacking
 # consideres it third-party
 from maas_common import get_auth_ref
+from maas_common import get_cinder_api_version
 from maas_common import get_keystone_client
 from maas_common import metric
 from maas_common import metric_bool
@@ -36,12 +36,13 @@
 def check(auth_ref, args):
     keystone = get_keystone_client(auth_ref)
     auth_token = keystone.auth_token
-
-    volume_endpoint = ('{protocol}://{ip}:{port}/v1/{tenant}'.format(
+    cinder_api_version = get_cinder_api_version()
+    volume_endpoint = ('{protocol}://{ip}:{port}/v{version}/{tenant}'.format(
         ip=args.ip,
         tenant=keystone.tenant_id,
         protocol=args.protocol,
-        port=args.port
+        port=args.port,
+        version=cinder_api_version
     ))
 
     s = requests.Session()

playbooks/files/rax-maas/plugins/cinder_service_check.py

@@ -19,6 +19,7 @@
 # Technically maas_common isn't third-party but our own thing but hacking
 # consideres it third-party
 from maas_common import get_auth_ref
+from maas_common import get_cinder_api_version
 from maas_common import get_keystone_client
 from maas_common import metric_bool
 from maas_common import print_output
@@ -35,12 +36,14 @@
 def check(auth_ref, args):
     keystone = get_keystone_client(auth_ref)
     auth_token = keystone.auth_token
+    cinder_api_version = get_cinder_api_version()
 
     VOLUME_ENDPOINT = (
-        '{protocol}://{hostname}:8776/v1/{tenant}'.format(
+        '{protocol}://{hostname}:8776/v{version}/{tenant}'.format(
             protocol=args.protocol,
             hostname=args.hostname,
-            tenant=keystone.tenant_id)
+            tenant=keystone.tenant_id,
+            version=cinder_api_version)
     )
 
     s = requests.Session()

playbooks/files/rax-maas/plugins/glance_api_local_check.py

@@ -21,6 +21,7 @@
 
 from glanceclient import exc as exc
 from maas_common import get_auth_ref
+from maas_common import get_glance_api_version
 from maas_common import get_glance_client
 from maas_common import metric
 from maas_common import metric_bool
@@ -33,11 +34,13 @@
 
 
 def check(auth_ref, args):
+    glance_api_version = get_glance_api_version()
     glance_endpoint = (
-        '{protocol}://{ip}:{port}/v1'.format(
+        '{protocol}://{ip}:{port}/v{version}'.format(
             ip=args.ip,
             protocol=args.protocol,
-            port=args.port
+            port=args.port,
+            version=glance_api_version
         )
     )
 

playbooks/files/rax-maas/plugins/maas_common.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 from __future__ import print_function
 
+from itertools import chain
 import contextlib
 import datetime
 import errno
@@ -37,7 +38,9 @@
                 'OS_PROJECT_NAME': 'admin',
                 'OS_ENDPOINT_TYPE': 'PublicURL',
                 'OS_IDENTITY_API_VERSION': None,
-                'OS_API_INSECURE': False}
+                'OS_API_INSECURE': False,
+                'OS_VOLUME_API_VERSION': None,
+                'OS_IMAGE_API_VERSION': None, }
 
 if 'Ubuntu' in platform.linux_distribution()[0]:
     AUTH_DETAILS.update({
@@ -57,6 +60,7 @@
     requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 OPENRC = '/root/openrc'
+MAASRC = '/root/maasrc'
 TOKEN_FILE = '/root/.auth_ref.json'
 
 
@@ -124,7 +128,12 @@ def get_glance_client(*args, **kwargs):
         metric_bool('client_success', False, m_name='maas_glance')
         status_err('Cannot import glanceclient', m_name='maas_glance')
 else:
-    def get_glance_client(token=None, endpoint=None, previous_tries=0):
+    def get_glance_client(token=None,
+                          endpoint=None,
+                          previous_tries=0,
+                          glance_api_version=os.getenv('OS_IMAGE_API_VERSION',
+                                                       '1')
+                          ):
         if previous_tries > 3:
             return None
 
@@ -142,7 +151,9 @@ def get_glance_client(token=None, endpoint=None, previous_tries=0):
                                                     get_endpoint_type(
                                                         auth_details))
 
-        glance = g_client.Client('1', endpoint=endpoint, token=token)
+        glance = g_client.Client(glance_api_version,
+                                 endpoint=endpoint,
+                                 token=token)
 
         try:
             # We don't want to be pulling massive lists of images every time we
@@ -634,15 +645,15 @@ def get_auth_from_file():
         status_err(str(e), m_name='maas_keystone')
 
 
-def get_auth_details(openrc_file=OPENRC):
+def get_auth_details(openrc_file=OPENRC, maasrc_file=MAASRC):
     auth_details = AUTH_DETAILS
     pattern = re.compile(
         '^(?:export\s)?(?P<key>\w+)(?:\s+)?=(?:\s+)?(?P<value>.*)$'
     )
 
     try:
-        with open(openrc_file) as openrc:
-            for line in openrc:
+        with open(openrc_file) as openrc, open(maasrc_file) as maasrc:
+            for line in chain(openrc, maasrc):
                 match = pattern.match(line)
                 if match is None:
                     continue
@@ -844,3 +855,21 @@ def print_output(print_telegraf=False):
                 print(STATUS)
             for metric in METRICS:
                 print(metric)
+
+
+def get_cinder_api_version():
+    auth_details = get_auth_details()
+    version = auth_details.get('OS_VOLUME_API_VERSION')
+    if not version:
+        version = os.getenv('OS_VOLUME_API_VERSION', '1')
+
+    return version
+
+
+def get_glance_api_version():
+    auth_details = get_auth_details()
+    version = auth_details.get('OS_IMAGE_API_VERSION')
+    if not version:
+        version = os.getenv('OS_IMAGE_API_VERSION', '1')
+
+    return version

playbooks/maas-agent-setup.yml

@@ -76,6 +76,7 @@
         cmd: assign_agent_to_entity
         entity: "{{ maas_entity_name }}"
         venv_bin: "{{ maas_venv_bin }}"
+        create_entity_if_not_exists: "{{ create_entity_if_not_exists | default(False) }}"
       when:
         - maas_use_api | bool
 

playbooks/maas-openstack-cinder.yml

@@ -19,6 +19,7 @@
   pre_tasks:
     - include: "common-tasks/maas_excluded_regex.yml"
     - include: "common-tasks/maas_get_openrc.yml"
+    - include: "common-tasks/maas_get_maasrc.yml"
     - name: Set the current group
       set_fact:
         maas_current_group: cinder_all

playbooks/maas-openstack-glance.yml

@@ -19,6 +19,7 @@
   pre_tasks:
     - include: "common-tasks/maas_excluded_regex.yml"
     - include: "common-tasks/maas_get_openrc.yml"
+    - include: "common-tasks/maas_get_maasrc.yml"
     - name: Set the current group
       set_fact:
         maas_current_group: glance_all

playbooks/maas-pre-flight.yml

@@ -56,3 +56,50 @@
         - rendered_maas_metata_test_tmpl | search('unknown')
   tags:
     - maas-pre-flight
+
+#
+# maasrc generation
+# (This following section could be removed when upstream decides to include
+# OS_IMAGE_API_VERSION and OS_VOLUME_API_VERSION in openrc
+#
+- name: Generate maasrc
+  hosts: utility_all
+  gather_facts: true
+  tasks:
+    - name: Get rendered test metadata template string
+      set_fact:
+        rendered_maas_metata_test_tmpl: "{{ lookup('template', playbook_dir + '/templates/rax-maas/test_metadata.yaml.j2') }}"
+      delegate_to: "{{ physical_host | default(ansible_host) }}"
+
+    - name: Check OSA version is defined
+      fail:
+        msg: "Unable to figure out OSA version"
+      when: (rendered_maas_metata_test_tmpl | from_yaml).metadata.osa_version == 'unknown'
+
+    - name: Set MaaS product metadata facts
+      set_fact:
+        maas_product: "{{ (rendered_maas_metata_test_tmpl | from_yaml).metadata.product }}"
+        maas_product_version: "{{ (rendered_maas_metata_test_tmpl | from_yaml).metadata.product_version }}"
+        maas_osa_version: "{{ (rendered_maas_metata_test_tmpl | from_yaml).metadata.osa_version }}"
+
+    # For Queens and beyond, use image api version 2 and volume api version 3
+    - name: Override Image and Volume API versions Queens and beyond
+      set_fact:
+        os_image_api_version: '2'
+        os_volume_api_version: '3'
+      when:
+        - ansible_version.full | version_compare('2.0.0', '>=')
+        - maas_osa_version is defined
+        - maas_osa_version is match('[0-9]+.[0-9]+.[0-9]+')
+        - maas_osa_version | version_compare('17.0.0', '>=', 'true')
+
+    - name: Generate maasrc file
+      template:
+        src: "templates/rax-maas/maasrc.j2"
+        dest: "/root/maasrc"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+
+  vars_files:
+    - vars/main.yml

playbooks/templates/common/macros.jinja

@@ -20,7 +20,7 @@ host {# we can change to hardware here if we want #}
 {% macro get_metadata(label) %}
 {% set get_product_version_cmd="(cd "+maas_product_dir|default('/opt/rpc-openstack')+"&& git describe --tags --abbrev=0) || echo 'unknown\'" %}
 {% set get_product_osa_version_cmd="(cd "+maas_product_osa_dir|default('/opt/rpc-openstack/openstack-ansible')+"&& git describe --tags --abbrev=0) || "+"(cd "+ maas_2nd_product_osa_dir|default('/opt/openstack-ansible')+ "&& git describe --tags --abbrev=0) || echo 'unknown\'" %}
-metadata          :
+metadata:
   product: "{{ maas_env_product | default('rpco') }}"
   rpc_env_identifier: "{{ maas_env_identifier | default('unknown')}}"
 {% if not(maas_env_product is defined and maas_env_product == 'osa') and not deploy_osp|default(false)  %}

playbooks/templates/rax-maas/maasrc.j2

@@ -0,0 +1,2 @@
+OS_IMAGE_API_VERSION={{ os_image_api_version }}
+OS_VOLUME_API_VERSION={{ os_volume_api_version }}

playbooks/vars/main.yml

@@ -367,6 +367,11 @@ cinder_local_api_protocol: 'http'
 #
 cinder_local_api_port: '8776'
 
+#
+# cinder api version
+#
+os_volume_api_version: '1'
+
 # octavia_api_port: Port number for the octavia service
 #
 octavia_api_port: '9876'
@@ -396,6 +401,11 @@ glance_local_api_port: '9292'
 #
 glance_local_api_protocol: 'http'
 
+#
+# glance api version
+#
+os_image_api_version: '1'
+
 #
 # heat_local_api_port: Port number for the local heat api service
 #

releasenotes/notes/TURTLES-1425-0af3967f1df02ec4.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fixes new queens gate

tests/aio-create.sh

@@ -70,9 +70,20 @@ function enable_ironic {
   if [[ ! -d "/etc/openstack_deploy" ]]; then
     mkdir -p /etc/openstack_deploy
   fi
+
+  # Beginning with queens, the key must be 'aio_lxc'
+  case $RE_JOB_SCENARIO in
+    kilo|liberty|mitaka|newton|ocata|pike)
+      key="aio"
+      ;;
+    *)
+      key="aio_lxc"
+      ;;
+  esac
+
   echo "
   confd_overrides:
-    aio:
+    $key:
       - name: cinder.yml.aio
       - name: glance.yml.aio
       - name: heat.yml.aio
@@ -202,6 +213,11 @@ pushd /opt/openstack-ansible
     echo "Flask==0.12.2" >> /opt/openstack-ansible/global-requirement-pins.txt
     enable_ironic
 
+  elif [ "${RE_JOB_SCENARIO}" == "queens" ]; then
+    git checkout "stable/queens"  # Branch checkout of Queens (Current Stable)
+    export ANSIBLE_INVENTORY="/opt/openstack-ansible/inventory"
+    enable_ironic
+
   else
     enable_ironic
   fi

tests/test-ansible-functional.sh

@@ -47,6 +47,10 @@ export ANSIBLE_LOG_DIR="${TESTING_HOME}/.ansible/logs"
 export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-functional.log"
 # Ansible Inventory will be set to OSA
 export ANSIBLE_INVENTORY="${ANSIBLE_INVENTORY:-/opt/openstack-ansible/playbooks/inventory}"
+if [ "${RE_JOB_SCENARIO}" == "queens" ]; then
+  export ANSIBLE_INVENTORY="/opt/openstack-ansible/inventory"
+fi
+
 
 # The maas_rally performance monitoring requires a modern (>1.9) version of
 # ansible that is not available in liberty and mitaka.  There is no reason