Add 2023.2 Bobcat support

* sync charm-helpers to classic charms
* change openstack-origin/source default to bobcat
* add mantic to metadata series
* align testing with bobcat
* add new bobcat bundles
* add bobcat bundles to tests.yaml
* add bobcat tests to osci.yaml
* update build-on and run-on bases
* drop kinetic

Change-Id: I0f57eb5912bbb21e29f2f4b72dd137342c1cf3fe

charmcraft.yaml

@@ -33,8 +33,8 @@ bases:
         channel: "22.04"
         architectures: [amd64, s390x, ppc64el, arm64]
       - name: ubuntu
-        channel: "22.10"
+        channel: "23.04"
         architectures: [amd64, s390x, ppc64el, arm64]
       - name: ubuntu
-        channel: "23.04"
+        channel: "23.10"
         architectures: [amd64, s390x, ppc64el, arm64]

charmhelpers/contrib/hahelpers/cluster.py

@@ -221,6 +221,13 @@ def https():
         return True
     if config_get('ssl_cert') and config_get('ssl_key'):
         return True
+    # Local import to avoid ciruclar dependency.
+    import charmhelpers.contrib.openstack.cert_utils as cert_utils
+    if (
+        cert_utils.get_certificate_request() and not
+        cert_utils.get_requests_for_local_unit("certificates")
+    ):
+        return False
     for r_id in relation_ids('certificates'):
         for unit in relation_list(r_id):
             ca = relation_get('ca', rid=r_id, unit=unit)

charmhelpers/contrib/openstack/cert_utils.py

@@ -409,13 +409,24 @@ def get_requests_for_local_unit(relation_name=None):
     relation_name = relation_name or 'certificates'
     bundles = []
     for rid in relation_ids(relation_name):
+        sent = relation_get(rid=rid, unit=local_unit())
+        legacy_keys = ['certificate_name', 'common_name']
+        is_legacy_request = set(sent).intersection(legacy_keys)
         for unit in related_units(rid):
             data = relation_get(rid=rid, unit=unit)
             if data.get(raw_certs_key):
                 bundles.append({
                     'ca': data['ca'],
                     'chain': data.get('chain'),
                     'certs': json.loads(data[raw_certs_key])})
+            elif is_legacy_request:
+                bundles.append({
+                    'ca': data['ca'],
+                    'chain': data.get('chain'),
+                    'certs': {sent['common_name']:
+                              {'cert': data.get(local_name + '.server.cert'),
+                               'key': data.get(local_name + '.server.key')}}})
+
     return bundles
 
 

charmhelpers/contrib/openstack/context.py

@@ -1748,6 +1748,9 @@ def __init__(self, name=None, script=None, admin_script=None,
 
     def __call__(self):
         total_processes = _calculate_workers()
+        enable_wsgi_rotation = config('wsgi-rotation')
+        if enable_wsgi_rotation is None:
+            enable_wsgi_rotation = True
         ctxt = {
             "service_name": self.service_name,
             "user": self.user,
@@ -1761,6 +1764,7 @@ def __call__(self):
             "public_processes": int(math.ceil(self.public_process_weight *
                                               total_processes)),
             "threads": 1,
+            "wsgi_rotation": enable_wsgi_rotation,
         }
         return ctxt
 

charmhelpers/contrib/openstack/deferred_events.py

@@ -127,7 +127,9 @@ def deferred_events():
     """
     events = []
     for defer_file in deferred_events_files():
-        events.append((defer_file, read_event_file(defer_file)))
+        event = read_event_file(defer_file)
+        if event.policy_requestor_name == hookenv.service_name():
+            events.append((defer_file, event))
     return events
 
 

charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -12,6 +12,8 @@ signing_dir = {{ signing_dir }}
 {% if service_type -%}
 service_type = {{ service_type }}
 {% endif -%}
+{% if admin_role -%}
 service_token_roles = {{ admin_role }}
 service_token_roles_required = True
 {% endif -%}
+{% endif -%}

charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -22,4 +22,8 @@ signing_dir = {{ signing_dir }}
 {% if use_memcache == true %}
 memcached_servers = {{ memcache_url }}
 {% endif -%}
+{% if admin_role -%}
+service_token_roles = {{ admin_role }}
+service_token_roles_required = True
+{% endif -%}
 {% endif -%}

charmhelpers/contrib/openstack/templates/section-service-user

@@ -3,8 +3,8 @@
 send_service_user_token = true
 auth_type = password
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
-project_domain_id = default
-user_domain_id = default
+project_domain_name = service_domain
+user_domain_name = service_domain
 project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}

charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -12,6 +12,12 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
+{% if wsgi_rotation -%}
+WSGISocketRotation On
+{% else -%}
+WSGISocketRotation Off
+{% endif -%}
+
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \

charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf

@@ -12,6 +12,12 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 {% endif -%}
 
+{% if wsgi_rotation -%}
+WSGISocketRotation On
+{% else -%}
+WSGISocketRotation Off
+{% endif -%}
+
 {% if port -%}
 <VirtualHost *:{{ port }}>
     WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \

charmhelpers/contrib/openstack/utils.py

@@ -160,6 +160,7 @@
     ('2022.1', 'yoga'),
     ('2022.2', 'zed'),
     ('2023.1', 'antelope'),
+    ('2023.2', 'bobcat'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -957,7 +958,7 @@ def os_requires_version(ostack_release, pkg):
     def wrap(f):
         @wraps(f)
         def wrapped_f(*args):
-            if os_release(pkg) < ostack_release:
+            if CompareOpenStackReleases(os_release(pkg)) < ostack_release:
                 raise Exception("This hook is not supported on releases"
                                 " before %s" % ostack_release)
             f(*args)

charmhelpers/contrib/storage/linux/ceph.py

@@ -28,7 +28,6 @@
 import shutil
 import json
 import time
-import uuid
 
 from subprocess import (
     check_call,
@@ -1677,6 +1676,10 @@ class CephBrokerRq(object):
     The API is versioned and defaults to version 1.
     """
 
+    # The below hash is the result of running
+    # `hashlib.sha1('[]'.encode()).hexdigest()`
+    EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c'
+
     def __init__(self, api_version=1, request_id=None, raw_request_data=None):
         """Initialize CephBrokerRq object.
 
@@ -1685,8 +1688,12 @@ def __init__(self, api_version=1, request_id=None, raw_request_data=None):
 
         :param api_version: API version for request (default: 1).
         :type api_version: Optional[int]
-        :param request_id: Unique identifier for request.
-                           (default: string representation of generated UUID)
+        :param request_id: Unique identifier for request. The identifier will
+                           be updated as ops are added or removed from the
+                           broker request. This ensures that Ceph will
+                           correctly process requests where operations are
+                           added after the initial request is processed.
+                           (default: sha1 of operations)
         :type request_id: Optional[str]
         :param raw_request_data: JSON-encoded string to build request from.
         :type raw_request_data: Optional[str]
@@ -1695,16 +1702,20 @@ def __init__(self, api_version=1, request_id=None, raw_request_data=None):
         if raw_request_data:
             request_data = json.loads(raw_request_data)
             self.api_version = request_data['api-version']
-            self.request_id = request_data['request-id']
             self.set_ops(request_data['ops'])
+            self.request_id = request_data['request-id']
         else:
             self.api_version = api_version
             if request_id:
                 self.request_id = request_id
             else:
-                self.request_id = str(uuid.uuid1())
+                self.request_id = CephBrokerRq.EMPTY_LIST_SHA
             self.ops = []
 
+    def _hash_ops(self):
+        """Return the sha1 of the requested Broker ops."""
+        return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest()
+
     def add_op(self, op):
         """Add an op if it is not already in the list.
 
@@ -1713,6 +1724,7 @@ def add_op(self, op):
         """
         if op not in self.ops:
             self.ops.append(op)
+            self.request_id = self._hash_ops()
 
     def add_op_request_access_to_group(self, name, namespace=None,
                                        permission=None, key_name=None,
@@ -1991,6 +2003,7 @@ def set_ops(self, ops):
         to allow comparisons to ensure validity.
         """
         self.ops = ops
+        self.request_id = self._hash_ops()
 
     @property
     def request(self):

charmhelpers/core/host_factory/ubuntu.py

@@ -32,6 +32,7 @@
     'jammy',
     'kinetic',
     'lunar',
+    'mantic',
 )
 
 

charmhelpers/fetch/ubuntu.py

@@ -238,6 +238,14 @@
     'antelope/proposed': 'jammy-proposed/antelope',
     'jammy-antelope/proposed': 'jammy-proposed/antelope',
     'jammy-proposed/antelope': 'jammy-proposed/antelope',
+    # bobcat
+    'bobcat': 'jammy-updates/bobcat',
+    'jammy-bobcat': 'jammy-updates/bobcat',
+    'jammy-bobcat/updates': 'jammy-updates/bobcat',
+    'jammy-updates/bobcat': 'jammy-updates/bobcat',
+    'bobcat/proposed': 'jammy-proposed/bobcat',
+    'jammy-bobcat/proposed': 'jammy-proposed/bobcat',
+    'jammy-proposed/bobcat': 'jammy-proposed/bobcat',
 
     # OVN
     'focal-ovn-22.03': 'focal-updates/ovn-22.03',
@@ -270,6 +278,7 @@
     'yoga',
     'zed',
     'antelope',
+    'bobcat',
 )
 
 
@@ -298,6 +307,7 @@
     ('jammy', 'yoga'),
     ('kinetic', 'zed'),
     ('lunar', 'antelope'),
+    ('mantic', 'bobcat'),
 ])
 
 
@@ -591,7 +601,7 @@ def _get_key_by_keyid(keyid):
     curl_cmd = ['curl', keyserver_url.format(keyid)]
     # use proxy server settings in order to retrieve the key
     return subprocess.check_output(curl_cmd,
-                                   env=env_proxy_settings(['https']))
+                                   env=env_proxy_settings(['https', 'no_proxy']))
 
 
 def _dearmor_gpg_key(key_asc):

charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -122,13 +122,12 @@ def dpkg_list(self, packages):
         :raises: subprocess.CalledProcessError
         """
         pkgs = {}
-        cmd = ['dpkg-query', '--list']
+        cmd = [
+            'dpkg-query', '--show',
+            '--showformat',
+            r'${db:Status-Abbrev}\t${Package}\t${Version}\t${Architecture}\t${binary:Summary}\n'
+        ]
         cmd.extend(packages)
-        if locale.getlocale() == (None, None):
-            # subprocess calls out to locale.getpreferredencoding(False) to
-            # determine encoding.  Workaround for Trusty where the
-            # environment appears to not be set up correctly.
-            locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
         try:
             output = subprocess.check_output(cmd,
                                              stderr=subprocess.STDOUT,
@@ -140,24 +139,17 @@ def dpkg_list(self, packages):
             if cp.returncode != 1:
                 raise
             output = cp.output
-        headings = []
         for line in output.splitlines():
-            if line.startswith('||/'):
-                headings = line.split()
-                headings.pop(0)
-                continue
-            elif (line.startswith('|') or line.startswith('+') or
-                  line.startswith('dpkg-query:')):
+            # only process lines for successfully installed packages
+            if not (line.startswith('ii ') or line.startswith('hi ')):
                 continue
-            else:
-                data = line.split(None, 4)
-                status = data.pop(0)
-                if status not in ('ii', 'hi'):
-                    continue
-                pkg = {}
-                pkg.update({k.lower(): v for k, v in zip(headings, data)})
-                if 'name' in pkg:
-                    pkgs.update({pkg['name']: pkg})
+            status, name, version, arch, desc = line.split('\t', 4)
+            pkgs[name] = {
+                'name': name,
+                'version': version,
+                'architecture': arch,
+                'description': desc,
+            }
         return pkgs
 
     def _apt_cache_show(self, packages):

config.yaml

@@ -9,7 +9,7 @@ options:
     description: Enable logging of all request headers.
   openstack-origin:
     type: string
-    default: antelope
+    default: bobcat
     description: |
       Repository from which to install. May be one of the following:
       distro (default), ppa:somecustom/ppa, a deb url sources entry,

metadata.yaml

@@ -19,8 +19,8 @@ tags:
 - cache-proxy
 series:
 - jammy
-- kinetic
 - lunar
+- mantic
 extra-bindings:
   public:
   admin:

osci.yaml

@@ -3,37 +3,47 @@
       - charm-unit-jobs-py310
     check:
       jobs:
-        - test-s3api-jammy-zed
         - test-s3api-jammy-antelope
+        - test-s3api-jammy-bobcat:
+            voting: false
         - test-s3api-lunar-antelope:
             voting: false
+        - test-s3api-mantic-bobcat:
+            voting: false
     vars:
       needs_charm_build: true
       charm_build_name: swift-proxy
       build_type: charmcraft
       charmcraft_channel: 2.1/stable
 
 - job:
-    name: test-s3api-jammy-zed
+    name: test-s3api-jammy-antelope
     parent: func-target
     dependencies:
       - charm-build
       - osci-lint
       - name: tox-py310
         soft: true
     vars:
-      tox_extra_args: '-- test-s3api:jammy-zed'
+      tox_extra_args: '-- test-s3api:jammy-antelope'
 - job:
-    name: test-s3api-jammy-antelope
+    name: test-s3api-jammy-bobcat
     parent: func-target
     dependencies:
-      - test-s3api-jammy-zed
+      - test-s3api-jammy-antelope
     vars:
-      tox_extra_args: '-- test-s3api:jammy-antelope'
+      tox_extra_args: '-- test-s3api:jammy-bobcat'
 - job:
     name: test-s3api-lunar-antelope
     parent: func-target
     dependencies:
-      - test-s3api-jammy-zed
+      - test-s3api-jammy-antelope
     vars:
       tox_extra_args: '-- test-s3api:lunar-antelope'
+- job:
+    name: test-s3api-mantic-bobcat
+    parent: func-target
+    dependencies:
+      - test-s3api-jammy-antelope
+    vars:
+      tox_extra_args: '-- test-s3api:mantic-bobcat'