Merge branch 'master' into copy_storage_files-fix-jonah

build/bin/build-collection.sh

@@ -13,8 +13,8 @@ cat $GITHUB_WORKSPACE/ibm/mas_devops/galaxy.yml
 
 
 # Update this when we have new catalog
-MAS_PREVIOUS_CATALOG='v9-240730-amd64'
-MAS_LATEST_CATALOG='v9-240827-amd64'
+MAS_PREVIOUS_CATALOG='v9-240827-amd64'
+MAS_LATEST_CATALOG='v9-241003-amd64'
 
 
 # Update all the placeholders in the playbooks

docs/playbooks/oneclick-core.md

@@ -56,12 +56,12 @@ The other values can be left at their defaults. Finally, click **Generate** and
 Storage class configuration is built into the collection and the playbook will auto-select the appropriate storage classes when it detects the presence of certain storage classes in your cluster (IBM Cloud Storage or OpenShift Container Storage). If you are running the install on a cluster that does not have these storage classes then you will also must configure the following environment variables:
 
 #### ReadWriteMany Access Mode
-Usually fulfilled by block storage classes:
+Usually fulfilled by file storage classes:
 
 - `PROMETHEUS_ALERTMGR_STORAGE_CLASS`
 
 #### ReadWriteOnce Access Mode
-Usually fulfilled by file storage classes:
+Usually fulfilled by block storage classes:
 
 - `PROMETHEUS_STORAGE_CLASS`
 - `PROMETHEUS_USERWORKLOAD_STORAGE_CLASS`

ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml

@@ -48,7 +48,10 @@
  masbr_job_data_list: "{{ masbr_job_data_list | default([], true) }}"
 
 - name: "Set fact: specified backup data"
- when: masbr_backup_data is defined and masbr_backup_data | length > 0
+ when:
+ - masbr_backup_data is defined
+ - masbr_backup_data | length > 0
+ - (_ignore_masbr_backup_data is not defined) or (_ignore_masbr_backup_data is defined and not _ignore_masbr_backup_data)
  block:
  - name: "Set fact: reset masbr_job_data_specified"
  set_fact:

ibm/mas_devops/common_vars/casebundles/v9-241003-amd64.yml

@@ -0,0 +1,100 @@
+---
+# Case bundle configuration for IBM Maximo Operator Catalog 240625
+# -----------------------------------------------------------------------------
+# In the future this won't be necessary as we'll be able to mirror from the
+# catalog itself, but not everything in the catalog supports this yet (including MAS)
+# so we need to use the CASE bundle mirror process still.
+
+catalog_digest: sha256:ba2237481b2ce7407698775a71f44daaecd2db6f74855f20829ac1bceeddb3d9
+
+# Dependencies
+# -----------------------------------------------------------------------------
+ibm_licensing_version: 4.2.7 # Operator version 4.2.7 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-licensing)
+common_svcs_version: 4.3.0 # check once why we have 4.3.0 in all previous versions since so long it's 4.3.0 # Operator version 4.3.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-common-services)
+cp4d_platform_version: 4.0.0+20231213.115030 # Operator version 5.0.0 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-cp-datacore/4.0.0%2B20231213.115030)
+
+db2u_version: 6.0.1+20240704.142950.9960 # Operator version 110509.0.2 to find the version 6.0.1, search CASE_VERSION in db2u, catalog.yaml into ibm-maximo-operator-catalog (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-db2uoperator)
+events_version: 5.0.1 # Operator version 5.0.1 (https://github.com/IBM/cloud-pak/tree/master/repo/case/ibm-events-operator)
+uds_version: 2.0.12 # Operator version 2.0.12 # sticking to 2.0.12 version # Please do Not Change
+sls_version: 3.10.0 # No Update # Operator version 3.10.0 (https://github.ibm.com/maximoappsuite/ibm-sls/releases)
+tsm_version: 1.5.4 # Operator version 1.5.4 (https://github.ibm.com/maximoappsuite/ibm-truststore-mgr/releases)
+dd_version: 1.1.12 # Updated # Operator version 1.1.11 (https://github.ibm.com/maximoappsuite/ibm-data-dictionary/releases)
+appconnect_version: 6.2.0 # Operator version 6.2.0 # sticking to 6.2.0 version # Please do Not Change
+wsl_version: 8.0.0 # Operator version 8.0.0
+wml_version: 8.0.0 # Operator version 5.0.0
+spark_version: 8.0.0 # Operator version 5.0.0
+cognos_version: 25.0.0 # Operator version 25.0.0
+couchdb_version: 1.0.13 # Operator version 2.2.1 (This is required for Assist 9.0, https://github.com/IBM/cloud-pak/blob/master/repo/case/ibm-couchdb/index.yaml)
+elasticsearch_version: 1.1.2238 # Operator version 1.1.2238
+
+
+# Maximo Application Suite
+# -----------------------------------------------------------------------------
+mas_core_version:
+ 9.0.x: 9.0.3 # Updated
+ 8.10.x: 8.10.18 # Updated
+ 8.11.x: 8.11.15 # Updated
+mas_assist_version:
+ 9.0.x: 9.0.2 # No Update
+ 8.10.x: 8.7.7 # No Update
+ 8.11.x: 8.8.6 # No Update
+mas_hputilities_version:
+ 9.0.x: "" # Not Supported
+ 8.10.x: 8.6.7 # No Update
+ 8.11.x: "" # Not Supported
+mas_iot_version:
+ 9.0.x: 9.0.3 # Updated
+ 8.10.x: 8.7.17 # Updated
+ 8.11.x: 8.8.13 # Updated
+mas_manage_version:
+ 9.0.x: 9.0.3 # Updated
+ 8.10.x: 8.6.18 # Updated
+ 8.11.x: 8.7.12 # Updated
+mas_monitor_version:
+ 9.0.x: 9.0.3 # Updated
+ 8.10.x: 8.10.14 # Updated
+ 8.11.x: 8.11.11 # Updated
+mas_optimizer_version:
+ 9.0.x: 9.0.3 # Updated
+ 8.10.x: 8.4.10 # Updated
+ 8.11.x: 8.5.9 # Updated
+mas_predict_version:
+ 9.0.x: 9.0.2 # Updated
+ 8.10.x: 8.8.3 # No Update
+ 8.11.x: 8.9.5 # Updated
+mas_visualinspection_version:
+ 9.0.x: 9.0.3 # Updated
+ 8.10.x: 8.8.4 # No update
+ 8.11.x: 8.9.6 # Updated
+
+# Extra Images for UDS
+# ------------------------------------------------------------------------------
+uds_extras_version: 1.5.0
+
+# Extra Images for Mongo
+# ------------------------------------------------------------------------------
+mongo_extras_version_default: 7.0.12
+mongo_extras_version: "{{ lookup('env', 'MONGODB_VERSION') | default(mongo_extras_version_default, True) }}"
+
+# Variables used to mirror additional mongo image versions
+mongo_extras_version_4: 4.4.21
+mongo_extras_version_5: 5.0.23
+mongo_extras_version_6: 6.0.12
+mongo_extras_version_7: 7.0.12
+
+# Extra Images for Db2u
+# ------------------------------------------------------------------------------
+db2u_extras_version: 1.0.6
+
+# Extra Images for IBM Watson Discovery
+# ------------------------------------------------------------------------------
+#wd_extras_version: 1.0.4
+
+# Extra Images for Amlen
+# ------------------------------------------------------------------------------
+amlen_extras_version: 1.1.1
+
+# Default Cloud Pak for Data version
+# ------------------------------------------------------------------------------
+cpd_product_version_default: 4.8.0
+cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') | default(cpd_product_version_default, true) }}"

ibm/mas_devops/common_vars/cp4d_supported_versions.yml

@@ -10,4 +10,6 @@ cpd_supported_versions:
 cpd_minor_version: "{{ cpd_product_version | regex_search('(?<=)(.*)(?=..)') }}" # extract the cpd minor version as there are specific steps to run depending on the cpd minor version defined
 cpd_46_or_lower: "{{ cpd_minor_version is version('4.6','<=') }}"
 cpd_48_or_higher: "{{ cpd_minor_version is version('4.8','>=') }}"
+cpd_48: "{{ cpd_minor_version is version('4.8') }}"
 cpd_50_or_higher: "{{ cpd_minor_version is version('5.0','>=') }}"
+cpd_50: "{{ cpd_minor_version is version('5.0') }}"

ibm/mas_devops/playbooks/mirror_dependencies.yml

@@ -35,6 +35,7 @@
  cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') }}"
  mirror_cp4d: "{{ lookup('env', 'MIRROR_CP4D') | default ('False', True) | bool }}"
  cpd_48_or_higher: "{{ cpd_product_version is defined and cpd_product_version is version('4.8.0','>=') | bool }}"
+ cpd_48: "{{ cpd_product_version is defined and cpd_product_version is version('4.8.0','==') | bool }}"
 
  # 7. Watson Studio Local
  # -------------------------------------------------------------------------
@@ -274,6 +275,23 @@
  exclude_images: []
  ibmpak_skip_dependencies: true
 
+ - name: ibm.mas_devops.mirror_extras_prepare
+ when:
+ - mirror_cp4d
+ - cpd_48
+ - mirror_mode != "from-filesystem"
+ vars:
+ extras_name: cp4d
+ extras_version: "4.8.0"
+
+ - name: ibm.mas_devops.mirror_images
+ when:
+ - mirror_cp4d
+ - cpd_48
+ vars:
+ manifest_name: extras_cp4d
+ manifest_version: "4.8.0"
+
  - name: ibm.mas_devops.mirror_images
  when: mirror_cp4d
  vars:

ibm/mas_devops/roles/cp4d/tasks/wait/wait-zenmetastore-edb.yml

@@ -11,93 +11,102 @@
  delay: 30
  until: zenmetastoreCluster.resources[0].status is defined
 
-# 2. Wait for zen metastore replica pods to become ready
-# -----------------------------------------------------------------------------
-- name: "wait-zenmetastore-edb : Wait for ZenMetastore pods to be become ready"
- k8s_info:
- kind: Cluster
- namespace: "{{ cpd_instance_namespace }}"
- name: "zen-metastore-edb"
- register: zenmetastoreCluster
- retries: 40 # Give 20 minutes for the pods to become ready
- delay: 30
- until: >-
- zenmetastoreCluster.resources[0].spec.instances is defined
- and zenmetastoreCluster.resources[0].status.readyInstances is defined
- and zenmetastoreCluster.resources[0].spec.instances == zenmetastoreCluster.resources[0].status.readyInstances
- #ignore-errors: true # If this fails then we restart pending pods below
- failed_when: false
+# 2. For V4.8, We need to patch the postgres licensing job acccording to https://www.ibm.com/support/pages/node/7158524
+- name: "wait-zenmetastore-edb : Check and display the create-postgres-license-config license expiry date"
+ when: cpd_48
+ block:
+ - name: "wait-zenmetastore-edb : Fetch the license expiry date"
+ kubernetes.core.k8s_info:
+ kind: Cluster
+ namespace: "{{ cpd_instance_namespace }}"
+ name: "zen-metastore-edb"
+ register: zenmetastoreCluster
+ until:
+ zenmetastoreCluster.resources[0].spec.instances is defined
+ and zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined
+ retries: 40 # Give 20 minutes
+ delay: 30
+ - debug:
+ msg:
+ - "License Expiration .. {{ zenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration | default ('') }}"
+ - "License Status ...... {{ zenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus}}"
 
-# 2. Restart any ZenMetastore pods that are still Pending
-# -----------------------------------------------------------------------------
-- set_fact:
- is_zenmetastore_ready: true
+- name: "wait-zenmetastore-edb : Update create-postgres-license-config job"
  when:
- zenmetastoreCluster.resources[0].spec.instances is defined
- and zenmetastoreCluster.resources[0].status.readyInstances is defined
- and zenmetastoreCluster.resources[0].spec.instances == zenmetastoreCluster.resources[0].status.readyInstances
-
-- name: "wait-zenmetastore-edb : Detecting and restarting pending ZenMetastore Pods"
- when: is_zenmetastore_ready is not defined
+ - cpd_48
+ - zenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration is not defined
  block:
- - name: "install-cp4d : Get pending ZenMetastore Pods"
+ - name: "wait-zenmetastore-edb : Wait for create-postgres-license-config Job to be Completed or Failed (10s delay)"
  kubernetes.core.k8s_info:
- api_version: v1
- kind: Pod
- label_selectors:
- - "app.kubernetes.io/component=zen-metastore-edb"
- field_selectors:
- - "status.phase=Pending"
- namespace: "{{ cpd_instance_namespace }}"
- register: pending_pod_lookup
+ api_version: batch/v1
+ kind: Job
+ name: "create-postgres-license-config"
+ namespace: "{{ cpd_operators_namespace }}"
+ register: _job_info
+ until:
+ - _job_info.resources is defined
+ - _job_info.resources | length > 0
+ - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or
+ (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1)
+ retries: 30
+ delay: 10
 
- - set_fact:
- pending_pod_names: "{{ pending_pod_lookup.resources | map(attribute='metadata.name') }}"
+ - name: "wait-zenmetastore-edb : Recreate the job with up to date license image"
+ shell: >-
+ oc get job create-postgres-license-config -n {{ cpd_operators_namespace }} -o yaml | \
+ sed -e 's/operator.ibm.com\/opreq-control: "true"/operator.ibm.com\/opreq-control: "false"/' \
+ -e 's|\(image: \).*|\1"cp.icr.io/cp/cpd/edb-postgres-license-provider@sha256:c1670e7dd93c1e65a6659ece644e44aa5c2150809ac1089e2fd6be37dceae4ce"|' \
+ -e '/controller-uid:/d' | \
+ oc replace --force -f -
+ register: _job_recreate_output
 
  - debug:
- msg: "Restarting pending ZenMetastore Pods: {{ pending_pod_names }}"
+ msg: "Recreate the job: {{ _job_recreate_output }}"
 
- - name: "wait-zenmetastore-edb : Restarting pending ZenMetastore Pods"
- kubernetes.core.k8s:
- state: absent
- api_version: v1
- kind: Pod
- namespace: "{{ cpd_instance_namespace }}"
- name: "{{ item }}"
- loop: "{{ pending_pod_names }}"
+ - name: "wait-zenmetastore-edb : Wait for create-postgres-license-config Job to be Completed or Failed (10s delay)"
+ kubernetes.core.k8s_info:
+ api_version: batch/v1
+ kind: Job
+ name: "create-postgres-license-config"
+ namespace: "{{ cpd_operators_namespace }}"
+ register: _job_info
+ until:
+ - _job_info.resources is defined
+ - _job_info.resources | length > 0
+ - (_job_info.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1) or
+ (_job_info.resources | json_query('[*].status.conditions[?type==`Failed`][].status') | select ('match','True') | list | length == 1)
+ retries: 30
+ delay: 10
 
- # 3. Wait again zenmetastore replica pods to become ready
- # -----------------------------------------------------------------------------
- - name: "wait-zenmetastore-edb : Wait for ZenMetastore pods to be become ready"
- k8s_info:
+ - name: "wait-zenmetastore-edb : Check and display the license expiry date"
+ kubernetes.core.k8s_info:
  kind: Cluster
  namespace: "{{ cpd_instance_namespace }}"
  name: "zen-metastore-edb"
- register: zenmetastoreCluster
- retries: 40 # Give another 20 minutes for the pods to become ready
+ register: newzenmetastoreCluster
+ until:
+ newzenmetastoreCluster.resources[0].spec.instances is defined
+ and newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus is defined
+ and "Valid license" in newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus
+ retries: 40 # Give 20 minutes
  delay: 30
- until: >-
- zenmetastoreCluster.resources[0].spec.instances is defined
- and zenmetastoreCluster.resources[0].status.readyInstances is defined
- and zenmetastoreCluster.resources[0].spec.instances == zenmetastoreCluster.resources[0].status.readyInstances
- failed_when: false # We handle and log the failure below.
 
- - name: "wait-zenmetastore-edb : Fail if ZenMetastore pods are not ready"
- block:
- - name: "install-cp4d : Get Pending ZenMetastore Pods"
- kubernetes.core.k8s_info:
- api_version: v1
- kind: Pod
- label_selectors:
- - "app.kubernetes.io/component=zen-metastore-edb"
- field_selectors:
- - "status.phase=Pending"
- namespace: "{{ cpd_instance_namespace }}"
- register: pending_pod_lookup
+ - debug:
+ msg:
+ - "License Expiration .. {{ newzenmetastoreCluster.resources[0].status.licenseStatus.licenseExpiration | default ('') }}"
+ - "License Status ...... {{ newzenmetastoreCluster.resources[0].status.licenseStatus.licenseStatus | default ('')}}"
 
- - fail:
- msg:
- - "ZenMetastore pods are not ready Instances required: {{ zenmetastoreCluster.resources[0].spec.instances }}, ready: {{ zenmetastoreCluster.resources[0].status.readyInstances }}"
- - "Pending ZenMetastore Pods: {{ pending_pod_lookup.resources | map(attribute='metadata.name') }}"
- when:
- zenmetastoreCluster.resources[0].spec.instances != zenmetastoreCluster.resources[0].status.readyInstances
+# 3. Wait for zen metastore replica pods to become ready
+# -----------------------------------------------------------------------------
+- name: "wait-zenmetastore-edb : Wait for ZenMetastore pods to be become ready"
+ k8s_info:
+ kind: Cluster
+ namespace: "{{ cpd_instance_namespace }}"
+ name: "zen-metastore-edb"
+ register: zenmetastoreCluster
+ retries: 40 # Give 20 minutes for the pods to become ready
+ delay: 30
+ until: >-
+ zenmetastoreCluster.resources[0].spec.instances is defined
+ and zenmetastoreCluster.resources[0].status.readyInstances is defined
+ and zenmetastoreCluster.resources[0].spec.instances == zenmetastoreCluster.resources[0].status.readyInstances

ibm/mas_devops/roles/cp4d_service/tasks/install.yml

@@ -49,12 +49,11 @@
  when:
  - existing_cpd_service_lookup.resources is defined
  - existing_cpd_service_lookup.resources | length > 0
- - existing_cpd_service_lookup.resources[0].status is defined
- - existing_cpd_service_lookup.resources[0].status.versions is defined or existing_cpd_service_lookup.resources[0].status.version is defined # spss has status.version...
- - existing_cpd_service_lookup.resources[0].status.versions.reconciled is defined or existing_cpd_service_lookup.resources[0].status.version is defined
+ - existing_cpd_service_lookup.resources[0].spec.version is defined
+ - existing_cpd_service_lookup.resources[0].spec.version != ""
  set_fact:
- existing_cpd_service_cr_version: "{{ existing_cpd_service_lookup.resources[0].status.versions.reconciled | default(existing_cpd_service_lookup.resources[0].status.version, true) }}"
- existing_cpd_service_minor_version: "{{ existing_cpd_service_lookup.resources[0].status.versions.reconciled | default(existing_cpd_service_lookup.resources[0].status.version, true) | regex_search('(?<=)(.*)(?=..)') }}"
+ existing_cpd_service_cr_version: "{{ existing_cpd_service_lookup.resources[0].spec.version }}"
+ existing_cpd_service_minor_version: "{{ existing_cpd_service_lookup.resources[0].spec.version | regex_search('(?<=)(.*)(?=..)') }}"
 
 - name: "Debug existing {{ cpd_service_name }} installed version"
  when: existing_cpd_service_cr_version is defined

ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-elasticsearch.yml

@@ -47,6 +47,8 @@
  kind: Job
  name: "elasticsearch-master-ibm-elasticsearch-create-snapshot-repo-job"
  namespace: "{{ cpd_instance_namespace }}"
+ ignore_errors: true # Other roles running at the same time may delete this and then we get a 404 Not Found error - if it dosn't exist that is fine
+
 
  - name: "wait/ccs : Delete elasticsearch-server-esnodes statefulset so next time it recreates with right imagePullPolicy"
  kubernetes.core.k8s:
@@ -56,6 +58,7 @@
  namespace: "{{ cpd_instance_namespace }}"
  label_selectors:
  - "app.kubernetes.io/instance=elasticsearch-master"
+ ignore_errors: true # Other roles running at the same time may delete this and then we get a 404 Not Found error - if it dosn't exist that is fine
 
  - name: "wait-css : Patch the elasticsearch-master custom resource to include right imagePullPolicy"
  kubernetes.core.k8s: