From bba58b39c8a20d7e5d7e15e20ad3c0a28f4fb4d3 Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Tue, 29 Oct 2024 12:27:37 +0100 Subject: [PATCH 1/3] RHOAIENG-2003: feat(metadata/codeserver): update VSCode imagestream annotations with installed software --- .../code-server-notebook-imagestream.yaml | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/manifests/base/code-server-notebook-imagestream.yaml b/manifests/base/code-server-notebook-imagestream.yaml index dffbdb0bb..fea8110eb 100644 --- a/manifests/base/code-server-notebook-imagestream.yaml +++ b/manifests/base/code-server-notebook-imagestream.yaml @@ -16,8 +16,25 @@ spec: tags: # N Version of the image - annotations: - opendatahub.io/notebook-software: '[{"name":"Python","version":"v3.11"}]' - opendatahub.io/notebook-python-dependencies: '[{"name":"code-server","version":"4.92"}]' + # language=json + opendatahub.io/notebook-software: | + [ + {"name": "code-server", "version": "4.92"}, + {"name": "Python", "version": "v3.11"} + ] + # language=json + opendatahub.io/notebook-python-dependencies: | + [ + {"name": "Boto3", "version": "1.34"}, + {"name": "Kafka-Python", "version": "2.0"}, + {"name": "Matplotlib", "version": "3.8"}, + {"name": "Numpy", "version": "1.26"}, + {"name": "Pandas", "version": "2.2"}, + {"name": "Scikit-learn", "version": "1.4"}, + {"name": "Scipy", "version": "1.12"}, + {"name": "Sklearn-onnx", "version": "1.16"}, + {"name": "ipykernel", "version": "6.29"} + ] openshift.io/imported-from: quay.io/opendatahub/workbench-images opendatahub.io/workbench-image-recommended: 'true' opendatahub.io/notebook-build-commit: $(odh-codeserver-notebook-image-commit-n) @@ -29,8 +46,26 @@ spec: type: Source # N-1 Version of the image - annotations: - opendatahub.io/notebook-software: '[{"name":"Python","version":"v3.9"}]' - opendatahub.io/notebook-python-dependencies: '[{"name":"code-server","version":"4.22"}]' + # language=json + opendatahub.io/notebook-software: | + [ + {"name": "code-server", "version": "4.22"}, + {"name": "Python", "version": "v3.9"} + ] + # language=json + opendatahub.io/notebook-python-dependencies: | + [ + {"name": "Boto3", "version": "1.34"}, + {"name": "Kafka-Python", "version": "2.0"}, + {"name": "Matplotlib", "version": "3.8"}, + {"name": "Numpy", "version": "1.26"}, + {"name": "Pandas", "version": "2.2"}, + {"name": "Plotly", "version": "5.19"}, + {"name": "Scikit-learn", "version": "1.4"}, + {"name": "Scipy", "version": "1.12"}, + {"name": "Sklearn-onnx", "version": "1.16"}, + {"name": "ipykernel", "version": "6.29"} + ] openshift.io/imported-from: quay.io/opendatahub/workbench-images opendatahub.io/workbench-image-recommended: 'false' opendatahub.io/notebook-build-commit: $(odh-codeserver-notebook-image-commit-n-1) From 001104d8bd2bbc5fb35e32c1bdc813e521ae6a61 Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Tue, 29 Oct 2024 13:21:43 +0100 Subject: [PATCH 2/3] NO-JIRA: feat(ci/check-json.sh): make the script macOS compatible --- ci/check-json.sh | 52 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/ci/check-json.sh b/ci/check-json.sh index 7a23336bb..175af19f6 100755 --- a/ci/check-json.sh +++ b/ci/check-json.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # This script serves to check YAML files in this repository that contain particular # key fields where JSON string is expected. Such JSON strings are extracted and @@ -10,7 +10,19 @@ # In case of the PR on GitHub, this check is tied to GitHub actions automatically, # see `.github/workflows` directory. -shopt -s globstar +if ! shopt -s globstar; then + echo "macOS ships bash-3.2 that does not know shopt -s globstar; install newer bash from homebrew" + exit 1 +fi + +# yq: `brew install yq` or `apt-get install yq` +# json_verify: `brew install yajl` or `apt-get install yajl-tools` +for dep in yq json_verify; do + if ! which -- ${dep} >/dev/null; then + echo "the dependency ${dep} is not installed; install it now" + exit 1 + fi +done function check_json() { local f="${1}" @@ -29,8 +41,9 @@ function check_json() { echo " ${json}" echo -n " > "; echo "${json}" | json_verify || ret_code="${?}" done <<< "${jsons}" + tmp_dir=$(mktemp --directory -t=check-jsons-in-file-) else - echo " Ignoring as this file doesn't contain necessary key field '${string}' for check" + echo " Ignoring as this file doesn't contain necessary key field '${string}' for check" fi return "${ret_code}" @@ -48,20 +61,27 @@ function split_yaml_file() { return 0 } -ret_code=0 +function main() { + local ret_code=0 + # Some yaml files can contain more definitions. + # This is a problem for `yq` tool so we need to split these into separate files. + local tmp_dir + tmp_dir=$(mktemp --directory -t check-json-XXXXXXXXXX-) + for f in **/*.yaml; do + echo "Splitting the '${f}' file." + split_yaml_file "${f}" "${tmp_dir}" || ret_code="${?}" + done -# Some yaml files can contain more definitions. -# This is a problem for `yq` tool so we need to split these into separate files. -tmp_dir=$(mktemp --directory --suffix=-check-json) -for f in **/*.yaml; do - echo "Splitting the '${f}' file." - split_yaml_file "${f}" "${tmp_dir}" || ret_code="${?}" -done + for f in "${tmp_dir}"/*; do + check_json "${f}" "opendatahub.io/notebook-software" || ret_code="${?}" + check_json "${f}" "opendatahub.io/notebook-python-dependencies" || ret_code="${?}" + done -for f in "${tmp_dir}"/*; do - check_json "${f}" "opendatahub.io/notebook-software" || ret_code="${?}" - check_json "${f}" "opendatahub.io/notebook-python-dependencies" || ret_code="${?}" -done + exit "${ret_code}" +} -exit "${ret_code}" +# allows sourcing the script into interactive session without executing it +if [[ "${0}" == "${BASH_SOURCE[0]}" ]]; then + main +fi From 01f4e64e475e0be4b2008a1ad13e356aaacbcc41 Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Tue, 29 Oct 2024 13:51:57 +0100 Subject: [PATCH 3/3] NO-JIRA: feat(ci/check-json.sh): ask yq to output each result into a separate file, in case json strings are multiline --- ci/check-json.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/ci/check-json.sh b/ci/check-json.sh index 175af19f6..c81d9c9b2 100755 --- a/ci/check-json.sh +++ b/ci/check-json.sh @@ -34,14 +34,17 @@ function check_json() { echo "Checking: '${f}' - for '${string}':" if grep --quiet --extended-regexp "${string}" "${f}"; then - #if $(grep -e "${string}" "${f}"); then - jsons=$(yq -r ".spec.tags[].annotations.\"${string}\"" "${f}") - - while IFS= read -r json; do - echo " ${json}" - echo -n " > "; echo "${json}" | json_verify || ret_code="${?}" - done <<< "${jsons}" - tmp_dir=$(mktemp --directory -t=check-jsons-in-file-) + local tmp_dir + tmp_dir=$(mktemp --directory -t check-jsons-in-file-XXXXXXXXXX-) + if ! (cd "${tmp_dir}"; yq --split-exp "\$index" --unwrapScalar ".spec.tags[].annotations.\"${string}\"" "${f}"); then + echo "yq failed to run" + return 1 + fi + + for json in "${tmp_dir}"/*.yml; do + echo " "; cat "${json}" + echo -n " > "; json_verify < "${json}" || ret_code="${?}" + done else echo " Ignoring as this file doesn't contain necessary key field '${string}' for check" fi