refactor: simplify components.json code

e2e/components/components.go

@@ -43,16 +43,18 @@ func GetExpectedPackageVersions(packageName, distro, release string) []string {
 	release = strings.ReplaceAll(release, ".", "\\.")
 
 	for _, packageItem := range packages.Array() {
-		// check if versionsV2 exists
-		if packageItem.Get(fmt.Sprintf("%s.%s.versionsV2", distro, release)).Exists() {
-			versions := packageItem.Get(fmt.Sprintf("%s.%s.versionsV2", distro, release))
-			for _, version := range versions.Array() {
-				// get versions.latestVersion and append to expectedVersions
-				expectedVersions = append(expectedVersions, version.Get("latestVersion").String())
-				// get versions.previousLatestVersion (if exists) and append to expectedVersions
-				if version.Get("previousLatestVersion").Exists() {
-					expectedVersions = append(expectedVersions, version.Get("previousLatestVersion").String())
-				}
+		// Check if versionsV2 exists. Assume the DEFAULT OS variant.
+		versions := packageItem.Get(fmt.Sprintf("%s/DEFAULT.%s.versionsV2", distro, release)).Array()
+		if len(versions) == 0 {
+			versions = packageItem.Get(fmt.Sprintf("%s.%s.versionsV2", distro, release)).Array()
+		}
+
+		for _, version := range versions {
+			// get versions.latestVersion and append to expectedVersions
+			expectedVersions = append(expectedVersions, version.Get("latestVersion").String())
+			// get versions.previousLatestVersion (if exists) and append to expectedVersions
+			if version.Get("previousLatestVersion").Exists() {
+				expectedVersions = append(expectedVersions, version.Get("previousLatestVersion").String())
 			}
 		}
 	}

parts/common/components.json

@@ -946,6 +946,16 @@
             ],
             "downloadURL": "https://github.com/oras-project/oras/releases/download/v${version}/oras_${version}_linux_${CPU_ARCH}.tar.gz"
           }
+        },
+        "azurelinux/OSGUARD": {
+          "v3.0": {
+            "versionsV2": [
+              {
+                "renovateTag": "<DO_NOT_UPDATE>",
+                "latestVersion": "<SKIP>"
+              }
+            ]
+          }
         }
       }
     },
@@ -1076,6 +1086,16 @@
             ]
           }
         },
+        "azurelinux/OSGUARD": {
+          "v3.0": {
+            "versionsV2": [
+              {
+                "renovateTag": "<DO_NOT_UPDATE>",
+                "latestVersion": "<SKIP>"
+              }
+            ]
+          }
+        },
         "azurelinuxkata": {
           "v3.0": {
             "versionsV2": [
@@ -1685,7 +1705,7 @@
             ]
           }
         },
-        "azurelinux": {
+        "azurelinux/DEFAULT": {
           "v3.0": {
             "versionsV2": [
               {
@@ -1719,7 +1739,7 @@
             ]
           }
         },
-        "azurelinux": {
+        "azurelinux/DEFAULT": {
           "v3.0": {
             "versionsV2": [
               {
@@ -1753,7 +1773,7 @@
             ]
           }
         },
-        "azurelinux": {
+        "azurelinux/DEFAULT": {
           "v3.0": {
             "versionsV2": [
               {

parts/linux/cloud-init/artifacts/README-COMPONENTS.md

@@ -138,6 +138,7 @@ Here are the explanation of the above schema.
      - In `MarinerOSDistro`, we only have `current` now, which implies that single configurations will be applied to all Mariner release versions. We can distinguish them in needed. Note, we confirmed with Mariner team, Azure Linux 2.0 is reporting itself as `mariner` in the file `/etc/os-release`. So for Azure Linux 2.0 case, it will still read the package versions from `mariner` block.
     - In `AzureLinuxOSDistro`, `v3.0` is for Azure Linux v3.0. `current` is for otherwise but we are not using it now. Azure Linux 2.0 case is described in the `MarinerOSDistro` above. 
     - `DefaultOSDistro` means the default case of OS Distro. If an OSDistro metadata is not defined, it will fetch it from `default`. For example, if a node is Ubuntu 20.04, but we don't specify `ubuntu` in components.json, then it will fetch `default.current`. For another example, if only `default.current` is specified in the components.json, No matter what OSDistro is the node running, it will only fetch `default.current` because it's the default metadata. This provides flexibility while elimiating unnecessary duplication when defining the metadata.
+1. The OSDistro can optionally be suffixed with the OS variant. `azurelinux/OSGUARD` and `azurelinux/DEFAULT` are currently supported. The latter specifically matches Azure Linux without a variant, i.e. not OS Guard.
 1. In `ReleaseDownloadURI`, you can see 2 keys.
     - `versionsV2`: This is updated from `versions`. You can define a list of `VersionV2` for a particular package. And in the codes, it's up to the feature developer to determine how to use the list. For example, install all versions in the list or just pick the latest one. Note that in package `containerd`, `marinerkata`, the `versionV2s` array is defined as `<SKIP>`. This is to tell the install-dependencies.sh not to install any `containerd` version for Kata SKU.
 	- `downloadURL`: you can define a downloadURL with unresolved variables. For example, `https://acs-mirror.azureedge.net/azure-cni/v${version}/binaries/azure-vnet-cni-linux-${CPU_ARCH}-v${version}.tgz`. But the feature developer needs to make sure all variables are resolvable in the codes. In this example, `${CPU_ARCH}` is resolvable as it's defined at global scope. `${version}` is resovled based on the `versions` list above.

parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh

@@ -77,7 +77,7 @@ installCredentialProviderFromPMC() {
         os_version="${OS_VERSION}"
     fi
    	PACKAGE_VERSION=""
-    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version"
+    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version" "${OS_VARIANT}"
     packageVersion=$(echo $PACKAGE_VERSION | cut -d "-" -f 1)
 	echo "installing azure-acr-credential-provider package version: $packageVersion"
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -871,88 +871,45 @@ updatePackageVersions() {
     local package="$1"
     local os="$2"
     local osVersion="$3"
+    local osVariant=${4:-DEFAULT}
     RELEASE="current"
     updateRelease "${package}" "${os}" "${osVersion}"
     local osLowerCase=$(echo "${os}" | tr '[:upper:]' '[:lower:]')
     PACKAGE_VERSIONS=()
 
-    # if .downloadURIs.${osLowerCase} doesn't exist, it will get the versions from .downloadURIs.default.
-    # Otherwise get the versions from .downloadURIs.${osLowerCase
-    if [ "$(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}" 2>/dev/null)" = "null" ]; then
-        osLowerCase="default"
-    fi
+    # Use the first match from these:
+    # .downloadURIs.${osLowerCase}/${osVariant}
+    # .downloadURIs.${osLowerCase}
+    # .downloadURIs.default
+    package=$(jq -r ".downloadURIs.\"${osLowerCase}/${osVariant}\" // .downloadURIs.${osLowerCase} // .downloadURIs.default" <<< "${package}")
 
-    # jq the versions from the package. If downloadURIs.$osLowerCase.$release.versionsV2 is not null, then get the versions from there.
-    # Otherwise get the versions from .downloadURIs.$osLowerCase.$release.versions
-    if [ "$(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}.${RELEASE}.versionsV2")" != "null" ]; then
-        local latestVersions=($(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}.${RELEASE}.versionsV2[] | select(.latestVersion != null) | .latestVersion"))
-        local previousLatestVersions=($(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}.${RELEASE}.versionsV2[] | select(.previousLatestVersion != null) | .previousLatestVersion"))
-        for version in "${latestVersions[@]}"; do
-            PACKAGE_VERSIONS+=("${version}")
-        done
-        for version in "${previousLatestVersions[@]}"; do
-            PACKAGE_VERSIONS+=("${version}")
-        done
-        return 0
-    fi
-
-    # Fallback to versions if versionsV2 is null
-    if [ "$(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}.${RELEASE}.versions")" = "null" ]; then
-        return 0
-    fi
-    local versions=($(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}.${RELEASE}.versions[]"))
-    for version in "${versions[@]}"; do
-        PACKAGE_VERSIONS+=("${version}")
-    done
+    readarray -t PACKAGE_VERSIONS < <(jq -r ".${RELEASE}.versionsV2 // [] | map(.latestVersion, .previousLatestVersion) | .[] | values" <<< "${package}")
     return 0
 }
 
-# sets MULTI_ARCH_VERSIONS to multiArchVersionsV2 if it exists, otherwise multiArchVersions
+# sets MULTI_ARCH_VERSIONS to multiArchVersionsV2
 updateMultiArchVersions() {
   local imageToBePulled="$1"
-
-  #jq the MultiArchVersions from the containerImages. If ContainerImages[i].multiArchVersionsV2 is not null, return that, else return ContainerImages[i].multiArchVersions
-  if [ "$(echo "${imageToBePulled}" | jq -r '.multiArchVersionsV2 // [] | select(. != null and . != [])')" ]; then
-    local latestVersions=($(echo "${imageToBePulled}" | jq -r ".multiArchVersionsV2[] | select(.latestVersion != null) | .latestVersion"))
-    local previousLatestVersions=($(echo "${imageToBePulled}" | jq -r ".multiArchVersionsV2[] | select(.previousLatestVersion != null) | .previousLatestVersion"))
-    for version in "${latestVersions[@]}"; do
-      MULTI_ARCH_VERSIONS+=("${version}")
-    done
-    for version in "${previousLatestVersions[@]}"; do
-      MULTI_ARCH_VERSIONS+=("${version}")
-    done
-    return
-  fi
-
-  # check if multiArchVersions not exists
-  if [ "$(echo "${imageToBePulled}" | jq -r '.multiArchVersions | if . == null then "null" else empty end')" = "null" ]; then
-    MULTI_ARCH_VERSIONS=()
-    return
-  fi
-
-  local versions=($(echo "${imageToBePulled}" | jq -r ".multiArchVersions[]"))
-  for version in "${versions[@]}"; do
-    MULTI_ARCH_VERSIONS+=("${version}")
-  done
+  readarray -t MULTI_ARCH_VERSIONS < <(jq -r ".multiArchVersionsV2 // [] | map(.latestVersion, .previousLatestVersion) | .[] | values" <<< "${imageToBePulled}")
+  return
 }
 
 updatePackageDownloadURL() {
     local package=$1
     local os=$2
     local osVersion=$3
+    local osVariant=${4:-DEFAULT}
     RELEASE="current"
     updateRelease "${package}" "${os}" "${osVersion}"
     local osLowerCase=$(echo "${os}" | tr '[:upper:]' '[:lower:]')
 
-    #if .downloadURIs.${osLowerCase} exist, then get the downloadURL from there.
-    #otherwise get the downloadURL from .downloadURIs.default
-    if [ "$(echo "${package}" | jq -r ".downloadURIs.${osLowerCase}")" != "null" ]; then
-        downloadURL=$(echo "${package}" | jq ".downloadURIs.${osLowerCase}.${RELEASE}.downloadURL" -r)
-        [ "${downloadURL}" = "null" ] && PACKAGE_DOWNLOAD_URL="" || PACKAGE_DOWNLOAD_URL="${downloadURL}"
-        return
-    fi
-    downloadURL=$(echo "${package}" | jq ".downloadURIs.default.${RELEASE}.downloadURL" -r)
-    [ "${downloadURL}" = "null" ] && PACKAGE_DOWNLOAD_URL="" || PACKAGE_DOWNLOAD_URL="${downloadURL}"
+    # Use the first match from these:
+    # .downloadURIs.${osLowerCase}/${osVariant}
+    # .downloadURIs.${osLowerCase}
+    # .downloadURIs.default
+    package=$(jq -r ".downloadURIs.\"${osLowerCase}/${osVariant}\" // .downloadURIs.${osLowerCase} // .downloadURIs.default" <<< "${package}")
+
+    PACKAGE_DOWNLOAD_URL=$(jq -r ".${RELEASE}.downloadURL | values" <<< "${package}")
     return
 }
 
@@ -962,12 +919,13 @@ getLatestPkgVersionFromK8sVersion() {
     local componentName="$2"
     local os="$3"
     local os_version="$4"
+    local os_variant=${5:-DEFAULT}
 
     k8sMajorMinorVersion="$(echo "$k8sVersion" | cut -d- -f1 | cut -d. -f1,2)"
 
     package=$(jq ".Packages" "$COMPONENTS_FILEPATH" | jq ".[] | select(.name == \"${componentName}\")")
     PACKAGE_VERSIONS=()
-    updatePackageVersions "${package}" "${os}" "${os_version}"
+    updatePackageVersions "${package}" "${os}" "${os_version}" "${os_variant}"
 
     # shellcheck disable=SC3010
     if [[ ${#PACKAGE_VERSIONS[@]} -eq 0 || ${PACKAGE_VERSIONS[0]} == "<SKIP>" ]]; then

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -66,7 +66,7 @@ installContainerdWithComponentsJson() {
     if isAzureLinux "${OS}" && [ "${IS_KATA}" = "true" ]; then
         os=${AZURELINUX_KATA_OS_NAME}
     fi
-    updatePackageVersions "${containerdPackage}" "${os}" "${os_version}"
+    updatePackageVersions "${containerdPackage}" "${os}" "${os_version}" "${OS_VARIANT}"
 
     #Containerd's versions array is expected to have only one element.
     #If it has more than one element, we will install the last element in the array.
@@ -413,7 +413,7 @@ installCNI() {
         os=${MARINER_KATA_OS_NAME}
     fi
     PACKAGE_VERSIONS=()
-    updatePackageVersions "${cniPackage}" "${os}" "${os_version}"
+    updatePackageVersions "${cniPackage}" "${os}" "${os_version}" "${OS_VARIANT}"
 
     #should change to ne
     # shellcheck disable=SC3010

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -183,7 +183,7 @@ installCredentialProviderFromPMC() {
         os_version="${OS_VERSION}"
     fi
    	PACKAGE_VERSION=""
-    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version"
+    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version" "${OS_VARIANT}"
     packageVersion=$(echo $PACKAGE_VERSION | cut -d "-" -f 1)
 	echo "installing azure-acr-credential-provider package version: $packageVersion"
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
@@ -282,7 +282,7 @@ installCredentialProviderPackageFromBootstrapProfileRegistry() {
         os_version="${OS_VERSION}"
     fi
     PACKAGE_VERSION=""
-    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version"
+    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version" "${OS_VARIANT}"
     packageVersion=$(echo $PACKAGE_VERSION | cut -d "-" -f 1)
     if [ -z "$packageVersion" ]; then
         packageVersion=$(echo "$CREDENTIAL_PROVIDER_DOWNLOAD_URL" | grep -oP 'v\d+(\.\d+)*' | sed 's/^v//' | head -n 1)

parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh

@@ -196,7 +196,7 @@ installCredentialProviderFromPMC() {
         os_version="${UBUNTU_RELEASE}"
     fi
     PACKAGE_VERSION=""
-    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version"
+    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version" "${OS_VARIANT}"
     packageVersion=$(echo $PACKAGE_VERSION | cut -d "-" -f 1)
     echo "installing azure-acr-credential-provider package version: $packageVersion"
     mkdir -p "${CREDENTIAL_PROVIDER_BIN_DIR}"
@@ -249,7 +249,7 @@ installCredentialProviderPackageFromBootstrapProfileRegistry() {
         os_version="${UBUNTU_RELEASE}"
     fi
     PACKAGE_VERSION=""
-    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version"
+    getLatestPkgVersionFromK8sVersion "$k8sVersion" "azure-acr-credential-provider-pmc" "$os" "$os_version" "${OS_VARIANT}"
     packageVersion=$(echo $PACKAGE_VERSION | cut -d "-" -f 1)
     if [ -z "$packageVersion" ]; then
         packageVersion=$(echo "$CREDENTIAL_PROVIDER_DOWNLOAD_URL" | grep -oP 'v\d+(\.\d+)*' | sed 's/^v//' | head -n 1)