From 190fd2c38ef31b2d2f3ee57729d9a870105fc4c4 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 7 Jul 2023 11:27:21 +0200 Subject: [PATCH 1/3] build_sysext: bugfixes, QoL changes, help updated This change improves build_sysext by sourcing a missing lib dependency, adding a number of comfort / quality-of-life options, and updating the output of '--help' accordingly. The OEM sysext finction in build_library/vm_image_util.sh is also updated to use new command line format. 1. Include missing dependency toolchain_util.sh to fix an error in board_options.sh (get_board_arch undefined). 2. Use positional parameters for mandatory arguments. build_dir and sysext_name are mandatory and are now positional arguments instead of options. binary_package is the third positional argument but can be omitted if --metapkgs was specified. 3. --squashfs_base is now guessed better and will use the most recent build by default. 4. A new boolean flag --ignore_version_mismatch for the more daring developer was added. The flag will cause the script to continue if a version mismatch between SDK board packages and squashfs base is detected. 5. Error messages were improved for when mandatory parameters were not provided. 6. The '--help' message was improved and adjusted to the new parameters. Signed-off-by: Thilo Fromm --- build_library/vm_image_util.sh | 3 +- build_sysext | 99 +++++++++++++++++++++------------- 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 4f3e3fcbe6f..960500efcd4 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -540,7 +540,6 @@ install_oem_sysext() { local metapkg="coreos-base/${oem_sysext}" local build_sysext_flags=( --board="${BOARD}" - --build_dir="${built_sysext_dir}" --squashfs_base="${VM_SRC_SYSEXT_IMG}" --metapkgs="${metapkg}" ) @@ -554,7 +553,7 @@ install_oem_sysext() { fi mkdir -p "${built_sysext_dir}" - sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" + sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${built_sysext_dir}" "${oem_sysext}" local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${version}" diff --git a/build_sysext b/build_sysext index 710049770cc..aa472c5789e 100755 --- a/build_sysext +++ b/build_sysext @@ -14,42 +14,55 @@ SCRIPT_ROOT=$(dirname "$(readlink -f "$0")") assert_inside_chroot assert_root_user +default_squashfs_base="$(readlink -f "${SCRIPT_ROOT}/../build/images/amd64-usr/latest/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}")" + # All these are used to set up the 'BUILD_DIR' variable DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build a sysext for." -DEFINE_string build_dir '' \ - "Directory used for building the sysext image, must exist, but should be empty." DEFINE_string metapkgs '' \ "Comma-separated list of meta-packages to build from source and install into sysext image." +DEFINE_string squashfs_base "${default_squashfs_base}" \ + "The path to the squashfs base image. Defaults to the most current image built." DEFINE_string manglefs_script '' \ "A path to executable that will customize the rootfs of the sysext image." -DEFINE_string squashfs_base '' \ - "The path to the squashfs base image." +DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ + "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS." -FLAGS_HELP="USAGE: build_sysext [flags] [sysext name] [binary packages to install into image]. +FLAGS_HELP="USAGE: build_sysext [flags] [ ...] This script is used to build a Flatcar sysext image. +The sysext will be based on an OS image build's sysext base squashfs, i.e. it is specific to a Flatcar build or release. +The base squashfs can either come from a local build or a downloaded from an official release. Examples: -Builds a sysext image named interpreters in the images -directory with dev-lang/python and dev-lang/perl packages for amd64: +Builds a sysext image named 'interpreters' in the 'images' +sub-directory with 'dev-lang/python' and 'dev-lang/perl' packages for the +most recent amd64 production image: -sudo build_sysext \ - --board=amd64-usr \ - --build_dir=images \ +sudo build_sysext \\ + --board=amd64-usr \\ + images \\ interpreters dev-lang/python dev-lang/perl -Builds a sysext image named oem-azure in the oem-images directory with -metapackage coreos-base/oem-azure for arm64: -sudo build_sysext \ - --board=arm64-usr \ - --build_dir=oem-images \ - --metapkgs=coreos-base/oem-azure \ - --mangle_fs=…/coreos-base/oem-azure/files/manglefs.sh \ +Builds a sysext image named 'oem-azure' in the 'oem-images' sub-directory with +metapackage 'coreos-base/oem-azure' for the arm64 squashfs base at +'build/artifacts/flatcar_production_image_sysext.squashfs': + +sudo build_sysext \\ + --board=arm64-usr \\ + --metapkgs=coreos-base/oem-azure \\ + --mangle_fs=sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh \\ + --squashfs_base=build/artifacts/flatcar_production_image_sysext.squashfs \\ + oem-images \\ oem-azure + +Mandatory command line parameters: + - build directory to build the sysext in. Must exist but should be empty. + - name of the sysext output file. + - List of existing binary packages to install. Can be omitted if --metapkgs was specified. " show_help_if_requested "$@" @@ -59,14 +72,28 @@ FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" -source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 -source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 +# Validate command line parameters -if [[ -z "${FLAGS_build_dir}" ]]; then - die "Need a build directory to be specified with a --build_dir option" +build_dir="${1:-}" +if [[ ! -d "${build_dir}" ]]; then + die "Build directory not provided or directory does not exist." fi +BUILD_DIR=$(realpath "${build_dir}") +shift -BUILD_DIR=$(realpath "${FLAGS_build_dir}") +SYSEXTNAME="${1:-}" +if [[ -z "${SYSEXTNAME}" ]]; then + die "No sysext name provided." +fi +shift + +if [[ ! -f "${FLAGS_squashfs_base}" ]] ; then + die "Squashfs base '${FLAGS_squashfs_base}' not found." +fi + +source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 +source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 # Architecture values are taken from systemd.unit(5). declare -A SYSEXT_ARCHES @@ -85,7 +112,6 @@ _get_sysext_arch() { fi } - set -euo pipefail cleanup() { @@ -99,21 +125,10 @@ cleanup() { rm -rf "${dirs[@]}" || true } -if [[ ${#} -lt 1 ]]; then - show_help_if_requested -h - die 'Expected at least one parameter for sysext image name' -fi - -SYSEXTNAME="${1}" -shift - # Set up trap to execute cleanup() on script exit trap cleanup EXIT ARCH=$(_get_sysext_arch "${FLAGS_board}") -if [[ -z "${FLAGS_squashfs_base}" ]]; then - FLAGS_squashfs_base="${BUILD_DIR}/flatcar_production_image_sysext.squashfs" -fi cleanup mkdir "${BUILD_DIR}/fs-root" @@ -123,10 +138,13 @@ mkdir "${BUILD_DIR}/workdir" mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root" VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-) if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then - echo "$VERSION_BOARD" - echo "$FLATCAR_VERSION" - echo "Version mismatch between board flatcar release and SDK container flatcar release" - exit 1 + warn "Base squashfs version: $VERSION_BOARD" + warn "SDK board packages version: $FLATCAR_VERSION" + if [[ "${FLAGS_ignore_version_mismatch}" = "${FLAGS_TRUE}" ]] ; then + warn "Ignoring version mismatch as requested." + else + die "Version mismatch between board flatcar release and SDK container flatcar release." + fi fi if [[ -n "${FLAGS_metapkgs}" ]]; then @@ -135,6 +153,11 @@ if [[ -n "${FLAGS_metapkgs}" ]]; then set -- "${metapkgs[@]}" "${@}" fi +if [[ ${#} -lt 1 ]]; then + error 'No packages or meta packages to install.' + show_help_if_requested -h +fi + for package; do echo "Installing package into sysext image: $package" FEATURES="-ebuild-locks" emerge \ From 657a276cc229cb54b285150f37e3be177b34c7bd Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 7 Jul 2023 13:26:37 +0200 Subject: [PATCH 2/3] build_sysext: fix hard-coded arch, add -noappend Signed-off-by: Thilo Fromm --- build_sysext | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/build_sysext b/build_sysext index aa472c5789e..1f698693767 100755 --- a/build_sysext +++ b/build_sysext @@ -14,15 +14,15 @@ SCRIPT_ROOT=$(dirname "$(readlink -f "$0")") assert_inside_chroot assert_root_user -default_squashfs_base="$(readlink -f "${SCRIPT_ROOT}/../build/images/amd64-usr/latest/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}")" +default_squashfs_template="$(readlink -f "${SCRIPT_ROOT}/../build/images")//latest/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}" # All these are used to set up the 'BUILD_DIR' variable DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build a sysext for." DEFINE_string metapkgs '' \ "Comma-separated list of meta-packages to build from source and install into sysext image." -DEFINE_string squashfs_base "${default_squashfs_base}" \ - "The path to the squashfs base image. Defaults to the most current image built." +DEFINE_string squashfs_base '' \ + "The path to the squashfs base image. Defaults to the most current image built in '${default_squashfs_template}'." DEFINE_string manglefs_script '' \ "A path to executable that will customize the rootfs of the sysext image." DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ @@ -87,8 +87,11 @@ if [[ -z "${SYSEXTNAME}" ]]; then fi shift +if [[ -z "$FLAGS_squashfs_base" ]] ; then + FLAGS_squashfs_base="$(readlink -f "${SCRIPT_ROOT}/../build/images/${FLAGS_board}/latest/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}")" +fi if [[ ! -f "${FLAGS_squashfs_base}" ]] ; then - die "Squashfs base '${FLAGS_squashfs_base}' not found." + die "Squashfs base '${FLAGS_squashfs_base}' not found." fi source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 @@ -197,7 +200,7 @@ all_fields=( "ARCHITECTURE=${ARCH}" ) printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}" -mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" +mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir} # Generate reports From 1f2706d44e2f51c037cb2a1ab89d50f5f34e3759 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 7 Jul 2023 14:49:50 +0200 Subject: [PATCH 3/3] build_sysext: guess image build dir and auto-create it Signed-off-by: Thilo Fromm --- build_library/vm_image_util.sh | 3 ++- build_sysext | 40 ++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 960500efcd4..a57f6cd6edc 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -541,6 +541,7 @@ install_oem_sysext() { local build_sysext_flags=( --board="${BOARD}" --squashfs_base="${VM_SRC_SYSEXT_IMG}" + --image_builddir="${built_sysext_dir}" --metapkgs="${metapkg}" ) local overlay_path mangle_fs @@ -553,7 +554,7 @@ install_oem_sysext() { fi mkdir -p "${built_sysext_dir}" - sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${built_sysext_dir}" "${oem_sysext}" + sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${version}" diff --git a/build_sysext b/build_sysext index 1f698693767..c783f57fa57 100755 --- a/build_sysext +++ b/build_sysext @@ -14,7 +14,7 @@ SCRIPT_ROOT=$(dirname "$(readlink -f "$0")") assert_inside_chroot assert_root_user -default_squashfs_template="$(readlink -f "${SCRIPT_ROOT}/../build/images")//latest/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}" +default_imagedir="$(readlink -f "${SCRIPT_ROOT}/../build/images")//latest/" # All these are used to set up the 'BUILD_DIR' variable DEFINE_string board "${DEFAULT_BOARD}" \ @@ -22,27 +22,29 @@ DEFINE_string board "${DEFAULT_BOARD}" \ DEFINE_string metapkgs '' \ "Comma-separated list of meta-packages to build from source and install into sysext image." DEFINE_string squashfs_base '' \ - "The path to the squashfs base image. Defaults to the most current image built in '${default_squashfs_template}'." + "The path to the squashfs base image. Defaults to the most current image built in '${default_imagedir}/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}'." +DEFINE_string image_builddir '' \ + "Custom directory to build the sysext in. Defaults to a 'sysext' sub-directory of the directory the squashfs base image resides in; '${default_imagedir}/sysext' by default." DEFINE_string manglefs_script '' \ "A path to executable that will customize the rootfs of the sysext image." DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS." -FLAGS_HELP="USAGE: build_sysext [flags] [ ...] +FLAGS_HELP="USAGE: build_sysext [flags] [ ...] This script is used to build a Flatcar sysext image. -The sysext will be based on an OS image build's sysext base squashfs, i.e. it is specific to a Flatcar build or release. -The base squashfs can either come from a local build or a downloaded from an official release. +The sysext will be based on an OS image build's sysext base squashfs, i.e. it is specific to +a Flatcar build or release. +The base squashfs can either come from a local build or downloaded from an official release. +By default, the sysext will be built in a 'sysext' sub-dir of the directory the squashfs base image +is in, but this can be changed with the --image_builddir option. Examples: -Builds a sysext image named 'interpreters' in the 'images' -sub-directory with 'dev-lang/python' and 'dev-lang/perl' packages for the -most recent amd64 production image: +Builds a sysext image named 'interpreters' with 'dev-lang/python' and 'dev-lang/perl' packages for the +most recent production image (default architecture, likely amd64) in the defaut build directory: sudo build_sysext \\ - --board=amd64-usr \\ - images \\ interpreters dev-lang/python dev-lang/perl @@ -55,12 +57,11 @@ sudo build_sysext \\ --metapkgs=coreos-base/oem-azure \\ --mangle_fs=sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh \\ --squashfs_base=build/artifacts/flatcar_production_image_sysext.squashfs \\ - oem-images \\ + --image_builddir=oem-images \\ oem-azure Mandatory command line parameters: - - build directory to build the sysext in. Must exist but should be empty. - name of the sysext output file. - List of existing binary packages to install. Can be omitted if --metapkgs was specified. " @@ -74,13 +75,6 @@ eval set -- "${FLAGS_ARGV}" # Validate command line parameters -build_dir="${1:-}" -if [[ ! -d "${build_dir}" ]]; then - die "Build directory not provided or directory does not exist." -fi -BUILD_DIR=$(realpath "${build_dir}") -shift - SYSEXTNAME="${1:-}" if [[ -z "${SYSEXTNAME}" ]]; then die "No sysext name provided." @@ -94,6 +88,12 @@ if [[ ! -f "${FLAGS_squashfs_base}" ]] ; then die "Squashfs base '${FLAGS_squashfs_base}' not found." fi +if [[ -z "${FLAGS_image_builddir}" ]]; then + FLAGS_image_builddir="$(dirname "${FLAGS_squashfs_base}")/sysext" +fi +BUILD_DIR=$(realpath "${FLAGS_image_builddir}") +mkdir -p "${BUILD_DIR}" + source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 @@ -161,6 +161,8 @@ if [[ ${#} -lt 1 ]]; then show_help_if_requested -h fi +info "Building '${SYSEXTNAME}' with (meta-)packages '${@}' in '${BUILD_DIR}'". + for package; do echo "Installing package into sysext image: $package" FEATURES="-ebuild-locks" emerge \