From c54955a31951272a6e142c5363d2c24adb173880 Mon Sep 17 00:00:00 2001 From: "Eric D. Helms" Date: Wed, 27 Sep 2023 16:38:40 -0400 Subject: [PATCH] Switch to building stage repository by combining production and diff of Copr When the version is nightly, this will not pull from production but instead treat what is in Copr as the source of truth. At the end, the output for a versioned repository is a list of unsigned packages. --- build_stage_repository | 203 ++++++++++++++++++++++++++++++----------- list_unsigned_rpms | 63 +++++++++++++ sign_stage_rpms | 12 +++ 3 files changed, 226 insertions(+), 52 deletions(-) create mode 100755 list_unsigned_rpms create mode 100755 sign_stage_rpms diff --git a/build_stage_repository b/build_stage_repository index d1a1626..517edee 100755 --- a/build_stage_repository +++ b/build_stage_repository @@ -11,33 +11,19 @@ import hashlib import sys import tempfile import dnf.comps +import argparse -def move_rpms_from_copr_to_stage(collection, version, src_folder, dest_folder, dist, arch, source=False): - if source: - print(f"Moving {collection} Source RPMs from Copr directory to stage repository") - else: - print(f"Moving {collection} RPMs from Copr directory to stage repository") - - if not os.path.exists(dest_folder): - os.mkdir(dest_folder) - - repo_folder = f"{src_folder}/{dist}-{collection}-{version}-{arch}" - packages_to_include = packages_from_comps(comps(collection, version, dist)) - - if source: - files = glob.glob(repo_folder + f"/**/*.src.rpm") - else: - files = glob.glob(repo_folder + f"/**/*.rpm") +def filter_packages(target_dir, packages): + files = glob.glob(f"{target_dir}/*.rpm") for file in files: file_name = os.path.basename(file) rpm_name = get_rpm_name(file) - if rpm_name in packages_to_include: - shutil.move(file, os.path.join(dest_folder, file_name)) - - shutil.rmtree(src_folder) + if rpm_name not in packages: + print(f"Removed {rpm_name}. Package not in comps.") + os.remove(os.path.join(target_dir, file_name)) def get_rpm_name(rpm): @@ -62,10 +48,12 @@ def modulemd_yaml(collection, version): def comps(collection, version, dist): branch = 'develop' if version == 'nightly' else version - if collection == 'foreman-katello': - name = 'katello' - elif collection == 'foreman-candlepin': + if collection == 'candlepin': name = 'katello-candlepin' + elif collection == 'client': + name = 'foreman-client' + elif collection == 'plugins': + name = 'foreman-plugins' else: name = collection @@ -121,28 +109,30 @@ def create_modulemd(collection, version, stage_dir): def create_repository(repo_dir): - check_output(['createrepo', repo_dir]) + check_output(['createrepo_c', repo_dir], stderr=STDOUT) -def sync_copr_repository(collection, version, target_dir, dist, arch, source=False): - if source: - print(f"Syncing {collection} {version} Source RPM repository from Copr") +def sync_prod_repository(collection, version, target_dir, dist, arch): + if arch == 'source': + print(f"Syncing {collection} {version} Source RPM repository from yum.theforeman.org") else: - print(f"Syncing {collection} {version} RPM repository from Copr") + print(f"Syncing {collection} {version} RPM repository from yum.theforeman.org") cmd = [ 'dnf', 'reposync', - '--newest-only', + '--refresh', + '--download-metadata', + '--norepopath', '--repo', f"{dist}-{collection}-{version}-{arch}", '--repofrompath', - f"{dist}-{collection}-{version}-{arch},https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}/", + f"{dist}-{collection}-{version}-{arch},{prod_repository(collection, version, dist, arch)}", '--download-path', target_dir ] - if source: + if arch == 'source': cmd.extend([ '--source', ]) @@ -172,40 +162,149 @@ def packages_from_comps(comps): return packages +def copr_repository(collection, version, dist, arch): + if collection == 'candlepin': + name = 'katello-candlepin' + elif collection == 'client': + name = 'foreman-client' + elif collection == 'plugins': + name = 'foreman-plugins' + else: + name = collection + + return f"https://download.copr.fedorainfracloud.org/results/@theforeman/{name}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}" + + +def prod_repository(collection, version, dist, arch): + if collection == "foreman" and version == "nightly": + return f"https://yum.theforeman.org/{version}/{dist}/{arch}" + else: + return f"https://yum.theforeman.org/{collection.replace('foreman-', '')}/{version}/{dist}/{arch}" + + +def copr_repository_urls(repository): + cmd = [ + 'dnf', + 'reposync', + '--urls', + '--refresh', + '--repofrompath', + f"copr,{repository}", + '--repo', + 'copr' + ] + + urls = check_output(cmd, universal_newlines=True, stderr=STDOUT) + return urls.split("\n") + + +def compare_repositories(new_repository, old_repository, source=False): + cmd = [ + 'dnf', + 'repodiff', + '--simple', + '--refresh', + '--compare-arch', + '--repofrompath', + f"new,{new_repository}", + '--repofrompath', + f"old,{old_repository}", + '--repo-old', + 'old', + '--repo-new', + 'new' + ] + + if not source: + cmd.extend(['--arch', 'noarch,x86_64']) + + print(' '.join(cmd)) + return check_output(cmd, universal_newlines=True, stderr=STDOUT) + + +def parse_repodiff(diff): + split_diff = diff.split("\n") + packages = [] + + for line in split_diff: + if line.startswith('Added package :'): + packages.append(line.replace('Added package : ', '')) + + if ' -> ' in line: + packages.append(line.split(' -> ')[1]) + + return packages + + +def download_copr_packages(packages, urls, repository, downloads_dir, included_packages): + for package in packages: + name = f"{package}.rpm".replace("-1:", "-") + + for url in urls: + if name in url: + download_url = url + + if not os.path.exists(f"{downloads_dir}/{name}"): + print(f"Downloading {repository}/{name} to {downloads_dir}") + urlretrieve(download_url, f"{downloads_dir}/{name}") + else: + print(f"Skipping {repository}/{name}. Already downloaded.") + + +def handle_args(): + parser = argparse.ArgumentParser(description='Generate a stage repository') + parser.add_argument( + 'collection', + help='Repository to generate for (e.g. foreman)' + ) + parser.add_argument( + 'version', + help='Version to generate the repository for (e.g. nightly)' + ) + parser.add_argument( + 'dist', + help='Dist to generate repository for (e.g. el8)' + ) + + return parser.parse_args() + + def main(): - try: - collection = sys.argv[1] - version = sys.argv[2] - dist = sys.argv[3] - arch = 'x86_64' - except IndexError: - raise SystemExit(f"Usage: {sys.argv[0]} collection version os") + args = handle_args() - base_dir = 'tmp' - rpm_sync_dir = f"{base_dir}/rpms" - srpm_sync_dir = f"{base_dir}/srpms" + collection = args.collection + version = args.version + dist = args.dist + arch = 'x86_64' - stage_dir = f"{base_dir}/{collection}/{version}/{dist}/" + base_dir = 'tmp' + stage_dir = f"{base_dir}/{collection}/{version}/{dist}" rpm_dir = f"{stage_dir}/{arch}" srpm_dir = f"{stage_dir}/source" - if not os.path.exists(rpm_sync_dir): - os.makedirs(rpm_sync_dir) - - if not os.path.exists(srpm_sync_dir): - os.makedirs(srpm_sync_dir) - if not os.path.exists(rpm_dir): os.makedirs(rpm_dir) if not os.path.exists(srpm_dir): os.makedirs(srpm_dir) - sync_copr_repository(collection, version, rpm_sync_dir, dist, arch) - sync_copr_repository(collection, version, srpm_sync_dir, dist, arch, source=True) + if version != 'nightly': + sync_prod_repository(collection, version, rpm_dir, dist, arch) + sync_prod_repository(collection, version, srpm_dir, dist, 'source') - move_rpms_from_copr_to_stage(collection, version, srpm_sync_dir, srpm_dir, dist, arch, source=True) - move_rpms_from_copr_to_stage(collection, version, rpm_sync_dir, rpm_dir, dist, arch) + packages_to_include = packages_from_comps(comps(collection, version, dist)) + copr_repo = copr_repository(collection, version, dist, arch) + copr_package_urls = copr_repository_urls(copr_repo) + + rpm_diff = compare_repositories(copr_repo, rpm_dir) + packages = parse_repodiff(rpm_diff) + download_copr_packages(packages, copr_package_urls, copr_repo, rpm_dir, packages_to_include) + filter_packages(rpm_dir, packages_to_include) + + srpm_diff = compare_repositories(copr_repo, srpm_dir, True) + srpm_packages = parse_repodiff(srpm_diff) + download_copr_packages(srpm_packages, copr_package_urls, copr_repo, srpm_dir, packages_to_include) + filter_packages(srpm_dir, packages_to_include) create_repository(rpm_dir) create_repository(srpm_dir) diff --git a/list_unsigned_rpms b/list_unsigned_rpms new file mode 100755 index 0000000..ebf8c7e --- /dev/null +++ b/list_unsigned_rpms @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +from subprocess import check_output, STDOUT, CalledProcessError +import argparse +import glob + + +def find_unsigned_packages(target_dir, gpgkey): + unsigned = [] + packages = glob.glob(f"{target_dir}/*.rpm") + + for package in packages: + cmd = [ + 'rpm', + '-qpi', + package + ] + + output = check_output(cmd, universal_newlines=True, stderr=STDOUT) + + if gpgkey.lower() not in output: + unsigned.append(package) + + return unsigned + + +def handle_args(): + parser = argparse.ArgumentParser(description='Sign unsigned RPMs in local stage repository') + parser.add_argument( + 'repository_path', + help='Path to repository to sign unsigned RPMs' + ) + parser.add_argument( + 'gpgkey', + help='Short form gpgkey for the version being generated, does not apply to nightly' + ) + + args = parser.parse_args() + + if args.gpgkey: + if len(args.gpgkey) != 8: + raise SystemExit("GPG key must be in short form and 8 characters long") + + return args + + +def main(): + args = handle_args() + + repository_path = args.repository_path + gpgkey = args.gpgkey + + unsigned_rpms = find_unsigned_packages(f"{repository_path}/x86_64", gpgkey) + unsigned_srpms = find_unsigned_packages(f"{repository_path}/source", gpgkey) + + for rpm in unsigned_rpms: + print(rpm) + for srpm in unsigned_srpms: + print(srpm) + + +if __name__ == '__main__': + main() diff --git a/sign_stage_rpms b/sign_stage_rpms new file mode 100755 index 0000000..c769e15 --- /dev/null +++ b/sign_stage_rpms @@ -0,0 +1,12 @@ +#!/bin/bash -e + +. settings + +echo "./list_unsigned_rpms $1 $GPGKEY" +UNSIGNED_RPMS=(`./list_unsigned_rpms $1 $GPGKEY`) + +for rpm_path in "${UNSIGNED_RPMS[@]}" +do + echo "Signing $rpm_path" + ./sign_rpms $rpm_path +done