diff --git a/build_stage_repository b/build_stage_repository index d1a1626..517edee 100755 --- a/build_stage_repository +++ b/build_stage_repository @@ -11,33 +11,19 @@ import hashlib import sys import tempfile import dnf.comps +import argparse -def move_rpms_from_copr_to_stage(collection, version, src_folder, dest_folder, dist, arch, source=False): - if source: - print(f"Moving {collection} Source RPMs from Copr directory to stage repository") - else: - print(f"Moving {collection} RPMs from Copr directory to stage repository") - - if not os.path.exists(dest_folder): - os.mkdir(dest_folder) - - repo_folder = f"{src_folder}/{dist}-{collection}-{version}-{arch}" - packages_to_include = packages_from_comps(comps(collection, version, dist)) - - if source: - files = glob.glob(repo_folder + f"/**/*.src.rpm") - else: - files = glob.glob(repo_folder + f"/**/*.rpm") +def filter_packages(target_dir, packages): + files = glob.glob(f"{target_dir}/*.rpm") for file in files: file_name = os.path.basename(file) rpm_name = get_rpm_name(file) - if rpm_name in packages_to_include: - shutil.move(file, os.path.join(dest_folder, file_name)) - - shutil.rmtree(src_folder) + if rpm_name not in packages: + print(f"Removed {rpm_name}. Package not in comps.") + os.remove(os.path.join(target_dir, file_name)) def get_rpm_name(rpm): @@ -62,10 +48,12 @@ def modulemd_yaml(collection, version): def comps(collection, version, dist): branch = 'develop' if version == 'nightly' else version - if collection == 'foreman-katello': - name = 'katello' - elif collection == 'foreman-candlepin': + if collection == 'candlepin': name = 'katello-candlepin' + elif collection == 'client': + name = 'foreman-client' + elif collection == 'plugins': + name = 'foreman-plugins' else: name = collection @@ -121,28 +109,30 @@ def create_modulemd(collection, version, stage_dir): def create_repository(repo_dir): - check_output(['createrepo', repo_dir]) + check_output(['createrepo_c', repo_dir], stderr=STDOUT) -def sync_copr_repository(collection, version, target_dir, dist, arch, source=False): - if source: - print(f"Syncing {collection} {version} Source RPM repository from Copr") +def sync_prod_repository(collection, version, target_dir, dist, arch): + if arch == 'source': + print(f"Syncing {collection} {version} Source RPM repository from yum.theforeman.org") else: - print(f"Syncing {collection} {version} RPM repository from Copr") + print(f"Syncing {collection} {version} RPM repository from yum.theforeman.org") cmd = [ 'dnf', 'reposync', - '--newest-only', + '--refresh', + '--download-metadata', + '--norepopath', '--repo', f"{dist}-{collection}-{version}-{arch}", '--repofrompath', - f"{dist}-{collection}-{version}-{arch},https://download.copr.fedorainfracloud.org/results/@theforeman/{collection}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}/", + f"{dist}-{collection}-{version}-{arch},{prod_repository(collection, version, dist, arch)}", '--download-path', target_dir ] - if source: + if arch == 'source': cmd.extend([ '--source', ]) @@ -172,40 +162,149 @@ def packages_from_comps(comps): return packages +def copr_repository(collection, version, dist, arch): + if collection == 'candlepin': + name = 'katello-candlepin' + elif collection == 'client': + name = 'foreman-client' + elif collection == 'plugins': + name = 'foreman-plugins' + else: + name = collection + + return f"https://download.copr.fedorainfracloud.org/results/@theforeman/{name}-{version}-staging/rhel-{dist.replace('el', '')}-{arch}" + + +def prod_repository(collection, version, dist, arch): + if collection == "foreman" and version == "nightly": + return f"https://yum.theforeman.org/{version}/{dist}/{arch}" + else: + return f"https://yum.theforeman.org/{collection.replace('foreman-', '')}/{version}/{dist}/{arch}" + + +def copr_repository_urls(repository): + cmd = [ + 'dnf', + 'reposync', + '--urls', + '--refresh', + '--repofrompath', + f"copr,{repository}", + '--repo', + 'copr' + ] + + urls = check_output(cmd, universal_newlines=True, stderr=STDOUT) + return urls.split("\n") + + +def compare_repositories(new_repository, old_repository, source=False): + cmd = [ + 'dnf', + 'repodiff', + '--simple', + '--refresh', + '--compare-arch', + '--repofrompath', + f"new,{new_repository}", + '--repofrompath', + f"old,{old_repository}", + '--repo-old', + 'old', + '--repo-new', + 'new' + ] + + if not source: + cmd.extend(['--arch', 'noarch,x86_64']) + + print(' '.join(cmd)) + return check_output(cmd, universal_newlines=True, stderr=STDOUT) + + +def parse_repodiff(diff): + split_diff = diff.split("\n") + packages = [] + + for line in split_diff: + if line.startswith('Added package :'): + packages.append(line.replace('Added package : ', '')) + + if ' -> ' in line: + packages.append(line.split(' -> ')[1]) + + return packages + + +def download_copr_packages(packages, urls, repository, downloads_dir, included_packages): + for package in packages: + name = f"{package}.rpm".replace("-1:", "-") + + for url in urls: + if name in url: + download_url = url + + if not os.path.exists(f"{downloads_dir}/{name}"): + print(f"Downloading {repository}/{name} to {downloads_dir}") + urlretrieve(download_url, f"{downloads_dir}/{name}") + else: + print(f"Skipping {repository}/{name}. Already downloaded.") + + +def handle_args(): + parser = argparse.ArgumentParser(description='Generate a stage repository') + parser.add_argument( + 'collection', + help='Repository to generate for (e.g. foreman)' + ) + parser.add_argument( + 'version', + help='Version to generate the repository for (e.g. nightly)' + ) + parser.add_argument( + 'dist', + help='Dist to generate repository for (e.g. el8)' + ) + + return parser.parse_args() + + def main(): - try: - collection = sys.argv[1] - version = sys.argv[2] - dist = sys.argv[3] - arch = 'x86_64' - except IndexError: - raise SystemExit(f"Usage: {sys.argv[0]} collection version os") + args = handle_args() - base_dir = 'tmp' - rpm_sync_dir = f"{base_dir}/rpms" - srpm_sync_dir = f"{base_dir}/srpms" + collection = args.collection + version = args.version + dist = args.dist + arch = 'x86_64' - stage_dir = f"{base_dir}/{collection}/{version}/{dist}/" + base_dir = 'tmp' + stage_dir = f"{base_dir}/{collection}/{version}/{dist}" rpm_dir = f"{stage_dir}/{arch}" srpm_dir = f"{stage_dir}/source" - if not os.path.exists(rpm_sync_dir): - os.makedirs(rpm_sync_dir) - - if not os.path.exists(srpm_sync_dir): - os.makedirs(srpm_sync_dir) - if not os.path.exists(rpm_dir): os.makedirs(rpm_dir) if not os.path.exists(srpm_dir): os.makedirs(srpm_dir) - sync_copr_repository(collection, version, rpm_sync_dir, dist, arch) - sync_copr_repository(collection, version, srpm_sync_dir, dist, arch, source=True) + if version != 'nightly': + sync_prod_repository(collection, version, rpm_dir, dist, arch) + sync_prod_repository(collection, version, srpm_dir, dist, 'source') - move_rpms_from_copr_to_stage(collection, version, srpm_sync_dir, srpm_dir, dist, arch, source=True) - move_rpms_from_copr_to_stage(collection, version, rpm_sync_dir, rpm_dir, dist, arch) + packages_to_include = packages_from_comps(comps(collection, version, dist)) + copr_repo = copr_repository(collection, version, dist, arch) + copr_package_urls = copr_repository_urls(copr_repo) + + rpm_diff = compare_repositories(copr_repo, rpm_dir) + packages = parse_repodiff(rpm_diff) + download_copr_packages(packages, copr_package_urls, copr_repo, rpm_dir, packages_to_include) + filter_packages(rpm_dir, packages_to_include) + + srpm_diff = compare_repositories(copr_repo, srpm_dir, True) + srpm_packages = parse_repodiff(srpm_diff) + download_copr_packages(srpm_packages, copr_package_urls, copr_repo, srpm_dir, packages_to_include) + filter_packages(srpm_dir, packages_to_include) create_repository(rpm_dir) create_repository(srpm_dir) diff --git a/list_unsigned_rpms b/list_unsigned_rpms new file mode 100755 index 0000000..ebf8c7e --- /dev/null +++ b/list_unsigned_rpms @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +from subprocess import check_output, STDOUT, CalledProcessError +import argparse +import glob + + +def find_unsigned_packages(target_dir, gpgkey): + unsigned = [] + packages = glob.glob(f"{target_dir}/*.rpm") + + for package in packages: + cmd = [ + 'rpm', + '-qpi', + package + ] + + output = check_output(cmd, universal_newlines=True, stderr=STDOUT) + + if gpgkey.lower() not in output: + unsigned.append(package) + + return unsigned + + +def handle_args(): + parser = argparse.ArgumentParser(description='Sign unsigned RPMs in local stage repository') + parser.add_argument( + 'repository_path', + help='Path to repository to sign unsigned RPMs' + ) + parser.add_argument( + 'gpgkey', + help='Short form gpgkey for the version being generated, does not apply to nightly' + ) + + args = parser.parse_args() + + if args.gpgkey: + if len(args.gpgkey) != 8: + raise SystemExit("GPG key must be in short form and 8 characters long") + + return args + + +def main(): + args = handle_args() + + repository_path = args.repository_path + gpgkey = args.gpgkey + + unsigned_rpms = find_unsigned_packages(f"{repository_path}/x86_64", gpgkey) + unsigned_srpms = find_unsigned_packages(f"{repository_path}/source", gpgkey) + + for rpm in unsigned_rpms: + print(rpm) + for srpm in unsigned_srpms: + print(srpm) + + +if __name__ == '__main__': + main() diff --git a/sign_stage_rpms b/sign_stage_rpms new file mode 100755 index 0000000..5416f65 --- /dev/null +++ b/sign_stage_rpms @@ -0,0 +1,12 @@ +#!/bin/bash -e + +. settings + +echo "./list_unsigned_rpms $1 $GPGKEY" +UNSIGNED_RPMS=($(./list_unsigned_rpms "$1" "$GPGKEY")) + +for rpm_path in "${UNSIGNED_RPMS[@]}" +do + echo "Signing $rpm_path" + ./sign_rpms "$rpm_path" +done diff --git a/upload_stage_rpms b/upload_stage_rpms new file mode 100755 index 0000000..6860564 --- /dev/null +++ b/upload_stage_rpms @@ -0,0 +1,8 @@ +#!/bin/bash -e + +. settings + +USER='yumrepostage' +HOST='web01.osuosl.theforeman.org' + +rsync --archive --verbose --partial --one-file-system --delete-after "$PROJECT/$VERSION" "$USER@$HOST:rsync_cache/$PROJECT"