From f7c78be5f155c9bfd050be978b47219f1060b1f2 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Apr 13 2019 21:28:08 +0000 Subject: Merge #8213 `Fixes for block_retired` Signed-off-by: Kevin Fenzi --- diff --git a/scripts/block_retired.py b/scripts/block_retired.py index 75a61f6..da80318 100755 --- a/scripts/block_retired.py +++ b/scripts/block_retired.py @@ -9,7 +9,6 @@ import subprocess import time import koji -import pkgdb2client import requests @@ -17,13 +16,10 @@ log = logging.getLogger(__name__) RETIRING_BRANCHES = ["el6", "epel7", "f30", "master"] PROD_ONLY_BRANCHES = ["el6", "epel7", "f30", "master"] -PRODUCTION_PKGDB = "https://admin.fedoraproject.org/pkgdb" -STAGING_PKGDB = "https://admin.stg.fedoraproject.org/pkgdb" - PRODUCTION_PDC = "https://pdc.fedoraproject.org" STAGING_PDC = "https://pdc.stg.fedoraproject.org" -# pkgdb default namespace +# default namespace DEFAULT_NS = "rpms" @@ -127,53 +123,34 @@ def unblocked_packages(branch="master", staging=False, namespace=DEFAULT_NS): return unblocked -def get_retired_packages(branch="master", staging=False, namespace=DEFAULT_NS, - source='pkgdb'): - retiredpkgs = [] - if source == 'pkgdb': - url = PRODUCTION_PKGDB if not staging else STAGING_PKGDB - pkgdb = pkgdb2client.PkgDB(url) - - try: - retiredresponse = pkgdb.get_packages( - branches=branch, page="all", status="Retired", - namespace=namespace) - except pkgdb2client.PkgDBException as e: - if "No packages found for these parameters" not in str(e): - raise - return [] - - retiredinfo = retiredresponse["packages"] - retiredpkgs = [p["name"] for p in retiredinfo] - elif source == 'pdc': - # PDC uses singular names such as rpm and container - if namespace.endswith('s'): - content_type = namespace[:-1] - else: - content_type = namespace - url = PRODUCTION_PDC if not staging else STAGING_PDC - url = ('{0}/rest_api/v1/component-branches/?name={1}&type={2}' - '&active=false&page_size=100'.format(url, branch, content_type)) - while True: - rv = requests.get(url) - if not rv.ok: - raise RuntimeError('Failed getting the retired packages from ' - 'PDC. The response was: {0}' - .format(rv.content)) - - rv_json = rv.json() - for branch in rv_json['results']: - retiredpkgs.append(branch['global_component']) - - if rv_json['next']: - url = rv_json['next'] - else: - break +def get_active_packages(branch="master", staging=False, namespace=DEFAULT_NS): + activepkgs = [] + # PDC uses singular names such as rpm and container + if namespace.endswith('s'): + content_type = namespace[:-1] else: - raise RuntimeError('An invalid source of "{0}" was provided' - .format(source)) + content_type = namespace + url = PRODUCTION_PDC if not staging else STAGING_PDC + url = ('{0}/rest_api/v1/component-branches/?name={1}&type={2}' + '&active=true&page_size=100&fields=global_component'.format( + url, branch, content_type)) + while True: + rv = requests.get(url) + if not rv.ok: + raise RuntimeError('Failed getting the active packages from ' + 'PDC. The response was: {0}' + .format(rv.content)) + + rv_json = rv.json() + for branch in rv_json['results']: + activepkgs.append(branch['global_component']) + + if rv_json['next']: + url = rv_json['next'] + else: + break - return retiredpkgs + return activepkgs def run_koji(koji_params, staging=False): @@ -227,14 +204,14 @@ def block_package(packages, branch="master", staging=False, namespace=DEFAULT_NS def block_all_retired(branches=RETIRING_BRANCHES, staging=False, - namespace=DEFAULT_NS, source='pkgdb'): + namespace=DEFAULT_NS): for branch in branches: log.debug("Processing branch %s", branch) if staging and branch in PROD_ONLY_BRANCHES: log.warning('%s in namespace "%s" not handled in staging..' % (branch, namespace)) continue - retired = get_retired_packages(branch, staging, namespace, source) + activepkgs = get_active_packages(branch, staging, namespace) unblocked = [] # Check which packages are included in a tag but not blocked, this @@ -242,8 +219,8 @@ def block_all_retired(branches=RETIRING_BRANCHES, staging=False, # blocked. Packages might not be in the rawhide tag if they are retired # too fast, e.g. because they are EPEL-only allunblocked = unblocked_packages(branch, staging, namespace) - for pkg in retired: - if pkg in allunblocked: + for pkg in allunblocked: + if pkg not in activepkgs: unblocked.append(pkg) errors = block_package(unblocked, branch, staging=staging, namespace=namespace) @@ -301,16 +278,13 @@ if __name__ == "__main__": help="Branch to retire specified packages on, default: %(default)s") parser.add_argument( "--staging", default=False, action="store_true", - help="Talk to staging services (pkgdb/pdc), instead of production") + help="Talk to staging services instead of production") parser.add_argument( "-p", "--profile", default="koji", help="Koji profile to use, default: %(default)s (ignored with --staging)") parser.add_argument( "--namespace", default=DEFAULT_NS, - help="pkgdb/pdc namespace to use, default: %(default)s") - parser.add_argument( - "--source", default='pdc', choices=['pkgdb', 'pdc'], - help="Source for retirement information, default: %(default)s") + help="pdc namespace to use, default: %(default)s") args = parser.parse_args() setup_logging(args.debug) @@ -318,8 +292,7 @@ if __name__ == "__main__": PRODUCTION_KOJI_PROFILE = args.profile STAGING_KOJI_PROFILE = "stg" if not args.packages: - block_all_retired(staging=args.staging, namespace=args.namespace, - source=args.source) + block_all_retired(staging=args.staging, namespace=args.namespace) else: block_package(args.packages, args.branch, staging=args.staging, namespace=args.namespace)