From a729fbbbd2e6c98f58385d7fd87a8f735450c62e Mon Sep 17 00:00:00 2001 From: Till Maas Date: Jul 09 2018 15:37:18 +0000 Subject: Store mass rebuild details in massrebuildsinfo.py This makes it unnecessary to maintain the same information in multiple files and allows in the future to query for old mass rebuilds easily. Signed-off-by: Till Maas --- diff --git a/scripts/find_FTBFS.py b/scripts/find_FTBFS.py index 03990e3..ca7a723 100755 --- a/scripts/find_FTBFS.py +++ b/scripts/find_FTBFS.py @@ -13,41 +13,35 @@ import argparse import operator import koji -branched_tag = 'f29' - -# Dates retrieved from mass-rebuild.py for the respective mass rebuild -f18_rebuild_start = '2012-07-17 14:18:03.000000' -f19_rebuild_start = '2013-01-24 12:37:15.000000' -f20_rebuild_start = '2013-07-25 00:00:00.000000' -f21_rebuild_start = '2014-06-06 00:00:00.000000' -# no F22 rebuild -f23_rebuild_start = '2015-06-16 00:00:00.000000' -f27_rebuild_start = '2017-07-31 11:20:00.000000' -f28_rebuild_start = '2018-02-06 01:20:06.000000' - -epoch = f27_rebuild_start +from massrebuildsinfo import MASSREBUILDS kojihub = 'https://koji.fedoraproject.org/kojihub' kojisession = koji.ClientSession(kojihub) parser = argparse.ArgumentParser() +parser.add_argument("--check-tag", default="f29", help="Tag to check") +parser.add_argument("--since-rebuild", default="f27", + help="Mass-rebuild to use as reference for cut-off date") parser.add_argument("packages", nargs="*", metavar="package", help="if specified, only check whether the specified " "packages were not rebuild") args = parser.parse_args() +massrebuild = MASSREBUILDS[args.since_rebuild] if args.packages: all_koji_pkgs = args.packages else: - all_koji_pkgs = kojisession.listPackages(branched_tag, inherited=True) + all_koji_pkgs = kojisession.listPackages(args.check_tag, inherited=True) unblocked = sorted([pkg for pkg in all_koji_pkgs if not pkg['blocked']], key=operator.itemgetter('package_name')) kojisession.multicall = True for pkg in unblocked: - kojisession.listBuilds(pkg['package_id'], state=1, createdAfter=epoch) + kojisession.listBuilds(pkg['package_id'], + state=koji.BUILD_STATES["COMPLETE"], + createdAfter=massrebuild['epoch']) builds = kojisession.multiCall() @@ -60,7 +54,7 @@ unbuilt = [x for (x, b) in name_map if b == [[]]] # remove packages that have never build, e.g. EPEL-only packages kojisession.multicall = True for pkg_name in unbuilt: - kojisession.getLatestRPMS(branched_tag, pkg_name) + kojisession.getLatestRPMS(args.check_tag, pkg_name) last_builds = kojisession.multiCall() last_builds_map = zip(unbuilt, last_builds) diff --git a/scripts/mass-rebuild.py b/scripts/mass-rebuild.py index d2cc487..1070413 100755 --- a/scripts/mass-rebuild.py +++ b/scripts/mass-rebuild.py @@ -16,20 +16,19 @@ import subprocess import sys import operator +# contains info about all rebuilds, add new rebuilds there and update rebuildid +# here +from massrebuildsinfo import MASSREBUILDS + # Set some variables # Some of these could arguably be passed in as args. -buildtag = 'f28-rebuild' # tag to build from -targets = ['f28-candidate', 'rawhide', 'f28'] # tag to build from -epoch = '2018-02-06 01:20:06.000000' # rebuild anything not built after this date +rebuildid = 'f28' +massrebuild = MASSREBUILDS[rebuildid] user = 'Fedora Release Engineering ' -comment = '- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild' +comment = '- Rebuilt for ' + massrebuild['wikipage'] workdir = os.path.expanduser('~/massbuild') enviro = os.environ -target = 'f28-rebuild' -pkg_skip_list = ['fedora-release', 'fedora-repos', 'fedora-modular-release', 'fedora-modular-repos', 'generic-release', - 'redhat-rpm-config', 'shim', 'shim-signed', 'shim-unsigned-aarch64', 'shim-unsigned-x64', 'kernel', - 'linux-firmware', 'grub2', 'openh264', 'glibc32'] # Define functions @@ -74,7 +73,7 @@ koji_bin = '/usr/bin/compose-koji' kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub') # Generate a list of packages to iterate over -pkgs = kojisession.listPackages(buildtag, inherited=True) +pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True) # reduce the list to those that are not blocked and sort by package name pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']], @@ -88,20 +87,20 @@ for pkg in pkgs: id = pkg['package_id'] # some package we just dont want to ever rebuild - if name in pkg_skip_list: + if name in massrebuild['pkg_skip_list']: print('Skipping %s, package is explicitely skipped') continue # Query to see if a build has already been attempted # this version requires newer koji: - builds = kojisession.listBuilds(id, createdAfter=epoch) + builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch']) newbuild = False # Check the builds to make sure they were for the target we care about for build in builds: try: buildtarget = kojisession.getTaskInfo(build['task_id'], request=True)['request'][1] - if buildtarget == target or buildtarget in targets: + if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']: # We've already got an attempt made, skip. newbuild = True break @@ -175,7 +174,7 @@ for pkg in pkgs: continue # build - build = [koji_bin, 'build', '--nowait', '--background', target, url] + build = [koji_bin, 'build', '--nowait', '--background', massrebuild['target'], url] print('Building %s' % name) runme(build, 'build', name, enviro, cwd=os.path.join(workdir, name)) diff --git a/scripts/mass_rebuild_file_bugs.py b/scripts/mass_rebuild_file_bugs.py index 591ef15..3b66c49 100755 --- a/scripts/mass_rebuild_file_bugs.py +++ b/scripts/mass_rebuild_file_bugs.py @@ -19,24 +19,20 @@ from bugzilla.rhbugzilla import RHBugzilla from xmlrpclib import Fault from find_failures import get_failed_builds -# Set some variables -# Some of these could arguably be passed in as args. -buildtag = 'f28-rebuild' # tag to check -desttag = 'f28' # Tag where fixed builds go -epoch = '2018-02-06 01:20:06.000000' # rebuild anything not built after this date +# contains info about all rebuilds, add new rebuilds there and update rebuildid +# here +from massrebuildsinfo import MASSREBUILDS + +rebuildid = 'f28' failures = {} # dict of owners to lists of packages that failed. failed = [] # raw list of failed packages -product = "Fedora" # for BZ product field -version = "28" # for BZ version field -tracking_bug = 1555378 # Tracking bug for mass build failures - - -def report_failure(product, component, version, summary, comment, logs): +def report_failure(tracking_bug, product, component, version, summary, comment, logs): """This function files a new bugzilla bug for component with given arguments Keyword arguments: + tracking_bug -- bug used to track failures product -- bugzilla product (usually Fedora) component -- component (package) to file bug against version -- component version to file bug for (usually rawhide for Fedora) @@ -87,17 +83,14 @@ def report_failure(product, component, version, summary, comment, logs): username = raw_input('Bugzilla username: ') bzclient.login(user=username, password=getpass.getpass()) - report_failure(product, component, version, summary, comment, logs) + report_failure(tracking_bug, product, component, version, summary, comment, logs) def get_filed_bugs(tracking_bug): """Query bugzilla if given bug has already been filed - Keyword arguments: - product -- bugzilla product (usually Fedora) - component -- component (package) to file bug against - version -- component version to file bug for (usually rawhide for Fedora) - summary -- short bug summary + arguments: + tracking_bug -- bug used to track failures """ query_data = {'blocks': tracking_bug} bzurl = 'https://bugzilla.redhat.com' @@ -111,22 +104,26 @@ def get_task_failed(kojisession, task_id): task_id of the first children that failed to build. ''' for child in kojisession.getTaskChildren(task_id): - if child['state'] == 5: # 5 == Failed + if child['state'] == koji.TASK_STATES["FAILED"]: # 5 == Failed return child['id'] if __name__ == '__main__': + massrebuild = MASSREBUILDS[rebuildid] + kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub') print('Getting the list of failed builds...') - failbuilds = get_failed_builds(kojisession, epoch, buildtag, desttag) + failbuilds = get_failed_builds(kojisession, massrebuild['epoch'], + massrebuild['buildtag'], + massrebuild['desttag']) print('Getting the list of filed bugs...') - filed_bugs = get_filed_bugs(tracking_bug) + filed_bugs = get_filed_bugs(massrebuild['tracking_bug']) filed_bugs_components = [bug.component for bug in filed_bugs] + product_version = "{product} {version}".format(**massrebuild) for build in failbuilds: - global product, version task_id = build['task_id'] component = build['package_name'] - summary = "%s: FTBFS in %s" % (component, 'F28') + summary = "%s: FTBFS in %s" % (component, product_version) work_url = 'https://kojipkgs.fedoraproject.org/work' child_id = get_task_failed(kojisession, task_id) @@ -142,17 +139,19 @@ if __name__ == '__main__': state_log = log_url + "state.log" logs = [build_log, root_log, state_log] - comment = """Your package %s failed to build from source in current F28. + comment = """Your package {component} failed to build from source in current {product_version} -https://koji.fedoraproject.org/koji/taskinfo?taskID=%s +https://koji.fedoraproject.org/koji/taskinfo?taskID={task_id} -For details on mass rebuild see https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild -""" % (component, task_id) +For details on mass rebuild see {wikipage} +""".format(component=component, task_id=task_id, + wikipage=massrebuild["wikipage"] product_version=product_version) if component not in filed_bugs_components: print("Filing bug for %s" % component) - report_failure( - product, component, version, summary, comment, logs=logs) + report_failure(massrebuild['tracking_bug'], massrebuild['product'], + component, massrebuild['version'], + summary, comment, logs=logs) filed_bugs_components.append(component) else: print("Skipping %s, bug already filed" % component) diff --git a/scripts/massrebuildsinfo.py b/scripts/massrebuildsinfo.py new file mode 100644 index 0000000..d7ae1b1 --- /dev/null +++ b/scripts/massrebuildsinfo.py @@ -0,0 +1,41 @@ +#!/usr/bin/python -tt +# vim: fileencoding=utf8 + + +PKG_SKIP_LIST = [ + 'fedora-modular-release', + 'fedora-modular-repos', + 'fedora-release', + 'fedora-repos', + 'generic-release', + 'glibc32' + 'grub2', + 'kernel', + 'linux-firmware', + 'openh264', + 'redhat-rpm-config', + 'shim', + 'shim-signed', + 'shim-unsigned-aarch64', + 'shim-unsigned-x64', +] + +MASSREBUILDS = { + "f28": + { + "buildtag": 'f28-rebuild', # tag to build from + "epoch": '2018-02-06 01:20:06.000000', # rebuild anything not built after this date + "targets": ['f28-candidate', 'rawhide', 'f28'], # build targets to check for existing builds to skip rebuild + "target": 'f28-rebuild', # target to build into + "desttag": 'f28', # Tag where fixed builds go + "product": "Fedora", # for BZ product field + "version": "28", # for BZ version field + "tracking_bug": 1555378, # Tracking bug for mass build failures + "wikipage": "https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild", + "pkg_skip_list": PKG_SKIP_LIST, # packages to skip in rebuild + }, + "f27": + { + "epoch": '2017-07-31 11:20:00.000000', + } +}