From c874f67c1ffe95d82487301f4be7959bfffa5d93 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Jun 25 2020 19:16:08 +0000 Subject: [PATCH 1/3] Add information for eln mass rebuild Signed-off-by: Stephen Gallagher --- diff --git a/scripts/massrebuildsinfo.py b/scripts/massrebuildsinfo.py index 7b6f729..fcb3bc6 100644 --- a/scripts/massrebuildsinfo.py +++ b/scripts/massrebuildsinfo.py @@ -23,6 +23,25 @@ PKG_SKIP_LIST = [ # keep this sorted new -> old MASSREBUILDS = { + #eln ftbfs bug tracker: 1830264 + "eln": + { + "buildtag": 'eln-rebuild', # tag to build from + "epoch": '2020-06-23 00:00:00.000000', # rebuild anything not built after this date + "module_mass_rebuild_epoch": '2020-01-28T03:30:00Z', # rebuild anything not built after this date for modules + "module_mass_branching_epoch": '2020-02-11T23:30:00Z', # rebuild anything not built after this date for modules + "module_mass_rebuild_platform": "f32", # rebuild all modules that has build time dependency on this platform, this is used during mass rebuild time + "module_mass_branching_platform": "f33", # rebuild all modules that has run time dependency on this platform, this is used during mass branching time + "targets": ['eln-candidate', 'eln'], # build targets to check for existing builds to skip rebuild + "target": 'eln-rebuild', # target to build into + "desttag": 'eln', # Tag where fixed builds go + "product": "Fedora", # for BZ product field + "version": "rawhide", # for BZ version field, rawhide before branching or xx after branching + "rawhide_version": "32", # for next version calculation and other comments + "tracking_bug": "1830264", # Tracking bug for mass build failures + "wikipage": "ELN Import", + "pkg_skip_list": PKG_SKIP_LIST, # packages to skip in rebuild + }, #f33 ftbfs bug tracker: 1803234 "f32": { From c8fc13a79a4b132e7328396e17c3cd8da471a557 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Jun 25 2020 19:16:09 +0000 Subject: [PATCH 2/3] mass-rebuild.py: Add argument processing to simplify usage Signed-off-by: Stephen Gallagher --- diff --git a/scripts/mass-rebuild.py b/scripts/mass-rebuild.py index 0cb222c..ccad8de 100755 --- a/scripts/mass-rebuild.py +++ b/scripts/mass-rebuild.py @@ -15,17 +15,15 @@ import os import subprocess import sys import operator +import click -# contains info about all rebuilds, add new rebuilds there and update rebuildid -# here +# contains info about all rebuilds, add new rebuilds there from massrebuildsinfo import MASSREBUILDS # Set some variables # Some of these could arguably be passed in as args. -rebuildid = 'f32' -massrebuild = MASSREBUILDS[rebuildid] -user = 'Fedora Release Engineering ' -comment = '- Rebuilt for ' + massrebuild['wikipage'] +git_user = 'Fedora Release Engineering' +git_email = 'releng@fedoraproject.org' workdir = os.path.expanduser('~/massbuild') enviro = os.environ @@ -65,116 +63,152 @@ def runmeoutput(cmd, action, pkg, env, cwd=workdir): return result -# Environment for using releng credentials for pushing and building -enviro['GIT_SSH'] = '/usr/local/bin/relengpush' -koji_bin = '/usr/bin/compose-koji' - -# Create a koji session -kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub') - -# Generate a list of packages to iterate over -pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True) - -# reduce the list to those that are not blocked and sort by package name -pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']], - key=operator.itemgetter('package_name')) - -print('Checking %s packages...' % len(pkgs)) - -# Loop over each package -for pkg in pkgs: - name = pkg['package_name'] - id = pkg['package_id'] - - # some package we just dont want to ever rebuild - if name in massrebuild['pkg_skip_list']: - print('Skipping %s, package is explicitely skipped') - continue - - # Query to see if a build has already been attempted - # this version requires newer koji: - builds = kojisession.listBuilds(id, createdAfter=massrebuild['epoch']) - newbuild = False - # Check the builds to make sure they were for the target we care about - for build in builds: - try: - buildtarget = kojisession.getTaskInfo(build['task_id'], - request=True)['request'][1] - if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']: - # We've already got an attempt made, skip. - newbuild = True - break - except: - print('Skipping %s, no taskinfo.' % name) +@click.command() +@click.option('--bump-spec/--no-bump-spec', + help='Whether to run rpmdev-bumpspec before building', + default=True, + show_default=True) +@click.option('--created-after', + type=click.DateTime(), + help='Rebuild any package not built after this date. ' + 'If not specified, use the value of "epoch" from massrebuildsinfo.py', + ) +@click.option('--fas-user', + help='The name of the FAS user account to run as', + show_default=True, + default='releng') +@click.option('--rebuild-id', + type=click.Choice(MASSREBUILDS.keys()), + help='The name of the release to build for', + default=sorted(MASSREBUILDS.keys())[0], + show_default=True) +@click.argument('infile', + type=click.File('r')) +def main(bump_spec, created_after, fas_user, rebuild_id, infile): + """Mass-rebuild all package listed in INFILE""" + massrebuild = MASSREBUILDS[rebuild_id] + git_comment = '- Rebuilt for ' + massrebuild['wikipage'] + + if created_after: + massrebuild['epoch'] = created_after.strftime('%Y-%m-%d %H:%M:%S.000000') + + massrebuild['pkg_include_list'] = list() + for package in infile: + massrebuild['pkg_include_list'].append(package.strip()) + + if fas_user == 'releng': + # Environment for using releng credentials for pushing and building + enviro['GIT_SSH'] = '/usr/local/bin/relengpush' + + # Create a koji session + kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub') + + # Generate a list of packages to iterate over + pkgs = kojisession.listPackages(massrebuild['buildtag'], inherited=True) + + # reduce the list to those that are not blocked and sort by package name + pkgs = sorted([pkg for pkg in pkgs if not pkg['blocked']], + key=operator.itemgetter('package_name')) + + print('Checking %s packages...' % len(pkgs)) + + # Loop over each package + for pkg in pkgs: + name = pkg['package_name'] + pkg_id = pkg['package_id'] + + # some package we just dont want to ever rebuild + if name not in massrebuild['pkg_include_list']: continue - if newbuild: - print('Skipping %s, already attempted.' % name) - continue - - # Check out git - fedpkgcmd = ['fedpkg', '--user', 'releng', 'clone', '--branch', 'master', name] - print('Checking out %s' % name) - if runme(fedpkgcmd, 'fedpkg', name, enviro): - continue - - # Check for a checkout - if not os.path.exists(os.path.join(workdir, name)): - sys.stderr.write('%s failed checkout.\n' % name) - continue - - # Check for a noautobuild file - if os.path.exists(os.path.join(workdir, name, 'noautobuild')): - # Maintainer does not want us to auto build. - print('Skipping %s due to opt-out' % name) - continue - - # Find the spec file - files = os.listdir(os.path.join(workdir, name)) - spec = '' - for file in files: - if file.endswith('.spec'): - spec = os.path.join(workdir, name, file) - break - - if not spec: - sys.stderr.write('%s failed spec check\n' % name) - continue - - # rpmdev-bumpspec - bumpspec = ['rpmdev-bumpspec', '-u', user, '-c', comment, - os.path.join(workdir, name, spec)] - print('Bumping %s' % spec) - if runme(bumpspec, 'bumpspec', name, enviro): - continue - - # Set the git user.name and user.email - set_name = ['git', 'config', 'user.name', 'Fedora Release Engineering'] - set_mail = ['git', 'config', 'user.email', 'releng@fedoraproject.org'] - print('Setting git user.name and user.email') - if runme(set_name, 'set_name', name, enviro, - cwd=os.path.join(workdir, name)): - continue - if runme(set_mail, 'set_mail', name, enviro, - cwd=os.path.join(workdir, name)): - continue - - # git commit - commit = ['fedpkg', 'commit', '-s', '-p', '-m', comment] - print('Committing changes for %s' % name) - if runme(commit, 'commit', name, enviro, - cwd=os.path.join(workdir, name)): - continue - - # get git url - urlcmd = ['fedpkg', 'giturl'] - print('Getting git url for %s' % name) - url = runmeoutput(urlcmd, 'giturl', name, enviro, - cwd=os.path.join(workdir, name)) - if not url: - continue - - # build - build = [koji_bin, 'build', '--nowait', '--background', massrebuild['target'], url] - print('Building %s' % name) - runme(build, 'build', name, enviro, - cwd=os.path.join(workdir, name)) + + # Query to see if a build has already been attempted + # this version requires newer koji: + builds = kojisession.listBuilds(pkg_id, createdAfter=massrebuild['epoch']) + newbuild = False + # Check the builds to make sure they were for the target we care about + for build in builds: + try: + buildtarget = kojisession.getTaskInfo(build['task_id'], + request=True)['request'][1] + if buildtarget == massrebuild['target'] or buildtarget in massrebuild['targets']: + # We've already got an attempt made, skip. + newbuild = True + break + except: + print('Skipping %s, no taskinfo.' % name) + continue + if newbuild: + print('Skipping %s, already attempted.' % name) + continue + + # Check out git + fedpkgcmd = ['fedpkg', '--user', fas_user, 'clone', '--branch', 'master', name] + print('Checking out %s' % name) + if runme(fedpkgcmd, 'fedpkg', name, enviro): + continue + + # Check for a checkout + if not os.path.exists(os.path.join(workdir, name)): + sys.stderr.write('%s failed checkout.\n' % name) + continue + + # Check for a noautobuild file + if os.path.exists(os.path.join(workdir, name, 'noautobuild')): + # Maintainer does not want us to auto build. + print('Skipping %s due to opt-out' % name) + continue + + if bump_spec: + # Find the spec file + files = os.listdir(os.path.join(workdir, name)) + spec = '' + for file in files: + if file.endswith('.spec'): + spec = os.path.join(workdir, name, file) + break + + if not spec: + sys.stderr.write('%s failed spec check\n' % name) + continue + + # rpmdev-bumpspec + bumpspec = ['rpmdev-bumpspec', '-u', user, '-c', comment, + os.path.join(workdir, name, spec)] + print('Bumping %s' % spec) + if runme(bumpspec, 'bumpspec', name, enviro): + continue + + # Set the git user.name and user.email + set_name = ['git', 'config', 'user.name', git_user] + set_mail = ['git', 'config', 'user.email', git_email] + print('Setting git user.name and user.email') + if runme(set_name, 'set_name', name, enviro, + cwd=os.path.join(workdir, name)): + continue + if runme(set_mail, 'set_mail', name, enviro, + cwd=os.path.join(workdir, name)): + continue + + # git commit + commit = ['fedpkg', 'commit', '-s', '-p', '-m', git_comment] + print('Committing changes for %s' % name) + if runme(commit, 'commit', name, enviro, + cwd=os.path.join(workdir, name)): + continue + + # get git url + urlcmd = ['fedpkg', 'giturl'] + print('Getting git url for %s' % name) + url = runmeoutput(urlcmd, 'giturl', name, enviro, + cwd=os.path.join(workdir, name)) + if not url: + continue + + # build + build = [koji_bin, 'build', '--nowait', '--background', massrebuild['target'], url] + print('Building %s' % name) + runme(build, 'build', name, enviro, + cwd=os.path.join(workdir, name)) + +if __name__ == '__main__': + main() From 913a11c5fa66886424e00ed644bab31dc56284cb Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Jun 26 2020 16:46:50 +0000 Subject: [PATCH 3/3] fixup! mass-rebuild.py: Add argument processing to simplify usage Use the regular koji commandline tool unless running as the releng user Signed-off-by: Stephen Gallagher --- diff --git a/scripts/mass-rebuild.py b/scripts/mass-rebuild.py index ccad8de..4dd9411 100755 --- a/scripts/mass-rebuild.py +++ b/scripts/mass-rebuild.py @@ -99,6 +99,9 @@ def main(bump_spec, created_after, fas_user, rebuild_id, infile): if fas_user == 'releng': # Environment for using releng credentials for pushing and building enviro['GIT_SSH'] = '/usr/local/bin/relengpush' + koji_bin = '/usr/bin/compose-koji' + else: + koji_bin = '/usr/bin/koji' # Create a koji session kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')