#7881 Add a script to send reminders to FTBFs bugzillas
Merged 5 years ago by kevin. Opened 5 years ago by churchyard.
churchyard/releng ftbfs_bugzilla  into  master

@@ -0,0 +1,135 @@ 

+ import configparser

+ import bugzilla

+ import pathlib

+ import sys

+ from click import progressbar

+ import logging

+ 

+ import massrebuildsinfo

+ 

+ assert sys.version_info[0] > 2, 'Needs Python 3'

+ 

+ LOGGER = logging.getLogger()

+ LOGGER.setLevel(logging.DEBUG)

+ fh = logging.FileHandler('bugzilla.log')

+ fh.setLevel(logging.DEBUG)

+ LOGGER.addHandler(fh)

+ 

+ 

+ # get the latest Fedora version and tracking bug

+ for key, value in massrebuildsinfo.MASSREBUILDS.items():

+     fedora = key.upper()

+     tracking = value['tracking_bug']

+     break

+ 

+ # Get credentials from config

+ config_path = pathlib.Path('./ftbfs.cfg')

+ if not config_path.exists():

+     config_path = pathlib.Path('/etc/ftbfs.cfg')

+ if not config_path.exists():

+     raise RuntimeError('Create a config file as ./ftbfs.cfg or /etc/ftbfs.cfg')

+ 

+ config = configparser.ConfigParser()

+ config.read(config_path)

+ 

+ URL = config['bugzilla'].get('url', 'https://bugzilla.redhat.com')

+ USERNAME = config['bugzilla']['username']

+ PASSWORD = config['bugzilla']['password']

+ FEDORA = config['bugzilla'].get('fedora', fedora)

+ TRACKING = config['bugzilla'].get('tracking', tracking)

+ 

+ 

+ TEMPLATE = f"""Dear Maintainer,

+ 

+ your package has not been built successfully in {FEDORA}. Action is required from you.

+ 

+ If you can fix your package to build, perform a build in koji, and either create

+ an update in bodhi, or close this bug without creating an update, if updating is

+ not appropriate [1]. If you are working on a fix, set the status to ASSIGNED to

+ acknowledge this. Following the latest policy for such packages [2], your package

+ can be orphaned if this bug remains in NEW state more than 8 weeks.

+ 

+ [1] https://fedoraproject.org/wiki/Updates_Policy

+ [2] https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/

+ """  # noqa

+ 

+ cache_dir = pathlib.Path('~/.cache/FTBFS_weekly_reminder/').expanduser()

+ cache_dir.mkdir(exist_ok=True)

+ ALREADY_FILED = cache_dir / 'ALREADY_FILED'

+ 

+ bzapi = bugzilla.Bugzilla(URL, user=USERNAME, password=PASSWORD)

+ 

+ failed = []

+ updated = []

+ 

+ 

+ def new_ftbfs_bugz(tracker=TRACKING, version='29'):

+     query = bzapi.build_query(product='Fedora', status='NEW', version=version)

+     query['blocks'] = tracker

+     return bzapi.query(query)

+ 

+ 

+ def needinfo(requestee):

+     return {

+         'name': 'needinfo',

+         'requestee': requestee,

+         'status': '?',

+     }

+ 

+ 

+ def send_reminder(bug, comment=TEMPLATE):

+     flags = [needinfo(bug.assigned_to)]

+     update = bzapi.build_update(comment=comment, flags=flags)

+     try:

+         bzapi.update_bugs([bug.id], update)

+     except Exception:

+         LOGGER.exception(bug.weburl)

+         failed.append(bug)

+     else:

+         updated.append(bug)

+         with open(ALREADY_FILED, 'a') as f:

+             print(bug.id, file=f)

+ 

+ 

+ if ALREADY_FILED.exists():

+     print(f'Loading bug IDs from {ALREADY_FILED}. Will not fill those. '

+           f'Remove {ALREADY_FILED} to stop this from happening.')

+     ignore = [

+         int(line.rstrip()) for line in ALREADY_FILED.read_text().splitlines()

+     ]

+ else:

+     ignore = []

+ 

+ 

+ print('Gathering bugz, this can take a while...')

+ 

+ bugz = new_ftbfs_bugz()

+ 

+ print(f'There are {len(bugz)} NEW bugz, will send a reminder')

+ if ignore:

+     print(f'Will ignore {len(ignore)} bugz from {ALREADY_FILED}')

+     print(f'Will update {len(set(b.id for b in bugz) - set(ignore))} bugz')

+ 

+ 

+ def _item_show_func(bug):

+     if bug is None:

+         return 'Finished!'

+     return bug.weburl

+ 

+ 

+ with progressbar(bugz, item_show_func=_item_show_func) as bugbar:

+     for bug in bugbar:

+         if bug.id not in ignore:

+             send_reminder(bug)

+ 

+ print(f'Updated {len(updated)} bugz')

+ 

+ if failed:

+     print(f'Failed to update {len(failed)} bugz', file=sys.stderr)

+     for bug in failed:

+         print(bug.weburl, file=sys.stderr)

+     sys.exit(1)

+ elif ALREADY_FILED.exists():

+     target = ALREADY_FILED.parent / f'~{ALREADY_FILED.name}'

+     print(f'Moving {ALREADY_FILED} to {target}, all bugz filed')

+     ALREADY_FILED.rename(target)

@@ -20,6 +20,8 @@ 

      'shim-unsigned-x64',

  ]

  

+ 

+ # keep this sorted new -> old

  MASSREBUILDS = {

      "f30":

      {

rebased onto ac82cf04ddc1b7c27febdd6049d21e8ba73236d9

5 years ago

Can these be parameterized as CLI arguments?

I'd rather not assume a particular tracker by default.

They certainly can. However this was supposed to be a N-time-off script (small N), not a well crafted tool for multiple purposes.

The tracking bug info is available from https://pagure.io/releng/blob/master/f/scripts/massrebuildsinfo.py - it would be best to query that for the latest rebuild and corresponding tracker bug.

It wouldbe better to use full URLs, such as https://bugzilla.redhat.com to ensure that HTTPS is used.

The Fedora version here should be taken from massrebuildsinfo as well to ensure it stays up-to-date.

IMHO it would be better to store this in ~/.cache/FTBFS_weekly_reminder/....In general it would be better to query the actual bug to see if a new reminder needs to be sent to ensure that it can run in cron. Otherwise it can only run once AFAICS.

Also it would be ALREADY_FILED from to file and not from to fill I suppose.

Thanks @till - I'll work on that, except querying the bug (that's slow and error prone I guess), the purpose of the file is to recover from failure, not to store it between individual weekly runs. E.g. once the bugs are filed (= the script exits cleanly), it is safe to delete the file.

rebased onto ff76ff0781d3b8d2f0420bd42e94f728de7cb9e7

5 years ago

1 new commit added

  • FTBFS weekly reminder: Exit with 1 on failed bugz
5 years ago

Rebased. Added individual commits.

Now the cache file stays as long as there was a problem or force exit (^C etc.), but is removed when everything went well.

Should be fine to run this in weekly cron, and if the log suggests and error, re-run it.

What remains to be done?

rebased onto 077a60c122f19b590dbc3f01cc81551e005a3cce

5 years ago

Can you rebase again. And poke me on irc to merge after you do so we don't wait 2 more months. ;)

rebased onto ef5240816785af0ea7a99f6347b7c0bc69516a1e

5 years ago

rebased onto 8f72c24

5 years ago

Can you rebase again. And poke me on irc to merge after you do so we don't wait 2 more months. ;)

Since a few releases Pagure let's you rebase before merging. If you click on the Merge button you will have the choice to do a rebase, then once the rebase is done you proceed with the merge :fireworks:

BTW I've changed the text to match recent policy update and Iv'e squashed the commits.

@cverna yeah, it wasn't working for some reason here... not sure why. ;(

@churchyard nice signed-of-by line.

Lets merge and iterate from here. Thanks so much for making this!

Pull-Request has been merged by kevin

5 years ago

@kevin Thanks for merging \o/

I appreciate that you've noticed the signed-of-by line.

Can you explain the cfg format a bit?

I have:

[bugzilla]
username=releng@fedoraproject.org
password=thepasswordnotit
tracking=1674516
fedora=f30

Is that right? That gives me:

Gathering bugz, this can take a while...
There are 0 NEW bugz, will send a reminder
[####################################] 100% Finished!
Updated 0 bugz

There seems to be a missing argument to bugz = new_ftbfs_bugz() in the script, that should be:

bugz = new_ftbfs_bugz(FEDORA.lstrip('f'))

Given that most of the bugs are still on rawhide, in fact, we should probably not include version in the query.

Try changing:

def new_ftbfs_bugz(tracker=TRACKING, version='29'):
    query = bzapi.build_query(product='Fedora', status='NEW', version=version)
    query['blocks'] = tracker
    return bzapi.query(query)

To:

def new_ftbfs_bugz(tracker=TRACKING):
    query = bzapi.build_query(product='Fedora', status='NEW')
    query['blocks'] = tracker
    return bzapi.query(query)

instead.

Yes, that worked. ;) It's processing away now...

Thanks!

I see some of the comments! Thanks.

Please, don't forget to commit and push the fix.