| |
@@ -25,7 +25,7 @@
|
| |
|
| |
"""A script for the new package review tickets page generation."""
|
| |
|
| |
- from datetime import datetime
|
| |
+ from datetime import datetime, timedelta
|
| |
from jinja2 import Environment, FileSystemLoader
|
| |
from pkg_resources import resource_filename
|
| |
import click
|
| |
@@ -33,37 +33,34 @@
|
| |
import sys
|
| |
import time
|
| |
|
| |
- from review_stats import log, __version__ as version
|
| |
+ from review_stats import comments, log, __version__ as version
|
| |
from review_stats.logging import setup_logging
|
| |
- from review_stats.utils import (
|
| |
- bug_time, bz_connect, copy_static_content, generate_page, get_bz_baseurl,
|
| |
- get_hidden, get_in_progress, get_reviewable, get_reviewable_epel, get_reviewers,
|
| |
- get_top_blockers, get_top_submitters, order_by_month, run_query)
|
| |
+ from review_stats import utils
|
| |
|
| |
|
| |
- @click.command()
|
| |
+ @click.group()
|
| |
@click.option('-c', '--config', 'configfile', type=click.File(),
|
| |
default='/etc/review-stats.cfg', show_default=True)
|
| |
- @click.option('-t', '--templatedir', 'templdir', type=click.Path(exists=True),
|
| |
- default=resource_filename('review_stats', 'templates/'), show_default=True)
|
| |
- @click.option('-s', '--staticfilesdir', 'staticdir', type=click.Path(exists=True),
|
| |
- default=resource_filename('review_stats', 'static/'), show_default=True)
|
| |
- @click.option('-d', '--destination', 'dirname', type=click.Path(exists=True, writable=True),
|
| |
- required=True)
|
| |
@click.option('-l', '--enable-systemd-log', 'systemd_log', is_flag=True)
|
| |
@click.option('-v', '--verbose', 'loglevel', flag_value='INFO')
|
| |
@click.option('-D', '--debug', 'loglevel', flag_value='DEBUG')
|
| |
- def make_page(configfile, templdir, staticdir, dirname, systemd_log, loglevel):
|
| |
- """Builds Fedora's new package review tickets page."""
|
| |
+ @click.pass_context
|
| |
+ def cli(ctx, configfile, systemd_log, loglevel):
|
| |
+ """Setup common environment for commands."""
|
| |
+ ctx.ensure_object(dict)
|
| |
+
|
| |
loglevel = loglevel or 'WARNING'
|
| |
+ ctx.obj['loglevel'] = loglevel
|
| |
|
| |
config = configparser.ConfigParser()
|
| |
config.read_file(configfile)
|
| |
+ ctx.obj['config'] = config
|
| |
|
| |
setup_logging(loglevel, config, systemd_log)
|
| |
|
| |
try:
|
| |
- bz = bz_connect(config)
|
| |
+ bz = utils.bz_connect(config)
|
| |
+ ctx.obj['bz'] = bz
|
| |
except configparser.NoOptionError as ex:
|
| |
log.critical(f'Missing required value in configuration file: {ex.section}:{ex.option}')
|
| |
sys.exit(1)
|
| |
@@ -72,111 +69,163 @@
|
| |
sys.exit(1)
|
| |
|
| |
try:
|
| |
- bz_tickets = run_query(bz)
|
| |
+ bz_tickets = utils.run_query(bz)
|
| |
+ ctx.obj['bz_tickets'] = bz_tickets
|
| |
except Exception:
|
| |
log.critical('Something went wrong while fetching data from Bugzilla. '
|
| |
'I can\'t proceed further.', exc_info=1)
|
| |
sys.exit(1)
|
| |
|
| |
+
|
| |
+ @cli.command()
|
| |
+ @click.option('-t', '--templatedir', 'templdir', type=click.Path(exists=True),
|
| |
+ default=resource_filename('review_stats', 'templates/'), show_default=True)
|
| |
+ @click.option('-s', '--staticfilesdir', 'staticdir', type=click.Path(exists=True),
|
| |
+ default=resource_filename('review_stats', 'static/'), show_default=True)
|
| |
+ @click.option('-d', '--destination', 'dirname', type=click.Path(exists=True, writable=True),
|
| |
+ required=True)
|
| |
+ @click.pass_context
|
| |
+ def make_pages(ctx, templdir, staticdir, dirname):
|
| |
+ """Build Fedora's new package review tickets pages."""
|
| |
env = Environment(loader=FileSystemLoader(templdir), autoescape=True)
|
| |
env.globals['now'] = datetime.utcnow
|
| |
env.globals['version'] = version
|
| |
- env.globals['bz_baseurl'] = get_bz_baseurl(config)
|
| |
- env.filters['bugtime'] = bug_time
|
| |
+ env.globals['bz_baseurl'] = utils.get_bz_baseurl(ctx.obj['config'])
|
| |
+ env.filters['bugtime'] = utils.bug_time
|
| |
|
| |
log.info('Starting HTML page rendering...')
|
| |
counters = {}
|
| |
t = time.time()
|
| |
|
| |
+ bz_tickets = ctx.obj['bz_tickets']
|
| |
+
|
| |
# Reviewable Fedora tickets
|
| |
- new_tickets = get_reviewable(bz_tickets)
|
| |
+ new_tickets = utils.get_reviewable(ctx.obj['bz_tickets'])
|
| |
counters['reviewable'] = len(new_tickets)
|
| |
- generate_page(env, 'by_month.html', dirname, 'reviewable.html',
|
| |
+ utils.generate_page(env, 'by_month.html', dirname, 'reviewable.html',
|
| |
page_title='New, reviewable Fedora package review tickets',
|
| |
- months=order_by_month(new_tickets),
|
| |
+ months=utils.order_by_month(new_tickets),
|
| |
count=counters['reviewable'])
|
| |
|
| |
# Reviewable EPEL tickets
|
| |
- new_epel_tickets = get_reviewable_epel(bz_tickets)
|
| |
+ new_epel_tickets = utils.get_reviewable_epel(ctx.obj['bz_tickets'])
|
| |
counters['reviewable_epel'] = len(new_epel_tickets)
|
| |
- generate_page(env, 'by_month.html', dirname, 'reviewable_epel.html',
|
| |
+ utils.generate_page(env, 'by_month.html', dirname, 'reviewable_epel.html',
|
| |
page_title='New, reviewable EPEL only package review tickets',
|
| |
- months=order_by_month(new_epel_tickets),
|
| |
+ months=utils.order_by_month(new_epel_tickets),
|
| |
count=counters['reviewable_epel'])
|
| |
|
| |
# Reviews in progress
|
| |
- in_progress_tickets = get_in_progress(bz_tickets)
|
| |
+ in_progress_tickets = utils.get_in_progress(ctx.obj['bz_tickets'])
|
| |
counters['in_progress'] = len(in_progress_tickets)
|
| |
- generate_page(env, 'in_progress.html', dirname, 'in_progress.html',
|
| |
+ utils.generate_page(env, 'in_progress.html', dirname, 'in_progress.html',
|
| |
page_title='All tickets currently under review',
|
| |
tickets=sorted(in_progress_tickets, key=lambda t: t.last_modified),
|
| |
count=counters['in_progress'])
|
| |
|
| |
# Hidden tickets
|
| |
- hidden_tickets = get_hidden(bz_tickets)
|
| |
+ hidden_tickets = utils.get_hidden(ctx.obj['bz_tickets'])
|
| |
counters['hidden'] = len(hidden_tickets)
|
| |
- generate_page(env, 'hidden.html', dirname, 'hidden.html',
|
| |
+ utils.generate_page(env, 'hidden.html', dirname, 'hidden.html',
|
| |
page_title='All tickets hidden from the main review queues',
|
| |
- months=order_by_month(hidden_tickets),
|
| |
+ months=utils.order_by_month(hidden_tickets),
|
| |
count=counters['hidden'])
|
| |
|
| |
# Inconsistent tickets
|
| |
- inconsistent_tickets = [tk for tk in bz_tickets if tk.has_inconsistent_state]
|
| |
+ inconsistent_tickets = [tk for tk in ctx.obj['bz_tickets'] if tk.has_inconsistent_state]
|
| |
counters['inconsistent'] = len(inconsistent_tickets)
|
| |
- generate_page(env, 'plain.html', dirname, 'inconsistent.html',
|
| |
+ utils.generate_page(env, 'plain.html', dirname, 'inconsistent.html',
|
| |
page_title='All review tickets which are in limbo',
|
| |
tickets=sorted(inconsistent_tickets, key=lambda t: t.last_modified),
|
| |
count=counters['inconsistent'])
|
| |
|
| |
# Needsponsor tickets
|
| |
# Some of them are in the in progress queue, but still waiting for a sponsor
|
| |
- needsponsor_tickets = [tk for tk in bz_tickets if not tk.hidden and tk.needsponsor]
|
| |
+ needsponsor_tickets = [tk for tk in ctx.obj['bz_tickets'] if not tk.hidden and tk.needsponsor]
|
| |
counters['needsponsor'] = len(needsponsor_tickets)
|
| |
- generate_page(env, 'plain.html', dirname, 'needsponsor.html',
|
| |
+ utils.generate_page(env, 'plain.html', dirname, 'needsponsor.html',
|
| |
page_title='All review tickets where a sponsor is required',
|
| |
tickets=sorted(needsponsor_tickets, key=lambda t: t.last_modified),
|
| |
count=counters['needsponsor'])
|
| |
|
| |
# Trivial tickets
|
| |
- trivial_tickets = [tk for tk in get_reviewable(bz_tickets) + get_reviewable_epel(bz_tickets)
|
| |
+ trivial_tickets = [tk for tk in utils.get_reviewable(ctx.obj['bz_tickets'])
|
| |
+ + utils.get_reviewable_epel(ctx.obj['bz_tickets'])
|
| |
if not tk.hidden and tk.trivial]
|
| |
counters['trivial'] = len(trivial_tickets)
|
| |
- generate_page(env, 'by_month.html', dirname, 'trivial.html',
|
| |
+ utils.generate_page(env, 'by_month.html', dirname, 'trivial.html',
|
| |
page_title='All tickets marked as trivial',
|
| |
- months=order_by_month(trivial_tickets),
|
| |
+ months=utils.order_by_month(trivial_tickets),
|
| |
count=counters['trivial'])
|
| |
|
| |
# Top blockers
|
| |
- top_blockers = get_top_blockers(bz_tickets)
|
| |
+ top_blockers = utils.get_top_blockers(ctx.obj['bz_tickets'])
|
| |
counters['blockers'] = len(top_blockers)
|
| |
- generate_page(env, 'blockers.html', dirname, 'blockers.html',
|
| |
+ utils.generate_page(env, 'blockers.html', dirname, 'blockers.html',
|
| |
page_title='All tickets which block other tickets',
|
| |
blockers=top_blockers,
|
| |
count=counters['blockers'])
|
| |
|
| |
# Top submitters
|
| |
- top_submitters = get_top_submitters(bz_tickets)
|
| |
+ top_submitters = utils.get_top_submitters(ctx.obj['bz_tickets'])
|
| |
counters['submitters'] = len(top_submitters)
|
| |
- generate_page(env, 'by_user.html', dirname, 'submitters.html',
|
| |
+ utils.generate_page(env, 'by_user.html', dirname, 'submitters.html',
|
| |
page_title='All review tickets sorted by submitter',
|
| |
users=top_submitters,
|
| |
count=counters['submitters'])
|
| |
|
| |
# Reviewers
|
| |
- reviewers = get_reviewers(bz_tickets)
|
| |
+ reviewers = utils.get_reviewers(ctx.obj['bz_tickets'])
|
| |
counters['reviewers'] = len(reviewers)
|
| |
- generate_page(env, 'by_user.html', dirname, 'reviewers.html',
|
| |
+ utils.generate_page(env, 'by_user.html', dirname, 'reviewers.html',
|
| |
page_title='Review tickets sorted by assigned reviewer',
|
| |
users=reviewers,
|
| |
count=counters['reviewers'])
|
| |
|
| |
# Finally, generate the index page
|
| |
- generate_page(env, 'index.html', dirname, 'index.html',
|
| |
+ utils.generate_page(env, 'index.html', dirname, 'index.html',
|
| |
counters=counters)
|
| |
log.info(f'Done HTML rendering, took {time.time() - t:.2f}s.')
|
| |
|
| |
- copy_static_content(staticdir, dirname)
|
| |
+ utils.copy_static_content(staticdir, dirname)
|
| |
+
|
| |
+
|
| |
+ @cli.command()
|
| |
+ @click.option('-d', '--dry-run', 'dry', is_flag=True)
|
| |
+ @click.pass_context
|
| |
+ def work_on_bugs(ctx, dry):
|
| |
+ """Operate on Bugzilla tickets."""
|
| |
+ # Request info for tickets which were not recently updated
|
| |
+ worker_config = ctx.obj['config']['review-stats-worker']
|
| |
+ not_updated_days = timedelta(days=int(worker_config['not_updated_days']))
|
| |
+ for tk in utils.get_not_recently_updated(ctx.obj['bz_tickets'], not_updated_days):
|
| |
+ if not tk.review_flag_status or (tk.review_flag_status == '?' and not tk.really_assigned):
|
| |
+ # No one is reviewing or review in progress but no assignee set
|
| |
+ log.debug(f'Setting NEEDINFO from submitter for {tk.bug.id}.')
|
| |
+ if not dry:
|
| |
+ utils.set_needinfo(ctx.obj['bz'], tk, tk.bug.creator,
|
| |
+ comments['needinfo_submitter'])
|
| |
+ elif tk.review_flag_status == '?' and tk.really_assigned:
|
| |
+ # Review in progress and we're sure to have an assignee
|
| |
+ log.debug(f'Setting NEEDINFO from reviewer for {tk.bug.id}.')
|
| |
+ if not dry:
|
| |
+ utils.set_needinfo(ctx.obj['bz'], tk, tk.bug.assigned_to,
|
| |
+ comments['needinfo_reviewer'])
|
| |
+
|
| |
+ not_approved_tickets = [tk for tk in ctx.obj['bz_tickets'] if tk.review_flag_status != '+']
|
| |
+ needinfo_waiting_days = timedelta(days=int(worker_config['needinfo_waiting_days']))
|
| |
+ # Close tickets with unanswered NEEDINFO from submitter
|
| |
+ for tk in utils.get_needinfo_since(not_approved_tickets, needinfo_waiting_days, 'submitter'):
|
| |
+ log.debug(f'Closing ticket {tk.bug.id} as dead review.')
|
| |
+ if not dry:
|
| |
+ utils.close_needinfo(ctx.obj['bz'], tk)
|
| |
+
|
| |
+ # Reset tickets with unanswered NEEDINFO from reviewer
|
| |
+ for tk in utils.get_needinfo_since(not_approved_tickets, needinfo_waiting_days, 'reviewer'):
|
| |
+ log.debug(f'Resetting ticket {tk.bug.id}.')
|
| |
+ if not dry:
|
| |
+ utils.reset_needinfo(ctx.obj['bz'], tk)
|
| |
|
| |
|
| |
if __name__ == '__main__':
|
| |
- make_page()
|
| |
+ cli()
|
| |
As per discussion on https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/R2I6GPYRMSUNV44VBILGIJUB52E2ABMW/ this PR will make review_stats able to operate on stale review tickets.
The cli command responsible to generate webpages has been transferred to a subcommand (
make-pages
) with its options, while the common options stay with the main command.A new subcommand
work-on-bugs
will operate on stale tickets. Currently it does:Things to do along with (or after, since the html page generation should still work) this code change:
[ ] The PR already modify the docker entrypoint for the make-pages command to run. We will need to add another entrypoint for the work-on-bugs subcommand and set a daily cronjob for that.
[X] We also need the package-review user to be able to operate on bugs, since it currently hasn't got any privilege for that (I suppose I need to open a request on Infra).
[ ] It would be nice to see logs produced by the script, but in Openshift it seems the logs are deleted just after the run (?)
Signed-off-by: Mattia Verga mattia.verga@protonmail.com