From 0cfc12520e8b0765666b4044dcd69c91e1e606f6 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: May 08 2020 18:53:22 +0000 Subject: Move module under src/, add pyproject.toml, drop pytest-cov This moves the Python module under src/ , for reasons discussed at https://hynek.me/articles/testing-packaging/ : essentially it's good practice to *not* have the module directly importable from the top-level working directory. We update the tox (CI) configuration to work with the change, by building an sdist and testing that, rather than testing the contents of the working directory: this is a stronger test (as we ultimately deploy via sdist, so we need to make sure the sdist we build works). We also update setup.py for the change. We add pyproject.toml to express that setuptools-git is needed to build the sdist correctly, and we switch to running coverage and having it call pytest versus using pytest-cov as this is necessary for coverage diffing to work correctly with the layout change. Signed-off-by: Adam Williamson --- diff --git a/README.md b/README.md index 9dde351..1e22641 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ python-wikitcms is packaged in the official Fedora and EPEL 7+ repositories: to You can visit [the python-wikitcms project page on Pagure][7], and clone with `git clone https://pagure.io/fedora-qa/python-wikitcms.git`. Tarballs are available [from PyPI][8]. -You can also use the library directly from the git checkout if you place your code in the top-level directory, and you can copy or symlink the `wikitcms` directory into other source trees to conveniently use the latest code for development or testing purposes. +You can also use the library directly from the `src/` directory or add it to the Python import path, and you can copy or symlink the `wikitcms` directory into other source trees to conveniently use the latest code for development or testing purposes. ## Bugs, pull requests etc. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e3cc500 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["setuptools>=40.6.0", "setuptools-git", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.coverage.run] +parallel = true +branch = true +source = ["wikitcms", "src/wikitcms"] + +[tool.coverage.paths] +source = ["src", ".tox/*/site-packages"] + +[tool.coverage.report] +show_missing = true diff --git a/setup.py b/setup.py index b7fc634..cb085f3 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( keywords="fedora qa mediawiki validation", url="https://pagure.io/fedora-qa/python-wikitcms", packages=["wikitcms"], + package_dir={"": "src"}, setup_requires=[ 'setuptools_git', ], diff --git a/src/wikitcms/__init__.py b/src/wikitcms/__init__.py new file mode 100644 index 0000000..f249c07 --- /dev/null +++ b/src/wikitcms/__init__.py @@ -0,0 +1,27 @@ +# Copyright (C) Red Hat Inc. +# +# python-wikitcms is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""Library for interacting with Fedora release validation and test day +wiki pages. +""" + +from __future__ import unicode_literals +from __future__ import print_function + +__version__ = "2.5.2" + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/event.py b/src/wikitcms/event.py new file mode 100644 index 0000000..d571d39 --- /dev/null +++ b/src/wikitcms/event.py @@ -0,0 +1,404 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""Classes that describe test events.""" + +from __future__ import unicode_literals +from __future__ import print_function + +import abc +import logging + +import fedfind.helpers +import fedfind.release +import mwclient.errors +from cached_property import cached_property + +from . import listing +from . import page +from . import helpers +from .exceptions import FedfindNotFoundError + +logger = logging.getLogger(__name__) + + +class ValidationEvent(object): + """A parent class for different types of release validation event. + site must be an instance of wikitcms.Wiki, already with + appropriate access rights for any actions to be performed (i.e. + things instantiating an Event are expected to do site.login + themselves if needed). Required attributes: shortver, + category_page. If modular is True, the event will be for a + Fedora-Modular compose, with 'Modular' in the page names, category + names and so on. + """ + __metaclass__ = abc.ABCMeta + + def __init__(self, site, release, milestone='', compose='', modular=False): + self.site = site + self.release = release + self.milestone = str(milestone) + try: + self.compose = fedfind.helpers.date_check( + compose, fail_raise=True, out='str') + except ValueError: + self.compose = str(compose) + self.modular = modular + self.version = "{0} {1} {2}".format( + self.release, self.milestone, self.compose) + # Sorting helpers. sortname is a string, sorttuple is a + # 4-tuple. sorttuple is more reliable. See the function docs. + self.sortname = helpers.fedora_release_sort(self.version) + self.sorttuple = helpers.triplet_sort( + self.release, self.milestone, self.compose) + + @abc.abstractproperty + def _current_content(self): + """The content for the CurrentFedoraCompose template for + this test event. + """ + pass + + @abc.abstractproperty + def _pagetype(self): + """The ValidationPage class to be used for this event's pages + (for use by valid_pages). + """ + pass + + @abc.abstractproperty + def category_page(self): + """The category page for this event. Is a property because + page instantiation requires a remote trip. + """ + pass + + @property + def result_pages(self): + """A list of wikitcms page objects for currently-existing + pages that are a part of this test event, according to the + naming convention. + """ + _dist = "Fedora" + if self.modular: + _dist = "Fedora Modular" + pages = self.site.allresults( + prefix="{0} {1} ".format(_dist, self.version)) + return [p for p in pages if isinstance(p, page.ValidationPage)] + + @property + def download_page(self): + """The DownloadPage for this event. Is a property because page + instantiation requires a remote trip. + """ + return page.DownloadPage(self.site, self, modular=self.modular) + + @property + def ami_page(self): + """The AMIPage for this event. Is a property because page + instantiation requires a remote trip. + """ + return page.AMIPage(self.site, self, modular=self.modular) + + @property + def parent_category_page(self): + """The parent category page for this event. Is a property for + the same reason as download_page. + """ + return listing.ValidationCategory(self.site, self.release, modular=self.modular) + + @property + def valid_pages(self): + """A list of the expected possible result pages (as + page.ValidationPage objects) for this test event, derived from + the available test types and the naming convention. + """ + if self.modular: + types = self.site.modular_testtypes + else: + types = self.site.testtypes + return [self._pagetype(self.site, self.release, typ, + milestone=self.milestone, + compose=self.compose, modular=self.modular) + for typ in types] + + @property + def summary_page(self): + """The page.SummaryPage object for the event's result summary + page. Very simple property, but not set in __init__ as the + summary page object does (slow) wiki roundtrips in __init__. + """ + return page.SummaryPage(self.site, self, modular=self.modular) + + @cached_property + def ff_release(self): + """A fedfind release object matching this event.""" + # note: fedfind has a hack that parses date and respin out + # of a dot-separated compose, since wikitcms smooshes them + # into the compose value. + dist = "Fedora" + if self.modular: + dist = "Fedora-Modular" + try: + return fedfind.release.get_release(release=self.release, + milestone=self.milestone, + compose=self.compose, + dist=dist) + except ValueError as err: + try: + if self._cid: + return fedfind.release.get_release(cid=self._cid) + except AttributeError: + raise FedfindNotFoundError(err) + raise FedfindNotFoundError(err) + + @property + def ff_release_images(self): + """A fedfind release object matching this event, that has + images. If we can't find one, raise an exception. For the + base class this just acts as a check on ff_release; it does + something more clever in ComposeEvent. + """ + rel = self.ff_release + if rel.all_images: + return rel + else: + raise FedfindNotFoundError("Could not find fedfind release with images for event" + "{0}".format(self.version)) + + def update_current(self): + """Make the CurrentFedoraCompose template on the wiki point to + this event. The template is used for the Current (testtype) + Test redirect pages which let testers find the current results + pages, and for other features of wikitcms/relval. Children + must define _current_content. + """ + content = "{{tempdoc}}\n{{#switch: {{{1|full}}}\n" + content += self._current_content + content += "}}\n[[Category: Fedora Templates]]" + if self.modular: + curr = self.site.pages['Template:CurrentFedoraModularCompose'] + else: + curr = self.site.pages['Template:CurrentFedoraCompose'] + curr.save(content, "relval: update to current event", createonly=None) + + def create(self, testtypes=None, force=False, current=True, check=False): + """Create the event, by creating its validation pages, + summary page, download page, category pages, and updating the + current redirects. 'testtypes' can be an iterable that limits + creation to the specified testtypes. If 'force' is True, pages + that already exist will be recreated (destroying any results + on them). If 'current' is False, the current redirect pages + will not be updated. If 'check' is true, we check if any + result page already exists first, and bail immediately if so + (so we don't start creating pages then hit one that exists and + fail half-way through, for things that don't want that.) 'cid' + can be set to a compose ID, this is for forcing the compose + location at event creation time when we know we're not going + to be able to find it any way and is a short-term hack that + will be removed. + """ + logger.info("Creating validation event %s", self.version) + createonly = True + if force: + createonly = None + pages = self.valid_pages + if testtypes: + logger.debug("Restricting to testtypes %s", ' '.join(testtypes)) + pages = [pag for pag in pages if pag.testtype in testtypes] + if not pages: + raise ValueError("No result pages to create! Wrong test type?") + if check: + if any(pag.text() for pag in pages): + raise ValueError("A result page already exists!") + + # NOTE: download page creation for ComposeEvents will only + # work if: + # * the compose has being synced to stage, OR + # * the compose has been imported to PDC, OR + # * you used get_validation_event and passed it a cid + # Otherwise, the event will be created, but the download page + # will not. + pages.extend((self.summary_page, self.download_page, self.ami_page, + self.category_page, self.parent_category_page)) + + def _handle_existing(err): + """We need this in two places, so.""" + if err.args[0] == 'articleexists': + # no problem, just move on. + logger.info("Page already exists, and forced write was not " + "requested! Not writing.") + else: + raise err + + for pag in pages: + try: + # stage 1 - create page + logger.info("Creating page %s", pag.name) + pag.write(createonly=createonly) + except mwclient.errors.APIError as err: + _handle_existing(err) + except FedfindNotFoundError: + # this happens if download page couldn't be created + # because fedfind release couldn't be found + logger.warning("Could not create download page for event %s as fedfind release " + "was not found!") + + # stage 2 - update current. this is split so if we hit + # 'page already exists', we don't skip update_current + if current and hasattr(pag, 'update_current'): + logger.info("Pointing Current redirect to above page") + pag.update_current() + + if current: + try: + # update CurrentFedoraCompose + logger.info("Updating CurrentFedoraCompose") + self.update_current() + except mwclient.errors.APIError as err: + _handle_existing(err) + + @classmethod + def from_page(cls, pageobj): + """Return the ValidationEvent object for a given ValidationPage + object. + """ + return cls(pageobj.site, pageobj.release, pageobj.milestone, + pageobj.compose, modular=pageobj.modular) + + +class ComposeEvent(ValidationEvent): + """An Event that describes a release validation event - that is, + the testing for a particular nightly, test compose or release + candidate build. + """ + def __init__(self, site, release, milestone, compose, modular=False, cid=''): + super(ComposeEvent, self).__init__( + site, release, milestone=milestone, compose=compose, modular=modular) + # this is a little hint that gets set via get_validation_event + # when getting a page or event by cid; it helps us find the + # fedfind release for the event if the compose is not yet in + # PDC or synced to stage + self._cid = cid + self.shortver = "{0} {1}".format(self.milestone, self.compose) + + @property + def category_page(self): + """The category page for this event. Is a property because + page instantiation requires a remote trip. + """ + return listing.ValidationCategory( + self.site, self.release, self.milestone, modular=self.modular) + + @property + def _current_content(self): + """The content for the CurrentFedoraCompose template for + this test event. + """ + tmpl = ("| full = {0}\n| release = {1}\n| milestone = {2}\n" + "| compose = {3}\n| date =\n") + return tmpl.format( + self.version, self.release, self.milestone, self.compose) + + @property + def _pagetype(self): + """For a ComposeEvent, obviously, ComposePage.""" + return page.ComposePage + + @cached_property + def creation_date(self): + """We need this for ordering and determining delta between + this event and a potential new nightly event, if this is the + current event. Return creation date of the first result page + for the event, or "" if it doesn't exist. + """ + try: + return self.result_pages[0].creation_date + except IndexError: + # event doesn't exist + return "" + + @property + def ff_release_images(self): + """A fedfind release object matching this event, that has + images. If we can't find one, raise an exception. Here, we + try getting the release by (release, milestone, compose), but + if that release has no images - which happens in the specific + case that we've just created an event for a candidate compose + which has not yet been synced to stage - and we have the cid + hint, we try getting a release by cid instead, which should + find the compose in kojipkgs (a fedfind Production rather than + Compose). + """ + rel = self.ff_release + if rel.all_images: + return rel + + if self._cid: + rel = fedfind.release.get_release(cid=self._cid) + if rel.all_images: + return rel + else: + raise FedfindNotFoundError("Could not find fedfind release with images for event " + "{0}".format(self.version)) + + +class NightlyEvent(ValidationEvent): + """An Event that describes a release validation event - that is, + the testing for a particular nightly, test compose or release + candidate build. Milestone should be 'Rawhide' or 'Branched'. + Note that a Fedora release number attached to a Rawhide nightly + compose is an artificial concept that can be considered a Wikitcms + artifact. Rawhide is a rolling distribution; its nightly composes + do not really have a release number. What we do when we attach + a release number to a Rawhide nightly validation test event is + *declare* that, with our knowledge of Fedora's development cycle, + we believe the testing of that Rawhide nightly constitutes a part + of the release validation testing for that future release. + """ + def __init__(self, site, release, milestone, compose, modular=False): + super(NightlyEvent, self).__init__( + site, release, milestone=milestone, compose=compose, modular=modular) + self.shortver = self.compose + self.creation_date = compose.split('.')[0] + + @property + def category_page(self): + """The category page for this event. Is a property because + page instantiation requires a remote trip. + """ + return listing.ValidationCategory( + self.site, self.release, nightly=True, modular=self.modular) + + @property + def _current_content(self): + """The content for the CurrentFedoraCompose template for + this test event. + """ + tmpl = ("| full = {0}\n| release = {1}\n| milestone = {2}\n" + "| compose =\n| date = {3}\n") + return tmpl.format( + self.version, self.release, self.milestone, self.compose) + + @property + def _pagetype(self): + """For a NightlyEvent, obviously, NightlyPage.""" + return page.NightlyPage + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/exceptions.py b/src/wikitcms/exceptions.py new file mode 100644 index 0000000..6c6c3a3 --- /dev/null +++ b/src/wikitcms/exceptions.py @@ -0,0 +1,49 @@ +# Copyright (C) 2015 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""Defines custom exceptions used by wikitcms.""" + +from __future__ import unicode_literals +from __future__ import print_function + + +class NoPageError(Exception): + """Page does not exist.""" + pass + + +class NotFoundError(Exception): + """Requested thing wasn't found.""" + pass + + +class TooManyError(Exception): + """Found too many of the thing you asked for.""" + pass + + +# this inherits from ValueError as the things that raise this may +# previously have passed along a ValueError from fedfind +class FedfindNotFoundError(ValueError, NotFoundError): + """Couldn't find a fedfind release (probably the fedfind release + that matches an event). + """ + pass + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/helpers.py b/src/wikitcms/helpers.py new file mode 100644 index 0000000..1dd2e7c --- /dev/null +++ b/src/wikitcms/helpers.py @@ -0,0 +1,229 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson +# +"""This file contains helper functions that don't strictly belong in +any particular class or even in another file but outside of a class.""" + +from __future__ import unicode_literals +from __future__ import print_function + +import os +import re +from collections import OrderedDict +from decimal import Decimal + +import fedfind.helpers +import fedfind.release + +MILESTONE_PAIRS = ( + ('Rawhide', '100'), + # We called Branched nightly pages 'Nightly' in F21 cycle. + ('Nightly', '149'), + ('Branched', '150'), + ('Pre-Alpha', '175'), + ('Alpha', '200'), + ('Pre-Beta', '375'), + ('Beta', '400'), + ('Preview', '600'), + ('Pre-Final', '775'), + ('Final', '800'), + ('Postrelease', '900'), +) + +COMPOSE_PAIRS = ( + # Some F12 crap + ('PreBeta', '100'), + ('TC', '200'), + # and this. + ('Pre-RC', '300'), + # The extra digit here is a dumb way to make sure RC1 sorts + # later than TC10 - otherwise you get 20010 vs. 6001, and 20010 + # wins. It might be better to treat the 'TC' / 'RC' as a separate + # element in triplet_sort, but that's a bit harder and makes its + # name a lie... + ('RC', '6000'), +) + +def fedora_release_sort(string): + """Fed a string that looks something like a Fedora pre-release / + compose version, this will output a modified version of the string + which should sort correctly against others like it. Handles stuff + like 'Preview' coming before 'Final', and 'TC' coming before 'RC'. + 'Alpha', 'Beta' and 'Final' sort correctly by pure chance, but we + handle them here anyway to make sure. wikitcms ValidationEvent and + ValidationPage objects have a 'sortname' property you can use + instead of calling this directly. Milestones have a letter so they + sort after nightlies (nightlies are usually going to be earlier). + NOTE: can only sort compose event versions, really. With this + function, '22 Alpha TC1' > '22 Alpha'. + """ + # Some MILESTONES are substrings of COMPOSES so we do Cs first + for (orig, repl) in COMPOSE_PAIRS + MILESTONE_PAIRS: + string = string.replace(orig, repl) + return string + +def triplet_sort(release, milestone, compose): + """Just like fedora_release_sort, but requires you to pass the + now-'standard' release, milestone, compose triplet of inputs. + This is a better way in most cases as you're going to have more + certainty about instantiating wikitcms/fedfind objects from it, + plus we can handle things like '23' being higher than '23 Beta' + or '23 Final TC1'. Expects the inputs to be strings. The elements + in the output tuple will be ints if possible as this gives a + better sort, but may be strings if we missed something; you're not + meant to *do* anything with the tuple but compare it to another + similar tuple. + """ + for (orig, repl) in MILESTONE_PAIRS: + milestone = milestone.replace(orig, repl) + if not milestone: + # ('23', 'Final', '') == ('23', '', '') + milestone = '800' + for (orig, repl) in COMPOSE_PAIRS: + compose = compose.replace(orig, repl) + if not compose: + compose = '999' + # We want to get numerical sorts if we possibly can, so e.g. + # TC10 (becomes 20010) > TC9 (becomes 2009). But just in case + # we get passed a character we don't substitute to a digit, + # check first. + release = str(release) + if release.isdigit(): + release = int(release) + if milestone.isdigit(): + milestone = int(milestone) + if compose.isdigit(): + compose = int(compose) + else: + # this is a bit magic but handles "TC1.1" (old-school), + # "1.1" (new-school candidates), and "20160314.n.0" (new- + # school nightlies) + (comp, respin) = (compose.split('.')[0], compose.split('.')[-1]) + if comp.isdigit() and respin.isdigit(): + compose = Decimal('.'.join((comp, respin))) + return (release, milestone, compose) + +def triplet_unsort(release, milestone, compose): + """Reverse of triplet_sort.""" + (release, milestone, compose) = (str(release), str(milestone), + str(compose)) + for (repl, orig) in MILESTONE_PAIRS: + milestone = milestone.replace(orig, repl) + if milestone == '800': + milestone = 'Final' + # We don't want to do the replace here if the compose is a date, + # because any of the values might happen to be in the date. This + # is a horrible hack that should be OK (until 2100, at least). + if fedfind.helpers.date_check(compose, fail_raise=False): + pass + elif fedfind.helpers.date_check(compose.split('.')[0], fail_raise=False): + # this is slightly magic but should do for now + (comp, respin) = compose.split('.') + compose = "{0}.n.{1}".format(comp, respin) + else: + for (repl, orig) in COMPOSE_PAIRS: + compose = compose.replace(orig, repl) + if compose == '999': + compose = '' + return (release, milestone, compose) + +def rreplace(string, old, new, occurrence): + """A version of the str.replace() method which works from the right. + Taken from https://stackoverflow.com/questions/2556108/ + """ + elems = string.rsplit(old, occurrence) + return new.join(elems) + +def normalize(text): + """Lower case and replace ' ' with '_' so I don't have to + keep retyping it. + """ + return text.lower().replace(' ', '_') + +def find_bugs(text): + """Find RH bug references in a given chunk of text. More than one + method does this, so we'll put the logic here and they can share + it. Copes with [[rhbug:(ID)]] links, {{bz|(ID)}} templates and + URLs that look like Bugzilla. Doesn't handle aliases (only matches + numeric IDs 6 or 7 digits long). Returns a set (so bugs that occur + multiple times in the text will only appear once in the output). + """ + bugs = set() + bzpatt = re.compile(r'({{bz *\| *|' + r'\[\[rhbug *: *|' + r'bugzilla\.redhat\.com/show_bug\.cgi\?id=)' + r'(\d{6,7})') + matches = bzpatt.finditer(text) + for match in matches: + bugs.add(match.group(2)) + # Filter out bug IDs usually used as examples + for bug in ('12345', '123456', '54321', '654321', '234567', '1234567', + '7654321'): + bugs.discard(bug) + return bugs + +def cid_to_event(cid): + """Given a Pungi 4 compose ID, figure out the appropriate wikitcms + (release, milestone, compose) triplet. Guesses the appropriate + release number to assign to the event for Rawhide composes. To + do the opposite, you can use the ff_release property of Validation + Event instances. + """ + parsed = fedfind.helpers.parse_cid(cid, dic=True) + dist = parsed['short'] + release = parsed['version'].lower() + vertype = parsed['version_type'] + date = parsed['date'] + typ = parsed['compose_type'] + respin = parsed['respin'] + if dist not in ("Fedora", "Fedora-Modular"): + # we only have validation events for Fedora and Modular + # composes ATM - not e.g. FedoraRespin or Fedora-Atomic ones + raise ValueError("No validation events for {0} composes!".format(dist)) + if vertype != 'ga': + # this ensures we don't create events for updates/u-t composes + raise ValueError("No validation events for updates composes!") + if typ == "production": + # we need to get the label and parse that + ffrel = fedfind.release.get_release(cid=cid) + if not ffrel.label: + raise ValueError("{0} looks like a production compose, but found " + "no label! Cannot determine event.".format(cid)) + (milestone, compose) = ffrel.label.rsplit('-', 1) + return (dist, release, milestone, compose) + + # FIXME: we have no idea yet what to do for 'test' composes + if not typ == "nightly": + raise ValueError( + "cid_to_event(): cannot guess event for 'test' compose yet!") + + # nightlies + if release == "rawhide": + # the release number for new Rawhide events should always be + # 1 above the highest current 'real' release (branched or + # stable). FIXME: this will not do the right thing for *old* + # composes, not sure if we can fix that sensibly. + release = str(fedfind.helpers.get_current_release(branched=True) + 1) + milestone = "Rawhide" + else: + milestone = "Branched" + compose = "{0}.n.{1}".format(date, respin) + return (dist, release, milestone, compose) + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/listing.py b/src/wikitcms/listing.py new file mode 100644 index 0000000..0cfd2f3 --- /dev/null +++ b/src/wikitcms/listing.py @@ -0,0 +1,314 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson +# +"""This file kind of shadows mwclient's listing.py, creating modified +versions of several of its classes. The point is to provide generators +similar to mwclient's own, but which return wikitcms page/category +instances when appropriate, falling through to mwclient instances +otherwise. +""" + +from __future__ import unicode_literals +from __future__ import print_function + +import re + +from mwclient import listing as mwl + +from . import page as pg + +# exceptions: the wikitcms getter raises these when it fails rather than +# just returning None, so the generators can use try/except blocks to +# handle both this case and the case (which shouldn't ever happen, but +# just in case) where they're being used on something other than a list +# of pages. + +class NoPageWarning(Exception): + """Exception raised when the tcmswiki getter can't find a matching + page. Not really an error, should always be handled. + """ + def __init__(self, page): + self.page = page + + def __str__(self): + return "Could not produce a wikitcms page for: {0}".format(self.page) + + +class PageCheckWarning(Exception): + """Exception raised when the wikitcms getter finds a matching page, + but the page name the class generators from the page's various + attributes doesn't match the page name the getter was given. Should + usually be handled (and an mwclient Page instance returned instead). + """ + def __init__(self, frompage, topage): + self.frompage = frompage + self.topage = topage + + def __str__(self): + return ("Expected page name {0} does not match source " + "page name {1}".format(self.frompage, self.topage)) + + +class TcmsGeneratorList(mwl.GeneratorList): + """A GeneratorList which returns wikitcms page (and category etc.) + instances when appropriate. _get_tcms is implemented as a separate + function so TcmsPageList can use the discovery logic. + """ + def __init__(self, site, list_name, prefix, *args, **kwargs): + super(TcmsGeneratorList, self).__init__( + site, list_name, prefix, *args, **kwargs) + + def __next__(self): + # We can't get the next entry from mwl.List ourselves, try and + # handle it, then pass it up to our parent if we can't, because + # parent's next() gets the next entry from mwl.List itself, so + # in that scenario, one list item gets skipped. Either we + # entirely clone next() with our own additions, or we let it + # fire and then override the result if we can. Using nxt._info + # is bad, but super.next() doesn't return that, and the page + # instance doesn't expose it any other way. We could just use + # the name, but if you don't pass info when instantiating a + # Page, it has to hit the API during init to reconstruct info, + # and that causes a massive performance hit. + nxt = super(TcmsGeneratorList, self).__next__() + try: + return self._get_tcms(nxt.name, nxt._info) + except (NoPageWarning, PageCheckWarning): + return nxt + + def next(self): + # for python2 compat + return self.__next__() + + def _check_page(self, name, page): + # convenience function for _get_tcms sanity check + if page.checkname == name: + return page + raise PageCheckWarning(page.checkname, name) + + def _get_tcms(self, name, info=()): + # this is the meat: it runs a bunch of string checks on page + # names, basically, and returns the appropriate wikitcms + # object if any matches. + if isinstance(name, int): + # we'll have to do a wiki roundtrip, as we need the text + # name. + page = pg.Page(self.site, name) + name = page.name + name = name.replace('_', ' ') + # quick non-RE check to see if we'll ever match (and filter + # out some 'known bad' pages) + if (name.startswith('Test Results:') or + (name.startswith('Test Day:') and not + name.endswith('/ru') and not + 'metadata' in name.lower() and not + 'rendercheck' in name.lower()) or + (name.startswith('Category:'))): + nightly_patt = re.compile(r'Test Results:Fedora (Modular )?(\d{1,3}) ' + r'(Rawhide|Nightly|Branched) ' + r'(\d{8,8}(\.n\.\d+)?|\d{4,4} \d{2,2}) ' + r'(.+)$') + accept_patt = re.compile(r'Test Results:Fedora (\d{1,3}) ' + r'([^ ]+?) (Rawhide |)Acceptance Test ' + r'(\d{1,2})$') + ms_patt = re.compile(r'Test Results:Fedora (Modular )?(\d{1,3}) ' + r'([^ ]+?) ([^ ]+?) (.+)$') + cat_patt = re.compile(r'Category:Fedora (Modular )?(\d{1,3}) ' + r'(.*?) *?Test Results$') + tdcat_patt = re.compile(r'Category:Fedora (\d{1,3}) Test Days$') + testday_patt = re.compile(r'Test Day:(\d{4}-\d{2}-\d{2}) *(.*)$') + # FIXME: There's a few like this, handle 'em sometime + #testday2_patt = re.compile(u'Test Day:(.+) (\d{4}-\d{2}-\d{2})$') + + # Modern standard nightly compose event pages, and F21-era + # monthly Rawhide/Branched test pages + match = nightly_patt.match(name) + if match: + if match.group(6) == 'Summary': + # we don't really ever need to do anything to existing + # summary pages, and instantiating one from here is kinda + # gross, so just fall through + raise NoPageWarning(name) + modular = False + if match.group(1): + modular = True + page = pg.NightlyPage( + self.site, release=match.group(2), testtype=match.group(6), + milestone=match.group(3), compose=match.group(4), + info=info, modular=modular) + return self._check_page(name, page) + + match = accept_patt.match(name) + if match: + # we don't handle these, yet. + raise NoPageWarning(name) + + # milestone compose event pages + match = ms_patt.match(name) + if match: + if match.group(5) == 'Summary': + raise NoPageWarning(name) + modular = False + if match.group(1): + modular = True + page = pg.ComposePage( + self.site, release=match.group(2), testtype=match.group(5), + milestone=match.group(3), compose=match.group(4), + info=info, modular=modular) + return self._check_page(name, page) + + # test result categories + match = cat_patt.match(name) + if match: + modular = False + if match.group(1): + modular = True + if not match.group(3): + page = ValidationCategory( + self.site, match.group(2), info=info, modular=modular) + return self._check_page(name, page) + elif match.group(3) == 'Nightly': + page = ValidationCategory(self.site, match.group(2), + nightly=True, info=info, modular=modular) + return self._check_page(name, page) + else: + page = ValidationCategory(self.site, match.group(2), + match.group(3), info=info, modular=modular) + return self._check_page(name, page) + + # Test Day categories + match = tdcat_patt.match(name) + if match: + page = TestDayCategory(self.site, match.group(1), info=info) + return self._check_page(name, page) + + # test days + match = testday_patt.match(name) + if match: + page = pg.TestDayPage(self.site, match.group(1), + match.group(2), info=info) + return self._check_page(name, page) + raise NoPageWarning(name) + + +class TcmsPageList(mwl.PageList, TcmsGeneratorList): + """A version of PageList which returns wikitcms page (and category + etc.) objects when appropriate. + """ + def get(self, name, info=()): + modname = name + if self.namespace: + modname = '{0}:{1}'.format(self.site.namespaces[self.namespace], + name) + try: + return self._get_tcms(modname, info) + except (NoPageWarning, PageCheckWarning): + return super(TcmsPageList, self).get(name, info) + + +class TcmsCategory(pg.Page, TcmsGeneratorList): + """A modified category class - just as mwclient's Category class + inherits from both its Page class and its GeneratorList class, + acting as both a page and a generator returning the members of + the category, so this inherits from wikitcms' Page and + TcmsGeneratorList. You can produce the page contents with pg.Page + write() method, and you can use it as a generator which returns + the category's members, as wikitcms class instances if appropriate + or mwclient class instances otherwise. It works recursively - if + a member of a ValidationCategory is itself a test category, you'll + get another ValidationCategory instance. There are sub-classes for + various particular types of category (Test Days, validation, etc.) + """ + def __init__(self, site, wikiname, info=None): + super(TcmsCategory, self).__init__(site, wikiname, info=info) + TcmsGeneratorList.__init__(self, site, 'categorymembers', 'cm', + gcmtitle=self.name) + + +class ValidationCategory(TcmsCategory): + """A category class (inheriting from TcmsCategory) for validation + test result category pages. If nightly is True, this will be a + category for test results from Rawhide or Branched nightly builds + for the given release. Otherwhise, if milestone is passed, this + will be a category for the given milestone, and if it isn't, it + will be the top-level category for the given release. + """ + + def __init__(self, site, release, milestone=None, nightly=False, + info=None, modular=False): + _dist = "Fedora" + if modular: + _dist = "Fedora Modular" + if nightly is True: + wikiname = ("Category:{0} {1} Nightly Test " + "Results").format(_dist, release) + if modular: + self.seedtext = ("{{{{Validation results milestone category|" + "release={0}|nightly=true|modular=true}}}}").format(release) + else: + self.seedtext = ("{{{{Validation results milestone category|" + "release={0}|nightly=true}}}}").format(release) + + self.summary = ("Relval bot-created validation result category " + "page for {0} {1} nightly " + "results").format(_dist, release) + elif milestone: + wikiname = "Category:{0} {1} {2} Test Results".format( + _dist, release, milestone) + if modular: + self.seedtext = ("{{{{Validation results milestone category" + "|release={0}|" + "milestone={1}|modular=true}}}}").format(release, milestone) + else: + self.seedtext = ("{{{{Validation results milestone category" + "|release={0}|" + "milestone={1}}}}}").format(release, milestone) + self.summary = ("Relval bot-created validation result category " + "page for {0} " + "{1} {2}").format(_dist, release, milestone) + else: + wikiname = "Category:{0} {1} Test Results".format(_dist, release) + if modular: + self.seedtext = ("{{{{Validation results milestone category" + "|release={0}|modular=true}}}}").format(release) + else: + self.seedtext = ("{{{{Validation results milestone category" + "|release={0}}}}}").format(release) + self.summary = ("Relval bot-created validation result category " + "page for {0} {1}").format(_dist, release) + + super(ValidationCategory, self).__init__(site, wikiname, info=info) + + +class TestDayCategory(TcmsCategory): + """A category class (inheriting from TcmsCategory) for Test Day + category pages. + """ + + def __init__(self, site, release, info=None): + wikiname = "Category:Fedora {0} Test Days".format(str(release)) + self.seedtext = ( + "This category contains all the Fedora {0} [[QA/Test_Days|Test " + "Day]] pages. A calendar of the Test Days can be found [" + "https://apps.fedoraproject.org/calendar/list/QA/?subject=Test+Day" + " here].\n\n[[Category:Test Days]]").format(str(release)) + self.summary = "Created page (via wikitcms)" + super(TestDayCategory, self).__init__(site, wikiname, info=info) + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/page.py b/src/wikitcms/page.py new file mode 100644 index 0000000..bf767e4 --- /dev/null +++ b/src/wikitcms/page.py @@ -0,0 +1,789 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""Classes that describe different types of pages we are interested +in, and attributes of pages like test results and test cases, are +defined in this file. +""" + +from __future__ import unicode_literals +from __future__ import print_function + +import datetime +import logging +import pytz +import re +import time +from collections import OrderedDict + +import fedfind.helpers +import fedfind.release +from cached_property import cached_property +from mwclient import errors as mwe +from mwclient import page as mwp + +from . import result as rs +from . import helpers +from .exceptions import NoPageError, NotFoundError, TooManyError + +logger = logging.getLogger(__name__) + + +class Page(mwp.Page): + """Parent class for all page classes. Can be instantiated directly + if you just want to take advantage of the convenience methods like + sections() and save(). Available attributes: seedtext, summary. + Note 'name' is defined by mwp.Page's __init__. + """ + + def __init__(self, site, wikiname, info=None, extra_properties=None): + super(Page, self).__init__(site, wikiname, info, extra_properties) + # Used for sanity check by the page generator + self.checkname = wikiname + self._sections = None + self.results_separators = list() + + @property + def sections(self): + """A list of the page's sections. Each section is represented + by a dict whose values provide various attributes of the + section. Returns an empty list for non-existent page (or any + other API error). Cached, cache cleared on each page save. + """ + # None == not yet retrieved or cache expired. [] == retrieved, + # but page is empty or something. + if self._sections is None: + try: + apiout = self.site.api( + 'parse', page=self.name, prop='sections') + self._sections = apiout['parse']['sections'] + except mwe.APIError: + self._sections = [] + return self._sections + + @property + def results_wikitext(self): + """Returns a string containing the wikitext for the page's + results section. Will be empty if no results are found. Relies + on the layout for result pages remaining consistent. Class + must override definition of self.results_separators or else + this will always return an empty string. + """ + pagetext = self.text() + comment = re.compile('', re.S) + pos = -1 + for sep in self.results_separators: + pos = pagetext.find(sep) + if pos > -1: + break + + if pos == -1: + return '' + text = pagetext[pos:] + text = comment.sub('', text) + return text + + @cached_property + def creation_date(self): + """Date the page was created. Used for sorting and seeing how + long it's been since the last event, when creating new events. + """ + revs = self.revisions(limit=1, dir='newer', prop='timestamp') + try: + origrev = next(revs) + except StopIteration: + # page doesn't exist + return "" + return time.strftime('%Y%m%d', origrev['timestamp']) + + def write(self, createonly=True): + """Create a page with its default content and summary. mwclient + exception will be raised on any page write failure. + """ + seedtext = getattr(self, 'seedtext', None) + summary = getattr(self, 'summary', None) + if seedtext is None or summary is None: + raise ValueError("wikitcms.Page.write(): both seedtext and summary needed!") + self.save(seedtext, summary, createonly=createonly) + + def save(self, *args, **kwargs): + """Same as the original, but will retry once on fail. If you + already retrieved the current text, you can pass it in as + oldtext, and we will check to see if oldtext and text are the + same. If they are, we return a dict with the key nochange set + to an empty string - this saves a needless extra remote round + trip. Of course you could do this in the caller instead, it's + just a convenience. Also clears the page sections cache. + """ + if 'oldtext' in kwargs and args[0] == kwargs['oldtext']: + return dict(nochange='') + + if 'oldtext' in kwargs: + # avoid mwclient save() warning about unknown kwarg + del kwargs['oldtext'] + + try: + ret = super(Page, self).save(*args, **kwargs) + except mwe.EditError as err: + logger.warning("Page %s edit failed! Trying again in 15 seconds", + self.name) + logger.debug("Error was: %s", err) + time.sleep(15) + ret = super(Page, self).save(*args, **kwargs) + # clear the caches + self._sections = None + return ret + + +class ValidationPage(Page): + """A parent class for different types of release validation event + pages, containing common properties and methods. Required + attributes: version, shortver, seedtext. If modular is True, the + page will be for a Fedora-Modular compose, with 'Modular' in the + page name, using the appropriate templates and template values, + etc. + """ + def __init__(self, site, release, testtype, milestone='', compose='', + info=None, modular=False): + self.release = release + self.milestone = str(milestone) + try: + self.compose = fedfind.helpers.date_check( + compose, fail_raise=True, out='str') + except ValueError: + self.compose = str(compose) + self.version = "{0} {1} {2}".format( + self.release, self.milestone, self.compose) + self.testtype = testtype + self.modular = modular + + # Wiki name the page should have, according to the naming + # convention. + _dist = "Fedora" + if modular: + _dist = "Fedora Modular" + wikiname = "Test Results:{0} {1} {2}".format(_dist, self.version, self.testtype) + super(ValidationPage, self).__init__(site, wikiname, info) + + # Edit summary to be used for clean page creation. + self.summary = ("Relval bot-created {0} validation results page for {1} " + "{2}").format(testtype, _dist, self.version) + self.results_separators = ( + "Test Matri", "Test Areas", "An unsupported test or configuration." + " No testing is required.") + # Sorting helpers. sortname is a string, sorttuple is a + # 4-tuple. sorttuple is more reliable. See the function docs. + self.sortname = helpers.fedora_release_sort( + ' '.join((self.version, self.testtype))) + self.sorttuple = helpers.triplet_sort( + self.release, self.milestone, self.compose) + (self.testtype,) + + @property + def results_sections(self): + """A list of the sections in the page which (most likely) + contain test results. Takes all the sections in the page, + finds the one one which looks like the first "test results" + section and returns that section and those that follow it - or + returns all sections after the Key section, if it can't find + one which looks like the first results section. + """ + secs = self.sections + if not secs: + # empty page or some other malarkey + return secs + first = None + for i, sec in enumerate(secs): + if 'Test Matri' in sec['line'] or 'Test Areas' in sec['line']: + first = i + break + elif 'Key' in sec['line']: + first = i+1 + return secs[first:] + + def get_resultrows(self, statuses=None, transferred=True): + """Returns the result.ResultRow objects representing all the + page's table rows containing test results. + """ + sections = self.results_sections + if not sections: + return list() + rows = list() + pagetext = self.text() + comment = re.compile('', re.S) + for i, sec in enumerate(sections): + try: + nextsec = sections[i+1] + except IndexError: + nextsec = None + section = sec['line'] + secid = sec['index'] + if nextsec: + sectext = pagetext[sec['byteoffset']:nextsec['byteoffset']] + else: + sectext = pagetext[sec['byteoffset']:] + # strip comments + sectext = comment.sub('', sectext) + newrows = rs.find_resultrows(sectext, section, secid, statuses, + transferred) + rows.extend(newrows) + return rows + + def find_resultrow(self, testcase='', section='', testname='', env=''): + """Return exactly one result row with the desired attributes, + or raise an exception (if more or less than one row is found). + The Installation page contains some rows in the same section + with the same testcase and testname, but each row provides + a different set of envs, so these can be uniquely identified + by specifying the desired env. + """ + rows = self.get_resultrows() + if not rows: + raise NoPageError("Page does not exist or has no result rows.") + + # Find the right row + nrml = helpers.normalize + rows = [r for r in rows if + nrml(testcase) in nrml(r.testcase) + or nrml(testcase) in nrml(r.name)] + if len(rows) > 1 and section: + rows = [r for r in rows if nrml(section) in nrml(r.section)] + if len(rows) > 1 and testname: + rows = [r for r in rows if nrml(testname) in nrml(r.name)] + if len(rows) > 1 and env: + # the way this match is done must be kept in line with the + # corresponding match in add_results, below + rows = [r for r in rows if nrml(env) in + [renv.lower() for renv in r.results.keys()]] + # try a more precise name match - e.g. "upgrade_dnf" vs. + # "upgrade_dnf_encrypted" + if len(rows) > 1: + rows = [r for r in rows if + nrml(testcase) == nrml(r.testcase) or + nrml(testcase) == nrml(r.name) or + nrml(testname) == nrml(r.name)] + if not rows: + raise NotFoundError("Specified row cannot be found.") + if len(rows) > 1: + raise TooManyError("More than one matching row found.") + return rows[0] + + def update_current(self): + """Make the Current convenience redirect page on the wiki for + the given test type point to this page. + """ + if self.modular: + curr = self.site.pages[ + 'Test Results:Current Modular {0} Test'.format(self.testtype)] + else: + curr = self.site.pages[ + 'Test Results:Current {0} Test'.format(self.testtype)] + curr.save("#REDIRECT [[{0}]]".format(self.name), + "relval: update to current event", createonly=None) + + def add_results(self, resultsdict, allowdupe=False): + """Adds multiple results to the page. Passed a dict whose + keys are ResultRow() instances and whose values are iterables + of (env, Result()) 2-tuples. Returns a list, which will be + empty unless allowdupe is False and any of the results is a + 'dupe' - i.e. the given test and environment already have a + result from the user. The return list contains a 3-tuple of + (row, env, result) for each dupe. + """ + # We need to sort the dict in a particular way: by the section + # ID of each row, in reverse order. This is so when we edit + # the page, we effectively do so backwards, and the byte + # offsets we use to find each section don't get thrown off + # along the way (we don't edit section 1 before section 3 and + # thus not quite slice the text correctly when we look for + # section 3). + resultsdict = OrderedDict(sorted(resultsdict.items(), + key=lambda x: int(x[0].secid), + reverse=True)) + nonetext = rs.Result().result_template + dupes = list() + newtext = oldtext = self.text() + for (row, results) in resultsdict.items(): + # It's possible that we have rows with identical text in + # different page sections; this is why 'secid' is an attr + # of ResultRows. To make sure we edit the correct row, + # we'll slice the text at the byteoffset of the row's + # section. We only do one replacement, so we don't need + # to bother finding the *end* of the section. + # We could just edit the page section-by-section, but that + # involves doing one remote roundtrip per section. + secoff = [sec['byteoffset'] for sec in self.sections if + sec['index'] == row.secid][0] + if secoff: + sectext = newtext[secoff:] + else: + sectext = newtext + oldrow = row.origtext + cells = oldrow.split('\n|') + + for (env, result) in results: + if not env in row.results: + # the env passed wasn't precisely one of the row's + # envs. let's see if we can make a safe guess. If + # there's only one env, it's easy... + if len(row.results) == 1: + env = list(row.results.keys())[0] + else: + # ...if not, we'll see if the passed env is + # a substring of only one of the envs, case- + # insensitively. + cands = [cand for cand in row.results.keys() if + env.lower() in cand.lower()] + if len(cands) == 1: + env = cands[0] + else: + # LOG: bad env + continue + if not allowdupe: + dupe = [r for r in row.results[env] if + r.user == result.user] + if dupe: + dupes.append((row, env, result)) + continue + restext = result.result_template + rescell = cells[row.columns.index(env)] + if nonetext in rescell: + rescell = rescell.replace(nonetext, restext) + elif '\n' in rescell: + rescell = rescell.replace('\n', restext+'\n') + else: + rescell = rescell + restext + cells[row.columns.index(env)] = rescell + + newrow = '\n|'.join(cells) + if newrow == oldrow: + # All dupes, or something. + continue + sectext = sectext.replace(oldrow, newrow, 1) + if secoff: + newtext = newtext[:secoff] + sectext + else: + newtext = sectext + + if len(resultsdict) > 3: + testtext = ', '.join(row.name for row in list(resultsdict.keys())[:3]) + testtext = '{0}...'.format(testtext) + else: + testtext = ', '.join(row.name for row in resultsdict.keys()) + summary = ("Result(s) for test(s): {0} filed via " + "relval").format(testtext) + self.save(newtext, summary, oldtext=oldtext, createonly=None) + return dupes + + def add_result(self, result, row, env, allowdupe=False): + """Adds a result to the page. Must be passed a Result(), the + result.ResultRow() object representing the row into which a + result will be added, and the name of the environment for + which the result is to be reported. Works by replacing the + first instance of the row's text encountered in the page or + page section. Expected to be used together with get_resultrows + which provides the ResultRow() objects. + """ + resdict = dict() + resdict[row] = ((env, result),) + return self.add_results(resdict, allowdupe=allowdupe) + + +class ComposePage(ValidationPage): + """A Page class that describes a single result page from a test + compose or release candidate validation test event. + """ + def __init__(self, site, release, testtype, milestone, compose, info=None, modular=False): + super(ComposePage, self).__init__( + site, release=release, milestone=milestone, compose=compose, + testtype=testtype, info=info, modular=modular) + self.shortver = "{0} {1}".format(self.milestone, self.compose) + + # String that will generate a clean copy of the page using the + # test page generation template system. + if self.modular: + seedtmpl = ("{{{{subst:Modular validation results|testtype={0}|release={1}|" + "milestone={2}|compose={3}}}}}") + else: + seedtmpl = ("{{{{subst:Validation results|testtype={0}|release={1}|" + "milestone={2}|compose={3}}}}}") + self.seedtext = seedtmpl.format( + testtype, self.release, self.milestone, self.compose) + + +class NightlyPage(ValidationPage): + """A Page class that describes a single result page from a nightly + validation test event. + """ + def __init__(self, site, release, testtype, milestone, compose, info=None, modular=False): + super(NightlyPage, self).__init__( + site, release=release, milestone=milestone, compose=compose, + testtype=testtype, info=info, modular=modular) + self.shortver = self.compose + # overridden for nightlies to avoid expensive roundtrips + if '.' in compose: + self.creation_date = compose.split('.')[0] + else: + self.creation_date = compose + + # String that will generate a clean copy of the page using the + # test page generation template system. + if self.modular: + seedtmpl = ("{{{{subst:Modular validation results|testtype={0}|release={1}|" + "milestone={2}|date={3}}}}}") + else: + seedtmpl = ("{{{{subst:Validation results|testtype={0}|release={1}|" + "milestone={2}|date={3}}}}}") + self.seedtext = seedtmpl.format( + testtype, self.release, self.milestone, self.compose) + + +class SummaryPage(Page): + """A Page class that describes the result summary page for a given + event. event is the parent Event() for the page; summary pages are + always considered to be a part of an Event. + """ + def __init__(self, site, event, info=None, modular=False): + self.modular = modular + wikiname = "Test Results:Fedora {0} Summary".format(event.version) + if modular: + wikiname = "Test Results:Fedora Modular {0} Summary".format(event.version) + super(SummaryPage, self).__init__(site, wikiname, info) + _dist = "Fedora" + if modular: + _dist = "Fedora Modular" + self.summary = ("Relval bot-created validation results summary for " + "{0} {1}").format(_dist, event.version) + self.seedtext = ( + "{0} {1} [[QA:Release validation test plan|release " + "validation]] summary. This page shows the results from all the " + "individual result pages for this compose together. You can file " + "results directly from this page and they will be saved into the " + "correct individual result page. To see test instructions, visit " + "any of the individual pages (the section titles are links). You " + "can find download links below.\n\n").format(_dist, event.version) + self.seedtext += "__TOC__\n\n" + self.seedtext += "== Downloads ==\n{{" + _dist + " " + event.version + " Download}}" + for testpage in event.valid_pages: + self.seedtext += "\n\n== [[" + testpage.name + "|" + self.seedtext += testpage.testtype + "]] ==\n{{" + self.seedtext += testpage.name + "}}" + + def update_current(self): + """Make the Current convenience redirect page on the wiki for the + event point to this page. + """ + if self.modular: + curr = self.site.pages['Test Results:Current Modular Summary'] + else: + curr = self.site.pages['Test Results:Current Summary'] + curr.save("#REDIRECT [[{0}]]".format(self.name), + "relval: update to current event", createonly=None) + + +class DownloadPage(Page): + """The page containing image download links for a ValidationEvent. + As with SummaryPage, is always associated with a specific event. + """ + def __init__(self, site, event, info=None, modular=False): + _dist = "Fedora" + if modular: + _dist = "Fedora Modular" + wikiname = "Template:{0} {1} Download".format(_dist, event.version) + self.summary = "Relval bot-created download page for {0} {1}".format( + _dist, event.version) + super(DownloadPage, self).__init__(site, wikiname, info) + self.event = event + + @property + def seedtext(self): + """A nicely formatted download table for the images for this + compose. Here be dragons (and wiki table syntax). What you get + from this is a table with one row for each unique 'image + identifier' - the subvariant plus the image type - and columns + for all arches in the entire image set; if there's an image + for the given image type and arch then there'll be a download + link in the appropriate column. + """ + # sorting score values (see below) + archscores = ( + (('x86_64', 'i386'), 2000), + ) + loadscores = ( + (('everything',), 300), + (('workstation',), 220), + (('server',), 210), + (('cloud', 'desktop', 'cloud_base', 'docker_base', 'atomic'), 200), + (('kde',), 190), + (('minimal',), 90), + (('xfce',), 80), + (('soas',), 73), + (('mate',), 72), + (('cinnamon',), 71), + (('lxde',), 70), + (('source',), -10), + ) + # Start by iterating over all images and grouping them by load + # (that's imagedict) and keeping a record of each arch we + # encounter (that's arches). + arches = set() + imagedict = dict() + for img in self.event.ff_release_images.all_images: + if img['arch']: + arches.add(img['arch']) + # simple human-readable identifier for the image + desc = ' '.join((img['subvariant'], img['type'])) + # assign a 'score' to the image; this will be used for + # ordering the download table's rows. + img['score'] = 0 + for (values, score) in archscores: + if img['arch'] in values: + img['score'] = score + for (values, score) in loadscores: + if img['subvariant'].lower() in values: + img['score'] += score + # The dict values are lists of images. We could use a + # DefaultDict here, but faking it is easy too. + if desc in imagedict: + imagedict[desc].append(img) + else: + imagedict[desc] = [img] + # Now we have our data, sort the dict using the weight we + # calculated earlier. We use the max score of all arches in + # each group of images. + imagedict = OrderedDict(sorted(imagedict.items(), + key=lambda x: max(img['score'] for img in x[1]), + reverse=True)) + # ...and sort the arches (just so they don't move around in + # each new page and confuse people). + arches = sorted(arches) + + # Now generate the table. + table = '{| class="wikitable sortable mw-collapsible" width=100%\n|-\n' + # Start of the header row... + table += '! Image' + for arch in arches: + # Add a column for each arch + table += ' !! {0}'.format(arch) + table += '\n' + for (subvariant, imgs) in imagedict.items(): + # Add a row for each subvariant + table += '|-\n' + table += '| {0}\n'.format(subvariant) + for arch in arches: + # Add a cell for each arch (whether we have an image + # or not) + table += '| ' + for img in imgs: + if img['arch'] == arch: + # Add a link to the image if we have one + table += '[{0} Download]'.format(img['url']) + table += '\n' + # Close out the table when we're done + table += '|-\n|}' + return table + + def update_current(self): + """Kind of a hack - relval needs this to exist as things + stand. I'll probably refactor it later. + """ + pass + + +class AMIPage(Page): + """A page containing EC2 AMI links for a given event. Is included + in the Cloud validation page to make it easy for people to find + the correct AMIs. + """ + def __init__(self, site, event, info=None, modular=False): + _dist = "Fedora" + if modular: + _dist = "Fedora Modular" + wikiname = "Template:{0} {1} AMI".format(_dist, event.version) + self.summary = "Relval bot-created AMI page for {0} {1}".format( + _dist, event.version) + super(AMIPage, self).__init__(site, wikiname, info) + self.event = event + + @property + def seedtext(self): + """A table of all the AMIs for the compose for this event. We + have to query this information out of datagrepper, that's the + only place where it's available. + """ + text = "" + # first, let's get the information out of datagrepper. We'll + # ask for messages up to 2 days after the event date. + date = fedfind.helpers.parse_cid(self.event.ff_release.cid, dic=True)['date'] + start = datetime.datetime.strptime(date, '%Y%m%d').replace(tzinfo=pytz.utc) + end = start + datetime.timedelta(days=2) + # convert to epoch (this is what datagrepper wants). In Python + # 3 we can use just .timestamp() but sadly not in Python 2. + # https://stackoverflow.com/questions/6999726 + epoch = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) + start = (start - epoch).total_seconds() + end = (end - epoch).total_seconds() + url = "https://apps.fedoraproject.org/datagrepper/raw" + url += "?topic=org.fedoraproject.prod.fedimg.image.publish" + url += "&start={0}&end={1}".format(start, end) + json = fedfind.helpers.download_json(url) + msgs = json['raw_messages'] + # handle pagination + for page in range(2, json['pages'] + 1): + newurl = url + "&page={0}".format(page) + newjson = fedfind.helpers.download_json(newurl) + msgs.extend(newjson['raw_messages']) + + # now let's find the messages for our event compose + ours = [msg['msg'] for msg in msgs if msg['msg']['compose'] == self.event.ff_release.cid] + + def _table_line(msg): + """Convenience function for generating a table line.""" + destination = msg['destination'] + ami = msg['extra']['id'] + url = "https://redirect.fedoraproject.org/console.aws.amazon.com/ec2/v2/home?" + url += "region={0}#LaunchInstanceWizard:ami={1}".format(destination, ami) + return "| {0}\n| {1}\n| [{2} Launch in EC2]\n|-\n".format(destination, ami, url) + + def _table(arch, virttype, voltype): + """Convenience function for adding a table.""" + ret = "== {0} {1} {2} AMIs ==\n\n".format(arch, virttype, voltype) + ret += '{| class="wikitable sortable mw-collapsible' + if arch != 'x86_64' or virttype != 'hvm' or voltype != 'standard': + # we expand the x86_64 hvm standard table by default + ret += ' mw-collapsed' + ret += '" width=100%\n|-\n' + ret += "! Region !! AMI ID !! Direct launch link\n|-\n" + # find the right messages for this arch and types + relevants = [msg for msg in ours if + msg['architecture'] == arch and + msg['extra']['virt_type'] == virttype and + msg['extra']['vol_type'] == voltype] + # sort the messages by region so the table is easier to scan + relevants.sort(key=lambda x:x['destination']) + for msg in relevants: + ret += _table_line(msg) + ret += "|}\n\n" + return ret + + # now let's create and populate the tables + for arch in ('x86_64', 'arm64'): + for virttype in ('hvm',): + for voltype in ('standard', 'gp2'): + text += _table(arch, virttype, voltype) + + return text + + +class TestDayPage(Page): + """A Test Day results page. Usually contains table(s) with test + cases as the column headers and users as the rows - each row is + one user's results for all of the test cases in the table. Note + this class is somewhat incomplete and really can only be used + for its own methods, do *not* try writing one of these to the + wiki. + """ + def __init__(self, site, date, subject, info=None): + # Handle names with no subject, e.g. Test_Day:2012-03-14 + wikiname = "Test Day:{0}".format(date) + if subject: + wikiname = "{0} {1}".format(wikiname, subject) + super(TestDayPage, self).__init__(site, wikiname, info) + self.date = date + self.subject = subject + self.results_separators = ('Test Results', 'Results') + + def write(self): + print("Creating Test Day pages is not yet supported.") + return + + @cached_property + def bugs(self): + """Returns a list of bug IDs referenced in the results section + (as strings). Will find bugs in {{result}} and {{bz}} + templates.""" + bugs = helpers.find_bugs(self.results_wikitext) + for res in rs.find_results_by_row(self.results_wikitext): + bugs.update(res.bugs) + return sorted(bugs) + + def fix_app_results(self): + """The test day app does its own bug references outside the + result template, instead of including them as the final + parameters to the template like it should. This fixes that, in + a fairly rough and ready way. + """ + badres = re.compile(r'({{result.*?)}} {0,2}' + r'({{bz\|\d{6,7}}}) ?' + r'({{bz\|\d{6,7}}})? ?' + r'({{bz\|\d{6,7}}})? ?' + r'({{bz\|\d{6,7}}})? ?' + r'({{bz\|\d{6,7}}})? ?' + r'({{bz\|\d{6,7}}})?') + text = oldtext = self.text() + oldtext = text + matches = badres.finditer(text) + for match in matches: + bugs = list() + groups = match.groups() + for group in groups[1:]: + if group: + bugs.append(group[10:-8]) + text = text.replace(match.group(0), + match.group(1) + '||' + '|'.join(bugs) + '}}') + return self.save(text, summary=u"Fix testday app-generated results to " + "use {{result}} template for bug references", + oldtext=oldtext) + + def long_refs(self): + """People tend to include giant essays as notes on test + day results, which really makes table rendering ugly when + they're dumped in the last column of the table. This finds all + notes over 150 characters long, moves them to the "long" + group, and adds a section at the end of the page with all the + "long" notes in it. The 'end of page discovery' is a bit + hacky, it just finds the last empty line in the page except + for trailing lines and sticks the section there, but that's + usually what we want - basically we want to make sure it + appears just above the category memberships at the bottom of + the page. It does go wrong *sometimes*, so good idea to check + the page after it's edited. + """ + text = oldtext = self.text() + if '' in text: + # Don't run if we've already been run on this page + return dict(nochange='') + refpatt = re.compile('(.+?)', re.S) + matches = refpatt.finditer(text) + found = False + for match in matches: + if len(match.group(0)) > 150: + found = True + text = text.replace(match.group(0), + '' + match.group(1)) + if found: + text = helpers.rreplace( + text.strip(), '\n\n', + '\n\n== Long comments ==\n\n\n', 1) + return self.save(text, summary=u"Move long comments to a separate " + "section at end of page", oldtext=oldtext) + else: + # If we didn't find any long refs, don't do anything + return dict(nochange='') + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/release.py b/src/wikitcms/release.py new file mode 100644 index 0000000..777526c --- /dev/null +++ b/src/wikitcms/release.py @@ -0,0 +1,60 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""Classes that describe distribution releases are defined here.""" + +from __future__ import unicode_literals +from __future__ import print_function + +from . import page as pg +from . import listing as li + +class Release(object): + """Class for a Fedora release. wiki is a wikitcms site object. + Release is a string containing a Fedora release version (e.g. 21). + """ + def __init__(self, release, wiki, modular=False): + self.release = release + self.modular = modular + dist = "Fedora" + if modular: + dist = "Fedora Modular" + self.category_name = "Category:{0} {1} Test Results".format( + dist, self.release) + self.site = wiki + + @property + def testday_pages(self): + """All Test Day pages for this release (as a list).""" + cat = self.site.pages[ + 'Category:Fedora {0} Test Days'.format(self.release)] + return [page for page in cat if isinstance(page, pg.TestDayPage)] + + def milestone_pages(self, milestone=None): + """If no milestone, will give all release validation pages for + this release (as a generator). If a milestone is given, will + give validation pages only for that milestone. Note that this + works by category; you may get somewhat different results by + using page name prefixes. + """ + cat = li.ValidationCategory(self.site, self.release, milestone, modular=self.modular) + pgs = self.site.walk_category(cat) + return (p for p in pgs if isinstance(p, pg.ValidationPage)) + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/result.py b/src/wikitcms/result.py new file mode 100644 index 0000000..051586e --- /dev/null +++ b/src/wikitcms/result.py @@ -0,0 +1,566 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""This file defines various classes and helper functions for working +with results.""" + +from __future__ import unicode_literals +from __future__ import print_function + +import re + +from collections import OrderedDict + +from wikitcms import helpers + +# These are used by multiple functions, so let's share them. +# Wiki table row separator +SEP_PATT = re.compile(r'\|[-\}].*?\n') +# Identifies an instance of the result template. Will break if the +# result contains another template, but don't do that. the lookahead +# is used to capture the 'comments' for the result: we keep matching +# until we hit the next instance of the template, or a cell or row +# separator (newline starting with a |). The re.S is vital. +RES_PATT = re.compile(r'{{result.+?}}.*?(?=\n*{{result|$|\n\|)', re.S) + +def _filter_results(results, statuses=None, transferred=True, bot=True): + """Filter results. Shared between next two functions.""" + # Drop example / sample results + results = [r for r in results if not r.user or + r.user.lower() not in + ('sampleuser', 'exampleuser', 'example', 'username', 'fasname')] + if statuses: + results = [r for r in results for s in statuses if r.status and + s in r.status] + if not transferred: + results = [r for r in results if not r.transferred] + if not bot: + results = [r for r in results if not r.bot] + return results + +def find_results(text, statuses=None, transferred=True, bot=True): + """Find test results in a given chunk of wiki text. Returns a list + of Result objects. If statuses is not None, it should be an + iterable of strings, and only result objects whose status matches + one of the given statuses will be returned. If transferred is + False, results like {{result|something|previous TC5 run}} will not + be included. If bot is False, results from automated test bots + (results with 'bot=true') will not be included. + """ + results = list() + # Identifies an instance of the old {{testresult template, in the + # same way as RES_PATT (above). + oldres_patt = re.compile(r'{{testresult.+?}}.*?(?={{testresult|$)', re.M) + for res in RES_PATT.findall(text): + results.append(Result.from_result_template(res)) + for oldres in oldres_patt.findall(text): + results.append(Result.from_testresult_template(oldres)) + + results = _filter_results(results, statuses, transferred, bot) + return results + +def find_results_by_row(text, statuses=None): + """Find test results using a row-by-row scan, guessing the user + for results which do not have one. Used for Test Day pages. Note: + doesn't bother handling {{testresult because AFAICT no Test Day + pages use that template. + """ + # Slightly slapdash way to identify the contents of a wiki table + # cell. Good enough to find the contents of the first cell in the + # row, which we do because it'll usually be the user name in a + # Test Day results table. + cell_patt = re.compile(r'\| *(.*?) *\n\|') + # Captures the user name from a wikilink to a user page (typically + # used to indicate the user who reports a result). + user_patt = re.compile(r'\[\[User: *(.*?) *[|\]]') + results = list() + + for row in SEP_PATT.split(text): + for match in RES_PATT.findall(row): + res = Result.from_result_template(match) + if res.status and not res.user: + # We'll try and find a [[User: wikilink; if we can't, + # we'll guess that the reporter's name is the contents + # of the first cell in the row. + pattuser = user_patt.search(row) + if pattuser: + res.user = pattuser.group(1).lower() + else: + celluser = cell_patt.search(row) + if celluser: + res.user = celluser.group(1).lower() + results.append(res) + + results = _filter_results(results, statuses) + return results + +def find_resultrows(text, section='', secid=0, statuses=None, transferred=True): + """Find result rows in a given chunk of wiki text. Returns a list + of ResultRow objects. 'statuses' and 'transferred' are passed all + the way through ResultRow to find_results() and behave as + described there, for the Result objects in each ResultRow. + """ + # identify all test case names, including old ones. modern ones + # match QA:Testcase.*, but older ones sometimes have QA/TestCase. + testcase_pattern = re.compile(r'(QA[:/]Test.+?)[\|\]\n]') + # row separator is |-, end of table is |} + columns = list() + resultrows = list() + rows = SEP_PATT.split(text) + for row in rows: + rowlines = row.split('\n') + for line in rowlines: + # check if this is a column header row, and update column + # names. Sometimes the header row doesn't have an explicit + # row separator so the 'row' might be polluted with + # preceding lines, so we split the row into lines and + # check each line in the row. + line = line.strip() + if line.find('!') == 0 and line.find('!!') > 0: + # column titles. note: mw syntax in fact allows for + # '! title\n! title\n! title' as well as '! title !! + # title !! title'. But we don't use that syntax. + columns = line.lstrip('!').split('!!') + for column in columns: + # sanitize names a bit + newcol = column.strip() + newcol = newcol.strip("'[]") + newcol = newcol.strip() + try: + # drop out any block + posa = newcol.index("") + posb = newcol.index("") + 6 # length + newcol = newcol[:posa] + newcol[posb:] + newcol = newcol.strip() + except ValueError: + pass + try: + newcol = newcol.split('|')[1] + except IndexError: + pass + if newcol != column: + columns.insert(columns.index(column), newcol) + columns.remove(column) + tcmatch = testcase_pattern.search(row) + if tcmatch: + # *may* be a result row - may also be a garbage 'row' + # between tables which happens to contain a test case + # name. So we get a ResultRow object but discard it if it + # doesn't contain any result cells. This test works even + # if the actual results are filtered by statuses= or + # Transferred=, because the resrow.results dict will + # always have a key for each result column, though its + # value may be an empty list. + resrow = ResultRow.from_wiki_row(tcmatch.group(1), columns, row, + section, secid, statuses, + transferred) + if resrow.results: + resultrows.append(resrow) + return resultrows + + +class Result(object): + """A class that represents a single test result. Note that a + 'none' result, as you get if you just instantiate this class + without arguments, is a thing, at least for wikitcms; when text + with {{result|none}} templates in it is parsed, such objects may + be created/returned, and you can produce the {{result|none}} text + as the result_template property of such an instance. + + You would usually instantiate this class directly to report a new + result. + + Methods that parse existing results will use one of the class + methods that returns a Result() with the appropriate attributes. + When one of those parsers produces an instance it will set the + attribute origtext to record the exact string parsed to produce + the instance. + + transferred, if True, indicates the result is of the "previous + (compose) run" type that is used to indicate where we think a + result from a previous compose is valid for a later one. + """ + def __init__( + self, status=None, user=None, bugs=None, comment='', bot=False): + self.status = status + self.user = user + self.bugs = bugs + if self.bugs: + self.bugs = [str(bug) for bug in self.bugs] + self.comment = comment + self.bot = bot + self.transferred = False + self.comment_bugs = helpers.find_bugs(self.comment) + + def __str__(self): + if not self.status: + return "Result placeholder - {{result|none}}" + if self.bot: + bot = 'BOT ' + else: + bot = '' + status = 'Result: ' + self.status.capitalize() + if self.transferred: + user = ' transferred: ' + self.user + elif self.user: + user = ' from ' + self.user + else: + user = '' + if self.bugs: + bugs = ', bugs: ' + ', '.join(self.bugs) + else: + bugs = '' + if self.comment: + comment = ', comment: ' + self.comment + # Don't display ref tags + refpatt = re.compile(r'') + comment = refpatt.sub('', comment) + else: + comment = '' + return bot + status + user + bugs + comment + + @property + def result_template(self): + """The {{result}} template string that would represent the + properties of this result in a wiki page. + """ + bugtext = '' + commtext = self.comment + usertext = '' + bottext = '' + if self.status is None: + status = 'none' + else: + status = self.status + if self.bugs: + bugtext = "|" + '|'.join(self.bugs) + if self.user: + usertext = "|" + self.user + if self.bot: + bottext = "|bot=true" + tmpl = ("{{{{result|{status}{usertext}{bugtext}{bottext}}}}}" + "{commtext}").format(status=status, usertext=usertext, + bugtext=bugtext, bottext=bottext, + commtext=commtext) + return tmpl + + @classmethod + def from_result_template(cls, string): + """Returns a Result object based on the {{result}} template. + The most complex result template you see might be: + {{ result | fail| bot =true| adamwill | 123456|654321|615243}} comment + We want the 'fail' and 'adamwill' bits separately and stripped, + and all the bug numbers in one chunk to be parsed later to + construct a list of bugs, and none of the pipes, brackets, or + whitespace. Mediawiki named parameters can occur anywhere in + the template and aren't counted in the numbered parameters, so + we need to find them and extract them first. We record the + comment exactly as is. + """ + template, comment = string.strip().split('}}', 1) + comment = comment.strip() + template = template.lstrip('{') + params = template.split('|') + namedpars = dict() + bot = False + + for param in params: + if '=' in param: + (par, val) = param.split('=', 1) + namedpars[par.strip()] = val.strip() + params.remove(param) + if 'bot' in namedpars and namedpars['bot']: + # This maybe doesn't do what you expect for 'bot=false', + # but we don't handle that in Mediawiki either and we want + # to stay consistent. + bot = True + + # 'params' now contains only numbered params + # Pad the non-existent parameters to make things cleaner later + while len(params) < 3: + params.append('') + + for i, param in enumerate(params): + params[i] = param.strip() + if params[i] == '': + params[i] = None + status, user = params[1:3] + bugs = params[3:] + if status and status.lower() == "none": + status = None + + if bugs: + bugs = [b.strip() for b in bugs if b and b.strip()] + for i, bug in enumerate(bugs): + # sometimes people write 123456#c7, remove the suffix + if '#' in bug: + newbug = bug.split('#')[0] + if newbug.isdigit(): + bugs[i] = newbug + + res = cls(status, user, bugs, comment, bot) + res.origtext = string + if user and "previous " in user: + res.transferred = True + return res + + @classmethod + def from_testresult_template(cls, string): + '''Returns a Result object based on the {{testresult}} template. + This was used in Fedora 12. It looks like this: + {{testresult/pass|FASName}} comment or bug + The bug handling here is very special-case - it relies on the + fact that bug IDs were always six-digit strings, at the time, + and on the template folks used to link to bug reports - but + should be good enough. + ''' + bug_patt = re.compile(r'({{bz.*?(\d{6,6}).*?}})') + emptyref_patt = re.compile(r' *?') + template, comment = string.strip().split('}}', 1) + template = template.lstrip('{') + template = template.split('/')[1] + params = template.split('|') + try: + status = params[0].strip().lower() + if status == "none": + status = None + except IndexError: + status = None + try: + user = params[1].strip().lower() + except IndexError: + user = None + bugs = [b[1] for b in bug_patt.findall(comment)] + if comment: + comment = bug_patt.sub('', comment) + comment = emptyref_patt.sub('', comment) + if comment.replace(' ', '') == '': + comment = '' + comment = comment.strip() + else: + pass + res = cls(status, user, bugs, comment) + res.origtext = string + if user and "previous " in user: + res.transferred = True + return res + + @classmethod + def from_qatracker(cls, result): + '''Converts a result object from the QA Tracker library to a + wikitcms-style Result. Returns a Result instance with origres + as an extra property that is a pointer to the qatracker result + object. + ''' + if result.result == 1: + status = 'pass' + elif result.result == 0: + status = 'fail' + else: + status = None + if result.reportername: + user = result.reportername + else: + user = None + if result.comment: + comment = result.comment + else: + comment = '' + # This produces an empty string if there are no bugs, a dict + # if there are bugs. FIXME: this is completely wrong, it's + # a JSON dict, we should parse it as JSON, but I can't be + # bothered fixing this Ubuntu stuff right now. Nothing uses + # it. + bugs = eval(result.bugs) + if bugs: + bugs = list(bugs.keys()) + else: + bugs = None + res = cls(status, user, bugs, comment) + res.origres = result + return res + +class TestInstance(object): + """Represents the broad concept of a 'test instance': that is, in + any test management system, the 'basic unit' of a single test for + which some results are expected to be reported. In 'Wikitcms', for + instance, this corresponds to a single row in a results table, and + that is what the ResultRow() subclass represents. A subclass for + QATracker would represent a single test in a build of a product. + + The 'testcase' is the basic identifier of a test instance. It will + not necessarily be unique, though - in any test management system + you may find multiple test instances for the same test case (in + different builds and different products). The concept of the + name derives from Wikitcms, where it is not uncommon for a set of + test instances to have the same 'testcase' but a different 'name', + which in that system is the link text: there will a column which + for each row contains [[testcase|name]], the testcase being the + same but the name being different. The concept doesn't seem + entirely specific to Wiki TCMS, though, so it's represented here. + Commonly the 'testcase' and 'name' will be the same, when each + instance within a set has a different 'testcase' the name should + be identical to the testcase. + + milestone is, roughly, the priority of the test: milestone is + slightly Fedora-specific language, a hangover from early wikitcms + versions which didn't consider other systems. For Fedora it will + be Alpha, Beta or Final, usually. For Ubuntu it may be + 'mandatory', 'optional' or possibly 'disabled'. + + results is required to be a dict of lists; the dict keys represent + the test's environments. If the test system does not have the + concept of environments, the dict can have a single key with some + sort of generic name (like 'Results'). The values must be lists of + instances of wikitcms.Result or a subclass of it. + """ + def __init__(self, testcase, milestone='', results=None): + self.testcase = testcase + self.name = testcase + self.milestone = milestone + if not results: + self.results = dict() + else: + self.results = results + + +class ResultRow(TestInstance): + """Represents the 'test instance' concept for Wikitcms, where it + is a result row from the tables used to contain results. Some + Wikitcms-specific properties are encoded here. columns is the list + of columns in the table in which the result was found (this is + needed to figure out the environments for the results, as the envs + are represented by table columns, and to know which cell to edit + when modifying results). origtext is the text which was parsed to + produce the instance, if it was produced by the from_wiki_row() + class method which parses wiki text to produce instances. section + and secid are the wiki page section in which the table from which + the row came is located; though these are in a way attributes of + the page, this is really another case where an MW attribute is + just a way of encoding information about a *test*. The splitting + of result pages into sections is a way of sub-grouping tests in + each page. So it's appropriate to store those attributes here. + + At present you typically get ResultRow instances by calling a + ComposePage's get_resultrows() method, which runs its text through + result.find_resultrows(), which isolates the result rows in the + wiki text and runs through through this class' from_wiki_row() + method. This will always provide instances with a full set of + the above-described attributes. + """ + def __init__(self, testcase, columns, section='', secid=None, milestone='', + origtext='', results=None): + super(ResultRow, self).__init__(testcase, milestone, results) + self.columns = columns + self.origtext = origtext + self.section = section + self.secid = secid + + def matches(self, other): + """This is roughly an 'equals' check for ResultRows; if all + identifying characteristics and the origtext match, this is + True, otherwise False. __dict__ match is too strong as the + 'results' attribute is a list of Result instances; each time + you instantiate the same ResultRow you get different Result + objects. We don't override __eq__ and use == because that has + icky implications for hashing, and we just want to stay away + from that mess: + https://docs.python.org/3/reference/datamodel.html#object.__hash__ + """ + if isinstance(other, self.__class__): + ours = (self.testcase, self.name, self.secid, self.origtext) + theirs = (other.testcase, other.name, other.secid, other.origtext) + return ours == theirs + return NotImplemented + + @classmethod + def from_wiki_row(cls, testcase, columns, text, section, secid, + statuses=None, transferred=True): + """Instantiate a ResultRow from some wikitext and some info + that is worked out from elsewhere in the page. + """ + results = OrderedDict() + # this is presumptuous, but holds up for every result page + # tested so far; there may be some with whitespace, and + # '| cell || cell || cell' is permitted as an alternative to + # '| cell\n| cell\n| cell' but we do not seem to use it. + cells = text.split('\n|') + milestone = '' + for mile in ('Alpha', 'Basic', 'Beta', 'Final', 'Optional', 'Tier1', + 'Tier2', 'Tier3'): + if mile in cells[0]: + milestone = mile + # we take the *first* milestone we find, so we treat + # e.g. "Basic / Final" as Basic + break + for i, cell in enumerate(cells): + if testcase in cell: + try: + # see if we can find some link text for the test + # case, and assume it's the test's "name" if so + altname = cell.strip().strip('[]').split('|')[1] + continue + except IndexError: + try: + altname = cell.strip().strip('[]').split(maxsplit=1)[1] + except IndexError: + altname = None + if '{{result' in cell or '{{testresult' in cell: + # any cell containing a result string is a 'result + # cell', and the index of the cell in columns will be + # the title of the column it is in. find_results() + # returns an empty list if all results are filtered + # out, so the results dict's keys will always + # represent the full set of environments for this test + try: + results[columns[i]] = find_results(cell, statuses, + transferred) + except IndexError: + # FIXME: log (messy table, see e.g. F15 'Multi + # Image') + pass + row = cls(testcase, columns, section, secid, milestone, text, results) + if altname: + row.name = altname + return row + + +class TrackerBuildTest(TestInstance): + """Represents a 'result instance' from QA Tracker: this is the + data associated with a single testcase in a single build. + """ + def __init__(self, tcname, tcid, milestone='', results=None): + super(TrackerBuildTest, self).__init__(tcname, milestone, results) + self.tcid = tcid + + @classmethod + def from_api(cls, testcase, build): + """I don't remember what the shit this does, I'm just making + pylint happy. + """ + results = dict() + results['Results'] = list() + tcname = testcase.title + tcid = testcase.id + milestone = testcase.status_string + for res in build.get_results(testcase): + results['Results'].append(Result.from_qatracker(res)) + return cls(tcname, tcid, milestone, results) + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/src/wikitcms/wiki.py b/src/wikitcms/wiki.py new file mode 100644 index 0000000..f092106 --- /dev/null +++ b/src/wikitcms/wiki.py @@ -0,0 +1,677 @@ +# Copyright (C) 2014 Red Hat +# +# This file is part of wikitcms. +# +# wikitcms is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""The Wiki class here extends mwclient's Site class with additional +Wikitcms-specific functionality, and convenience features like stored +user credentials. +""" + +from __future__ import unicode_literals +from __future__ import print_function + +from collections import namedtuple +import datetime +import re + +import fedfind.helpers +import mwclient +from productmd.composeinfo import get_date_type_respin + +from . import page as pg +from . import event as ev +from . import helpers as hl +from . import listing as li +from . import result as rs +from .exceptions import NoPageError, NotFoundError, TooManyError + +try: + from openidc_client import OpenIDCClient +except ImportError: + OpenIDCClient = None +try: + from openidc_client.requestsauth import OpenIDCClientAuther +except ImportError: + OpenIDCClientAuther = None + +# I'd really like to use namedlist.namedtuple, but it isn't widely +# available. +class ResTuple(namedtuple('ResTuple', 'testtype release milestone compose ' + 'testcase section testname env status user bugs ' + 'comment bot cid modular')): + """namedtuple (with default values) used for report_validation_ + results(). See that method's docstring for details. + """ + def __new__(cls, testtype, release='', milestone='', compose='', + testcase='', section='', testname='', env='', status='', + user='', bugs='', comment='', bot=False, cid='', modular=False): + return super(ResTuple, cls).__new__( + cls, testtype, release, milestone, compose, testcase, section, + testname, env, status, user, bugs, comment, bot, cid, modular) + +class Wiki(mwclient.Site): + """Extends the mwclient.Site class with some extra capabilities.""" + # parent class has a whole bunch of args, so just pass whatever through. + # always init this the same as a regular mwclient.Site instance. + def __init__(self, host='fedoraproject.org', *args, **kwargs): + super(Wiki, self).__init__(host, *args, **kwargs) + # override the 'pages' property so it returns wikitcms Pages when + # appropriate + self.pages = li.TcmsPageList(self) + + @property + def current_compose(self): + """A dict of the key / value pairs from the CurrentFedora + Compose page which is the canonical definition of the 'current' + primary arch validation testing compose. You can usually expect + keys full, release, date, milestone, and compose. The page is + normally written by ValidationEvent.update_current(). + """ + currdict = dict() + valpatt = re.compile(r'^\| *?(\w+?) *?= *([\w .]*?) *$', re.M) + page = self.pages['Template:CurrentFedoraCompose'] + for match in valpatt.finditer(page.text()): + currdict[match.group(1)] = match.group(2) + return currdict + + @property + def current_event(self): + """The current event, as a ValidationEvent instance. Will be a + ComposeEvent or a NightlyEvent.""" + curr = self.current_compose + # Use of 'max' plus get_validation_event handles getting us + # the right kind of event. + return self.get_validation_event( + release=curr['release'], milestone=curr['milestone'], + compose=max(curr['date'], curr['compose'])) + + @property + def current_modular_compose(self): + """A dict of the key / value pairs from the CurrentFedora + ModularCompose page which is the canonical definition of the + 'current' modular primary arch validation testing compose. You + can usually expect keys full, release, date, milestone, and + compose. The page is normally written by + ValidationEvent.update_current(). + """ + currdict = dict() + valpatt = re.compile(r'^\| *?(\w+?) *?= *([\w .]*?) *$', re.M) + page = self.pages['Template:CurrentFedoraModularCompose'] + for match in valpatt.finditer(page.text()): + currdict[match.group(1)] = match.group(2) + return currdict + + @property + def current_modular_event(self): + """The current modular event, as a ValidationEvent instance. + Will be a ComposeEvent or a NightlyEvent.""" + curr = self.current_modular_compose + # Use of 'max' plus get_validation_event handles getting us + # the right kind of event. + return self.get_validation_event( + release=curr['release'], milestone=curr['milestone'], + compose=max(curr['date'], curr['compose']), modular=True) + + @property + def matrices(self): + """A list of dicts representing pages in the test matrix + template category. These are the canonical definition of + 'existing' test types. Done this way - rather than using the + category object's ability to act as an iterator over its member + page objects - because this method respects the sort order of + the member pages, whereas the other does not. The sort order is + used in creating the overview summary page. + """ + category = self.pages['Category:QA test matrix templates'] + return category.members(generator=False) + + @property + def testtypes(self): + """Test types, derived from the matrix page names according to + a naming convention. A list of strings. + """ + return [m['title'].replace('Template:', '') + .replace(' test matrix', '') for m in self.matrices] + + @property + def modular_matrices(self): + """A list of dicts representing pages in the modular test + matrix template category. These are the canonical definition of + 'existing' test types. Done this way - rather than using the + category object's ability to act as an iterator over its member + page objects - because this method respects the sort order of + the member pages, whereas the other does not. The sort order is + used in creating the overview summary page. + """ + category = self.pages['Category:QA modular test matrix templates'] + return category.members(generator=False) + + @property + def modular_testtypes(self): + """Test types, derived from the matrix page names according to + a naming convention. A list of strings. + """ + return [m['title'].replace('Template:', '') + .replace(' modular test matrix', '') for m in self.modular_matrices] + + def login(self, *args, **kwargs): + """Login method, overridden to use openidc auth when necessary. + This will open a browser window and run through FAS auth on + the first use, then a token will be saved that will allow auth + for a while, when the token expires, the web auth process will + pop up again. + """ + use_openidc = False + # This probably breaks on private wikis, but wikitcms isn't + # ever used with any of those, AFAIK. + host = self.host + if isinstance(host, (list, tuple)): + host = host[1] + if host.endswith('fedoraproject.org') and self.version[:2] >= (1, 29): + use_openidc = True + if not use_openidc: + # just work like mwclient + return super(Wiki, self).login(*args, **kwargs) + + # Fedora wiki since upgrade to 1.29 doesn't allow native + # mediawiki auth with FAS creds any more, it only allows + # auth via OpenID Connect. For this case we set up an + # openidc auther, call site_init() to trigger auth and + # update the site properties, and return. + if OpenIDCClient is None: + raise ImportError('python-openidc-client is needed for OIDC') + if OpenIDCClientAuther is None: + raise ImportError('python-openidc-client 0.4.0 or higher is ' + 'required for OIDC') + client = OpenIDCClient( + app_identifier='wikitcms', + id_provider='https://id.{0}/openidc/'.format(host), + id_provider_mapping={'Token': 'Token', + 'Authorization': 'Authorization'}, + client_id='wikitcms', + client_secret='notsecret', + useragent='wikitcms') + + auther = OpenIDCClientAuther(client, + ['openid', 'https://fedoraproject.org/wiki/api']) + + self.connection.auth = auther + self.site_init() + + def add_to_category(self, page_name, category_name, summary=''): + """Add a given page to a given category if it is not already a + member. Takes strings for the names of the page and the + category, not mwclient objects. + """ + page = self.pages[page_name] + text = page.text() + if category_name not in text: + text += "\n[[{0}]]".format(category_name) + page.save(text, summary, createonly=False) + + def walk_category(self, category): + """Simple recursive category walk. Returns a list of page + objects that are members of the parent category or its + sub-categories, to any level of recursion. 14 is the Category: + namespace. + """ + pages = dict() + for page in category: + if page.namespace == 14: + sub_pages = self.walk_category(page) + for sub_page in sub_pages: + pages[sub_page.name] = sub_page + else: + pages[page.name] = page + pages = pages.values() + return pages + + def allresults(self, prefix=None, start=None, redirects='all', end=None): + """A generator for pages in the Test Results: namespace, + similar to mwclient's allpages, allcategories etc. generators. + This is a TcmsPageList, so it returns wikitcms objects when + appropriate. Note, if passing prefix, start or end, leave out + the "Test Results:" part of the name. + """ + gen = li.TcmsPageList(self, prefix=prefix, start=start, + namespace=116, redirects=redirects, end=end) + return gen + + def alltestdays(self, prefix=None, start=None, redirects='all', end=None): + """A generator for pages in the Test Day: namespace, + similar to mwclient's allpages, allcategories etc. generators. + This is a TcmsPageList, so it returns wikitcms objects when + appropriate. Note, if passing prefix, start or end, leave out + the "Test Day:" part of the name. + """ + gen = li.TcmsPageList(self, prefix=prefix, start=start, + namespace=114, redirects=redirects, end=end) + return gen + + def _check_compose(self, compose): + """Trivial checker shared between following two methods.""" + date = fedfind.helpers.date_check(compose, out='obj', fail_raise=False) + # all nightlies after F24 branch are Pungi 4-style; plain date + # is never a valid 'compose' value after that date, 2016-02-23 + if date and date < datetime.datetime(2016, 2, 23): + return 'date' + else: + # check if we have a valid Pungi4-style identifier + try: + (date, typ, respin) = get_date_type_respin(compose) + if date and typ and respin is not None: + if fedfind.helpers.date_check(date, fail_raise=False): + return 'date' + except ValueError: + pass + # regex to match TC/RC names: TC1, RC10, RC23.6 + patt = re.compile(r'[TR]C\d+\.?\d*') + if patt.match(compose.upper()): + return 'compose' + # regex for Pungi 4 milestone composes: 1.1, 1.2 ... 10.10 ... + patt = re.compile(r'\d+\.\d+') + if patt.match(compose): + return 'compose' + else: + raise ValueError( + "Compose must be a TC/RC identifier (TC1, RC3...) for pre-" + "Fedora 24 milestone composes, a Pungi 4 milestone compose" + " identifier (1.1, 10.10...) for post-Fedora 23 milestone " + "composes, a date in YYYYMMDD format (for pre-Fedora 24 " + "nightlies) or a Pungi 4 nightly identifier (20160308.n.0" + ", 20160310.n.2) for post-Fedora 23 nightlies.") + + def get_validation_event(self, release='', milestone='', compose='', + cid='', modular=False): + """Get an appropriate ValidationEvent object for the values + given. As with get_validation_page(), this method is for + sloppy instantiation of pages that follow the rules. This + method has no required arguments and tries to figure out + what you want from what you give it. It will raise errors + if what you give it is impossible to interpret or if it + tries and comes up with an inconsistent-seeming result. + + If modular is True, it will look for a Fedora Modular event + with the relevant version attributes. If you pass a compose ID + as cid, any value you pass for modular will be ignored; we'll + instead parse the modular value out of the compose ID. + + If you pass a numeric release, a milestone, and a valid + compose (TC/RC or date), it will give you the appropriate + event, whether it exists or not. All it really does in this + case is pick NightlyEvent or ComposeEvent for you. If you + don't fulfill any of those conditions, it'll need to do + some guessing/assumptions, and in some of those cases it + will only return an Event *that actually exists*, and may + raise exceptions if you passed a particularly pathological + set of values. + + If you don't pass a compose argument it will get the current + event; if you passed either of the other arguments and they + don't match the current event, it will raise an error. It + follows that calling this with no arguments just gives you + current_event. + + If you pass a date as compose with no milestone, it will see + if there's a Rawhide nightly and return it if so, otherwise it + will see if there's a Branched nightly and return that if so, + otherwise raise an error. It follows that you can't get the + page for an event that *doesn't exist yet* this way: you must + instantiate it directly or call this method with a milestone. + + It will not attempt to guess a milestone for TC/RC composes; + it will raise an exception in this case. + + The guessing bits require wiki roundtrips, so they will be + slower than instantiating a class directly or using this + method with sufficient information to avoid guessing. + """ + if cid: + (dist, release, milestone, compose) = hl.cid_to_event(cid) + modular = bool(dist == 'Fedora-Modular') + if not compose or not release: + # Can't really make an educated guess without a compose + # and release, so just get the current event and return it + # if it matches any other values passed. + if modular: + event = self.current_modular_event + else: + event = self.current_event + if release and event.release != release: + raise ValueError( + "get_validation_event(): Guessed event release {0} does " + "not match requested release {1}".format( + event.release, release)) + if milestone and event.milestone != milestone: + raise ValueError( + "get_validation_event(): Guessed event milestone {0} " + "does not match specified milestone {1}".format( + event.milestone, milestone)) + # all checks OK + return event + + if self._check_compose(compose) == 'date': + if milestone: + return ev.NightlyEvent( + self, release=release, milestone=milestone, + compose=compose, modular=modular) + else: + # we have a date and no milestone. Try both and return + # whichever exists. We check whether the first result + # page has any contents so that if someone mistakenly + # creates the wrong event, we can clean up by blanking + # the pages, rather than by getting an admin to + # actually *delete* them. + rawev = ev.NightlyEvent(self, release, 'Rawhide', compose, modular=modular) + pgs = rawev.result_pages + if pgs and pgs[0].text(): + return rawev + brev = ev.NightlyEvent(self, release, 'Branched', compose, modular=modular) + pgs = brev.result_pages + if pgs and pgs[0].text(): + return brev + # Here, we failed to guess. Boohoo. + raise ValueError( + "get_validation_event(): Could not find any event for " + "release {0} and date {1}.".format(release, compose)) + + elif self._check_compose(compose) == 'compose': + compose = str(compose).upper() + if not milestone: + raise ValueError( + "get_validation_event(): For a TC/RC compose, a milestone " + "- Alpha, Beta, or Final - must be specified.") + # With Pungi 4, the 'Final' milestone became 'RC', let's + # be nice and convert it + if int(release) > 23 and milestone.lower() == 'final': + milestone = 'RC' + return ev.ComposeEvent(self, release, milestone, compose, modular=modular, cid=cid) + else: + # We should never get here, but just in case. + raise ValueError( + "get_validation_event(): Something very strange happened.") + + def get_validation_page(self, testtype, release='', milestone='', + compose='', cid='', modular=False): + """Get an appropriate ValidationPage object for the values + given. As with get_validation_event(), this method is for + sloppy instantiation of pages that follow the rules. This + method has no required arguments except the testtype and tries + to figure out what you want from what you give it. It will + raise errors if what you give it is impossible to interpret or + if it tries and comes up with an inconsistent-seeming result. + + If modular is True, it will look for a Fedora Modular page + with the relevant version attributes. If you pass a compose ID + as cid, any value you pass for modular will be ignored; we'll + instead parse the modular value out of the compose ID. + + If you pass a numeric release, a milestone, and a valid + compose (TC/RC or date), it will give you the appropriate + event, whether it exists or not. All it really does in this + case is pick NightlyEvent or ComposeEvent for you. If you + don't fulfill any of those conditions, it'll need to do + some guessing/assumptions, and in some of those cases it + will only return an Event *that actually exists*, and may + raise exceptions if you passed a particularly pathological + set of values. + + If you don't pass a compose argument it will get the page for + the current event; if you passed either of the other + arguments and they don't match the current event, it will + raise an error. It follows that calling this with no arguments + just gives you the page of the specified test type for the + current event. + + If you pass a date as compose with no milestone, it will see + if there's a Rawhide nightly and return it if so, otherwise it + will see if there's a Branched nightly and return that if so, + otherwise raise an error. It follows that you can't get the + page for an event that *doesn't exist yet* this way: you must + instantiate it directly or call this method with a milestone. + + It will not attempt to guess a milestone for TC/RC composes; + it will raise an exception in this case. + + The guessing bits require wiki roundtrips, so they will be + slower than instantiating a class directly or using this + method with sufficient information to avoid guessing. + """ + if cid: + (dist, release, milestone, compose) = hl.cid_to_event(cid) + modular = bool(dist == 'Fedora-Modular') + if not compose or not release: + # Can't really make an educated guess without a compose + # and release, so just get the current event and return it + # if it matches any other values passed. + curr = self.current_compose + page = self.get_validation_page( + testtype, release=curr['release'], milestone=curr['milestone'], + compose=max(curr['compose'], curr['date']), modular=modular) + if release and page.release != release: + raise ValueError( + "get_validation_page(): Guessed page release {0} does " + "not match requested release {1}".format( + page.release, release)) + if milestone and page.milestone != milestone: + raise ValueError( + "get_validation_page(): Guessed page milestone {0} " + "does not match specified milestone {1}".format( + page.milestone, milestone)) + return page + + if self._check_compose(compose) == 'date': + if milestone: + return pg.NightlyPage( + self, release, testtype, milestone, compose, modular=modular) + else: + rawpg = pg.NightlyPage( + self, release, testtype, 'Rawhide', compose, modular=modular) + if rawpg.exists: + return rawpg + brpg = pg.NightlyPage( + self, release, testtype, 'Branched', compose, modular=modular) + if brpg.exists: + return brpg + # Here, we failed to guess. Boohoo. + raise ValueError( + "get_validation_page(): Could not find any event for " + "release {0} and date {1}.".format(release, compose)) + + elif self._check_compose(compose) == 'compose': + if not milestone: + raise ValueError( + "get_validation_page(): For a milestone compose, a " + " milestone - Alpha, Beta, or Final - must be specified.") + # With Pungi 4, the 'Final' milestone became 'RC', let's + # be nice and convert it + if int(release) > 23 and milestone.lower() == 'final': + milestone = 'RC' + return pg.ComposePage( + self, release, testtype, milestone, compose, modular=modular) + else: + # We should never get here, but just in case. + raise ValueError( + "get_validation_page(): Something very strange happened.") + + def report_validation_results(self, reslist, allowdupe=False): + """High-level result reporting function. Pass it an iterable + of objects identifying results. It's pretty forgiving about + what these can be. They can be any kind of sequence or + iterable containing up to 15 values in the following order: + (testtype, release, milestone, compose, testcase, section, + testname, env, status, user, bugs, comment, bot, cid, modular). + They can also be any mapping type (e.g. a dict) with enough of + the 15 keys set (see below for requirements). + + You may find it convenient to import the ResTuple class from + this module and pass your results as instances of it: it is + a namedtuple with default values which uses the names given + above, so you can avoid having to pad values you don't need to + set and conveniently read back items from the tuple after + you've created it, if you like. + + Any value of the result item can be absent, or set to + something empty or falsey. However, to be successfully + reported, each item must meet these conditions: + + * 'testtype', 'release', 'milestone', 'compose', 'modular' and + 'cid' must identify a single validation page + using get_validation_page() (at least 'testtype' must always + be set). + * 'testcase', 'section' and 'testname' must identify a single + test instance using ValidationPage.find_resultrow() + * 'env' must uniquely identify one of the test instance's + 'environments' (the columns into which results can be entered; + it can be empty if there is only one for this test instance) + *'status' must indicate the result status. + + 'user', 'bugs', and 'comment' can always be left empty if + desired; if 'user' is not specified, the mediawiki username + (lower-cased) will be used. + + All values should be strings, except 'bugs', which should be + an iterable of digit strings (though iterable of ints will be + tolerated), and 'bot' which should be True if the result is + from some sort of automated testing system (not a human). + + Returns a 2-tuple of lists of objects indicating any failures. + The first list is of submissions that failed because there was + insufficient identifying data. The second list is of + submissions that failed because they were 'dupes' - the given + user has already reported a result for the given test and env. + If allowdupe is True, duplicate reports will be allowed and + this list will always be empty. + + The items in each list have the same basic layout as the input + results items. The 'insufficients' list will contain the exact + same objects that were provided as input. The 'duplicates' + list is reconstructed and will often contain more or corrected + data compared to the input tuples. Its members are *always* + instances of the ResTuple namedtuple class, which provides + access to the fields by the names given above. + + Uses get_validation_page() to guess the desired page and then + constructs a result dict to pass to the ValidationPage + add_results() method. + """ + pagedict = dict() # KEY: (testtype, release, milestone, compose, cid, modular) + insufficients = list() + dupes = list() + + for resitem in reslist: + # Convert the item into a ResTuple. + try: + restup = ResTuple(**resitem) + except TypeError: + restup = ResTuple(*resitem) + if not restup.status: + # It doesn't make sense to allow {{result|none}} + # from this high-level function. + insufficients.append(resitem) + continue + user = restup.user + if not user: + # If no username was given, guess at the wiki account + # name, lower-cased. + user = self.username.lower() + key = (restup.testtype, restup.release, restup.milestone, + restup.compose, restup.cid, restup.modular) + # We construct a dict to sort the results by page. The + # value for each page is a 2-tuple containing the actual + # ValidationPage object and the 'results dictionary' we + # will pass to page.add_results() once we have iterated + # through the full result list (unless the page cannot be + # found, in which case the value is None.) + if key not in pagedict: + # We haven't yet encountered this page, so we'll try + # and find it and add it to the dict. + pagedict[key] = dict() + try: + pagedict[key] = (self.get_validation_page(*key), dict()) + except ValueError: + # This means we couldn't find a page from the info + # provided; set the pagedict entry for this key to + # (None, None) to cache that information, add this + # result to the 'insufficients' list, and move on + # to the next. + pagedict[key] = (None, None) + insufficients.append(resitem) + continue + elif pagedict[key] == (None, None): + # This is when we've already tried to find a page from + # the same (testtype, release, milestone, compose) and + # come up empty, so append the restup to the insuffs + # list and move on. + insufficients.append(resitem) + continue + + # The code from here on gets hit whenever we can find the + # correct page for a restup. We need to find the correct + # ResultRow from the testcase, section, testname and + # env, and produce a Result from status, username, bugs, + # and comment. The keys in resdict are ResultRow + # instances, so we check if there's already one that's + # 'equal' to the ResultRow for this restup. If so, we + # append the (env, Result) tuple to the list that is the + # value for that key. If not, we add a new entry to the + # resdict. It's vital that the resdict contain only *one* + # entry for any given ResultRow, or else the edit will go + # squiffy. + (page, resdict) = pagedict[key] + try: + myrow = page.find_resultrow( + restup.testcase, restup.section, restup.testname, + restup.env) + myres = rs.Result( + restup.status, user, restup.bugs, + restup.comment, restup.bot) + except (NoPageError, NotFoundError, TooManyError): + # We couldn't find precisely one result row from the + # provided information. + insufficients.append(resitem) + continue + + done = False + for row in resdict.keys(): + if myrow.matches(row): + resdict[row].append((restup.env, myres)) + done = True + break + if not done: + resdict[myrow] = [(restup.env, myres)] + + # Finally, we've sorted our restups into the right shape. Now + # all we do is throw each page's resdict at its add_results() + # method and grab the return value, which is a list of tuples + # representing results that were 'dupes'. We then reconstruct + # a ResTuple for each dupe, and return the lists of insuffs + # and dupes. + for (page, resdict) in pagedict.values(): + if page: + _dupes = (page.add_results(resdict, allowdupe)) + for (row, env, result) in _dupes: + dupes.append(ResTuple( + page.testtype, page.release, page.milestone, + page.compose, row.testcase, row.section, row.name, + env, result.status, result.user, result.bugs, + result.comment, result.bot, '', page.modular)) + + return (insufficients, dupes) + +# vim: set textwidth=100 ts=8 et sw=4: diff --git a/tox.ini b/tox.ini index fa2cc8c..33c11f7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,15 @@ [tox] envlist = py27,py34,py35,py36,py37,py38 skip_missing_interpreters=true +isolated_build=true + [testenv] deps=-r{toxinidir}/install.requires -r{toxinidir}/tests.requires -r{toxinidir}/tox.requires -commands=py.test --cov-report term-missing --cov-report xml --cov wikitcms +commands=coverage run -m pytest {posargs} + coverage combine + coverage report + coverage xml diff-cover coverage.xml --fail-under=90 diff-quality --violations=pylint --fail-under=90 -setenv = - PYTHONPATH = {toxinidir} diff --git a/tox.requires b/tox.requires index c245df4..5b015e9 100644 --- a/tox.requires +++ b/tox.requires @@ -1,4 +1,4 @@ coverage diff-cover pylint -pytest-cov +toml diff --git a/wikitcms/__init__.py b/wikitcms/__init__.py deleted file mode 100644 index f249c07..0000000 --- a/wikitcms/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) Red Hat Inc. -# -# python-wikitcms is free software; you can redistribute it -# and/or modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation, either version 3 of -# the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""Library for interacting with Fedora release validation and test day -wiki pages. -""" - -from __future__ import unicode_literals -from __future__ import print_function - -__version__ = "2.5.2" - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/event.py b/wikitcms/event.py deleted file mode 100644 index d571d39..0000000 --- a/wikitcms/event.py +++ /dev/null @@ -1,404 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""Classes that describe test events.""" - -from __future__ import unicode_literals -from __future__ import print_function - -import abc -import logging - -import fedfind.helpers -import fedfind.release -import mwclient.errors -from cached_property import cached_property - -from . import listing -from . import page -from . import helpers -from .exceptions import FedfindNotFoundError - -logger = logging.getLogger(__name__) - - -class ValidationEvent(object): - """A parent class for different types of release validation event. - site must be an instance of wikitcms.Wiki, already with - appropriate access rights for any actions to be performed (i.e. - things instantiating an Event are expected to do site.login - themselves if needed). Required attributes: shortver, - category_page. If modular is True, the event will be for a - Fedora-Modular compose, with 'Modular' in the page names, category - names and so on. - """ - __metaclass__ = abc.ABCMeta - - def __init__(self, site, release, milestone='', compose='', modular=False): - self.site = site - self.release = release - self.milestone = str(milestone) - try: - self.compose = fedfind.helpers.date_check( - compose, fail_raise=True, out='str') - except ValueError: - self.compose = str(compose) - self.modular = modular - self.version = "{0} {1} {2}".format( - self.release, self.milestone, self.compose) - # Sorting helpers. sortname is a string, sorttuple is a - # 4-tuple. sorttuple is more reliable. See the function docs. - self.sortname = helpers.fedora_release_sort(self.version) - self.sorttuple = helpers.triplet_sort( - self.release, self.milestone, self.compose) - - @abc.abstractproperty - def _current_content(self): - """The content for the CurrentFedoraCompose template for - this test event. - """ - pass - - @abc.abstractproperty - def _pagetype(self): - """The ValidationPage class to be used for this event's pages - (for use by valid_pages). - """ - pass - - @abc.abstractproperty - def category_page(self): - """The category page for this event. Is a property because - page instantiation requires a remote trip. - """ - pass - - @property - def result_pages(self): - """A list of wikitcms page objects for currently-existing - pages that are a part of this test event, according to the - naming convention. - """ - _dist = "Fedora" - if self.modular: - _dist = "Fedora Modular" - pages = self.site.allresults( - prefix="{0} {1} ".format(_dist, self.version)) - return [p for p in pages if isinstance(p, page.ValidationPage)] - - @property - def download_page(self): - """The DownloadPage for this event. Is a property because page - instantiation requires a remote trip. - """ - return page.DownloadPage(self.site, self, modular=self.modular) - - @property - def ami_page(self): - """The AMIPage for this event. Is a property because page - instantiation requires a remote trip. - """ - return page.AMIPage(self.site, self, modular=self.modular) - - @property - def parent_category_page(self): - """The parent category page for this event. Is a property for - the same reason as download_page. - """ - return listing.ValidationCategory(self.site, self.release, modular=self.modular) - - @property - def valid_pages(self): - """A list of the expected possible result pages (as - page.ValidationPage objects) for this test event, derived from - the available test types and the naming convention. - """ - if self.modular: - types = self.site.modular_testtypes - else: - types = self.site.testtypes - return [self._pagetype(self.site, self.release, typ, - milestone=self.milestone, - compose=self.compose, modular=self.modular) - for typ in types] - - @property - def summary_page(self): - """The page.SummaryPage object for the event's result summary - page. Very simple property, but not set in __init__ as the - summary page object does (slow) wiki roundtrips in __init__. - """ - return page.SummaryPage(self.site, self, modular=self.modular) - - @cached_property - def ff_release(self): - """A fedfind release object matching this event.""" - # note: fedfind has a hack that parses date and respin out - # of a dot-separated compose, since wikitcms smooshes them - # into the compose value. - dist = "Fedora" - if self.modular: - dist = "Fedora-Modular" - try: - return fedfind.release.get_release(release=self.release, - milestone=self.milestone, - compose=self.compose, - dist=dist) - except ValueError as err: - try: - if self._cid: - return fedfind.release.get_release(cid=self._cid) - except AttributeError: - raise FedfindNotFoundError(err) - raise FedfindNotFoundError(err) - - @property - def ff_release_images(self): - """A fedfind release object matching this event, that has - images. If we can't find one, raise an exception. For the - base class this just acts as a check on ff_release; it does - something more clever in ComposeEvent. - """ - rel = self.ff_release - if rel.all_images: - return rel - else: - raise FedfindNotFoundError("Could not find fedfind release with images for event" - "{0}".format(self.version)) - - def update_current(self): - """Make the CurrentFedoraCompose template on the wiki point to - this event. The template is used for the Current (testtype) - Test redirect pages which let testers find the current results - pages, and for other features of wikitcms/relval. Children - must define _current_content. - """ - content = "{{tempdoc}}\n{{#switch: {{{1|full}}}\n" - content += self._current_content - content += "}}\n[[Category: Fedora Templates]]" - if self.modular: - curr = self.site.pages['Template:CurrentFedoraModularCompose'] - else: - curr = self.site.pages['Template:CurrentFedoraCompose'] - curr.save(content, "relval: update to current event", createonly=None) - - def create(self, testtypes=None, force=False, current=True, check=False): - """Create the event, by creating its validation pages, - summary page, download page, category pages, and updating the - current redirects. 'testtypes' can be an iterable that limits - creation to the specified testtypes. If 'force' is True, pages - that already exist will be recreated (destroying any results - on them). If 'current' is False, the current redirect pages - will not be updated. If 'check' is true, we check if any - result page already exists first, and bail immediately if so - (so we don't start creating pages then hit one that exists and - fail half-way through, for things that don't want that.) 'cid' - can be set to a compose ID, this is for forcing the compose - location at event creation time when we know we're not going - to be able to find it any way and is a short-term hack that - will be removed. - """ - logger.info("Creating validation event %s", self.version) - createonly = True - if force: - createonly = None - pages = self.valid_pages - if testtypes: - logger.debug("Restricting to testtypes %s", ' '.join(testtypes)) - pages = [pag for pag in pages if pag.testtype in testtypes] - if not pages: - raise ValueError("No result pages to create! Wrong test type?") - if check: - if any(pag.text() for pag in pages): - raise ValueError("A result page already exists!") - - # NOTE: download page creation for ComposeEvents will only - # work if: - # * the compose has being synced to stage, OR - # * the compose has been imported to PDC, OR - # * you used get_validation_event and passed it a cid - # Otherwise, the event will be created, but the download page - # will not. - pages.extend((self.summary_page, self.download_page, self.ami_page, - self.category_page, self.parent_category_page)) - - def _handle_existing(err): - """We need this in two places, so.""" - if err.args[0] == 'articleexists': - # no problem, just move on. - logger.info("Page already exists, and forced write was not " - "requested! Not writing.") - else: - raise err - - for pag in pages: - try: - # stage 1 - create page - logger.info("Creating page %s", pag.name) - pag.write(createonly=createonly) - except mwclient.errors.APIError as err: - _handle_existing(err) - except FedfindNotFoundError: - # this happens if download page couldn't be created - # because fedfind release couldn't be found - logger.warning("Could not create download page for event %s as fedfind release " - "was not found!") - - # stage 2 - update current. this is split so if we hit - # 'page already exists', we don't skip update_current - if current and hasattr(pag, 'update_current'): - logger.info("Pointing Current redirect to above page") - pag.update_current() - - if current: - try: - # update CurrentFedoraCompose - logger.info("Updating CurrentFedoraCompose") - self.update_current() - except mwclient.errors.APIError as err: - _handle_existing(err) - - @classmethod - def from_page(cls, pageobj): - """Return the ValidationEvent object for a given ValidationPage - object. - """ - return cls(pageobj.site, pageobj.release, pageobj.milestone, - pageobj.compose, modular=pageobj.modular) - - -class ComposeEvent(ValidationEvent): - """An Event that describes a release validation event - that is, - the testing for a particular nightly, test compose or release - candidate build. - """ - def __init__(self, site, release, milestone, compose, modular=False, cid=''): - super(ComposeEvent, self).__init__( - site, release, milestone=milestone, compose=compose, modular=modular) - # this is a little hint that gets set via get_validation_event - # when getting a page or event by cid; it helps us find the - # fedfind release for the event if the compose is not yet in - # PDC or synced to stage - self._cid = cid - self.shortver = "{0} {1}".format(self.milestone, self.compose) - - @property - def category_page(self): - """The category page for this event. Is a property because - page instantiation requires a remote trip. - """ - return listing.ValidationCategory( - self.site, self.release, self.milestone, modular=self.modular) - - @property - def _current_content(self): - """The content for the CurrentFedoraCompose template for - this test event. - """ - tmpl = ("| full = {0}\n| release = {1}\n| milestone = {2}\n" - "| compose = {3}\n| date =\n") - return tmpl.format( - self.version, self.release, self.milestone, self.compose) - - @property - def _pagetype(self): - """For a ComposeEvent, obviously, ComposePage.""" - return page.ComposePage - - @cached_property - def creation_date(self): - """We need this for ordering and determining delta between - this event and a potential new nightly event, if this is the - current event. Return creation date of the first result page - for the event, or "" if it doesn't exist. - """ - try: - return self.result_pages[0].creation_date - except IndexError: - # event doesn't exist - return "" - - @property - def ff_release_images(self): - """A fedfind release object matching this event, that has - images. If we can't find one, raise an exception. Here, we - try getting the release by (release, milestone, compose), but - if that release has no images - which happens in the specific - case that we've just created an event for a candidate compose - which has not yet been synced to stage - and we have the cid - hint, we try getting a release by cid instead, which should - find the compose in kojipkgs (a fedfind Production rather than - Compose). - """ - rel = self.ff_release - if rel.all_images: - return rel - - if self._cid: - rel = fedfind.release.get_release(cid=self._cid) - if rel.all_images: - return rel - else: - raise FedfindNotFoundError("Could not find fedfind release with images for event " - "{0}".format(self.version)) - - -class NightlyEvent(ValidationEvent): - """An Event that describes a release validation event - that is, - the testing for a particular nightly, test compose or release - candidate build. Milestone should be 'Rawhide' or 'Branched'. - Note that a Fedora release number attached to a Rawhide nightly - compose is an artificial concept that can be considered a Wikitcms - artifact. Rawhide is a rolling distribution; its nightly composes - do not really have a release number. What we do when we attach - a release number to a Rawhide nightly validation test event is - *declare* that, with our knowledge of Fedora's development cycle, - we believe the testing of that Rawhide nightly constitutes a part - of the release validation testing for that future release. - """ - def __init__(self, site, release, milestone, compose, modular=False): - super(NightlyEvent, self).__init__( - site, release, milestone=milestone, compose=compose, modular=modular) - self.shortver = self.compose - self.creation_date = compose.split('.')[0] - - @property - def category_page(self): - """The category page for this event. Is a property because - page instantiation requires a remote trip. - """ - return listing.ValidationCategory( - self.site, self.release, nightly=True, modular=self.modular) - - @property - def _current_content(self): - """The content for the CurrentFedoraCompose template for - this test event. - """ - tmpl = ("| full = {0}\n| release = {1}\n| milestone = {2}\n" - "| compose =\n| date = {3}\n") - return tmpl.format( - self.version, self.release, self.milestone, self.compose) - - @property - def _pagetype(self): - """For a NightlyEvent, obviously, NightlyPage.""" - return page.NightlyPage - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/exceptions.py b/wikitcms/exceptions.py deleted file mode 100644 index 6c6c3a3..0000000 --- a/wikitcms/exceptions.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2015 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""Defines custom exceptions used by wikitcms.""" - -from __future__ import unicode_literals -from __future__ import print_function - - -class NoPageError(Exception): - """Page does not exist.""" - pass - - -class NotFoundError(Exception): - """Requested thing wasn't found.""" - pass - - -class TooManyError(Exception): - """Found too many of the thing you asked for.""" - pass - - -# this inherits from ValueError as the things that raise this may -# previously have passed along a ValueError from fedfind -class FedfindNotFoundError(ValueError, NotFoundError): - """Couldn't find a fedfind release (probably the fedfind release - that matches an event). - """ - pass - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/helpers.py b/wikitcms/helpers.py deleted file mode 100644 index 1dd2e7c..0000000 --- a/wikitcms/helpers.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson -# -"""This file contains helper functions that don't strictly belong in -any particular class or even in another file but outside of a class.""" - -from __future__ import unicode_literals -from __future__ import print_function - -import os -import re -from collections import OrderedDict -from decimal import Decimal - -import fedfind.helpers -import fedfind.release - -MILESTONE_PAIRS = ( - ('Rawhide', '100'), - # We called Branched nightly pages 'Nightly' in F21 cycle. - ('Nightly', '149'), - ('Branched', '150'), - ('Pre-Alpha', '175'), - ('Alpha', '200'), - ('Pre-Beta', '375'), - ('Beta', '400'), - ('Preview', '600'), - ('Pre-Final', '775'), - ('Final', '800'), - ('Postrelease', '900'), -) - -COMPOSE_PAIRS = ( - # Some F12 crap - ('PreBeta', '100'), - ('TC', '200'), - # and this. - ('Pre-RC', '300'), - # The extra digit here is a dumb way to make sure RC1 sorts - # later than TC10 - otherwise you get 20010 vs. 6001, and 20010 - # wins. It might be better to treat the 'TC' / 'RC' as a separate - # element in triplet_sort, but that's a bit harder and makes its - # name a lie... - ('RC', '6000'), -) - -def fedora_release_sort(string): - """Fed a string that looks something like a Fedora pre-release / - compose version, this will output a modified version of the string - which should sort correctly against others like it. Handles stuff - like 'Preview' coming before 'Final', and 'TC' coming before 'RC'. - 'Alpha', 'Beta' and 'Final' sort correctly by pure chance, but we - handle them here anyway to make sure. wikitcms ValidationEvent and - ValidationPage objects have a 'sortname' property you can use - instead of calling this directly. Milestones have a letter so they - sort after nightlies (nightlies are usually going to be earlier). - NOTE: can only sort compose event versions, really. With this - function, '22 Alpha TC1' > '22 Alpha'. - """ - # Some MILESTONES are substrings of COMPOSES so we do Cs first - for (orig, repl) in COMPOSE_PAIRS + MILESTONE_PAIRS: - string = string.replace(orig, repl) - return string - -def triplet_sort(release, milestone, compose): - """Just like fedora_release_sort, but requires you to pass the - now-'standard' release, milestone, compose triplet of inputs. - This is a better way in most cases as you're going to have more - certainty about instantiating wikitcms/fedfind objects from it, - plus we can handle things like '23' being higher than '23 Beta' - or '23 Final TC1'. Expects the inputs to be strings. The elements - in the output tuple will be ints if possible as this gives a - better sort, but may be strings if we missed something; you're not - meant to *do* anything with the tuple but compare it to another - similar tuple. - """ - for (orig, repl) in MILESTONE_PAIRS: - milestone = milestone.replace(orig, repl) - if not milestone: - # ('23', 'Final', '') == ('23', '', '') - milestone = '800' - for (orig, repl) in COMPOSE_PAIRS: - compose = compose.replace(orig, repl) - if not compose: - compose = '999' - # We want to get numerical sorts if we possibly can, so e.g. - # TC10 (becomes 20010) > TC9 (becomes 2009). But just in case - # we get passed a character we don't substitute to a digit, - # check first. - release = str(release) - if release.isdigit(): - release = int(release) - if milestone.isdigit(): - milestone = int(milestone) - if compose.isdigit(): - compose = int(compose) - else: - # this is a bit magic but handles "TC1.1" (old-school), - # "1.1" (new-school candidates), and "20160314.n.0" (new- - # school nightlies) - (comp, respin) = (compose.split('.')[0], compose.split('.')[-1]) - if comp.isdigit() and respin.isdigit(): - compose = Decimal('.'.join((comp, respin))) - return (release, milestone, compose) - -def triplet_unsort(release, milestone, compose): - """Reverse of triplet_sort.""" - (release, milestone, compose) = (str(release), str(milestone), - str(compose)) - for (repl, orig) in MILESTONE_PAIRS: - milestone = milestone.replace(orig, repl) - if milestone == '800': - milestone = 'Final' - # We don't want to do the replace here if the compose is a date, - # because any of the values might happen to be in the date. This - # is a horrible hack that should be OK (until 2100, at least). - if fedfind.helpers.date_check(compose, fail_raise=False): - pass - elif fedfind.helpers.date_check(compose.split('.')[0], fail_raise=False): - # this is slightly magic but should do for now - (comp, respin) = compose.split('.') - compose = "{0}.n.{1}".format(comp, respin) - else: - for (repl, orig) in COMPOSE_PAIRS: - compose = compose.replace(orig, repl) - if compose == '999': - compose = '' - return (release, milestone, compose) - -def rreplace(string, old, new, occurrence): - """A version of the str.replace() method which works from the right. - Taken from https://stackoverflow.com/questions/2556108/ - """ - elems = string.rsplit(old, occurrence) - return new.join(elems) - -def normalize(text): - """Lower case and replace ' ' with '_' so I don't have to - keep retyping it. - """ - return text.lower().replace(' ', '_') - -def find_bugs(text): - """Find RH bug references in a given chunk of text. More than one - method does this, so we'll put the logic here and they can share - it. Copes with [[rhbug:(ID)]] links, {{bz|(ID)}} templates and - URLs that look like Bugzilla. Doesn't handle aliases (only matches - numeric IDs 6 or 7 digits long). Returns a set (so bugs that occur - multiple times in the text will only appear once in the output). - """ - bugs = set() - bzpatt = re.compile(r'({{bz *\| *|' - r'\[\[rhbug *: *|' - r'bugzilla\.redhat\.com/show_bug\.cgi\?id=)' - r'(\d{6,7})') - matches = bzpatt.finditer(text) - for match in matches: - bugs.add(match.group(2)) - # Filter out bug IDs usually used as examples - for bug in ('12345', '123456', '54321', '654321', '234567', '1234567', - '7654321'): - bugs.discard(bug) - return bugs - -def cid_to_event(cid): - """Given a Pungi 4 compose ID, figure out the appropriate wikitcms - (release, milestone, compose) triplet. Guesses the appropriate - release number to assign to the event for Rawhide composes. To - do the opposite, you can use the ff_release property of Validation - Event instances. - """ - parsed = fedfind.helpers.parse_cid(cid, dic=True) - dist = parsed['short'] - release = parsed['version'].lower() - vertype = parsed['version_type'] - date = parsed['date'] - typ = parsed['compose_type'] - respin = parsed['respin'] - if dist not in ("Fedora", "Fedora-Modular"): - # we only have validation events for Fedora and Modular - # composes ATM - not e.g. FedoraRespin or Fedora-Atomic ones - raise ValueError("No validation events for {0} composes!".format(dist)) - if vertype != 'ga': - # this ensures we don't create events for updates/u-t composes - raise ValueError("No validation events for updates composes!") - if typ == "production": - # we need to get the label and parse that - ffrel = fedfind.release.get_release(cid=cid) - if not ffrel.label: - raise ValueError("{0} looks like a production compose, but found " - "no label! Cannot determine event.".format(cid)) - (milestone, compose) = ffrel.label.rsplit('-', 1) - return (dist, release, milestone, compose) - - # FIXME: we have no idea yet what to do for 'test' composes - if not typ == "nightly": - raise ValueError( - "cid_to_event(): cannot guess event for 'test' compose yet!") - - # nightlies - if release == "rawhide": - # the release number for new Rawhide events should always be - # 1 above the highest current 'real' release (branched or - # stable). FIXME: this will not do the right thing for *old* - # composes, not sure if we can fix that sensibly. - release = str(fedfind.helpers.get_current_release(branched=True) + 1) - milestone = "Rawhide" - else: - milestone = "Branched" - compose = "{0}.n.{1}".format(date, respin) - return (dist, release, milestone, compose) - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/listing.py b/wikitcms/listing.py deleted file mode 100644 index 0cfd2f3..0000000 --- a/wikitcms/listing.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson -# -"""This file kind of shadows mwclient's listing.py, creating modified -versions of several of its classes. The point is to provide generators -similar to mwclient's own, but which return wikitcms page/category -instances when appropriate, falling through to mwclient instances -otherwise. -""" - -from __future__ import unicode_literals -from __future__ import print_function - -import re - -from mwclient import listing as mwl - -from . import page as pg - -# exceptions: the wikitcms getter raises these when it fails rather than -# just returning None, so the generators can use try/except blocks to -# handle both this case and the case (which shouldn't ever happen, but -# just in case) where they're being used on something other than a list -# of pages. - -class NoPageWarning(Exception): - """Exception raised when the tcmswiki getter can't find a matching - page. Not really an error, should always be handled. - """ - def __init__(self, page): - self.page = page - - def __str__(self): - return "Could not produce a wikitcms page for: {0}".format(self.page) - - -class PageCheckWarning(Exception): - """Exception raised when the wikitcms getter finds a matching page, - but the page name the class generators from the page's various - attributes doesn't match the page name the getter was given. Should - usually be handled (and an mwclient Page instance returned instead). - """ - def __init__(self, frompage, topage): - self.frompage = frompage - self.topage = topage - - def __str__(self): - return ("Expected page name {0} does not match source " - "page name {1}".format(self.frompage, self.topage)) - - -class TcmsGeneratorList(mwl.GeneratorList): - """A GeneratorList which returns wikitcms page (and category etc.) - instances when appropriate. _get_tcms is implemented as a separate - function so TcmsPageList can use the discovery logic. - """ - def __init__(self, site, list_name, prefix, *args, **kwargs): - super(TcmsGeneratorList, self).__init__( - site, list_name, prefix, *args, **kwargs) - - def __next__(self): - # We can't get the next entry from mwl.List ourselves, try and - # handle it, then pass it up to our parent if we can't, because - # parent's next() gets the next entry from mwl.List itself, so - # in that scenario, one list item gets skipped. Either we - # entirely clone next() with our own additions, or we let it - # fire and then override the result if we can. Using nxt._info - # is bad, but super.next() doesn't return that, and the page - # instance doesn't expose it any other way. We could just use - # the name, but if you don't pass info when instantiating a - # Page, it has to hit the API during init to reconstruct info, - # and that causes a massive performance hit. - nxt = super(TcmsGeneratorList, self).__next__() - try: - return self._get_tcms(nxt.name, nxt._info) - except (NoPageWarning, PageCheckWarning): - return nxt - - def next(self): - # for python2 compat - return self.__next__() - - def _check_page(self, name, page): - # convenience function for _get_tcms sanity check - if page.checkname == name: - return page - raise PageCheckWarning(page.checkname, name) - - def _get_tcms(self, name, info=()): - # this is the meat: it runs a bunch of string checks on page - # names, basically, and returns the appropriate wikitcms - # object if any matches. - if isinstance(name, int): - # we'll have to do a wiki roundtrip, as we need the text - # name. - page = pg.Page(self.site, name) - name = page.name - name = name.replace('_', ' ') - # quick non-RE check to see if we'll ever match (and filter - # out some 'known bad' pages) - if (name.startswith('Test Results:') or - (name.startswith('Test Day:') and not - name.endswith('/ru') and not - 'metadata' in name.lower() and not - 'rendercheck' in name.lower()) or - (name.startswith('Category:'))): - nightly_patt = re.compile(r'Test Results:Fedora (Modular )?(\d{1,3}) ' - r'(Rawhide|Nightly|Branched) ' - r'(\d{8,8}(\.n\.\d+)?|\d{4,4} \d{2,2}) ' - r'(.+)$') - accept_patt = re.compile(r'Test Results:Fedora (\d{1,3}) ' - r'([^ ]+?) (Rawhide |)Acceptance Test ' - r'(\d{1,2})$') - ms_patt = re.compile(r'Test Results:Fedora (Modular )?(\d{1,3}) ' - r'([^ ]+?) ([^ ]+?) (.+)$') - cat_patt = re.compile(r'Category:Fedora (Modular )?(\d{1,3}) ' - r'(.*?) *?Test Results$') - tdcat_patt = re.compile(r'Category:Fedora (\d{1,3}) Test Days$') - testday_patt = re.compile(r'Test Day:(\d{4}-\d{2}-\d{2}) *(.*)$') - # FIXME: There's a few like this, handle 'em sometime - #testday2_patt = re.compile(u'Test Day:(.+) (\d{4}-\d{2}-\d{2})$') - - # Modern standard nightly compose event pages, and F21-era - # monthly Rawhide/Branched test pages - match = nightly_patt.match(name) - if match: - if match.group(6) == 'Summary': - # we don't really ever need to do anything to existing - # summary pages, and instantiating one from here is kinda - # gross, so just fall through - raise NoPageWarning(name) - modular = False - if match.group(1): - modular = True - page = pg.NightlyPage( - self.site, release=match.group(2), testtype=match.group(6), - milestone=match.group(3), compose=match.group(4), - info=info, modular=modular) - return self._check_page(name, page) - - match = accept_patt.match(name) - if match: - # we don't handle these, yet. - raise NoPageWarning(name) - - # milestone compose event pages - match = ms_patt.match(name) - if match: - if match.group(5) == 'Summary': - raise NoPageWarning(name) - modular = False - if match.group(1): - modular = True - page = pg.ComposePage( - self.site, release=match.group(2), testtype=match.group(5), - milestone=match.group(3), compose=match.group(4), - info=info, modular=modular) - return self._check_page(name, page) - - # test result categories - match = cat_patt.match(name) - if match: - modular = False - if match.group(1): - modular = True - if not match.group(3): - page = ValidationCategory( - self.site, match.group(2), info=info, modular=modular) - return self._check_page(name, page) - elif match.group(3) == 'Nightly': - page = ValidationCategory(self.site, match.group(2), - nightly=True, info=info, modular=modular) - return self._check_page(name, page) - else: - page = ValidationCategory(self.site, match.group(2), - match.group(3), info=info, modular=modular) - return self._check_page(name, page) - - # Test Day categories - match = tdcat_patt.match(name) - if match: - page = TestDayCategory(self.site, match.group(1), info=info) - return self._check_page(name, page) - - # test days - match = testday_patt.match(name) - if match: - page = pg.TestDayPage(self.site, match.group(1), - match.group(2), info=info) - return self._check_page(name, page) - raise NoPageWarning(name) - - -class TcmsPageList(mwl.PageList, TcmsGeneratorList): - """A version of PageList which returns wikitcms page (and category - etc.) objects when appropriate. - """ - def get(self, name, info=()): - modname = name - if self.namespace: - modname = '{0}:{1}'.format(self.site.namespaces[self.namespace], - name) - try: - return self._get_tcms(modname, info) - except (NoPageWarning, PageCheckWarning): - return super(TcmsPageList, self).get(name, info) - - -class TcmsCategory(pg.Page, TcmsGeneratorList): - """A modified category class - just as mwclient's Category class - inherits from both its Page class and its GeneratorList class, - acting as both a page and a generator returning the members of - the category, so this inherits from wikitcms' Page and - TcmsGeneratorList. You can produce the page contents with pg.Page - write() method, and you can use it as a generator which returns - the category's members, as wikitcms class instances if appropriate - or mwclient class instances otherwise. It works recursively - if - a member of a ValidationCategory is itself a test category, you'll - get another ValidationCategory instance. There are sub-classes for - various particular types of category (Test Days, validation, etc.) - """ - def __init__(self, site, wikiname, info=None): - super(TcmsCategory, self).__init__(site, wikiname, info=info) - TcmsGeneratorList.__init__(self, site, 'categorymembers', 'cm', - gcmtitle=self.name) - - -class ValidationCategory(TcmsCategory): - """A category class (inheriting from TcmsCategory) for validation - test result category pages. If nightly is True, this will be a - category for test results from Rawhide or Branched nightly builds - for the given release. Otherwhise, if milestone is passed, this - will be a category for the given milestone, and if it isn't, it - will be the top-level category for the given release. - """ - - def __init__(self, site, release, milestone=None, nightly=False, - info=None, modular=False): - _dist = "Fedora" - if modular: - _dist = "Fedora Modular" - if nightly is True: - wikiname = ("Category:{0} {1} Nightly Test " - "Results").format(_dist, release) - if modular: - self.seedtext = ("{{{{Validation results milestone category|" - "release={0}|nightly=true|modular=true}}}}").format(release) - else: - self.seedtext = ("{{{{Validation results milestone category|" - "release={0}|nightly=true}}}}").format(release) - - self.summary = ("Relval bot-created validation result category " - "page for {0} {1} nightly " - "results").format(_dist, release) - elif milestone: - wikiname = "Category:{0} {1} {2} Test Results".format( - _dist, release, milestone) - if modular: - self.seedtext = ("{{{{Validation results milestone category" - "|release={0}|" - "milestone={1}|modular=true}}}}").format(release, milestone) - else: - self.seedtext = ("{{{{Validation results milestone category" - "|release={0}|" - "milestone={1}}}}}").format(release, milestone) - self.summary = ("Relval bot-created validation result category " - "page for {0} " - "{1} {2}").format(_dist, release, milestone) - else: - wikiname = "Category:{0} {1} Test Results".format(_dist, release) - if modular: - self.seedtext = ("{{{{Validation results milestone category" - "|release={0}|modular=true}}}}").format(release) - else: - self.seedtext = ("{{{{Validation results milestone category" - "|release={0}}}}}").format(release) - self.summary = ("Relval bot-created validation result category " - "page for {0} {1}").format(_dist, release) - - super(ValidationCategory, self).__init__(site, wikiname, info=info) - - -class TestDayCategory(TcmsCategory): - """A category class (inheriting from TcmsCategory) for Test Day - category pages. - """ - - def __init__(self, site, release, info=None): - wikiname = "Category:Fedora {0} Test Days".format(str(release)) - self.seedtext = ( - "This category contains all the Fedora {0} [[QA/Test_Days|Test " - "Day]] pages. A calendar of the Test Days can be found [" - "https://apps.fedoraproject.org/calendar/list/QA/?subject=Test+Day" - " here].\n\n[[Category:Test Days]]").format(str(release)) - self.summary = "Created page (via wikitcms)" - super(TestDayCategory, self).__init__(site, wikiname, info=info) - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/page.py b/wikitcms/page.py deleted file mode 100644 index bf767e4..0000000 --- a/wikitcms/page.py +++ /dev/null @@ -1,789 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""Classes that describe different types of pages we are interested -in, and attributes of pages like test results and test cases, are -defined in this file. -""" - -from __future__ import unicode_literals -from __future__ import print_function - -import datetime -import logging -import pytz -import re -import time -from collections import OrderedDict - -import fedfind.helpers -import fedfind.release -from cached_property import cached_property -from mwclient import errors as mwe -from mwclient import page as mwp - -from . import result as rs -from . import helpers -from .exceptions import NoPageError, NotFoundError, TooManyError - -logger = logging.getLogger(__name__) - - -class Page(mwp.Page): - """Parent class for all page classes. Can be instantiated directly - if you just want to take advantage of the convenience methods like - sections() and save(). Available attributes: seedtext, summary. - Note 'name' is defined by mwp.Page's __init__. - """ - - def __init__(self, site, wikiname, info=None, extra_properties=None): - super(Page, self).__init__(site, wikiname, info, extra_properties) - # Used for sanity check by the page generator - self.checkname = wikiname - self._sections = None - self.results_separators = list() - - @property - def sections(self): - """A list of the page's sections. Each section is represented - by a dict whose values provide various attributes of the - section. Returns an empty list for non-existent page (or any - other API error). Cached, cache cleared on each page save. - """ - # None == not yet retrieved or cache expired. [] == retrieved, - # but page is empty or something. - if self._sections is None: - try: - apiout = self.site.api( - 'parse', page=self.name, prop='sections') - self._sections = apiout['parse']['sections'] - except mwe.APIError: - self._sections = [] - return self._sections - - @property - def results_wikitext(self): - """Returns a string containing the wikitext for the page's - results section. Will be empty if no results are found. Relies - on the layout for result pages remaining consistent. Class - must override definition of self.results_separators or else - this will always return an empty string. - """ - pagetext = self.text() - comment = re.compile('', re.S) - pos = -1 - for sep in self.results_separators: - pos = pagetext.find(sep) - if pos > -1: - break - - if pos == -1: - return '' - text = pagetext[pos:] - text = comment.sub('', text) - return text - - @cached_property - def creation_date(self): - """Date the page was created. Used for sorting and seeing how - long it's been since the last event, when creating new events. - """ - revs = self.revisions(limit=1, dir='newer', prop='timestamp') - try: - origrev = next(revs) - except StopIteration: - # page doesn't exist - return "" - return time.strftime('%Y%m%d', origrev['timestamp']) - - def write(self, createonly=True): - """Create a page with its default content and summary. mwclient - exception will be raised on any page write failure. - """ - seedtext = getattr(self, 'seedtext', None) - summary = getattr(self, 'summary', None) - if seedtext is None or summary is None: - raise ValueError("wikitcms.Page.write(): both seedtext and summary needed!") - self.save(seedtext, summary, createonly=createonly) - - def save(self, *args, **kwargs): - """Same as the original, but will retry once on fail. If you - already retrieved the current text, you can pass it in as - oldtext, and we will check to see if oldtext and text are the - same. If they are, we return a dict with the key nochange set - to an empty string - this saves a needless extra remote round - trip. Of course you could do this in the caller instead, it's - just a convenience. Also clears the page sections cache. - """ - if 'oldtext' in kwargs and args[0] == kwargs['oldtext']: - return dict(nochange='') - - if 'oldtext' in kwargs: - # avoid mwclient save() warning about unknown kwarg - del kwargs['oldtext'] - - try: - ret = super(Page, self).save(*args, **kwargs) - except mwe.EditError as err: - logger.warning("Page %s edit failed! Trying again in 15 seconds", - self.name) - logger.debug("Error was: %s", err) - time.sleep(15) - ret = super(Page, self).save(*args, **kwargs) - # clear the caches - self._sections = None - return ret - - -class ValidationPage(Page): - """A parent class for different types of release validation event - pages, containing common properties and methods. Required - attributes: version, shortver, seedtext. If modular is True, the - page will be for a Fedora-Modular compose, with 'Modular' in the - page name, using the appropriate templates and template values, - etc. - """ - def __init__(self, site, release, testtype, milestone='', compose='', - info=None, modular=False): - self.release = release - self.milestone = str(milestone) - try: - self.compose = fedfind.helpers.date_check( - compose, fail_raise=True, out='str') - except ValueError: - self.compose = str(compose) - self.version = "{0} {1} {2}".format( - self.release, self.milestone, self.compose) - self.testtype = testtype - self.modular = modular - - # Wiki name the page should have, according to the naming - # convention. - _dist = "Fedora" - if modular: - _dist = "Fedora Modular" - wikiname = "Test Results:{0} {1} {2}".format(_dist, self.version, self.testtype) - super(ValidationPage, self).__init__(site, wikiname, info) - - # Edit summary to be used for clean page creation. - self.summary = ("Relval bot-created {0} validation results page for {1} " - "{2}").format(testtype, _dist, self.version) - self.results_separators = ( - "Test Matri", "Test Areas", "An unsupported test or configuration." - " No testing is required.") - # Sorting helpers. sortname is a string, sorttuple is a - # 4-tuple. sorttuple is more reliable. See the function docs. - self.sortname = helpers.fedora_release_sort( - ' '.join((self.version, self.testtype))) - self.sorttuple = helpers.triplet_sort( - self.release, self.milestone, self.compose) + (self.testtype,) - - @property - def results_sections(self): - """A list of the sections in the page which (most likely) - contain test results. Takes all the sections in the page, - finds the one one which looks like the first "test results" - section and returns that section and those that follow it - or - returns all sections after the Key section, if it can't find - one which looks like the first results section. - """ - secs = self.sections - if not secs: - # empty page or some other malarkey - return secs - first = None - for i, sec in enumerate(secs): - if 'Test Matri' in sec['line'] or 'Test Areas' in sec['line']: - first = i - break - elif 'Key' in sec['line']: - first = i+1 - return secs[first:] - - def get_resultrows(self, statuses=None, transferred=True): - """Returns the result.ResultRow objects representing all the - page's table rows containing test results. - """ - sections = self.results_sections - if not sections: - return list() - rows = list() - pagetext = self.text() - comment = re.compile('', re.S) - for i, sec in enumerate(sections): - try: - nextsec = sections[i+1] - except IndexError: - nextsec = None - section = sec['line'] - secid = sec['index'] - if nextsec: - sectext = pagetext[sec['byteoffset']:nextsec['byteoffset']] - else: - sectext = pagetext[sec['byteoffset']:] - # strip comments - sectext = comment.sub('', sectext) - newrows = rs.find_resultrows(sectext, section, secid, statuses, - transferred) - rows.extend(newrows) - return rows - - def find_resultrow(self, testcase='', section='', testname='', env=''): - """Return exactly one result row with the desired attributes, - or raise an exception (if more or less than one row is found). - The Installation page contains some rows in the same section - with the same testcase and testname, but each row provides - a different set of envs, so these can be uniquely identified - by specifying the desired env. - """ - rows = self.get_resultrows() - if not rows: - raise NoPageError("Page does not exist or has no result rows.") - - # Find the right row - nrml = helpers.normalize - rows = [r for r in rows if - nrml(testcase) in nrml(r.testcase) - or nrml(testcase) in nrml(r.name)] - if len(rows) > 1 and section: - rows = [r for r in rows if nrml(section) in nrml(r.section)] - if len(rows) > 1 and testname: - rows = [r for r in rows if nrml(testname) in nrml(r.name)] - if len(rows) > 1 and env: - # the way this match is done must be kept in line with the - # corresponding match in add_results, below - rows = [r for r in rows if nrml(env) in - [renv.lower() for renv in r.results.keys()]] - # try a more precise name match - e.g. "upgrade_dnf" vs. - # "upgrade_dnf_encrypted" - if len(rows) > 1: - rows = [r for r in rows if - nrml(testcase) == nrml(r.testcase) or - nrml(testcase) == nrml(r.name) or - nrml(testname) == nrml(r.name)] - if not rows: - raise NotFoundError("Specified row cannot be found.") - if len(rows) > 1: - raise TooManyError("More than one matching row found.") - return rows[0] - - def update_current(self): - """Make the Current convenience redirect page on the wiki for - the given test type point to this page. - """ - if self.modular: - curr = self.site.pages[ - 'Test Results:Current Modular {0} Test'.format(self.testtype)] - else: - curr = self.site.pages[ - 'Test Results:Current {0} Test'.format(self.testtype)] - curr.save("#REDIRECT [[{0}]]".format(self.name), - "relval: update to current event", createonly=None) - - def add_results(self, resultsdict, allowdupe=False): - """Adds multiple results to the page. Passed a dict whose - keys are ResultRow() instances and whose values are iterables - of (env, Result()) 2-tuples. Returns a list, which will be - empty unless allowdupe is False and any of the results is a - 'dupe' - i.e. the given test and environment already have a - result from the user. The return list contains a 3-tuple of - (row, env, result) for each dupe. - """ - # We need to sort the dict in a particular way: by the section - # ID of each row, in reverse order. This is so when we edit - # the page, we effectively do so backwards, and the byte - # offsets we use to find each section don't get thrown off - # along the way (we don't edit section 1 before section 3 and - # thus not quite slice the text correctly when we look for - # section 3). - resultsdict = OrderedDict(sorted(resultsdict.items(), - key=lambda x: int(x[0].secid), - reverse=True)) - nonetext = rs.Result().result_template - dupes = list() - newtext = oldtext = self.text() - for (row, results) in resultsdict.items(): - # It's possible that we have rows with identical text in - # different page sections; this is why 'secid' is an attr - # of ResultRows. To make sure we edit the correct row, - # we'll slice the text at the byteoffset of the row's - # section. We only do one replacement, so we don't need - # to bother finding the *end* of the section. - # We could just edit the page section-by-section, but that - # involves doing one remote roundtrip per section. - secoff = [sec['byteoffset'] for sec in self.sections if - sec['index'] == row.secid][0] - if secoff: - sectext = newtext[secoff:] - else: - sectext = newtext - oldrow = row.origtext - cells = oldrow.split('\n|') - - for (env, result) in results: - if not env in row.results: - # the env passed wasn't precisely one of the row's - # envs. let's see if we can make a safe guess. If - # there's only one env, it's easy... - if len(row.results) == 1: - env = list(row.results.keys())[0] - else: - # ...if not, we'll see if the passed env is - # a substring of only one of the envs, case- - # insensitively. - cands = [cand for cand in row.results.keys() if - env.lower() in cand.lower()] - if len(cands) == 1: - env = cands[0] - else: - # LOG: bad env - continue - if not allowdupe: - dupe = [r for r in row.results[env] if - r.user == result.user] - if dupe: - dupes.append((row, env, result)) - continue - restext = result.result_template - rescell = cells[row.columns.index(env)] - if nonetext in rescell: - rescell = rescell.replace(nonetext, restext) - elif '\n' in rescell: - rescell = rescell.replace('\n', restext+'\n') - else: - rescell = rescell + restext - cells[row.columns.index(env)] = rescell - - newrow = '\n|'.join(cells) - if newrow == oldrow: - # All dupes, or something. - continue - sectext = sectext.replace(oldrow, newrow, 1) - if secoff: - newtext = newtext[:secoff] + sectext - else: - newtext = sectext - - if len(resultsdict) > 3: - testtext = ', '.join(row.name for row in list(resultsdict.keys())[:3]) - testtext = '{0}...'.format(testtext) - else: - testtext = ', '.join(row.name for row in resultsdict.keys()) - summary = ("Result(s) for test(s): {0} filed via " - "relval").format(testtext) - self.save(newtext, summary, oldtext=oldtext, createonly=None) - return dupes - - def add_result(self, result, row, env, allowdupe=False): - """Adds a result to the page. Must be passed a Result(), the - result.ResultRow() object representing the row into which a - result will be added, and the name of the environment for - which the result is to be reported. Works by replacing the - first instance of the row's text encountered in the page or - page section. Expected to be used together with get_resultrows - which provides the ResultRow() objects. - """ - resdict = dict() - resdict[row] = ((env, result),) - return self.add_results(resdict, allowdupe=allowdupe) - - -class ComposePage(ValidationPage): - """A Page class that describes a single result page from a test - compose or release candidate validation test event. - """ - def __init__(self, site, release, testtype, milestone, compose, info=None, modular=False): - super(ComposePage, self).__init__( - site, release=release, milestone=milestone, compose=compose, - testtype=testtype, info=info, modular=modular) - self.shortver = "{0} {1}".format(self.milestone, self.compose) - - # String that will generate a clean copy of the page using the - # test page generation template system. - if self.modular: - seedtmpl = ("{{{{subst:Modular validation results|testtype={0}|release={1}|" - "milestone={2}|compose={3}}}}}") - else: - seedtmpl = ("{{{{subst:Validation results|testtype={0}|release={1}|" - "milestone={2}|compose={3}}}}}") - self.seedtext = seedtmpl.format( - testtype, self.release, self.milestone, self.compose) - - -class NightlyPage(ValidationPage): - """A Page class that describes a single result page from a nightly - validation test event. - """ - def __init__(self, site, release, testtype, milestone, compose, info=None, modular=False): - super(NightlyPage, self).__init__( - site, release=release, milestone=milestone, compose=compose, - testtype=testtype, info=info, modular=modular) - self.shortver = self.compose - # overridden for nightlies to avoid expensive roundtrips - if '.' in compose: - self.creation_date = compose.split('.')[0] - else: - self.creation_date = compose - - # String that will generate a clean copy of the page using the - # test page generation template system. - if self.modular: - seedtmpl = ("{{{{subst:Modular validation results|testtype={0}|release={1}|" - "milestone={2}|date={3}}}}}") - else: - seedtmpl = ("{{{{subst:Validation results|testtype={0}|release={1}|" - "milestone={2}|date={3}}}}}") - self.seedtext = seedtmpl.format( - testtype, self.release, self.milestone, self.compose) - - -class SummaryPage(Page): - """A Page class that describes the result summary page for a given - event. event is the parent Event() for the page; summary pages are - always considered to be a part of an Event. - """ - def __init__(self, site, event, info=None, modular=False): - self.modular = modular - wikiname = "Test Results:Fedora {0} Summary".format(event.version) - if modular: - wikiname = "Test Results:Fedora Modular {0} Summary".format(event.version) - super(SummaryPage, self).__init__(site, wikiname, info) - _dist = "Fedora" - if modular: - _dist = "Fedora Modular" - self.summary = ("Relval bot-created validation results summary for " - "{0} {1}").format(_dist, event.version) - self.seedtext = ( - "{0} {1} [[QA:Release validation test plan|release " - "validation]] summary. This page shows the results from all the " - "individual result pages for this compose together. You can file " - "results directly from this page and they will be saved into the " - "correct individual result page. To see test instructions, visit " - "any of the individual pages (the section titles are links). You " - "can find download links below.\n\n").format(_dist, event.version) - self.seedtext += "__TOC__\n\n" - self.seedtext += "== Downloads ==\n{{" + _dist + " " + event.version + " Download}}" - for testpage in event.valid_pages: - self.seedtext += "\n\n== [[" + testpage.name + "|" - self.seedtext += testpage.testtype + "]] ==\n{{" - self.seedtext += testpage.name + "}}" - - def update_current(self): - """Make the Current convenience redirect page on the wiki for the - event point to this page. - """ - if self.modular: - curr = self.site.pages['Test Results:Current Modular Summary'] - else: - curr = self.site.pages['Test Results:Current Summary'] - curr.save("#REDIRECT [[{0}]]".format(self.name), - "relval: update to current event", createonly=None) - - -class DownloadPage(Page): - """The page containing image download links for a ValidationEvent. - As with SummaryPage, is always associated with a specific event. - """ - def __init__(self, site, event, info=None, modular=False): - _dist = "Fedora" - if modular: - _dist = "Fedora Modular" - wikiname = "Template:{0} {1} Download".format(_dist, event.version) - self.summary = "Relval bot-created download page for {0} {1}".format( - _dist, event.version) - super(DownloadPage, self).__init__(site, wikiname, info) - self.event = event - - @property - def seedtext(self): - """A nicely formatted download table for the images for this - compose. Here be dragons (and wiki table syntax). What you get - from this is a table with one row for each unique 'image - identifier' - the subvariant plus the image type - and columns - for all arches in the entire image set; if there's an image - for the given image type and arch then there'll be a download - link in the appropriate column. - """ - # sorting score values (see below) - archscores = ( - (('x86_64', 'i386'), 2000), - ) - loadscores = ( - (('everything',), 300), - (('workstation',), 220), - (('server',), 210), - (('cloud', 'desktop', 'cloud_base', 'docker_base', 'atomic'), 200), - (('kde',), 190), - (('minimal',), 90), - (('xfce',), 80), - (('soas',), 73), - (('mate',), 72), - (('cinnamon',), 71), - (('lxde',), 70), - (('source',), -10), - ) - # Start by iterating over all images and grouping them by load - # (that's imagedict) and keeping a record of each arch we - # encounter (that's arches). - arches = set() - imagedict = dict() - for img in self.event.ff_release_images.all_images: - if img['arch']: - arches.add(img['arch']) - # simple human-readable identifier for the image - desc = ' '.join((img['subvariant'], img['type'])) - # assign a 'score' to the image; this will be used for - # ordering the download table's rows. - img['score'] = 0 - for (values, score) in archscores: - if img['arch'] in values: - img['score'] = score - for (values, score) in loadscores: - if img['subvariant'].lower() in values: - img['score'] += score - # The dict values are lists of images. We could use a - # DefaultDict here, but faking it is easy too. - if desc in imagedict: - imagedict[desc].append(img) - else: - imagedict[desc] = [img] - # Now we have our data, sort the dict using the weight we - # calculated earlier. We use the max score of all arches in - # each group of images. - imagedict = OrderedDict(sorted(imagedict.items(), - key=lambda x: max(img['score'] for img in x[1]), - reverse=True)) - # ...and sort the arches (just so they don't move around in - # each new page and confuse people). - arches = sorted(arches) - - # Now generate the table. - table = '{| class="wikitable sortable mw-collapsible" width=100%\n|-\n' - # Start of the header row... - table += '! Image' - for arch in arches: - # Add a column for each arch - table += ' !! {0}'.format(arch) - table += '\n' - for (subvariant, imgs) in imagedict.items(): - # Add a row for each subvariant - table += '|-\n' - table += '| {0}\n'.format(subvariant) - for arch in arches: - # Add a cell for each arch (whether we have an image - # or not) - table += '| ' - for img in imgs: - if img['arch'] == arch: - # Add a link to the image if we have one - table += '[{0} Download]'.format(img['url']) - table += '\n' - # Close out the table when we're done - table += '|-\n|}' - return table - - def update_current(self): - """Kind of a hack - relval needs this to exist as things - stand. I'll probably refactor it later. - """ - pass - - -class AMIPage(Page): - """A page containing EC2 AMI links for a given event. Is included - in the Cloud validation page to make it easy for people to find - the correct AMIs. - """ - def __init__(self, site, event, info=None, modular=False): - _dist = "Fedora" - if modular: - _dist = "Fedora Modular" - wikiname = "Template:{0} {1} AMI".format(_dist, event.version) - self.summary = "Relval bot-created AMI page for {0} {1}".format( - _dist, event.version) - super(AMIPage, self).__init__(site, wikiname, info) - self.event = event - - @property - def seedtext(self): - """A table of all the AMIs for the compose for this event. We - have to query this information out of datagrepper, that's the - only place where it's available. - """ - text = "" - # first, let's get the information out of datagrepper. We'll - # ask for messages up to 2 days after the event date. - date = fedfind.helpers.parse_cid(self.event.ff_release.cid, dic=True)['date'] - start = datetime.datetime.strptime(date, '%Y%m%d').replace(tzinfo=pytz.utc) - end = start + datetime.timedelta(days=2) - # convert to epoch (this is what datagrepper wants). In Python - # 3 we can use just .timestamp() but sadly not in Python 2. - # https://stackoverflow.com/questions/6999726 - epoch = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) - start = (start - epoch).total_seconds() - end = (end - epoch).total_seconds() - url = "https://apps.fedoraproject.org/datagrepper/raw" - url += "?topic=org.fedoraproject.prod.fedimg.image.publish" - url += "&start={0}&end={1}".format(start, end) - json = fedfind.helpers.download_json(url) - msgs = json['raw_messages'] - # handle pagination - for page in range(2, json['pages'] + 1): - newurl = url + "&page={0}".format(page) - newjson = fedfind.helpers.download_json(newurl) - msgs.extend(newjson['raw_messages']) - - # now let's find the messages for our event compose - ours = [msg['msg'] for msg in msgs if msg['msg']['compose'] == self.event.ff_release.cid] - - def _table_line(msg): - """Convenience function for generating a table line.""" - destination = msg['destination'] - ami = msg['extra']['id'] - url = "https://redirect.fedoraproject.org/console.aws.amazon.com/ec2/v2/home?" - url += "region={0}#LaunchInstanceWizard:ami={1}".format(destination, ami) - return "| {0}\n| {1}\n| [{2} Launch in EC2]\n|-\n".format(destination, ami, url) - - def _table(arch, virttype, voltype): - """Convenience function for adding a table.""" - ret = "== {0} {1} {2} AMIs ==\n\n".format(arch, virttype, voltype) - ret += '{| class="wikitable sortable mw-collapsible' - if arch != 'x86_64' or virttype != 'hvm' or voltype != 'standard': - # we expand the x86_64 hvm standard table by default - ret += ' mw-collapsed' - ret += '" width=100%\n|-\n' - ret += "! Region !! AMI ID !! Direct launch link\n|-\n" - # find the right messages for this arch and types - relevants = [msg for msg in ours if - msg['architecture'] == arch and - msg['extra']['virt_type'] == virttype and - msg['extra']['vol_type'] == voltype] - # sort the messages by region so the table is easier to scan - relevants.sort(key=lambda x:x['destination']) - for msg in relevants: - ret += _table_line(msg) - ret += "|}\n\n" - return ret - - # now let's create and populate the tables - for arch in ('x86_64', 'arm64'): - for virttype in ('hvm',): - for voltype in ('standard', 'gp2'): - text += _table(arch, virttype, voltype) - - return text - - -class TestDayPage(Page): - """A Test Day results page. Usually contains table(s) with test - cases as the column headers and users as the rows - each row is - one user's results for all of the test cases in the table. Note - this class is somewhat incomplete and really can only be used - for its own methods, do *not* try writing one of these to the - wiki. - """ - def __init__(self, site, date, subject, info=None): - # Handle names with no subject, e.g. Test_Day:2012-03-14 - wikiname = "Test Day:{0}".format(date) - if subject: - wikiname = "{0} {1}".format(wikiname, subject) - super(TestDayPage, self).__init__(site, wikiname, info) - self.date = date - self.subject = subject - self.results_separators = ('Test Results', 'Results') - - def write(self): - print("Creating Test Day pages is not yet supported.") - return - - @cached_property - def bugs(self): - """Returns a list of bug IDs referenced in the results section - (as strings). Will find bugs in {{result}} and {{bz}} - templates.""" - bugs = helpers.find_bugs(self.results_wikitext) - for res in rs.find_results_by_row(self.results_wikitext): - bugs.update(res.bugs) - return sorted(bugs) - - def fix_app_results(self): - """The test day app does its own bug references outside the - result template, instead of including them as the final - parameters to the template like it should. This fixes that, in - a fairly rough and ready way. - """ - badres = re.compile(r'({{result.*?)}} {0,2}' - r'({{bz\|\d{6,7}}}) ?' - r'({{bz\|\d{6,7}}})? ?' - r'({{bz\|\d{6,7}}})? ?' - r'({{bz\|\d{6,7}}})? ?' - r'({{bz\|\d{6,7}}})? ?' - r'({{bz\|\d{6,7}}})?') - text = oldtext = self.text() - oldtext = text - matches = badres.finditer(text) - for match in matches: - bugs = list() - groups = match.groups() - for group in groups[1:]: - if group: - bugs.append(group[10:-8]) - text = text.replace(match.group(0), - match.group(1) + '||' + '|'.join(bugs) + '}}') - return self.save(text, summary=u"Fix testday app-generated results to " - "use {{result}} template for bug references", - oldtext=oldtext) - - def long_refs(self): - """People tend to include giant essays as notes on test - day results, which really makes table rendering ugly when - they're dumped in the last column of the table. This finds all - notes over 150 characters long, moves them to the "long" - group, and adds a section at the end of the page with all the - "long" notes in it. The 'end of page discovery' is a bit - hacky, it just finds the last empty line in the page except - for trailing lines and sticks the section there, but that's - usually what we want - basically we want to make sure it - appears just above the category memberships at the bottom of - the page. It does go wrong *sometimes*, so good idea to check - the page after it's edited. - """ - text = oldtext = self.text() - if '' in text: - # Don't run if we've already been run on this page - return dict(nochange='') - refpatt = re.compile('(.+?)', re.S) - matches = refpatt.finditer(text) - found = False - for match in matches: - if len(match.group(0)) > 150: - found = True - text = text.replace(match.group(0), - '' + match.group(1)) - if found: - text = helpers.rreplace( - text.strip(), '\n\n', - '\n\n== Long comments ==\n\n\n', 1) - return self.save(text, summary=u"Move long comments to a separate " - "section at end of page", oldtext=oldtext) - else: - # If we didn't find any long refs, don't do anything - return dict(nochange='') - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/release.py b/wikitcms/release.py deleted file mode 100644 index 777526c..0000000 --- a/wikitcms/release.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""Classes that describe distribution releases are defined here.""" - -from __future__ import unicode_literals -from __future__ import print_function - -from . import page as pg -from . import listing as li - -class Release(object): - """Class for a Fedora release. wiki is a wikitcms site object. - Release is a string containing a Fedora release version (e.g. 21). - """ - def __init__(self, release, wiki, modular=False): - self.release = release - self.modular = modular - dist = "Fedora" - if modular: - dist = "Fedora Modular" - self.category_name = "Category:{0} {1} Test Results".format( - dist, self.release) - self.site = wiki - - @property - def testday_pages(self): - """All Test Day pages for this release (as a list).""" - cat = self.site.pages[ - 'Category:Fedora {0} Test Days'.format(self.release)] - return [page for page in cat if isinstance(page, pg.TestDayPage)] - - def milestone_pages(self, milestone=None): - """If no milestone, will give all release validation pages for - this release (as a generator). If a milestone is given, will - give validation pages only for that milestone. Note that this - works by category; you may get somewhat different results by - using page name prefixes. - """ - cat = li.ValidationCategory(self.site, self.release, milestone, modular=self.modular) - pgs = self.site.walk_category(cat) - return (p for p in pgs if isinstance(p, pg.ValidationPage)) - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/result.py b/wikitcms/result.py deleted file mode 100644 index 051586e..0000000 --- a/wikitcms/result.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""This file defines various classes and helper functions for working -with results.""" - -from __future__ import unicode_literals -from __future__ import print_function - -import re - -from collections import OrderedDict - -from wikitcms import helpers - -# These are used by multiple functions, so let's share them. -# Wiki table row separator -SEP_PATT = re.compile(r'\|[-\}].*?\n') -# Identifies an instance of the result template. Will break if the -# result contains another template, but don't do that. the lookahead -# is used to capture the 'comments' for the result: we keep matching -# until we hit the next instance of the template, or a cell or row -# separator (newline starting with a |). The re.S is vital. -RES_PATT = re.compile(r'{{result.+?}}.*?(?=\n*{{result|$|\n\|)', re.S) - -def _filter_results(results, statuses=None, transferred=True, bot=True): - """Filter results. Shared between next two functions.""" - # Drop example / sample results - results = [r for r in results if not r.user or - r.user.lower() not in - ('sampleuser', 'exampleuser', 'example', 'username', 'fasname')] - if statuses: - results = [r for r in results for s in statuses if r.status and - s in r.status] - if not transferred: - results = [r for r in results if not r.transferred] - if not bot: - results = [r for r in results if not r.bot] - return results - -def find_results(text, statuses=None, transferred=True, bot=True): - """Find test results in a given chunk of wiki text. Returns a list - of Result objects. If statuses is not None, it should be an - iterable of strings, and only result objects whose status matches - one of the given statuses will be returned. If transferred is - False, results like {{result|something|previous TC5 run}} will not - be included. If bot is False, results from automated test bots - (results with 'bot=true') will not be included. - """ - results = list() - # Identifies an instance of the old {{testresult template, in the - # same way as RES_PATT (above). - oldres_patt = re.compile(r'{{testresult.+?}}.*?(?={{testresult|$)', re.M) - for res in RES_PATT.findall(text): - results.append(Result.from_result_template(res)) - for oldres in oldres_patt.findall(text): - results.append(Result.from_testresult_template(oldres)) - - results = _filter_results(results, statuses, transferred, bot) - return results - -def find_results_by_row(text, statuses=None): - """Find test results using a row-by-row scan, guessing the user - for results which do not have one. Used for Test Day pages. Note: - doesn't bother handling {{testresult because AFAICT no Test Day - pages use that template. - """ - # Slightly slapdash way to identify the contents of a wiki table - # cell. Good enough to find the contents of the first cell in the - # row, which we do because it'll usually be the user name in a - # Test Day results table. - cell_patt = re.compile(r'\| *(.*?) *\n\|') - # Captures the user name from a wikilink to a user page (typically - # used to indicate the user who reports a result). - user_patt = re.compile(r'\[\[User: *(.*?) *[|\]]') - results = list() - - for row in SEP_PATT.split(text): - for match in RES_PATT.findall(row): - res = Result.from_result_template(match) - if res.status and not res.user: - # We'll try and find a [[User: wikilink; if we can't, - # we'll guess that the reporter's name is the contents - # of the first cell in the row. - pattuser = user_patt.search(row) - if pattuser: - res.user = pattuser.group(1).lower() - else: - celluser = cell_patt.search(row) - if celluser: - res.user = celluser.group(1).lower() - results.append(res) - - results = _filter_results(results, statuses) - return results - -def find_resultrows(text, section='', secid=0, statuses=None, transferred=True): - """Find result rows in a given chunk of wiki text. Returns a list - of ResultRow objects. 'statuses' and 'transferred' are passed all - the way through ResultRow to find_results() and behave as - described there, for the Result objects in each ResultRow. - """ - # identify all test case names, including old ones. modern ones - # match QA:Testcase.*, but older ones sometimes have QA/TestCase. - testcase_pattern = re.compile(r'(QA[:/]Test.+?)[\|\]\n]') - # row separator is |-, end of table is |} - columns = list() - resultrows = list() - rows = SEP_PATT.split(text) - for row in rows: - rowlines = row.split('\n') - for line in rowlines: - # check if this is a column header row, and update column - # names. Sometimes the header row doesn't have an explicit - # row separator so the 'row' might be polluted with - # preceding lines, so we split the row into lines and - # check each line in the row. - line = line.strip() - if line.find('!') == 0 and line.find('!!') > 0: - # column titles. note: mw syntax in fact allows for - # '! title\n! title\n! title' as well as '! title !! - # title !! title'. But we don't use that syntax. - columns = line.lstrip('!').split('!!') - for column in columns: - # sanitize names a bit - newcol = column.strip() - newcol = newcol.strip("'[]") - newcol = newcol.strip() - try: - # drop out any block - posa = newcol.index("") - posb = newcol.index("") + 6 # length - newcol = newcol[:posa] + newcol[posb:] - newcol = newcol.strip() - except ValueError: - pass - try: - newcol = newcol.split('|')[1] - except IndexError: - pass - if newcol != column: - columns.insert(columns.index(column), newcol) - columns.remove(column) - tcmatch = testcase_pattern.search(row) - if tcmatch: - # *may* be a result row - may also be a garbage 'row' - # between tables which happens to contain a test case - # name. So we get a ResultRow object but discard it if it - # doesn't contain any result cells. This test works even - # if the actual results are filtered by statuses= or - # Transferred=, because the resrow.results dict will - # always have a key for each result column, though its - # value may be an empty list. - resrow = ResultRow.from_wiki_row(tcmatch.group(1), columns, row, - section, secid, statuses, - transferred) - if resrow.results: - resultrows.append(resrow) - return resultrows - - -class Result(object): - """A class that represents a single test result. Note that a - 'none' result, as you get if you just instantiate this class - without arguments, is a thing, at least for wikitcms; when text - with {{result|none}} templates in it is parsed, such objects may - be created/returned, and you can produce the {{result|none}} text - as the result_template property of such an instance. - - You would usually instantiate this class directly to report a new - result. - - Methods that parse existing results will use one of the class - methods that returns a Result() with the appropriate attributes. - When one of those parsers produces an instance it will set the - attribute origtext to record the exact string parsed to produce - the instance. - - transferred, if True, indicates the result is of the "previous - (compose) run" type that is used to indicate where we think a - result from a previous compose is valid for a later one. - """ - def __init__( - self, status=None, user=None, bugs=None, comment='', bot=False): - self.status = status - self.user = user - self.bugs = bugs - if self.bugs: - self.bugs = [str(bug) for bug in self.bugs] - self.comment = comment - self.bot = bot - self.transferred = False - self.comment_bugs = helpers.find_bugs(self.comment) - - def __str__(self): - if not self.status: - return "Result placeholder - {{result|none}}" - if self.bot: - bot = 'BOT ' - else: - bot = '' - status = 'Result: ' + self.status.capitalize() - if self.transferred: - user = ' transferred: ' + self.user - elif self.user: - user = ' from ' + self.user - else: - user = '' - if self.bugs: - bugs = ', bugs: ' + ', '.join(self.bugs) - else: - bugs = '' - if self.comment: - comment = ', comment: ' + self.comment - # Don't display ref tags - refpatt = re.compile(r'') - comment = refpatt.sub('', comment) - else: - comment = '' - return bot + status + user + bugs + comment - - @property - def result_template(self): - """The {{result}} template string that would represent the - properties of this result in a wiki page. - """ - bugtext = '' - commtext = self.comment - usertext = '' - bottext = '' - if self.status is None: - status = 'none' - else: - status = self.status - if self.bugs: - bugtext = "|" + '|'.join(self.bugs) - if self.user: - usertext = "|" + self.user - if self.bot: - bottext = "|bot=true" - tmpl = ("{{{{result|{status}{usertext}{bugtext}{bottext}}}}}" - "{commtext}").format(status=status, usertext=usertext, - bugtext=bugtext, bottext=bottext, - commtext=commtext) - return tmpl - - @classmethod - def from_result_template(cls, string): - """Returns a Result object based on the {{result}} template. - The most complex result template you see might be: - {{ result | fail| bot =true| adamwill | 123456|654321|615243}} comment - We want the 'fail' and 'adamwill' bits separately and stripped, - and all the bug numbers in one chunk to be parsed later to - construct a list of bugs, and none of the pipes, brackets, or - whitespace. Mediawiki named parameters can occur anywhere in - the template and aren't counted in the numbered parameters, so - we need to find them and extract them first. We record the - comment exactly as is. - """ - template, comment = string.strip().split('}}', 1) - comment = comment.strip() - template = template.lstrip('{') - params = template.split('|') - namedpars = dict() - bot = False - - for param in params: - if '=' in param: - (par, val) = param.split('=', 1) - namedpars[par.strip()] = val.strip() - params.remove(param) - if 'bot' in namedpars and namedpars['bot']: - # This maybe doesn't do what you expect for 'bot=false', - # but we don't handle that in Mediawiki either and we want - # to stay consistent. - bot = True - - # 'params' now contains only numbered params - # Pad the non-existent parameters to make things cleaner later - while len(params) < 3: - params.append('') - - for i, param in enumerate(params): - params[i] = param.strip() - if params[i] == '': - params[i] = None - status, user = params[1:3] - bugs = params[3:] - if status and status.lower() == "none": - status = None - - if bugs: - bugs = [b.strip() for b in bugs if b and b.strip()] - for i, bug in enumerate(bugs): - # sometimes people write 123456#c7, remove the suffix - if '#' in bug: - newbug = bug.split('#')[0] - if newbug.isdigit(): - bugs[i] = newbug - - res = cls(status, user, bugs, comment, bot) - res.origtext = string - if user and "previous " in user: - res.transferred = True - return res - - @classmethod - def from_testresult_template(cls, string): - '''Returns a Result object based on the {{testresult}} template. - This was used in Fedora 12. It looks like this: - {{testresult/pass|FASName}} comment or bug - The bug handling here is very special-case - it relies on the - fact that bug IDs were always six-digit strings, at the time, - and on the template folks used to link to bug reports - but - should be good enough. - ''' - bug_patt = re.compile(r'({{bz.*?(\d{6,6}).*?}})') - emptyref_patt = re.compile(r' *?') - template, comment = string.strip().split('}}', 1) - template = template.lstrip('{') - template = template.split('/')[1] - params = template.split('|') - try: - status = params[0].strip().lower() - if status == "none": - status = None - except IndexError: - status = None - try: - user = params[1].strip().lower() - except IndexError: - user = None - bugs = [b[1] for b in bug_patt.findall(comment)] - if comment: - comment = bug_patt.sub('', comment) - comment = emptyref_patt.sub('', comment) - if comment.replace(' ', '') == '': - comment = '' - comment = comment.strip() - else: - pass - res = cls(status, user, bugs, comment) - res.origtext = string - if user and "previous " in user: - res.transferred = True - return res - - @classmethod - def from_qatracker(cls, result): - '''Converts a result object from the QA Tracker library to a - wikitcms-style Result. Returns a Result instance with origres - as an extra property that is a pointer to the qatracker result - object. - ''' - if result.result == 1: - status = 'pass' - elif result.result == 0: - status = 'fail' - else: - status = None - if result.reportername: - user = result.reportername - else: - user = None - if result.comment: - comment = result.comment - else: - comment = '' - # This produces an empty string if there are no bugs, a dict - # if there are bugs. FIXME: this is completely wrong, it's - # a JSON dict, we should parse it as JSON, but I can't be - # bothered fixing this Ubuntu stuff right now. Nothing uses - # it. - bugs = eval(result.bugs) - if bugs: - bugs = list(bugs.keys()) - else: - bugs = None - res = cls(status, user, bugs, comment) - res.origres = result - return res - -class TestInstance(object): - """Represents the broad concept of a 'test instance': that is, in - any test management system, the 'basic unit' of a single test for - which some results are expected to be reported. In 'Wikitcms', for - instance, this corresponds to a single row in a results table, and - that is what the ResultRow() subclass represents. A subclass for - QATracker would represent a single test in a build of a product. - - The 'testcase' is the basic identifier of a test instance. It will - not necessarily be unique, though - in any test management system - you may find multiple test instances for the same test case (in - different builds and different products). The concept of the - name derives from Wikitcms, where it is not uncommon for a set of - test instances to have the same 'testcase' but a different 'name', - which in that system is the link text: there will a column which - for each row contains [[testcase|name]], the testcase being the - same but the name being different. The concept doesn't seem - entirely specific to Wiki TCMS, though, so it's represented here. - Commonly the 'testcase' and 'name' will be the same, when each - instance within a set has a different 'testcase' the name should - be identical to the testcase. - - milestone is, roughly, the priority of the test: milestone is - slightly Fedora-specific language, a hangover from early wikitcms - versions which didn't consider other systems. For Fedora it will - be Alpha, Beta or Final, usually. For Ubuntu it may be - 'mandatory', 'optional' or possibly 'disabled'. - - results is required to be a dict of lists; the dict keys represent - the test's environments. If the test system does not have the - concept of environments, the dict can have a single key with some - sort of generic name (like 'Results'). The values must be lists of - instances of wikitcms.Result or a subclass of it. - """ - def __init__(self, testcase, milestone='', results=None): - self.testcase = testcase - self.name = testcase - self.milestone = milestone - if not results: - self.results = dict() - else: - self.results = results - - -class ResultRow(TestInstance): - """Represents the 'test instance' concept for Wikitcms, where it - is a result row from the tables used to contain results. Some - Wikitcms-specific properties are encoded here. columns is the list - of columns in the table in which the result was found (this is - needed to figure out the environments for the results, as the envs - are represented by table columns, and to know which cell to edit - when modifying results). origtext is the text which was parsed to - produce the instance, if it was produced by the from_wiki_row() - class method which parses wiki text to produce instances. section - and secid are the wiki page section in which the table from which - the row came is located; though these are in a way attributes of - the page, this is really another case where an MW attribute is - just a way of encoding information about a *test*. The splitting - of result pages into sections is a way of sub-grouping tests in - each page. So it's appropriate to store those attributes here. - - At present you typically get ResultRow instances by calling a - ComposePage's get_resultrows() method, which runs its text through - result.find_resultrows(), which isolates the result rows in the - wiki text and runs through through this class' from_wiki_row() - method. This will always provide instances with a full set of - the above-described attributes. - """ - def __init__(self, testcase, columns, section='', secid=None, milestone='', - origtext='', results=None): - super(ResultRow, self).__init__(testcase, milestone, results) - self.columns = columns - self.origtext = origtext - self.section = section - self.secid = secid - - def matches(self, other): - """This is roughly an 'equals' check for ResultRows; if all - identifying characteristics and the origtext match, this is - True, otherwise False. __dict__ match is too strong as the - 'results' attribute is a list of Result instances; each time - you instantiate the same ResultRow you get different Result - objects. We don't override __eq__ and use == because that has - icky implications for hashing, and we just want to stay away - from that mess: - https://docs.python.org/3/reference/datamodel.html#object.__hash__ - """ - if isinstance(other, self.__class__): - ours = (self.testcase, self.name, self.secid, self.origtext) - theirs = (other.testcase, other.name, other.secid, other.origtext) - return ours == theirs - return NotImplemented - - @classmethod - def from_wiki_row(cls, testcase, columns, text, section, secid, - statuses=None, transferred=True): - """Instantiate a ResultRow from some wikitext and some info - that is worked out from elsewhere in the page. - """ - results = OrderedDict() - # this is presumptuous, but holds up for every result page - # tested so far; there may be some with whitespace, and - # '| cell || cell || cell' is permitted as an alternative to - # '| cell\n| cell\n| cell' but we do not seem to use it. - cells = text.split('\n|') - milestone = '' - for mile in ('Alpha', 'Basic', 'Beta', 'Final', 'Optional', 'Tier1', - 'Tier2', 'Tier3'): - if mile in cells[0]: - milestone = mile - # we take the *first* milestone we find, so we treat - # e.g. "Basic / Final" as Basic - break - for i, cell in enumerate(cells): - if testcase in cell: - try: - # see if we can find some link text for the test - # case, and assume it's the test's "name" if so - altname = cell.strip().strip('[]').split('|')[1] - continue - except IndexError: - try: - altname = cell.strip().strip('[]').split(maxsplit=1)[1] - except IndexError: - altname = None - if '{{result' in cell or '{{testresult' in cell: - # any cell containing a result string is a 'result - # cell', and the index of the cell in columns will be - # the title of the column it is in. find_results() - # returns an empty list if all results are filtered - # out, so the results dict's keys will always - # represent the full set of environments for this test - try: - results[columns[i]] = find_results(cell, statuses, - transferred) - except IndexError: - # FIXME: log (messy table, see e.g. F15 'Multi - # Image') - pass - row = cls(testcase, columns, section, secid, milestone, text, results) - if altname: - row.name = altname - return row - - -class TrackerBuildTest(TestInstance): - """Represents a 'result instance' from QA Tracker: this is the - data associated with a single testcase in a single build. - """ - def __init__(self, tcname, tcid, milestone='', results=None): - super(TrackerBuildTest, self).__init__(tcname, milestone, results) - self.tcid = tcid - - @classmethod - def from_api(cls, testcase, build): - """I don't remember what the shit this does, I'm just making - pylint happy. - """ - results = dict() - results['Results'] = list() - tcname = testcase.title - tcid = testcase.id - milestone = testcase.status_string - for res in build.get_results(testcase): - results['Results'].append(Result.from_qatracker(res)) - return cls(tcname, tcid, milestone, results) - -# vim: set textwidth=100 ts=8 et sw=4: diff --git a/wikitcms/wiki.py b/wikitcms/wiki.py deleted file mode 100644 index f092106..0000000 --- a/wikitcms/wiki.py +++ /dev/null @@ -1,677 +0,0 @@ -# Copyright (C) 2014 Red Hat -# -# This file is part of wikitcms. -# -# wikitcms is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Author: Adam Williamson - -"""The Wiki class here extends mwclient's Site class with additional -Wikitcms-specific functionality, and convenience features like stored -user credentials. -""" - -from __future__ import unicode_literals -from __future__ import print_function - -from collections import namedtuple -import datetime -import re - -import fedfind.helpers -import mwclient -from productmd.composeinfo import get_date_type_respin - -from . import page as pg -from . import event as ev -from . import helpers as hl -from . import listing as li -from . import result as rs -from .exceptions import NoPageError, NotFoundError, TooManyError - -try: - from openidc_client import OpenIDCClient -except ImportError: - OpenIDCClient = None -try: - from openidc_client.requestsauth import OpenIDCClientAuther -except ImportError: - OpenIDCClientAuther = None - -# I'd really like to use namedlist.namedtuple, but it isn't widely -# available. -class ResTuple(namedtuple('ResTuple', 'testtype release milestone compose ' - 'testcase section testname env status user bugs ' - 'comment bot cid modular')): - """namedtuple (with default values) used for report_validation_ - results(). See that method's docstring for details. - """ - def __new__(cls, testtype, release='', milestone='', compose='', - testcase='', section='', testname='', env='', status='', - user='', bugs='', comment='', bot=False, cid='', modular=False): - return super(ResTuple, cls).__new__( - cls, testtype, release, milestone, compose, testcase, section, - testname, env, status, user, bugs, comment, bot, cid, modular) - -class Wiki(mwclient.Site): - """Extends the mwclient.Site class with some extra capabilities.""" - # parent class has a whole bunch of args, so just pass whatever through. - # always init this the same as a regular mwclient.Site instance. - def __init__(self, host='fedoraproject.org', *args, **kwargs): - super(Wiki, self).__init__(host, *args, **kwargs) - # override the 'pages' property so it returns wikitcms Pages when - # appropriate - self.pages = li.TcmsPageList(self) - - @property - def current_compose(self): - """A dict of the key / value pairs from the CurrentFedora - Compose page which is the canonical definition of the 'current' - primary arch validation testing compose. You can usually expect - keys full, release, date, milestone, and compose. The page is - normally written by ValidationEvent.update_current(). - """ - currdict = dict() - valpatt = re.compile(r'^\| *?(\w+?) *?= *([\w .]*?) *$', re.M) - page = self.pages['Template:CurrentFedoraCompose'] - for match in valpatt.finditer(page.text()): - currdict[match.group(1)] = match.group(2) - return currdict - - @property - def current_event(self): - """The current event, as a ValidationEvent instance. Will be a - ComposeEvent or a NightlyEvent.""" - curr = self.current_compose - # Use of 'max' plus get_validation_event handles getting us - # the right kind of event. - return self.get_validation_event( - release=curr['release'], milestone=curr['milestone'], - compose=max(curr['date'], curr['compose'])) - - @property - def current_modular_compose(self): - """A dict of the key / value pairs from the CurrentFedora - ModularCompose page which is the canonical definition of the - 'current' modular primary arch validation testing compose. You - can usually expect keys full, release, date, milestone, and - compose. The page is normally written by - ValidationEvent.update_current(). - """ - currdict = dict() - valpatt = re.compile(r'^\| *?(\w+?) *?= *([\w .]*?) *$', re.M) - page = self.pages['Template:CurrentFedoraModularCompose'] - for match in valpatt.finditer(page.text()): - currdict[match.group(1)] = match.group(2) - return currdict - - @property - def current_modular_event(self): - """The current modular event, as a ValidationEvent instance. - Will be a ComposeEvent or a NightlyEvent.""" - curr = self.current_modular_compose - # Use of 'max' plus get_validation_event handles getting us - # the right kind of event. - return self.get_validation_event( - release=curr['release'], milestone=curr['milestone'], - compose=max(curr['date'], curr['compose']), modular=True) - - @property - def matrices(self): - """A list of dicts representing pages in the test matrix - template category. These are the canonical definition of - 'existing' test types. Done this way - rather than using the - category object's ability to act as an iterator over its member - page objects - because this method respects the sort order of - the member pages, whereas the other does not. The sort order is - used in creating the overview summary page. - """ - category = self.pages['Category:QA test matrix templates'] - return category.members(generator=False) - - @property - def testtypes(self): - """Test types, derived from the matrix page names according to - a naming convention. A list of strings. - """ - return [m['title'].replace('Template:', '') - .replace(' test matrix', '') for m in self.matrices] - - @property - def modular_matrices(self): - """A list of dicts representing pages in the modular test - matrix template category. These are the canonical definition of - 'existing' test types. Done this way - rather than using the - category object's ability to act as an iterator over its member - page objects - because this method respects the sort order of - the member pages, whereas the other does not. The sort order is - used in creating the overview summary page. - """ - category = self.pages['Category:QA modular test matrix templates'] - return category.members(generator=False) - - @property - def modular_testtypes(self): - """Test types, derived from the matrix page names according to - a naming convention. A list of strings. - """ - return [m['title'].replace('Template:', '') - .replace(' modular test matrix', '') for m in self.modular_matrices] - - def login(self, *args, **kwargs): - """Login method, overridden to use openidc auth when necessary. - This will open a browser window and run through FAS auth on - the first use, then a token will be saved that will allow auth - for a while, when the token expires, the web auth process will - pop up again. - """ - use_openidc = False - # This probably breaks on private wikis, but wikitcms isn't - # ever used with any of those, AFAIK. - host = self.host - if isinstance(host, (list, tuple)): - host = host[1] - if host.endswith('fedoraproject.org') and self.version[:2] >= (1, 29): - use_openidc = True - if not use_openidc: - # just work like mwclient - return super(Wiki, self).login(*args, **kwargs) - - # Fedora wiki since upgrade to 1.29 doesn't allow native - # mediawiki auth with FAS creds any more, it only allows - # auth via OpenID Connect. For this case we set up an - # openidc auther, call site_init() to trigger auth and - # update the site properties, and return. - if OpenIDCClient is None: - raise ImportError('python-openidc-client is needed for OIDC') - if OpenIDCClientAuther is None: - raise ImportError('python-openidc-client 0.4.0 or higher is ' - 'required for OIDC') - client = OpenIDCClient( - app_identifier='wikitcms', - id_provider='https://id.{0}/openidc/'.format(host), - id_provider_mapping={'Token': 'Token', - 'Authorization': 'Authorization'}, - client_id='wikitcms', - client_secret='notsecret', - useragent='wikitcms') - - auther = OpenIDCClientAuther(client, - ['openid', 'https://fedoraproject.org/wiki/api']) - - self.connection.auth = auther - self.site_init() - - def add_to_category(self, page_name, category_name, summary=''): - """Add a given page to a given category if it is not already a - member. Takes strings for the names of the page and the - category, not mwclient objects. - """ - page = self.pages[page_name] - text = page.text() - if category_name not in text: - text += "\n[[{0}]]".format(category_name) - page.save(text, summary, createonly=False) - - def walk_category(self, category): - """Simple recursive category walk. Returns a list of page - objects that are members of the parent category or its - sub-categories, to any level of recursion. 14 is the Category: - namespace. - """ - pages = dict() - for page in category: - if page.namespace == 14: - sub_pages = self.walk_category(page) - for sub_page in sub_pages: - pages[sub_page.name] = sub_page - else: - pages[page.name] = page - pages = pages.values() - return pages - - def allresults(self, prefix=None, start=None, redirects='all', end=None): - """A generator for pages in the Test Results: namespace, - similar to mwclient's allpages, allcategories etc. generators. - This is a TcmsPageList, so it returns wikitcms objects when - appropriate. Note, if passing prefix, start or end, leave out - the "Test Results:" part of the name. - """ - gen = li.TcmsPageList(self, prefix=prefix, start=start, - namespace=116, redirects=redirects, end=end) - return gen - - def alltestdays(self, prefix=None, start=None, redirects='all', end=None): - """A generator for pages in the Test Day: namespace, - similar to mwclient's allpages, allcategories etc. generators. - This is a TcmsPageList, so it returns wikitcms objects when - appropriate. Note, if passing prefix, start or end, leave out - the "Test Day:" part of the name. - """ - gen = li.TcmsPageList(self, prefix=prefix, start=start, - namespace=114, redirects=redirects, end=end) - return gen - - def _check_compose(self, compose): - """Trivial checker shared between following two methods.""" - date = fedfind.helpers.date_check(compose, out='obj', fail_raise=False) - # all nightlies after F24 branch are Pungi 4-style; plain date - # is never a valid 'compose' value after that date, 2016-02-23 - if date and date < datetime.datetime(2016, 2, 23): - return 'date' - else: - # check if we have a valid Pungi4-style identifier - try: - (date, typ, respin) = get_date_type_respin(compose) - if date and typ and respin is not None: - if fedfind.helpers.date_check(date, fail_raise=False): - return 'date' - except ValueError: - pass - # regex to match TC/RC names: TC1, RC10, RC23.6 - patt = re.compile(r'[TR]C\d+\.?\d*') - if patt.match(compose.upper()): - return 'compose' - # regex for Pungi 4 milestone composes: 1.1, 1.2 ... 10.10 ... - patt = re.compile(r'\d+\.\d+') - if patt.match(compose): - return 'compose' - else: - raise ValueError( - "Compose must be a TC/RC identifier (TC1, RC3...) for pre-" - "Fedora 24 milestone composes, a Pungi 4 milestone compose" - " identifier (1.1, 10.10...) for post-Fedora 23 milestone " - "composes, a date in YYYYMMDD format (for pre-Fedora 24 " - "nightlies) or a Pungi 4 nightly identifier (20160308.n.0" - ", 20160310.n.2) for post-Fedora 23 nightlies.") - - def get_validation_event(self, release='', milestone='', compose='', - cid='', modular=False): - """Get an appropriate ValidationEvent object for the values - given. As with get_validation_page(), this method is for - sloppy instantiation of pages that follow the rules. This - method has no required arguments and tries to figure out - what you want from what you give it. It will raise errors - if what you give it is impossible to interpret or if it - tries and comes up with an inconsistent-seeming result. - - If modular is True, it will look for a Fedora Modular event - with the relevant version attributes. If you pass a compose ID - as cid, any value you pass for modular will be ignored; we'll - instead parse the modular value out of the compose ID. - - If you pass a numeric release, a milestone, and a valid - compose (TC/RC or date), it will give you the appropriate - event, whether it exists or not. All it really does in this - case is pick NightlyEvent or ComposeEvent for you. If you - don't fulfill any of those conditions, it'll need to do - some guessing/assumptions, and in some of those cases it - will only return an Event *that actually exists*, and may - raise exceptions if you passed a particularly pathological - set of values. - - If you don't pass a compose argument it will get the current - event; if you passed either of the other arguments and they - don't match the current event, it will raise an error. It - follows that calling this with no arguments just gives you - current_event. - - If you pass a date as compose with no milestone, it will see - if there's a Rawhide nightly and return it if so, otherwise it - will see if there's a Branched nightly and return that if so, - otherwise raise an error. It follows that you can't get the - page for an event that *doesn't exist yet* this way: you must - instantiate it directly or call this method with a milestone. - - It will not attempt to guess a milestone for TC/RC composes; - it will raise an exception in this case. - - The guessing bits require wiki roundtrips, so they will be - slower than instantiating a class directly or using this - method with sufficient information to avoid guessing. - """ - if cid: - (dist, release, milestone, compose) = hl.cid_to_event(cid) - modular = bool(dist == 'Fedora-Modular') - if not compose or not release: - # Can't really make an educated guess without a compose - # and release, so just get the current event and return it - # if it matches any other values passed. - if modular: - event = self.current_modular_event - else: - event = self.current_event - if release and event.release != release: - raise ValueError( - "get_validation_event(): Guessed event release {0} does " - "not match requested release {1}".format( - event.release, release)) - if milestone and event.milestone != milestone: - raise ValueError( - "get_validation_event(): Guessed event milestone {0} " - "does not match specified milestone {1}".format( - event.milestone, milestone)) - # all checks OK - return event - - if self._check_compose(compose) == 'date': - if milestone: - return ev.NightlyEvent( - self, release=release, milestone=milestone, - compose=compose, modular=modular) - else: - # we have a date and no milestone. Try both and return - # whichever exists. We check whether the first result - # page has any contents so that if someone mistakenly - # creates the wrong event, we can clean up by blanking - # the pages, rather than by getting an admin to - # actually *delete* them. - rawev = ev.NightlyEvent(self, release, 'Rawhide', compose, modular=modular) - pgs = rawev.result_pages - if pgs and pgs[0].text(): - return rawev - brev = ev.NightlyEvent(self, release, 'Branched', compose, modular=modular) - pgs = brev.result_pages - if pgs and pgs[0].text(): - return brev - # Here, we failed to guess. Boohoo. - raise ValueError( - "get_validation_event(): Could not find any event for " - "release {0} and date {1}.".format(release, compose)) - - elif self._check_compose(compose) == 'compose': - compose = str(compose).upper() - if not milestone: - raise ValueError( - "get_validation_event(): For a TC/RC compose, a milestone " - "- Alpha, Beta, or Final - must be specified.") - # With Pungi 4, the 'Final' milestone became 'RC', let's - # be nice and convert it - if int(release) > 23 and milestone.lower() == 'final': - milestone = 'RC' - return ev.ComposeEvent(self, release, milestone, compose, modular=modular, cid=cid) - else: - # We should never get here, but just in case. - raise ValueError( - "get_validation_event(): Something very strange happened.") - - def get_validation_page(self, testtype, release='', milestone='', - compose='', cid='', modular=False): - """Get an appropriate ValidationPage object for the values - given. As with get_validation_event(), this method is for - sloppy instantiation of pages that follow the rules. This - method has no required arguments except the testtype and tries - to figure out what you want from what you give it. It will - raise errors if what you give it is impossible to interpret or - if it tries and comes up with an inconsistent-seeming result. - - If modular is True, it will look for a Fedora Modular page - with the relevant version attributes. If you pass a compose ID - as cid, any value you pass for modular will be ignored; we'll - instead parse the modular value out of the compose ID. - - If you pass a numeric release, a milestone, and a valid - compose (TC/RC or date), it will give you the appropriate - event, whether it exists or not. All it really does in this - case is pick NightlyEvent or ComposeEvent for you. If you - don't fulfill any of those conditions, it'll need to do - some guessing/assumptions, and in some of those cases it - will only return an Event *that actually exists*, and may - raise exceptions if you passed a particularly pathological - set of values. - - If you don't pass a compose argument it will get the page for - the current event; if you passed either of the other - arguments and they don't match the current event, it will - raise an error. It follows that calling this with no arguments - just gives you the page of the specified test type for the - current event. - - If you pass a date as compose with no milestone, it will see - if there's a Rawhide nightly and return it if so, otherwise it - will see if there's a Branched nightly and return that if so, - otherwise raise an error. It follows that you can't get the - page for an event that *doesn't exist yet* this way: you must - instantiate it directly or call this method with a milestone. - - It will not attempt to guess a milestone for TC/RC composes; - it will raise an exception in this case. - - The guessing bits require wiki roundtrips, so they will be - slower than instantiating a class directly or using this - method with sufficient information to avoid guessing. - """ - if cid: - (dist, release, milestone, compose) = hl.cid_to_event(cid) - modular = bool(dist == 'Fedora-Modular') - if not compose or not release: - # Can't really make an educated guess without a compose - # and release, so just get the current event and return it - # if it matches any other values passed. - curr = self.current_compose - page = self.get_validation_page( - testtype, release=curr['release'], milestone=curr['milestone'], - compose=max(curr['compose'], curr['date']), modular=modular) - if release and page.release != release: - raise ValueError( - "get_validation_page(): Guessed page release {0} does " - "not match requested release {1}".format( - page.release, release)) - if milestone and page.milestone != milestone: - raise ValueError( - "get_validation_page(): Guessed page milestone {0} " - "does not match specified milestone {1}".format( - page.milestone, milestone)) - return page - - if self._check_compose(compose) == 'date': - if milestone: - return pg.NightlyPage( - self, release, testtype, milestone, compose, modular=modular) - else: - rawpg = pg.NightlyPage( - self, release, testtype, 'Rawhide', compose, modular=modular) - if rawpg.exists: - return rawpg - brpg = pg.NightlyPage( - self, release, testtype, 'Branched', compose, modular=modular) - if brpg.exists: - return brpg - # Here, we failed to guess. Boohoo. - raise ValueError( - "get_validation_page(): Could not find any event for " - "release {0} and date {1}.".format(release, compose)) - - elif self._check_compose(compose) == 'compose': - if not milestone: - raise ValueError( - "get_validation_page(): For a milestone compose, a " - " milestone - Alpha, Beta, or Final - must be specified.") - # With Pungi 4, the 'Final' milestone became 'RC', let's - # be nice and convert it - if int(release) > 23 and milestone.lower() == 'final': - milestone = 'RC' - return pg.ComposePage( - self, release, testtype, milestone, compose, modular=modular) - else: - # We should never get here, but just in case. - raise ValueError( - "get_validation_page(): Something very strange happened.") - - def report_validation_results(self, reslist, allowdupe=False): - """High-level result reporting function. Pass it an iterable - of objects identifying results. It's pretty forgiving about - what these can be. They can be any kind of sequence or - iterable containing up to 15 values in the following order: - (testtype, release, milestone, compose, testcase, section, - testname, env, status, user, bugs, comment, bot, cid, modular). - They can also be any mapping type (e.g. a dict) with enough of - the 15 keys set (see below for requirements). - - You may find it convenient to import the ResTuple class from - this module and pass your results as instances of it: it is - a namedtuple with default values which uses the names given - above, so you can avoid having to pad values you don't need to - set and conveniently read back items from the tuple after - you've created it, if you like. - - Any value of the result item can be absent, or set to - something empty or falsey. However, to be successfully - reported, each item must meet these conditions: - - * 'testtype', 'release', 'milestone', 'compose', 'modular' and - 'cid' must identify a single validation page - using get_validation_page() (at least 'testtype' must always - be set). - * 'testcase', 'section' and 'testname' must identify a single - test instance using ValidationPage.find_resultrow() - * 'env' must uniquely identify one of the test instance's - 'environments' (the columns into which results can be entered; - it can be empty if there is only one for this test instance) - *'status' must indicate the result status. - - 'user', 'bugs', and 'comment' can always be left empty if - desired; if 'user' is not specified, the mediawiki username - (lower-cased) will be used. - - All values should be strings, except 'bugs', which should be - an iterable of digit strings (though iterable of ints will be - tolerated), and 'bot' which should be True if the result is - from some sort of automated testing system (not a human). - - Returns a 2-tuple of lists of objects indicating any failures. - The first list is of submissions that failed because there was - insufficient identifying data. The second list is of - submissions that failed because they were 'dupes' - the given - user has already reported a result for the given test and env. - If allowdupe is True, duplicate reports will be allowed and - this list will always be empty. - - The items in each list have the same basic layout as the input - results items. The 'insufficients' list will contain the exact - same objects that were provided as input. The 'duplicates' - list is reconstructed and will often contain more or corrected - data compared to the input tuples. Its members are *always* - instances of the ResTuple namedtuple class, which provides - access to the fields by the names given above. - - Uses get_validation_page() to guess the desired page and then - constructs a result dict to pass to the ValidationPage - add_results() method. - """ - pagedict = dict() # KEY: (testtype, release, milestone, compose, cid, modular) - insufficients = list() - dupes = list() - - for resitem in reslist: - # Convert the item into a ResTuple. - try: - restup = ResTuple(**resitem) - except TypeError: - restup = ResTuple(*resitem) - if not restup.status: - # It doesn't make sense to allow {{result|none}} - # from this high-level function. - insufficients.append(resitem) - continue - user = restup.user - if not user: - # If no username was given, guess at the wiki account - # name, lower-cased. - user = self.username.lower() - key = (restup.testtype, restup.release, restup.milestone, - restup.compose, restup.cid, restup.modular) - # We construct a dict to sort the results by page. The - # value for each page is a 2-tuple containing the actual - # ValidationPage object and the 'results dictionary' we - # will pass to page.add_results() once we have iterated - # through the full result list (unless the page cannot be - # found, in which case the value is None.) - if key not in pagedict: - # We haven't yet encountered this page, so we'll try - # and find it and add it to the dict. - pagedict[key] = dict() - try: - pagedict[key] = (self.get_validation_page(*key), dict()) - except ValueError: - # This means we couldn't find a page from the info - # provided; set the pagedict entry for this key to - # (None, None) to cache that information, add this - # result to the 'insufficients' list, and move on - # to the next. - pagedict[key] = (None, None) - insufficients.append(resitem) - continue - elif pagedict[key] == (None, None): - # This is when we've already tried to find a page from - # the same (testtype, release, milestone, compose) and - # come up empty, so append the restup to the insuffs - # list and move on. - insufficients.append(resitem) - continue - - # The code from here on gets hit whenever we can find the - # correct page for a restup. We need to find the correct - # ResultRow from the testcase, section, testname and - # env, and produce a Result from status, username, bugs, - # and comment. The keys in resdict are ResultRow - # instances, so we check if there's already one that's - # 'equal' to the ResultRow for this restup. If so, we - # append the (env, Result) tuple to the list that is the - # value for that key. If not, we add a new entry to the - # resdict. It's vital that the resdict contain only *one* - # entry for any given ResultRow, or else the edit will go - # squiffy. - (page, resdict) = pagedict[key] - try: - myrow = page.find_resultrow( - restup.testcase, restup.section, restup.testname, - restup.env) - myres = rs.Result( - restup.status, user, restup.bugs, - restup.comment, restup.bot) - except (NoPageError, NotFoundError, TooManyError): - # We couldn't find precisely one result row from the - # provided information. - insufficients.append(resitem) - continue - - done = False - for row in resdict.keys(): - if myrow.matches(row): - resdict[row].append((restup.env, myres)) - done = True - break - if not done: - resdict[myrow] = [(restup.env, myres)] - - # Finally, we've sorted our restups into the right shape. Now - # all we do is throw each page's resdict at its add_results() - # method and grab the return value, which is a list of tuples - # representing results that were 'dupes'. We then reconstruct - # a ResTuple for each dupe, and return the lists of insuffs - # and dupes. - for (page, resdict) in pagedict.values(): - if page: - _dupes = (page.add_results(resdict, allowdupe)) - for (row, env, result) in _dupes: - dupes.append(ResTuple( - page.testtype, page.release, page.milestone, - page.compose, row.testcase, row.section, row.name, - env, result.status, result.user, result.bugs, - result.comment, result.bot, '', page.modular)) - - return (insufficients, dupes) - -# vim: set textwidth=100 ts=8 et sw=4: