From 2eb89386fe01f9c2f3e79f3e4492748814710eef Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Feb 02 2017 12:45:37 +0000 Subject: Release 1.0.0 --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..038d7e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +**/*.pyc +**/*.pyo +__pycache__/ +dist/ +build/ +resultsdb_conventions.egg-info/ +.cache/ +.tox/ diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..7615b86 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[FORMAT] +max-line-length=120 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..838c065 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +## Changelog + +### 1.0.0 - 2017-02-02 + +* [resultsdb_conventions-1.0.0.tar.xz](https://releases.pagure.org/fedora-qa/resultsdb_conventions/resultsdb_conventions-1.0.0.tar.xz) + +1. Initial release of resultsdb_conventions diff --git a/README.md b/README.md new file mode 100644 index 0000000..1a5174e --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# resultsdb_conventions + +resultsdb_conventions is a Python library that represents various conventions for reporting test results to [ResultsDB][1]. It allows you to report results easily and without a lot of boilerplate, and be relatively confident their ResultsDB metadata will be consistent with other results of the same basic nature. + +## Installation + +resultsdb_conventions is not yet packaged for distributions, but will be shortly. In the meantime, you can simply make the library (currently just a single module file) available in the Python path for your consumer in some way, or to install the library systemwide, just run `sudo python setup.py install`. You will need the `cached-property` and `fedfind` libraries as well (these are both packaged for Fedora and EPEL). For actually submitting results, you will need the `ResultsDBapi` class from the `resultsdb_api` module. + +resultsdb_conventions is intended to be compatible with Python 2.6+ and current Python 3. Please report bugs if you find compatibility problems. + +## Use + +The simplest way to use resultsdb_conventions is to pick the `Result` subclass that most closely represents the kind of result you wish to submit, instantiate it with appropriate arguments, get an instance of `ResultsDBapi`, and run the `report()` method. This will apply the 'default' metadata for the result (based on the kind of result and the args used for instantiation), and submit it to whichever ResultsDB you got an API instance for. The `Result` subclasses should all document their required and optional arguments. + +For simple modifications of the submitted result, you can simply adjust the `extradata` property (which is just a dict of arbitrary string key:value pairs that are passed to ResultsDB and stored as-is) after getting the instance but before running `report()`. You can also cause the result to be added to more groups by including an iterable of group dicts or UUID strings as the `groups` arg when instantiating the result class, or by adjusting the instance's `groups` property directly. + +For more complex changes to the behaviour, you can of course start from the most relevant class and create a subclass, then adjust things as appropriate. The important conventions for how subclasses should be implemented are documented in the `Result` class. If your subclass is likely to have utility outside your project, you may want to submit a pull request for it, so other projects can conveniently report results according to the same conventions. + +A simple validation mechanism has been included, but currently none of the included classes implements any significant validation. The validation is intended to enforce the convention being encoded, not to do fundamental checks on the validity of the result in ResultsDB terms; ResultsDB will reject any outright invalid submission. Please consider implementing validation for any pull requests you submit. + +## Bugs, pull requests etc. + +You can file issues and pull requests on the [resultsdb_conventions project][2] in Pagure. + +## Credits + +Jan Sedlak and Josef Skladanka contributed valuable inspiration, ideas and reviews. + +## Licensing + +resultsdb_conventions is available under the GPL, version 3 or any later version. A copy is included as COPYING. + +[1]: https://fedoraproject.org/wiki/ResultsDB +[2]: https://pagure.io/fedora-qa/resultsdb_conventions diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..704946d --- /dev/null +++ b/release.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ "$#" != "1" ]; then + echo "Must pass release version!" + exit 1 +fi + +version=$1 +name=resultsdb_conventions +sed -i -e "s,version = \".*\",version = \"${version}\", g" setup.py +git add setup.py +git commit -m "release ${version}" +git push +git tag -m "release ${version}" -a ${version} +git push origin ${version} +# FIXME: this is wrong, but hopefully works until i can find the right +# version of this damn script +git archive --format=tar --prefix=${name}-${version}/ ${version} > dist/${name}-${version}.tar +xz dist/${name}-${version}.tar +# FIXME: need scp upload for Pagure: +# https://pagure.io/pagure/issue/851 +# Uploading manually via web UI for now diff --git a/resultsdb_conventions.py b/resultsdb_conventions.py new file mode 100644 index 0000000..f661dab --- /dev/null +++ b/resultsdb_conventions.py @@ -0,0 +1,263 @@ +# Copyright (C) Adam Williamson +# +# resultsdb_conventions is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""OO representation of conventions for reporting various types of +result to ResultsDB. +""" + +from __future__ import unicode_literals +from __future__ import print_function + +import logging +import uuid + +from cached_property import cached_property +import fedfind.helpers +import fedfind.release + +# pylint:disable=invalid-name +logger = logging.getLogger(__name__) + +# UTILITY FUNCTIONS + +def uuid_namespace(name, namespace=None): + """Create a UUID using the provided name and namespace (or default + DNS namespace), handling string type and encoding ickiness on both + Py2 and Py3. + """ + # so the deal here is the passed name may be a string or a unicode + # in Python 2 (for Python 3 we assume it's a string), and what we + # need back is a string - not a bytestring on Python 3, or a + # unicode on Python 2, as uuid doesn't accept either - with non- + # ASCII characters stripped (as uuid doesn't accept those either). + # This magic formula seems to work and produce the same UUID on + # both. + if not namespace: + namespace = uuid.NAMESPACE_DNS + return uuid.uuid5(namespace, str(name.encode('ascii', 'ignore').decode())) + +# EXCEPTIONS + +class ValidationError(Exception): + """Validation error class.""" + def __init__(self, prop, value, desc): + self.property = prop + self.value = value + self.desc = desc + super(ValidationError, self).__init__(str(self)) + + def __str__(self): + return "Value {0} for property {1} is {2}".format(self.value, self.property, self.desc) + +# RESULT CLASSES + + +class Result(object): + """Base class for a single result. The signature follows the API + create_result class as closely as possible. outcome is the actual + result of the test ('PASSED', 'FAILED' etc.) tc_name is the test + case name. groups may be an iterable of UUID strings or ResultsDB + group instances; if set, the Result will be added to all the + groups. note is the freeform text note. ref_url is the URL for + this specific result, tc_url is the general URL for the testcase. + source is the source of the result - something like 'openqa' or + 'autocloud'. + """ + def __init__(self, outcome, tc_name, groups=None, note='', ref_url='', tc_url='', source=''): + self.outcome = outcome + self.tc_name = tc_name + self.note = note + self.ref_url = ref_url + self.tc_url = tc_url + self.source = source + self.extradata = {} + self.groups = [] + if groups: + self.groups.extend(groups) + + @property + def testcase_object(self): + """The testcase object for this result.""" + return { + "name": self.tc_name, + "ref_url": self.tc_url, + } + + def validate(self): + """Check if the contents of the result are valid. We do not + actually do any validation at this level - we cannot logically + declare any conventions beyond what ResultsDB will accept, and + the API will refuse any flat out invalid result. + """ + pass + + def default_extradata(self): + """If we have a source, add it to extradata.""" + if self.source: + extradata = {'source': self.source} + # doing things this way around means we don't override + # existing values, only add new ones + extradata.update(self.extradata) + self.extradata = extradata + + def add_group(self, namespace, group_name, **extraparams): + """Create a group dict and add it to the instance group list, + using the normal convention for creating ResultsDB groups. + The description of the group will be 'namespace.group' and + the UUIDs are created from those values in a kinda agreed-upon + way. Any extra params for the group can be passed in.""" + uuidns = uuid_namespace(namespace) + groupdict = { + 'uuid': str(uuid_namespace(group_name, uuidns)), + 'description': '.'.join((namespace, group_name)) + } + groupdict.update(**extraparams) + self.groups.append(groupdict) + + def default_groups(self): + """If we have a source, add a generic source group.""" + # NOTE: we could add a generic test case group, like there is + # for Taskotron results, but I don't think it's any use + if self.source: + self.add_group('source', self.source) + + def report(self, rdbinstance, default_extradata=True, default_groups=True): + """Report this result to ResultsDB. rdbinstance is an instance + of ResultsDBapi. + """ + self.validate() + if default_extradata: + self.default_extradata() + if default_groups: + self.default_groups() + logger.debug("Result: %s", self.outcome) + logger.debug("Testcase object: %s", self.testcase_object) + logger.debug("Groups: %s", self.groups) + logger.debug("Job link (ref_url): %s", self.ref_url) + logger.debug("Extradata: %s", self.extradata) + if rdbinstance: + rdbinstance.create_result(outcome=self.outcome, testcase=self.testcase_object, groups=self.groups, + note=self.note, ref_url=self.ref_url, **self.extradata) + + +class ComposeResult(Result): + """Result from testing of a distribution compose. cid is the + compose id. May raise ValueError via parse_cid. See Result for + required args. + """ + def __init__(self, cid, *args, **kwargs): + super(ComposeResult, self).__init__(*args, **kwargs) + self.cid = cid + # item is always the compose ID (unless subclass overrides) + self.extradata.update({ + 'item': cid, + 'type': 'compose', + }) + + def default_extradata(self): + """Get productmd compose info via fedfind and update extradata.""" + super(ComposeResult, self).default_extradata() + (dist, release, date, imgtype, respin) = fedfind.helpers.parse_cid(self.cid, dist=True) + extradata = { + 'productmd.compose.name': dist, + 'productmd.compose.version': release, + 'productmd.compose.date': date, + 'productmd.compose.type': imgtype, + 'productmd.compose.respin': respin, + 'productmd.compose.id': self.cid, + } + extradata.update(self.extradata) + self.extradata = extradata + + def default_groups(self): + """Add to generic result group for this compose.""" + super(ComposeResult, self).default_groups() + extraparams = {} + # get compose location if we can + try: + rel = fedfind.release.get_release(cid=self.cid) + extraparams['ref_url'] = rel.location + except ValueError: + logger.warning("fedfind found no release for compose ID %s, compose group will have no URL", self.cid) + self.add_group('compose', self.cid, **extraparams) + if self.source: + # We cannot easily do a URL here, unless we start having + # a store of 'known' sources and how URLs are built for + # them, or using callbacks, or something. I think we might + # just ask downstreams to get the group from the group + # list and add the URL to it? + self.add_group(self.source, self.cid) + + +class FedoraImageResult(ComposeResult): + """Result from testing a specific image from a compose. filename + is the image filename. May raise ValueError via get_release + or directly. See Result for required args. + """ + def __init__(self, cid, filename, *args, **kwargs): + super(FedoraImageResult, self).__init__(cid, *args, **kwargs) + self.filename = filename + # when we have an image, item is always the filename + self.extradata.update({ + 'item': filename, + }) + + @cached_property + def ffrel(self): + """Cached instance of fedfind release object.""" + return fedfind.release.get_release(cid=self.cid) + + @cached_property + def ffimg(self): + """Cached instance of fedfind image dict.""" + try: + # this just gets the first image found by the expression, + # we expect there to be maximum one (per dgilmore, image + # filenames are unique at least until koji namespacing) + return next(_img for _img in self.ffrel.all_images if _img['path'].endswith(self.filename)) + except StopIteration: + # this happens if the expression find *no* images + raise ValueError("Can't find image {0} in release {1}".format(self.filename, self.cid)) + + def default_extradata(self): + """Populate extradata from compose ID and filename.""" + super(FedoraImageResult, self).default_extradata() + extradata = { + 'productmd.image.arch': self.ffimg['arch'], + 'productmd.image.disc_number': str(self.ffimg['disc_number']), + 'productmd.image.format': self.ffimg['format'], + 'productmd.image.subvariant': self.ffimg['subvariant'], + 'productmd.image.type': self.ffimg['type'], + } + extradata.update(self.extradata) + self.extradata = extradata + + def default_groups(self): + """Add to generic result group for this image.""" + super(FedoraImageResult, self).default_groups() + imgid = '.'.join((self.ffimg['subvariant'], self.ffimg['type'], self.ffimg['format'], + self.ffimg['arch'], str(self.ffimg['disc_number']))).lower() + self.add_group('image', imgid) + if self.source: + # We cannot easily do a URL here, unless we start having + # a store of 'known' sources and how URLs are built for + # them, or using callbacks, or something. I think we might + # just ask downstreams to get the group from the group + # list and add the URL to it? + self.add_group(self.source, imgid) + +# vim: set textwidth=120 ts=8 et sw=4: diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5fb6425 --- /dev/null +++ b/setup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# Copyright (C) Red Hat, Inc. +# +# resultsdb_conventions is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Author: Adam Williamson + +import os +import sys +from setuptools import setup, find_packages +from setuptools.command.test import test as TestCommand + +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + self.test_suite = 'tests' + + def run_tests(self): + #import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(self.pytest_args.split()) + sys.exit(errno) + +# Utility function to read the README file. +# Used for the long_description. It's nice, because now 1) we have a top level +# README file and 2) it's easier to type in the README file than to put a raw +# string in below. Stolen from +# https://pythonhosted.org/an_example_pypi_project/setuptools.html +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + +setup( + name = "resultsdb_conventions", + version = "1.0.0", + py_modules = ['resultsdb_conventions'], + author = "Adam Williamson", + author_email = "awilliam@redhat.com", + description = "Module for conveniently reporting to ResultsDB following conventions", + license = "GPLv3+", + keywords = "fedora rhel epel resultsdb test taskotron", + url = "https://pagure.io/fedora-qa/resultsdb_conventions", + install_requires = ['cached-property', 'fedfind', 'resultsdb_api'], + tests_require=['pytest', 'mock'], + cmdclass = {'test': PyTest}, + long_description=read('README.md'), + classifiers=[ + "Development Status :: 3 - Alpha", + "Topic :: Utilities", + "License :: OSI Approved :: GNU General Public License v3 or later " + "(GPLv3+)", + ], +) + +# vim: set textwidth=120 ts=8 et sw=4: diff --git a/test_resultsdb_conventions.py b/test_resultsdb_conventions.py new file mode 100644 index 0000000..9bea070 --- /dev/null +++ b/test_resultsdb_conventions.py @@ -0,0 +1,164 @@ +# Copyright (C) Adam Williamson +# +# resultsdb_conventions is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: Adam Williamson + +"""Tests for resultsdb_conventions.""" + +from __future__ import unicode_literals +from __future__ import print_function + +import pytest + +import mock +import uuid + +import resultsdb_conventions as rdc + +# fedfind image dict we use to avoid a round trip to get the real one. +FFIMG01 = { + 'variant': 'Server', + 'checksums': {'sha256': 'a7cd606a44b40a3e82cc71079f80b6928e155e112d0788e5cc80a6f3d4cbe6a3'}, + 'arch': 'x86_64', + 'path': 'Server/x86_64/iso/Fedora-Server-dvd-x86_64-Rawhide-20170131.n.1.iso', + 'bootable': True, + 'size': 3009413120, + 'implant_md5': '962ec7863a78607ba4e3fc7cda01cc46', + 'mtime': 1485870969, + 'disc_count': 1, + 'format': 'iso', + 'volume_id': 'Fedora-S-dvd-x86_64-rawh', + 'subvariant': 'Server', + 'disc_number': 1, + 'type': 'dvd' +} + +class TestHelpers: + """Tests for the helper functions.""" + + def test_uuid_namespace_default(self): + """Test the uuid_namespace helper works correctly with the + default namespace (which should be uuid.NAMESPACE_DNS). + """ + # this is a native 'str' in py2 and py3 + res = rdc.uuid_namespace(str('test')) + # this is a unicode in py2, native 'str' in py3 + res2 = rdc.uuid_namespace('test') + assert type(res) is uuid.UUID + assert str(res) == '4be0643f-1d98-573b-97cd-ca98a65347dd' + assert type(res2) is uuid.UUID + assert str(res2) == '4be0643f-1d98-573b-97cd-ca98a65347dd' + + def test_uuid_unicode(self): + """Test the uuid_namespace helper works with a name containing + Unicode characters. + """ + res = rdc.uuid_namespace('\u4500foobar') + assert type(res) is uuid.UUID + assert str(res) == 'a050b517-6677-5119-9a77-2d26bbf30507' + + def test_uuid_namespace(self): + """Test the uuid_namespace helper works with a specified + namespace. + """ + ns = rdc.uuid_namespace('test') + res = rdc.uuid_namespace('test', ns) + assert type(res) is uuid.UUID + assert str(res) == '18ce9adf-9d2e-57a3-9374-076282f3d95b' + +class TestRelease: + """Tests for the Release class.""" + # FIXME: write tests + pass + +class TestFunc: + """Some functional tests.""" + + @mock.patch.object(rdc.FedoraImageResult, 'ffimg', FFIMG01) + def test_basic(self): + """This is just a pretty basic overall test that instantiates + the most complex class, produces a result and checks its + properties. + """ + res = rdc.FedoraImageResult( + cid='Fedora-Rawhide-20170131.n.1', + filename='Fedora-Server-dvd-x86_64-Rawhide-20170131.n.1.iso', + outcome='PASSED', + tc_name='compose.some.test', + note='note here', + ref_url='https://www.job.link/', + tc_url='https://www.test.case/', + source='testsource' + ) + # Report to a MagicMock, so we can check create_result args + fakeapi = mock.MagicMock() + res.report(fakeapi) + + # basic attribute checks + assert res.outcome == 'PASSED' + assert res.tc_name == 'compose.some.test' + assert res.note == 'note here' + assert res.ref_url == 'https://www.job.link/' + assert res.tc_url == 'https://www.test.case/' + assert res.source == 'testsource' + + # check the testcase object + assert res.testcase_object == { + 'name': 'compose.some.test', + 'ref_url': 'https://www.test.case/', + } + + # check the extradata + assert res.extradata == { + 'item': 'Fedora-Server-dvd-x86_64-Rawhide-20170131.n.1.iso', + 'productmd.compose.date': '20170131', + 'productmd.compose.id': 'Fedora-Rawhide-20170131.n.1', + 'productmd.compose.name': 'Fedora', + 'productmd.compose.respin': 1, + 'productmd.compose.type': 'nightly', + 'productmd.compose.version': 'rawhide', + 'productmd.image.arch': 'x86_64', + 'productmd.image.disc_number': '1', + 'productmd.image.format': 'iso', + 'productmd.image.subvariant': 'Server', + 'productmd.image.type': 'dvd', + 'source': 'testsource', + 'type': 'compose' + } + + # check the groups + assert res.groups == [ + { + 'description': 'source.testsource', + 'uuid': 'ddf8c194-5e34-50ec-b0e8-205c63b0dfc1' + }, + { + 'description': 'compose.Fedora-Rawhide-20170131.n.1', + 'ref_url': 'https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20170131.n.1/compose', + 'uuid': '8f6a8786-7b02-5ec0-9ac4-086ef3e33515' + }, + { + 'description': 'testsource.Fedora-Rawhide-20170131.n.1', + 'uuid': 'b42e8fbf-b74a-5ee3-8d1d-e59e17220fce' + }, + { + 'description': 'image.server.dvd.iso.x86_64.1', + 'uuid': '65b7f973-d87b-5008-a6e7-f431155b9a00' + }, + { + 'description': 'testsource.server.dvd.iso.x86_64.1', + 'uuid': 'f5969a48-5c55-5d16-abb8-a94d54aacf22' + } + ]