From 0a1389a3316e8d79a42c8f3f1182b97d7e14a357 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Jun 16 2014 22:45:39 +0000 Subject: Updating master from develop for 0.3.0 release --- diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/docs/source/conf.py b/docs/source/conf.py index f0f75eb..414c0ee 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,9 +51,9 @@ copyright = u'2014, Fedora QA Devel' # built documents. # # The short X.Y version. -version = '0.1.0' +version = '0.3.0' # The full version, including alpha/beta/rc tags. -release = '0.1.0' +release = '0.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/devguide.rst b/docs/source/devguide.rst index 5b64c50..df33aba 100644 --- a/docs/source/devguide.rst +++ b/docs/source/devguide.rst @@ -258,9 +258,12 @@ mentioned, follow the `Python documentation style guide Building documentation ---------------------- -The documentation is easy to build once deps are installed:: +The documentation is easy to build once deps are installed. Because system-installed +Sphinx is not able to import libraries from virtualenv (`bug 1454 `_), +it needs to be installed in virtualenv as well:: - yum install python-sphinx + pip install Sphinx To actually build the documentation:: diff --git a/docs/source/taskyaml.rst b/docs/source/taskyaml.rst index ca03152..540cf18 100644 --- a/docs/source/taskyaml.rst +++ b/docs/source/taskyaml.rst @@ -49,14 +49,16 @@ order to verify that the task can be run .. code-block:: yaml input: - args: arch, envr + args: + - arch + - koji_build Valid args include: - * arch - * envr - * bodhi_id - * koji_tag + * arch (e.g.: ``x86_64``, ``[i386, x86_64]``) + * koji_build (e.g.: ``xchat-0:2.8.8-19.fc19``, ``xchat-2.8.8-19.fc19``) + * bodhi_id (e.g.: ``FEDORA-2013-10476``, ``xchat-2.8.8-19.fc19``) + * koji_tag (e.g.: ``f20-updates-testing-pending``) Dependencies ------------ @@ -96,31 +98,20 @@ export of data. - name: using directivename for something directivename: arg1: value1 - arg2: "{{ some_variable }}" + arg2: ${some_variable} export: somestep_output Variable Storage and Syntax --------------------------- -The task yaml file uses standard `jinja2 `_ syntax for -variables. Variables can be created during task execution or provided by default -by the task runner. +The task yaml file uses standard `string.Template`_ syntax for variables. You +can use ``$variable`` or ``${variable}`` format, and you need to make the dollar +sign doubled if you want to include it literally (``$$not_a_variable``). +Variables can be created during task execution or provided by default by the +task runner. -.. note:: - - An unfortunate side-effect of using jinja2 syntax with yaml files is that the - yaml parser tends to view variables as invalid dictionaries. If the variable - is directly following a colon, make sure to surround it in quotes so that the - yaml parser interprets the variable as a string. - - .. code-block:: yaml - - # this will not parse well - arg1: {{ some_variable }} - - # this will parse properly - arg1: "{{ some_variable }}" +.. _string.Template: https://docs.python.org/2.7/library/string.html?highlight=string#template-strings Provided Variables ------------------ @@ -128,8 +119,8 @@ Provided Variables The task runner provides the following variables without a need for them to be explicitly specified: - * workdir - * jobid + * ``workdir`` + * ``jobid`` .. note:: @@ -155,7 +146,7 @@ used later, it must first be exported. - name: second step directive_two: - arg1: "{{ firststep_output }}" + arg1: ${firststep_output} In this example, the output from ``first step`` is stored as ``firststep_output`` and later used by ``second step``. Once the ``first step`` is executed, its @@ -190,7 +181,7 @@ for the update. bodhi ----- -This directive is used to download specific packages from +This directive is used to download specific packages from `bodhi `_. createrepo @@ -201,15 +192,15 @@ Takes a path from yaml input and creates a repository on that path. dummy ----- -Primarily a testing directive, it simply returns a status and -optionally a message. Also works as a useful placeholder while +Primarily a testing directive, it simply returns a status and +optionally a message. Also works as a useful placeholder while writing new tasks. koji ---- Provides an interface to `koji `_ for -downloading rpms with a specific envr or rpms with a specific tag. +downloading rpms with a specific NEVR or rpms with a specific tag. mash ---- diff --git a/docs/source/writingtasks.rst b/docs/source/writingtasks.rst index 0881f62..9de6410 100644 --- a/docs/source/writingtasks.rst +++ b/docs/source/writingtasks.rst @@ -51,13 +51,6 @@ rpmlint on it, reporting results as TAP13. Creating a New Task =================== -.. warning:: - - With the current libtaskotron code, the following example code does run but - its output does not match what is shown here. We are aware of the issue and - will be fixing it soon. See the `phabricator task - `_ for details. - While running rpmlint is an easy and trivial example, that doesn't help with creating something new. To write a new task, you'll need some information: @@ -100,7 +93,9 @@ non-executed task data needed in :file:`mytask.yml`: maintainer: somefasuser input: - args: arch, bodhi_id + args: + - arch + - bodhi_id environment: rpm: @@ -125,7 +120,7 @@ Let's have this new task do the following: # this is a trivial example that doesn't do everything we want it to do, more # details will be added later in the example def run_mytask(rpmfiles, bodhi_id): - print "Running mytask on %s" % (str(bodhi_id)) + print "Running mytask on %s" % bodhi_id .. code-block:: yaml @@ -137,8 +132,8 @@ Let's have this new task do the following: - name: download bodhi update bodhi: action: download - bodhi_id: "{{ bodhi_id }}" - arch: "{{ arch }}" + bodhi_id: ${bodhi_id} + arch: ${arch} export: bodhi_downloads @@ -149,19 +144,19 @@ Let's have this new task do the following: python: file: mytask.py callable: run_mytask - rpmfiles: "{{ bodhi_downloads }}" - bodhi_id: "{{ bodhi_id }}" + rpmfiles: ${bodhi_downloads} + bodhi_id: ${bodhi_id} export: mytask_output # this assumes that the output from mytask is valid TAP - name: report mytask results to resultsdb resultsdb: - results: "{{ mytask_output }}" + results: ${mytask_output} checkname: 'mytask' .. this needs to be fixed, this output is doctored since it doesn't actually - work. + work. We can execute this new task using:: @@ -209,7 +204,7 @@ Our original task looked like: # details will be added later in the example def run_mytask(rpmfiles): for rpmfile in rpmfiles: - print "Running mytask on %s" % (str(rpmfile)) + print "Running mytask on %s" % rpmfile To create valid TAP, we want to populate a :py:class:`libtaskotron.check.CheckDetail` object and use :py:meth:`libtaskotron.check.export_TAP` to generate valid TAP13 @@ -222,12 +217,12 @@ with the data required for reporting. def run_mytask(rpmfiles, bodhi_id): """run through all passed in rpmfiles and emit TAP13 saying they all passed""" - print "Running mytask on %s" % (str(bodhi_id)) + print "Running mytask on %s" % bodhi_id details = [] result = 'PASSED' - summary = 'mycheck %s for %s' % (result, str(bodhi_id)) - detail = check.CheckDetail(rpmfile, check.ReportType.BODHI_UPDATE, + summary = 'mycheck %s for %s' % (result, bodhi_id) + detail = check.CheckDetail(bodhi_id, check.ReportType.BODHI_UPDATE, result, summary) for rpmfile in rpmfiles: detail.store(rpmfile, False) @@ -244,7 +239,7 @@ Now if we run the task we get output that ends with:: ok - $CHECKNAME for Bodhi update openvswitch-2.1.2-1.fc19 --- details: - output: " foo-1.2-3.fc99.x86_64.rpm" + output: "foo-1.2-3.fc99.x86_64.rpm" item: foo-1.2-3.fc99 outcome: PASSED summary: mycheck PASSED for foo-1.2-3.fc99 @@ -253,7 +248,7 @@ Now if we run the task we get output that ends with:: .. note:: Reporting is disabled in the default configuration for Taskotron. While it is - possible to set up a local resultsdb instance to check reporting, we want + possible to set up a local resultsdb instance to check reporting, we want to make it easier than that and will update these docs when that easier method is available. diff --git a/libtaskotron.spec b/libtaskotron.spec index 6b93179..0ffc5ba 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -3,7 +3,7 @@ %{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} Name: libtaskotron -Version: 0.1.0 +Version: 0.3.0 Release: 1%{?dist} Summary: Taskotron Support Library @@ -19,8 +19,7 @@ Requires: rpm-python Requires: pyOpenSSL Requires: python-pycurl Requires: python-urlgrabber -Requires: python-jinja2 -Requires: pytap13 +Requires: pytap13 >= 0.1.0 Requires: resultsdb_api Requires: python-bayeux Requires: resultsdb_api @@ -29,15 +28,15 @@ Requires: python-bunch Requires: python-fedora Requires: createrepo Requires: mash +Requires: libtaskotron-config BuildRequires: python-devel BuildRequires: python-setuptools BuildRequires: pytest BuildRequires: python-bunch BuildRequires: python-dingus BuildRequires: python-urlgrabber -BuildRequires: python-jinja2 BuildRequires: koji -BuildRequires: pytap13 +BuildRequires: pytap13 >= 0.1.0 BuildRequires: python-bayeux BuildRequires: bodhi-client BuildRequires: resultsdb_api @@ -75,7 +74,7 @@ install conf/yumrepoinfo.conf.example %{buildroot}%{_sysconfdir}/taskotron/yumre %files -%doc readme.rst +%doc readme.rst LICENSE %{python_sitelib}/libtaskotron %{python_sitelib}/*.egg-info @@ -89,6 +88,19 @@ install conf/yumrepoinfo.conf.example %{buildroot}%{_sysconfdir}/taskotron/yumre %changelog +* Mon Jun 16 2014 Tim Flink - 0.3.0-1 +- Added sphinx to requirements.txt + +* Fri Jun 13 2014 Tim Flink - 0.2.1-1 +- documentation improvements, added LICENSE file +- better support for depcheck, srpm downloading +- improved logging configuration + +* Wed May 28 2014 Tim Flink - 0.1.1-1 +- adding libtaskotron-config as requires for libtaskotron +- changing variable syntax to $var and ${var} +- add yumrepoinfo directive, other bugfixes + * Fri May 16 2014 Tim Flink - 0.1.0-1 - Releasing libtaskotron 0.1 diff --git a/libtaskotron/__init__.py b/libtaskotron/__init__.py index b794fd4..014150b 100644 --- a/libtaskotron/__init__.py +++ b/libtaskotron/__init__.py @@ -1 +1,7 @@ -__version__ = '0.1.0' +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import +__version__ = '0.3.0' diff --git a/libtaskotron/bodhi_utils.py b/libtaskotron/bodhi_utils.py index e73a61d..71396d9 100644 --- a/libtaskotron/bodhi_utils.py +++ b/libtaskotron/bodhi_utils.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- -# Copyright 2010-2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Utility functions for dealing with Bodhi''' +from __future__ import absolute_import import fedora.client from .logger import log @@ -92,9 +94,9 @@ class BodhiUtils(object): :raise TaskotronValueError: if ``builds`` type is incorrect ''' # validate input params - if not python_utils.listlike(builds): - raise exc.TaskotronValueError("Param 'builds' must be iterable, " - "and yours was: %s" % type(builds)) + if not python_utils.iterable(builds, basestring): + raise exc.TaskotronValueError("Param 'builds' must be an iterable " + "of strings, and yours was: %s" % type(builds)) build2update = {} failures = {} diff --git a/libtaskotron/buildbot_utils.py b/libtaskotron/buildbot_utils.py new file mode 100644 index 0000000..94f463c --- /dev/null +++ b/libtaskotron/buildbot_utils.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Copyright 2010-2014, Red Hat, Inc. +# License: GNU General Public License version 2 or later + +'''Utility functions for dealing with Buildbot''' + +from __future__ import absolute_import +from libtaskotron.logger import log + + +def parse_jobid(jobid): + """ Parse the incoming jobid which should either be '-1' to indicate + that dummy values should be used or in the form of /. + + If the passed in format is invalid, no exceptions will be raised but a + warning message will be emitted to logs + + :param str jobid: jobid from runner + :returns: (builder, buildid) + """ + if jobid != "-1": + try: + builder, buildid = jobid.split('/') + return (builder, buildid) + except ValueError, e: + log.debug(e) + log.warning( + 'Invalid jobid format detected, resetting to default ' + 'value. jobid must be in builder/buildid format, ' + 'found %s' % jobid) + + return "default", "-1" + + +def get_urls(jobid, masterurl, task_stepname): + """ Form url to the job and its logs based on given jobid. + + :param str jobid: jobid from runner + :param str masterurl: taskotron master url + :param str task_stepname: taskotron master step name + :returns: (job_url, log_url) + """ + builder, buildid = parse_jobid(jobid) + + job_url = '%s/builders/%s/builds/%s' % (masterurl, builder, buildid) + log_url = '%s/steps/%s/logs/stdio' % (job_url, task_stepname) + + return job_url, log_url diff --git a/libtaskotron/check.py b/libtaskotron/check.py index d87c6c3..b771cd4 100644 --- a/libtaskotron/check.py +++ b/libtaskotron/check.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Helper tools for managing check status, outcome, and output.''' +from __future__ import absolute_import import pprint import collections @@ -88,12 +90,13 @@ class CheckDetail(object): def __init__(self, item, report_type=None, outcome=None, summary='', output=None, keyvals=None): # validate input - if output is not None and not python_utils.listlike(output): - raise exc.TaskotronValueError("'output' parameter must be a list. " - "Yours was: %s" % type(output)) + if (output is not None and + not python_utils.sequence(output, basestring, mutable=True)): + raise exc.TaskotronValueError("'output' parameter must be a " + "mutable sequence of strings. Yours was: %s" % type(output)) if keyvals is not None and not isinstance(keyvals, collections.Mapping): - raise exc.TaskotronValueError("'keyvals' parameter must behave like" - " a dictionary. Yours was: %s" % type(keyvals)) + raise exc.TaskotronValueError("'keyvals' parameter must be a " + "mapping. Yours was: %s" % type(keyvals)) self.item = item self.report_type = report_type @@ -200,12 +203,12 @@ class CheckDetail(object): if len(outcomes) <= 0: return '' - if isinstance(iter(outcomes).next(), CheckDetail): + if python_utils.iterable(outcomes, CheckDetail): all_outcomes = [detail.outcome for detail in outcomes] else: # list of strings - if not python_utils.listlike(outcomes): + if not python_utils.iterable(outcomes, basestring): raise exc.TaskotronValueError("'outcomes' parameter type is " - 'incorrect: %s' % type(outcomes)) + 'incorrect: %s' % type(outcomes)) all_outcomes = outcomes # create the summary @@ -235,10 +238,15 @@ class ReportType(object): YUM_REPOSITORY = 'yum_repository' #: -def export_TAP(*check_details): +def export_TAP(check_details): '''Generate TAP output used for reporting to ResultsDB. - :param check_details: any number of :class:`CheckDetail` instances + Note: You need to provide all your :class:`CheckDetail`s in a single pass + in order to generate a valid TAP output. You can't call this method several + times and then simply join the outputs simply as strings. + + :param check_details: iterable of :class:`CheckDetail` instances or single + instance of :class:`CheckDetail` :return: TAP output with results for every :class:`CheckDetail` instance provided :rtype: str @@ -263,6 +271,11 @@ def export_TAP(*check_details): ... ''' + # if check_details is single CheckDetail create a list with one element + if isinstance(check_details, CheckDetail): + check_details = [check_details] + + # validate input for detail in check_details: if not detail.item: @@ -342,12 +355,14 @@ def import_TAP(source): summary = y.get('summary', '') details = y.get('details', {}) output = details.get('output', None) + if output is not None: + output = [output] # TODO: is there a better way to do this? other_keys = set(y.keys()) - set(RESERVED_KEYS) keyvals = dict([(k,y[k]) for k in other_keys]) - cd = CheckDetail(item, report_type, outcome, summary, [output]) + cd = CheckDetail(item, report_type, outcome, summary, output) cd.keyvals = keyvals check_details.append(cd) diff --git a/libtaskotron/config.py b/libtaskotron/config.py index 38a2b9c..1b7ae6d 100644 --- a/libtaskotron/config.py +++ b/libtaskotron/config.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- -# Copyright 2010-2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Global configuration for Taskotron and relevant helper methods.''' +from __future__ import absolute_import import os import yaml +import collections import libtaskotron @@ -174,9 +177,9 @@ def _load_file(conf_file): # check correct types # we should receive a single dictionary with keyvals - if not isinstance(conf_obj, dict): + if not isinstance(conf_obj, collections.Mapping): raise exc.TaskotronConfigError('The config file %s does not have ' - 'a valid structure. Instead of a dictionary, it is recognized as: %s' % + 'a valid structure. Instead of a mapping, it is recognized as: %s' % (filename, type(conf_obj))) default_conf = Config() diff --git a/libtaskotron/config_defaults.py b/libtaskotron/config_defaults.py index b4c3763..5efe668 100644 --- a/libtaskotron/config_defaults.py +++ b/libtaskotron/config_defaults.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''This includes the default values for Taskotron configuration. This is automatically loaded by config.py and then overridden by values from config files available in system-wide location.''' +from __future__ import absolute_import import pprint diff --git a/libtaskotron/directives/__init__.py b/libtaskotron/directives/__init__.py index 96370fa..2d6a792 100644 --- a/libtaskotron/directives/__init__.py +++ b/libtaskotron/directives/__init__.py @@ -1,3 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import + class BaseDirective(object): def __init__(self): diff --git a/libtaskotron/directives/bodhi_comment_directive.py b/libtaskotron/directives/bodhi_comment_directive.py index b2741ff..5a72060 100644 --- a/libtaskotron/directives/bodhi_comment_directive.py +++ b/libtaskotron/directives/bodhi_comment_directive.py @@ -1,26 +1,11 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Authors: -# Tim Flink -# Josef Skladanka +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing -directive_class = 'BodhiCommentDirective' +from __future__ import absolute_import +directive_class = 'BodhiCommentDirective' from libtaskotron.directives import BaseDirective @@ -28,6 +13,7 @@ from libtaskotron.logger import log from libtaskotron import check from libtaskotron import config from libtaskotron import bodhi_utils +from libtaskotron import buildbot_utils from libtaskotron.exceptions import TaskotronDirectiveError, TaskotronValueError from libtaskotron.rpm_utils import basearch @@ -69,9 +55,8 @@ class BodhiUpdateState: def add_result(self, new_result): ''' Update the current state with a new result. - Args: - new_result: a dictionary containing the following keys: - time (datetime object), testname, result, arch + :param dict new_result: a dictionary containing the following keys: + ``time`` (datetime object), ``testname``, ``result``, ``arch`` ''' self.result_history.append(new_result) @@ -135,14 +120,16 @@ class BodhiUpdateState: def _is_comment_email_needed(update_name, parsed_comments, new_result, kojitag): ''' - Determines whether or not to send an email with the comment posted to bodhi. - Uses previous comments on update in order to determine current state - - Args: - update_name -- name of the update to be tested - parsed_comments -- already existing AutoQA comments for the update - new_result -- the new result to be posted - kojitag -- the koji tag being tested (affects the expected tests) + Determines whether or not to send an email with the comment posted to bodhi. + Uses previous comments on update in order to determine current state + + :param str update_name: name of the update to be tested + :param parsed_comments: already existing Taskotron comments for + the update + :type parsed_comments: list of dict + :param dict new_result: the new result to be posted + :param str kojitag: the koji tag being tested (affects the expected tests) + :rtype: bool ''' Config = config.get_config() @@ -169,10 +156,10 @@ def _is_comment_email_needed(update_name, parsed_comments, new_result, def _parse_result_from_comment(comment): ''' - Parses timestamp and results from bodhi comment + Parses timestamp and results from bodhi comment - Args: - comment -- the 'comment' part of bodhi update object + :param dict comment: the ``comment`` part of bodhi update object + :rtype: dict ''' comment_time = datetime.datetime.strptime(comment['timestamp'], '%Y-%m-%d %H:%M:%S') @@ -201,26 +188,24 @@ def _parse_result_from_comment(comment): def _already_commented(bodhi_api, update, testname, arch): '''Check if Taskotron comment is already posted. - Args: - update -- Bodhi update object --or-- update title --or-- update ID --or-- package NVR - testname -- the name of the test - arch -- tested architecture + :param update: Bodhi update object --or-- update title --or-- update ID + --or-- package NVR + :param str testname: the name of the test + :param str arch: tested architecture Note: Only NVR allowed, not ENVR. See https://fedorahosted.org/bodhi/ticket/592. - Returns: - Tuple containing old result and time when the last comment was posted. - If no comment is posted yet, or it is, but the update - has been modified since, tuple will contain two empty strings. + :return: Tuple containing old result and time when the last comment was posted. + If no comment is posted yet, or it is, but the update + has been modified since, tuple will contain two empty strings. - Throws: - TaskotronValueError -- if no such update can be found + :raises TaskotronValueError: if no such update can be found ''' Config = config.get_config() # if we received update title or ID, let's convert it to update object first - if isinstance(update, unicode) or isinstance(update, str): + if isinstance(update, basestring): u = bodhi_api.query_update(update) if u: update = u @@ -247,14 +232,13 @@ def _already_commented(bodhi_api, update, testname, arch): def _is_comment_needed(old_result, comment_time, result, time_span = None): '''Check if the comment is meant to be posted. - Args: - old_result -- the result of the last test - comment_time -- the comment time of the last test - result -- the result of the test - time_span -- waiting period (in minutes) before posting the same comment + :param str old_result: the result of the last test + :param str comment_time: the comment time of the last test + :param str result: the result of the test + :param int time_span: waiting period (in minutes) before posting the same comment - Returns: - True if the comment will be posted, False otherwise. + :return: ``True`` if the comment will be posted, ``False`` otherwise. + :rtype: bool ''' # the first comment or a comment with different result, post it if not old_result or old_result != result: @@ -293,19 +277,18 @@ def _post_testresult(bodhi_api, update, testname, result, url, arch = 'noarch', karma = 0, doreport='onchange'): '''Post comment and karma to bodhi - Args: - update -- the *title* of the update comment on - testname -- the name of the test - result -- the result of the test - url -- url of the result of the test - arch -- tested architecture (default 'noarch') - karma -- karma points (default 0) - doreport -- set to 'all' to force posting bodhi comment - - Returns: - True if comment was posted successfully or comment wasn't meant to be - posted (either posting is turned off or comment was already posted), - False otherwise. + :param str update: the **title** of the update comment on + :param str testname: the name of the test + :param str result: the result of the test + :param str url: url of the result of the test + :param str arch: tested architecture (default ``noarch``) + :param int karma: karma points (default ``0``) + :param str doreport: set to 'all' to force posting bodhi comment + + :return: ``True`` if comment was posted successfully or comment wasn't + meant to be posted (either posting is turned off or comment was + already posted), ``False`` otherwise. + :rtype: bool ''' # TODO when new bodhi releases, add update identification by UPDATEID support @@ -366,12 +349,15 @@ def _post_testresult(bodhi_api, update, testname, result, url, class BodhiCommentDirective(BaseDirective): """The bodhi_comment directive interfaces with Bodhi to create - a comment in Bodhi + a comment in Bodhi + + Format:: - format: "bodhi_comment: + bodhi_comment: results: - doreport: [all, onchange]" - Also note, that 'checkname' needs to be present in env_data. + doreport: [all, onchange] + + Also note, that ``checkname`` needs to be present in ``env_data``. """ def __init__(self, bodhi_api=None): @@ -418,12 +404,15 @@ class BodhiCommentDirective(BaseDirective): # ? Log when no results of the type are found ? + _, log_url = buildbot_utils.get_urls(env_data['jobid'], + Config.taskotron_master, + Config.buildbot_task_step) + for detail in check_details: - #TODO: replace url with proper URL outcome = _post_testresult(self.bodhi_api, detail.item, env_data['checkname'], detail.outcome, - url = "http://example.com", + url = log_url, doreport = input_data['doreport'], arch = detail.keyvals.get('arch', 'noarch'), ) diff --git a/libtaskotron/directives/bodhi_directive.py b/libtaskotron/directives/bodhi_directive.py index 877d3f9..f6e3383 100644 --- a/libtaskotron/directives/bodhi_directive.py +++ b/libtaskotron/directives/bodhi_directive.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing +from __future__ import absolute_import import libtaskotron.bodhi_utils as bodhi from libtaskotron.koji_utils import KojiClient from libtaskotron.directives import BaseDirective @@ -15,18 +17,23 @@ class BodhiDirective(BaseDirective): """The bodhi directive interfaces with Bodhi to facilitate various bodhi actions. - format: "bodhi: + Format:: + + bodhi: action: [download] - " + Valid Commands: download Downloads rpms of builds from update specified by its name or id - format: "bodhi: + + Format:: + + bodhi: action: download bodhi_id: - arch=" + arch= """ def __init__(self, bodhi_api=None, koji_session=None): @@ -78,10 +85,10 @@ class BodhiDirective(BaseDirective): workdir = env_data['workdir'] update = input_data['bodhi_id'] - if input_data['arch'] == 'all': + if 'all' in input_data['arch']: arches = ['i386', 'x86_64', 'noarch', 'armhfp'] else: - arches = [arch.strip() for arch in input_data['arch'].split(',')] + arches = input_data['arch'] if 'noarch' not in arches and arches != ['src']: arches.append('noarch') diff --git a/libtaskotron/directives/createrepo_directive.py b/libtaskotron/directives/createrepo_directive.py index f5aba2b..d623561 100644 --- a/libtaskotron/directives/createrepo_directive.py +++ b/libtaskotron/directives/createrepo_directive.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import import subprocess as sub from libtaskotron.directives import BaseDirective from libtaskotron.logger import log @@ -9,7 +15,7 @@ directive_class = 'CreaterepoDirective' class CreaterepoDirective(BaseDirective): def process(self, input_data, env_data): """For the createrepo directive, we're expecting a yaml declaration of - the form "createrepo: repodir=/path/to/some/dir" + the form ``createrepo: repodir=/path/to/some/dir`` """ repodir = input_data['repodir'] diff --git a/libtaskotron/directives/dummy_directive.py b/libtaskotron/directives/dummy_directive.py index d0df674..4c6aaf7 100644 --- a/libtaskotron/directives/dummy_directive.py +++ b/libtaskotron/directives/dummy_directive.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import from libtaskotron.directives import BaseDirective from libtaskotron.exceptions import TaskotronDirectiveError @@ -11,7 +17,8 @@ class DummyDirective(BaseDirective): It is primarially meant for testing the runner or as a placeholder while writing new tasks. - format: + Format:: + dummy: result= [msg=] required params: result optional params: msg diff --git a/libtaskotron/directives/koji_directive.py b/libtaskotron/directives/koji_directive.py index d789eb4..3bbc3bf 100644 --- a/libtaskotron/directives/koji_directive.py +++ b/libtaskotron/directives/koji_directive.py @@ -1,7 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import + from libtaskotron.koji_utils import KojiClient from libtaskotron.logger import log from libtaskotron.directives import BaseDirective from libtaskotron.exceptions import TaskotronDirectiveError +from libtaskotron import rpm_utils directive_class = 'KojiDirective' @@ -9,18 +17,31 @@ class KojiDirective(BaseDirective): """The koji directive interfaces with Koji to facilitate various koji actions. - format: "koji: target_dir= command=[download, download_tag] + Format:: + + koji: target_dir= command=[download, download_tag] Valid Commands: download - Downloads the envr specified as "envr" - format: - "koji: command=download envr= arch= + Downloads the koji build specified as ``koji_build`` + + Format:: + + koji: command=download koji_build= arch= + download_tag - Downloads rpms tagged by "tag" - format: - "koji: command=download_tag tag= arch= + Downloads rpms tagged by ``koji_tag`` + + Format:: + + koji: command=download_tag koji_tag= arch= + + + Additional supported ``arch`` argument value: ``src`` + + Note about the ``arch`` argument: if not requested, ``noarch`` is added to the list + of arches (unless ``src`` is the only one requested) """ def __init__(self, koji_session=None): @@ -52,38 +73,38 @@ class KojiDirective(BaseDirective): else: self.target_dir = input_data['target_dir'] - self.arches = [arch.strip() for arch in input_data['arch'].split(',')] + self.arches = input_data['arch'] - # https://phab.qadevel.cloud.fedoraproject.org/T155 - if 'noarch' not in self.arches: + if 'noarch' not in self.arches and not self.arches == ['src']: self.arches.append('noarch') if action == 'download': - if 'envr' not in input_data: + if 'koji_build' not in input_data: detected_args = ', '.join(input_data.keys()) raise TaskotronDirectiveError( - "The koji directive requires 'envr' for the 'download'" + "The koji directive requires 'koji_build' for the 'download'" "action. Detected arguments: %s" % detected_args) - envr = input_data['envr'] + nvr = rpm_utils.rpmformat(input_data['koji_build'], 'nvr') - log.info("getting koji builds for %s (%s) and downloading to %s", - envr, str(self.arches), self.target_dir) - output_data['downloaded_rpms'] = self.koji.get_nvr_rpms(envr, + log.info("Getting RPMs for koji build %s (%s) and downloading to %s", + nvr, str(self.arches), self.target_dir) + output_data['downloaded_rpms'] = self.koji.get_nvr_rpms(nvr, self.target_dir, - self.arches) + arches=self.arches, + src=('src' in self.arches)) elif action == 'download_tag': - if 'tag' not in input_data: + if 'koji_tag' not in input_data: detected_args = ', '.join(input_data.keys()) raise TaskotronDirectiveError( - "The koji directive requires 'tag' for the 'download_tag'" + "The koji directive requires 'koji_tag' for the 'download_tag'" "action. Detected arguments: %s" % detected_args) - tag = input_data['tag'] + koji_tag = input_data['koji_tag'] - log.info("getting koji builds for tag '%s', arches '%s' and " - "downloading to %s", tag, str(self.arches), self.target_dir) - output_data['downloaded_rpms'] = self.koji.get_tagged_rpms(tag, + log.info("Getting koji builds for koji tag '%s', arches '%s' and " + "downloading to %s", koji_tag, str(self.arches), self.target_dir) + output_data['downloaded_rpms'] = self.koji.get_tagged_rpms(koji_tag, self.target_dir, self.arches) diff --git a/libtaskotron/directives/mash_directive.py b/libtaskotron/directives/mash_directive.py index ae00bc5..8b40cd2 100644 --- a/libtaskotron/directives/mash_directive.py +++ b/libtaskotron/directives/mash_directive.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import import subprocess as sub import mash import mash.config @@ -11,7 +17,28 @@ from libtaskotron.exceptions import TaskotronDirectiveError directive_class = 'MashDirective' class MashDirective(BaseDirective): + """This will create a YUM repository in ``rpmdir`` from all RPM packages + inside the same directory. It will call ``mash`` tool to perform this task. + + The directive returns None. + + For the mash directive, we're expecting a yaml declaration of the form: + + mash: + rpmdir=/path/to/some/dir + dodelta=[True, False] + arch=["i386", "x86_64", "armhfp", "noarch"] + """ + def do_mash(self, rpmdir, dodelta, arch): + """Set up a mash object with an ad-hoc config + + :param str rpmdir: directory containing rpms to mash + :param bool dodelta: create drpms during mash + :param str arch: arch of the rpms + :returns: return code of doMultilib, 0 if it went fine, 1 othwerise + """ + # copied over from the old depcheck distname = 'depcheck' @@ -58,15 +85,6 @@ class MashDirective(BaseDirective): return rc def process(self, input_data, env_data): - """For the mash directive, we're expecting a yaml declaration of - the form: - - mash: - rpmdir=/path/to/some/dir - dodelta=[True, False] - arch=["i386", "x86_64", "armhfp", "noarch"] - """ - if 'rpmdir' not in input_data or 'arch' not in input_data: detected_args = ', '.join(input_data.keys()) raise TaskotronDirectiveError( diff --git a/libtaskotron/directives/python_directive.py b/libtaskotron/directives/python_directive.py index f403f9c..2cf2256 100644 --- a/libtaskotron/directives/python_directive.py +++ b/libtaskotron/directives/python_directive.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import import imp import os @@ -9,9 +15,11 @@ directive_class = 'PythonDirective' class PythonDirective(BaseDirective): """The python directive is designed to execute a piece of python code as - part of task execution. Given a callable name 'dothis_thing', possible + part of task execution. Given a callable name ``dothis_thing``, possible implementations in the loaded code could include: + :: + class TaskClass(object): def my_task(self): return "I'm a task class method!" @@ -20,7 +28,9 @@ class PythonDirective(BaseDirective): task_class_target = _instantiated_class.my_task - In this case, you could pass in 'task_class_target' as the callable arg + In this case, you could pass in ``task_class_target`` as the callable arg + + :: class EmbeddedCallClass(object): def __call__(self): @@ -28,27 +38,31 @@ class PythonDirective(BaseDirective): embedded_task_target = EmbeddedCallClass() - In this case, you could pass in 'embedded_task_target' as the callable arg + In this case, you could pass in ``embedded_task_target`` as the callable arg + + :: def task_method(): return "I'm a task method!" - And in this third case, you would pass in 'task_method' as the callable arg + And in this third case, you would pass in ``task_method`` as the callable arg + + Format:: - format: python: - file= callable= [=' - required params: , callable - optional params: farther key/value pairs, to be passed into the callable - as kwargs + file= callable= [=] + + * required params: ````, ``callable`` + * optional params: farther key/value pairs, to be passed into the callable + as kwargs """ def _do_getattr(self, module, name): """Isolation of getattr to make the code more easily testable - @param module module from which to getattr from - @param name name of object to getattr - @returns object retrieved from module + :param module module: module from which to getattr from + :param str name: name of object to getattr + :returns: object retrieved from module """ return getattr(module, name) @@ -56,22 +70,27 @@ class PythonDirective(BaseDirective): def execute(self, task_module, method_name, kwargs): """Execute a callable in the specified module - @param task_module module containing the specified callable - @param method_name name of callable to execute - @param kwargs kwargs to pass as parameters into the callable - @return output from the executed method + :param module task_module: module containing the specified callable + :param str method_name: name of callable to execute + :param dict kwargs: kwargs to pass as parameters into the callable + :returns: output from the executed method """ task_method = self._do_getattr(task_module, method_name) output = task_method(**kwargs) + if output is not None and not isinstance(output, basestring): + raise TaskotronDirectiveError( + "Callable task method must return string or None, actual " + "returned type: %s" % type(output)) + return output def load_pyfile(self, filename): """Import python code specified - @param filename absolute path to the python file + :param str filename: absolute path to the python file """ task_importname = 'runtask_%s' % os.path.basename(filename).rstrip( @@ -83,8 +102,8 @@ class PythonDirective(BaseDirective): def checkfile(self, filename): """Check to see if the file exists - @param filename abspath to the filename - @throws TaskotronDirectiveError if the file does not exist + :param str filename: abspath to the filename + :raises TaskotronDirectiveError: if the file does not exist """ if not os.path.exists(filename): @@ -94,13 +113,17 @@ class PythonDirective(BaseDirective): def process(self, input_data, env_data): """For the python directive, we're expecting a yaml declaration of the form - "python: - file= callable= [kwarg1=something ...]" - @param input_data dictionary of all kwargs from yaml declaration, + :: + + python: + file= callable= [kwarg1=something ...] + + :param dict input_data: dictionary of all kwargs from yaml declaration, including callable and file - @param env_data information on the environment in which we're running - @return output from the method specified, this is expected to be TAP + :param dict env_data: information on the environment in which we're + running + :returns: output from the method specified, this is expected to be TAP output """ diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index 209ee47..2a8b519 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -1,26 +1,14 @@ -# Copyright 2014, Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Authors: -# Josef Skladanka +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing +from __future__ import absolute_import from libtaskotron.directives import BaseDirective from libtaskotron import check from libtaskotron import config +from libtaskotron import buildbot_utils from libtaskotron.exceptions import TaskotronDirectiveError, TaskotronValueError from libtaskotron.logger import log @@ -32,7 +20,12 @@ directive_class = 'ResultsdbDirective' class ResultsdbDirective(BaseDirective): """The resultsdb directive interfaces with ResultsDB to store results. - format: "resultsdb: results= + When reporting is disabled, the directive checks whether the input is a valid + TAP and lists out the details that would have been reported. + + Format:: + + resultsdb: results= """ def __init__(self, resultsdb = None): @@ -85,7 +78,7 @@ class ResultsdbDirective(BaseDirective): return jobdata def complete_resultsdb_job(self, jobid): - """ Change the resultsdb job to a status of 'COMPLETED', indicating + """ Change the resultsdb job to a status of ``COMPLETED``, indicating that the reporting is complete. :param int jobid: The resultsdb jobid to be modified @@ -93,36 +86,7 @@ class ResultsdbDirective(BaseDirective): """ return self.resultsdb.update_job(jobid, status='COMPLETED') - def parse_jobid(self, jobid): - """ Parse the incoming jobid which should either be '-1' to indicate - that dummy values should be used or in the form of /. - - If the passed in format is invalid, no exceptions will be raised but a - warning message will be emitted to logs - - :param str jobid: jobid from runner - :returns: (builder, buildid) - """ - if jobid != "-1": - try: - builder, buildid = jobid.split('/') - return (builder, buildid) - except ValueError, e: - log.debug(e) - log.warning( - 'Invalid jobid format detected, resetting to default ' - 'value. jobid must be in builder/buildid format, ' - 'found %s' % jobid) - - return "default", "-1" - def process(self, input_data, env_data): - - conf = config.get_config() - if not (conf.reporting_enabled and conf.report_to_resultsdb): - log.info("Reporting to ResultsDB is disabled.") - return - #TODO: Replace with proper location and adjust exc. text # we're creating the jobid in the directive for now, so adjust the check @@ -130,24 +94,34 @@ class ResultsdbDirective(BaseDirective): raise TaskotronDirectiveError("The resultsdb directive requires "\ "resultsdb_job_id and checkname.") + # checking if reporting is enabled is done after importing tap which + # serves as validation of input results try: check_details = check.import_TAP(input_data['results']) except TaskotronValueError as e: raise TaskotronDirectiveError("Failed to load 'results': %s" % e.message) - builder, buildid = self.parse_jobid(env_data['jobid']) + conf = config.get_config() + if not (conf.reporting_enabled and conf.report_to_resultsdb): + log.info("TAP is OK.") + log.info("Reporting to ResultsDB is disabled.") + log.info("Once enabled, the following would be reported:") + log.info('\n'.join([str(detail) for detail in check_details])) + return # for now, we're creating the resultsdb job at reporting time # the job needs to be 'RUNNING' in order to append any results - job_url = '%s/builders/%s/builds/%s' % (self.masterurl, builder, buildid) - job_data = self.create_resultsdb_job(env_data['checkname'], refurl=job_url) + job_url, log_url = buildbot_utils.get_urls(env_data['jobid'], + self.masterurl, + self.task_stepname) + job_data = self.create_resultsdb_job(env_data['checkname'], + refurl=job_url) self.ensure_testcase_exists(env_data['checkname']) output = [] for detail in check_details: - log_url = '%s/steps/%s/logs/stdio' % (job_url, self.task_stepname) try: result = self.resultsdb.create_result( job_id = job_data['id'], diff --git a/libtaskotron/directives/yumrepoinfo_directive.py b/libtaskotron/directives/yumrepoinfo_directive.py new file mode 100644 index 0000000..f3dc36c --- /dev/null +++ b/libtaskotron/directives/yumrepoinfo_directive.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import +from libtaskotron.directives import BaseDirective + +from libtaskotron import config +from libtaskotron import yumrepoinfo + +from libtaskotron.exceptions import TaskotronDirectiveError +from libtaskotron.logger import log + +directive_class = 'YumrepoinfoDirective' + +class YumrepoinfoDirective(BaseDirective): + """A directive for translating koji tag into a set of yum repos. + The directive returns a dictionary having ``koji_tag`` as the key, and + URL for the respective repo as a value. + If the koji tag specified by ``koji_tag`` argument has any parent repos + (e.g. f20-updates is a child of f20), all the repos from the ``koji_tag`` + and up are returned. + If the ``koji_tag`` is ``-pending``, the ``-pending`` is stripped (as it is + not a repo), and then the code works the same. + It is also possible to assign ``koji_tag`` to ``rawhide``. + + Format:: + + yumrepoinfo: + koji_tag= + arch= + """ + + def __init__(self, repoinfo = None): + self.repoinfo = repoinfo + + def process(self, input_data, env_data): + + if 'koji_tag' not in input_data or 'arch' not in input_data: + raise TaskotronDirectiveError("The yumrepoinfo directive requires "\ + "koji_tag and arch arguments.") + + + arches = input_data['arch'] + + processed_arches = [arch for arch in arches if arch not in ['noarch', 'all', 'src']] + if len(processed_arches) == 0: + raise TaskotronDirectiveError("No valid yumrepo arches supplied to " + "yumrepoinfo directive. Recieved %r" % arches) + + if len(processed_arches) > 1: + raise TaskotronDirectiveError("Yumrepoinfo requires a single arch " + "but multiple arches were submitted: %r" % arches) + + arch = processed_arches[0] + + koji_tag = input_data['koji_tag'] + + output = {} + + if self.repoinfo is None: + self.repoinfo = yumrepoinfo.get_yumrepoinfo(arch) + + if koji_tag.endswith('-pending'): + koji_tag = koji_tag[:-len('-pending')] + + if koji_tag == 'rawhide': + koji_tag = self.repoinfo.repo('rawhide')['tag'] + + while koji_tag: + repo = self.repoinfo.repo_by_tag(koji_tag) + if repo is None: + raise TaskotronDirectiveError('Repo with tag'\ + '%r not found.' % koji_tag) + + output[repo['name']] = repo['url'] + koji_tag = repo['parent'] + + log.debug("Found %s repos for %s: %r" % (len(output), input_data['koji_tag'], output)) + return output + diff --git a/libtaskotron/exceptions.py b/libtaskotron/exceptions.py index badd267..61e57b4 100644 --- a/libtaskotron/exceptions.py +++ b/libtaskotron/exceptions.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- -# Copyright 2010-2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''This module contains custom Taskotron exceptions''' +from __future__ import absolute_import class TaskotronError(Exception): '''Common ancestor for Taskotron related exceptions''' diff --git a/libtaskotron/file_utils.py b/libtaskotron/file_utils.py index 7af09d8..e778194 100644 --- a/libtaskotron/file_utils.py +++ b/libtaskotron/file_utils.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import import os import sys import urlgrabber.grabber @@ -91,7 +97,7 @@ def download(url, dest, overwrite=False, grabber=None): log.debug('Already downloaded: %s', os.path.basename(dest)) return dest - log.info('Downloading: %s', url) + log.debug('Downloading: %s', url) try: grabber.urlgrab(url, dest) except urlgrabber.grabber.URLGrabError, e: diff --git a/libtaskotron/koji_utils.py b/libtaskotron/koji_utils.py index f1752ac..736ebe6 100644 --- a/libtaskotron/koji_utils.py +++ b/libtaskotron/koji_utils.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing ''' Utility methods related to Koji ''' +from __future__ import absolute_import +import os + import collections import koji import hawkey @@ -77,14 +81,42 @@ class KojiClient(object): :rtype: list of str :raise TaskotronRemoteError: if downloading failed ''' + + # Create the rpm_dir if if does not exist + # as fileutils.download requires existing directory + # If this is not present, then the first rpm to be downloaded + # will fail to download, because of how file_utils.download() + # is currently written. On line #79, the `dest` is not a dir + # (because it is not yet created), so it is created on line #83 + # but as an effect, `dest` is still containing just the dirname + # instead of dirname+rpmname (because line #80 was not executed + # so the lines #89-94 end up going through the else branch + # thus not downloading the first file. + # All subsequent downloads are OK, since the rpm_dir (aka dest) + # already exists. + # YAY for two hours of debugging! + # FIXME: this should be solved better, somehow... + if os.path.exists(rpm_dir): + if not os.path.isdir(rpm_dir): + raise exc.TaskotronRemoteError( + "Can't create directory: %r It is an already " + "existing file.", rpm_dir) + else: + try: + file_utils.makedirs(rpm_dir) + except OSError, e: + log.exception("Can't create directory: %s", rpm_dir) + raise TaskotronRemoteError(e) + + rpm_urls = self.nvr_to_urls(nvr, arches=arches, debuginfo=debuginfo, src=src) rpm_files = [] - log.info('Fetching RPMs for: %s', nvr) + log.info('Fetching %s RPMs for: %s', len(rpm_urls), nvr) for url in rpm_urls: - log.info(' fetching %s', url) + log.debug(' Fetching %s', url) rpm_file = file_utils.download(url, rpm_dir) rpm_files.append(rpm_file) @@ -94,14 +126,23 @@ class KojiClient(object): def get_tagged_rpms(self, tag, dest, arches): '''Downloads all RPMs of all NVRs tagged by a specific Koji tag. + :param str tag: Koji tag to be queried for available builds, e.g. + ``f20-updates-pending`` :param str dest: destination directory :param arches: list of architectures :type arches: list of str + :return: list of local filenames of the grabbed RPMs + :rtype: list of str :raise TaskotronRemoteError: if downloading failed ''' + log.debug('Querying Koji for tag: %s' % tag) tag_data = self.session.listTagged(tag) + nvrs = ["%(nvr)s" % x for x in tag_data] rpms = [] + log.info("Fetching %s builds for tag: %s", len(nvrs), tag) + log.debug('Builds to be downloaded:\n %s', '\n '.join(nvrs)) + for nvr in nvrs: rpms.append(self.get_nvr_rpms(nvr, dest, arches)) diff --git a/libtaskotron/logger.py b/libtaskotron/logger.py index 49a3ba0..0b99321 100644 --- a/libtaskotron/logger.py +++ b/libtaskotron/logger.py @@ -1,23 +1,9 @@ -#!/usr/bin/python -# -# Copyright 2014, Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Author: Martin Krizek +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing +from __future__ import absolute_import import sys import logging import logging.handlers @@ -35,7 +21,7 @@ def _log_excepthook(*exc_info): def init(name='libtaskotron', level=logging.INFO, stream=True, syslog=False, - filelog=None): + filelog=None, set_rootlogger=False): """Setup a logger :param str name: name of the logger, 'libtaskotron' is default @@ -44,12 +30,14 @@ def init(name='libtaskotron', level=logging.INFO, stream=True, syslog=False, :param bool stream: enables logging to stderr :param bool syslog: enables logging to syslog :param str filelog: enables logging to a file, the value is the file name + :param bool set_rootlogger: adds output handlers to root logger """ # overwrite default except hook sys.excepthook = _log_excepthook logger = logging.getLogger(name) + rootlogger = logging.getLogger() logger.setLevel(level) logger.addHandler(logging.NullHandler()) @@ -61,21 +49,24 @@ def init(name='libtaskotron', level=logging.INFO, stream=True, syslog=False, if stream: stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) + if set_rootlogger: + rootlogger.addHandler(stream_handler) logger.debug("doing stream logging") if syslog: syslog_handler = logging.handlers.SysLogHandler(address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_LOCAL4) syslog_handler.setFormatter(formatter) - logger.addHandler(syslog_handler) + if set_rootlogger: + rootlogger.addHandler(syslog_handler) logger.debug("doing syslog logging") if filelog: file_handler = logging.handlers.RotatingFileHandler(filelog, maxBytes=500000, backupCount=5) file_handler.setFormatter(formatter) - logger.addHandler(file_handler) + if set_rootlogger: + rootlogger.addHandler(file_handler) logger.debug('doing file logging to %s', filelog) return logger diff --git a/libtaskotron/python_utils.py b/libtaskotron/python_utils.py index 0ef6094..c684451 100644 --- a/libtaskotron/python_utils.py +++ b/libtaskotron/python_utils.py @@ -1,21 +1,59 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''A collection of convenience methods related to Python base libraries.''' +from __future__ import absolute_import import collections +import libtaskotron.exceptions as exc -def listlike(value): - '''Return whether ``value`` is iterable (``list``, ``tuple``, ``set``, - etc), but not ``str`` (which is iterable, but we don't often want to - consider it as such). This method is useful e.g. if you want to check that a - variable contains a list/set/etc of strings. - :param any value: any object or literal you want to check - :return: whether ``value`` is iterable but not string +def iterable(obj, item_type=None): + '''Decide whether ``obj`` is an :class:`~collections.Iterable` (you can + traverse through its elements - e.g. ``list``, ``tuple``, ``set``, even + ``dict``), but not ``basestring`` (which satisfies most collections' + requirements, but we don't often want to consider as a collection). You can + also verify the types of items in the collection. + + :param any obj: any object you want to check + :param type item_type: all items in ``obj`` must be instances of this + provided type. If ``None``, no check is performed. + :return: whether ``obj`` is iterable but not a string, and whether ``obj`` + contains only ``item_type`` items :rtype: bool + :raise TaskotronValueError: if you provide invalid parameter value + ''' + return _collection_of(obj, collections.Iterable, item_type) + + +def sequence(obj, item_type=None, mutable=False): + '''This has the same functionality and basic arguments as :func:`iterable` + (read its documentation), but decides whether ``obj`` is a + :class:`~collections.Sequence` (ordered and indexable collection - e.g. + ``list`` or ``tuple``, but not ``set`` or ``dict``). + + :param bool mutable: if ``True``, the ``obj`` must be a mutable sequence + (e.g. ``list``, but not ``tuple``) ''' - return (isinstance(value, collections.Iterable) and - not isinstance(value, basestring)) + col = collections.MutableSequence if mutable else collections.Sequence + return _collection_of(obj, col, item_type) + + +def _collection_of(obj, collection_cls, item_type=None): + '''The same as :func:`iterable` or :func:`sequence`, but the abstract + collection class can be specified dynamically with ``collection_cls``. + ''' + if not isinstance(obj, collection_cls) or isinstance(obj, basestring): + return False + + if item_type is not None and isinstance(obj, collections.Iterable): + try: + return all([isinstance(item, item_type) for item in obj]) + except TypeError as e: + raise exc.TaskotronValueError("'item_type' must be a type " + "definition, not '%r': %s" % (item_type, e)) + + return True diff --git a/libtaskotron/rpm_utils.py b/libtaskotron/rpm_utils.py index 670f1ce..f3083f1 100644 --- a/libtaskotron/rpm_utils.py +++ b/libtaskotron/rpm_utils.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing ''' Utility methods related to RPM ''' +from __future__ import absolute_import import os import hawkey diff --git a/libtaskotron/runner.py b/libtaskotron/runner.py index 2a6db69..44f9d0a 100644 --- a/libtaskotron/runner.py +++ b/libtaskotron/runner.py @@ -1,10 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import import logging import tempfile import os.path import argparse import imp -from jinja2 import Environment import copy +import collections from libtaskotron import taskyaml from libtaskotron import logger @@ -13,13 +19,17 @@ from libtaskotron import config from libtaskotron.logger import log from libtaskotron.exceptions import TaskotronYamlError + +# The list of accepted item types on the command line (--type option) +_ITEM_TYPES = ["bodhi_id", "koji_build", "koji_tag"] + + class Runner(object): def __init__(self, taskdata, argdata, workdir=None): self.taskdata = taskdata self.envdata = argdata self.working_data = {} self.directives = {} - self.jinja_env = None self.workdir = workdir or tempfile.mkdtemp(prefix="task-", dir=config.get_config().tmpdir) @@ -46,28 +56,22 @@ class Runner(object): loaded_directive = imp.load_source(real_name, directive_file) self.directives[directive_name] = loaded_directive - def _get_jinja_env(self): - if not self.jinja_env: - self.jinja_env = Environment() - self.jinja_env.globals = self.envdata - return self.jinja_env - def _render_action(self, action): + '''Take an action and replace all included variables with actual values + from :attr:`env_data` and :attr:`working_data`. See :meth:`do_actions` + to see how an action looks like. + + :param dict action: An action specification parsed from the task yaml + file + :return: a rendered action + :rtype: dict + ''' # copy the input so that we don't disrupt what we're processing rendered_action = copy.deepcopy(action) - # any action is going to be a dict with 0 or more embedded dicts - for action_key in action: - jinja_env = self._get_jinja_env() - - if isinstance(action[action_key], dict): - action_line = action[action_key] - for keyval in action_line: - input_action = action[action_key][keyval] - arg_template = jinja_env.from_string(input_action) - - rendered_line = arg_template.render(self.working_data) - rendered_action[action_key][keyval] = rendered_line + variables = copy.deepcopy(self.envdata) + variables.update(self.working_data) + taskyaml.replace_vars_in_action(rendered_action, variables) return rendered_action @@ -79,6 +83,12 @@ class Runner(object): str(action)) def do_single_action(self, action): + '''Execute a single action from the task. See :meth:`do_actions` to see + how an action looks like. + + :param dict action: An action specification parsed from the task yaml + file + ''' directive_name = self._extract_directive_from_action(action) rendered_action = self._render_action(action) @@ -97,6 +107,15 @@ class Runner(object): self.working_data[action['export']] = output def do_actions(self): + '''Sequentially run all actions for a task. An 'action' is a single step + under the ``task:`` key. An example action looks like:: + + - name: download rpms from koji + koji: + action: download + koji_build: $koji_build + arch: $arch + ''' if 'task' not in self.taskdata or not self.taskdata['task']: raise TaskotronYamlError("At least one task should be specified" " in input yaml file") @@ -108,21 +127,21 @@ class Runner(object): if 'input' not in self.taskdata: return - if not isinstance(self.taskdata['input'], dict): - raise TaskotronYamlError("Input yaml should contain correct" - " 'input' section") + if not isinstance(self.taskdata['input'], collections.Mapping): + raise TaskotronYamlError("Input yaml should contain correct 'input'" + "section (a mapping). Yours was: %s" % type( + self.taskdata['input'])) - if ('args' not in self.taskdata['input']) or not isinstance( - self.taskdata['input']['args'], str): - raise TaskotronYamlError("Input yaml should contain correct" - " 'args' section") + required_args = self.taskdata['input'].get('args', None) - required_input = self.taskdata['input']['args'].split(',') + if not python_utils.iterable(required_args): + raise TaskotronYamlError("Input yaml should contain correct 'args' " + "section (an iterable). Yours was: %s" % type(required_args)) - for arg in required_input: + for arg in required_args: if not arg in self.envdata: - raise TaskotronYamlError('Required input arg %s ' - 'was not defined' % arg) + raise TaskotronYamlError("Required input arg '%s' " + "was not defined" % arg) def _validate_env(self): # TODO: implement this @@ -140,7 +159,7 @@ def get_argparser(): "defaults to noarch. 'all' specifies all known architectures.") parser.add_argument("-i", "--item", help="item to be checked") parser.add_argument("-t", "--type", - choices=["bodhi_id", "koji_build", "koji_tag"], + choices=_ITEM_TYPES, help="type of --item argument") parser.add_argument("-j", "--jobid", default="-1", help="optional job identifier used to render log urls") @@ -157,19 +176,12 @@ def process_args(args): """ # process item + type - if args['type'] == 'koji_build': - args['envr'] = args['item'] - if args['type'] == 'bodhi_id': - args['bodhi_id'] = args['item'] - if args['type'] == 'koji_tag': - args['tag'] = args['item'] + if args['type'] in _ITEM_TYPES: + args[args['type']] = args['item'] # process arch if args['arch'] is None: args['arch'] = ['noarch'] - # since lists don't play nice with yaml, convert arch to csv - # this is a workaround for https://phab.qadevel.cloud.fedoraproject.org/T148 - args['arch'] = ','.join(args['arch']) return args @@ -178,7 +190,7 @@ def main(): parser = get_argparser() args = parser.parse_args() - logger.init(level=logging.DEBUG) + logger.init(level=logging.DEBUG, set_rootlogger=True) arg_data = process_args(vars(args)) arg_data['taskfile'] = args.task[0] @@ -191,8 +203,6 @@ def main(): task_runner = Runner(task_data, arg_data) task_runner.run() - for output in task_runner.working_data: - if python_utils.listlike(task_runner.working_data[output]): - log.info("\n" + "\n".join(task_runner.working_data[output])) - else: - log.info("\n" + task_runner.working_data[output]) + log.info('Check execution finished. Showing stored variables:') + for name, value in task_runner.working_data.items(): + log.info("${%s}:\n%s" % (name, value)) diff --git a/libtaskotron/taskyaml.py b/libtaskotron/taskyaml.py index 0669e63..eaf3a17 100644 --- a/libtaskotron/taskyaml.py +++ b/libtaskotron/taskyaml.py @@ -1,4 +1,17 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +'''Methods for operating with a task YAML file''' + +from __future__ import absolute_import +import collections +import string + import yaml +from libtaskotron import exceptions as exc +from libtaskotron.logger import log def parse_yaml_from_file(filename): @@ -9,3 +22,98 @@ def parse_yaml_from_file(filename): def parse_yaml(contents): return yaml.safe_load(contents) + +def _replace_vars(text, variables): + '''Go through ``text`` and replace all variables (in the form of what is + supported by :class:`string.Template`) with their values, as provided in + ``variables``. This is used for variable expansion in the task yaml file. + + :param str text: input text where to search for variables + :param dict variables: names (keys) and values of variables to replace + :return: if ``text`` contains just a single variable and nothing else, the + variable value is directly returned (i.e. with matching type, not + cast to ``str``). If ``text`` contains something else as well + (other variables or text), a string is returned. + :raise TaskotronYamlError: if ``text`` contains a variable that is not + present in ``variables``, or if the variable + syntax is incorrect + ''' + + try: + # try to find the first match + match = string.Template.pattern.search(text) + + if not match: + return text + + # There are 4 groups in the pattern: 1 - escaped, 2 - named, 3 - braced, + # 4 - invalid. Group 0 returns the whole match. + if match.group(0) == text and (match.group(2) or match.group(3)): + # We found a single variable and nothing more. We shouldn't return + # a string, but the exact value, so that we don't lose value type. + # This makes it possible to pass lists, dicts, etc as variables. + var_name = match.group(2) or match.group(3) + return variables[var_name] + + # Now it's clear there's also something else in `text` than just a + # single variable. We will replace all variables and return a string + # again. + output = string.Template(text).substitute(variables) + return output + + except KeyError as e: + raise exc.TaskotronYamlError("The task yaml file includes a variable, " + "but no value has been provided for it: %s" % e) + + except ValueError as e: + raise exc.TaskotronYamlError("The task yaml file includes an incorrect " + "variable definition. Dollar signs must be doubled if they " + "shouldn't be considered as a variable denotation.\n" + "Error: %s\n" + "Text: %s" % (e, text)) + + +def replace_vars_in_action(action, variables): + '''Find all variables that are leaves (in a tree sense) in an action. Leaves + are variables which can no longer be traversed, i.e. "primitive" types like + ``str`` or ``int``. Non-leaves are containers like ``dict`` or ``list``. + + For all leaves, call :func:`_replace_vars` and update their value with + the function's output. + + :param dict action: An action specification parsed from the task yaml file. + See :meth:`.Runner.do_actions` to see what an action + looks like. + :param dict variables: names (keys) and values of variables to replace + :raise TaskotronYamlError: if ``text`` contains a variable that is not + present in ``variables``, or if the variable + syntax is incorrect + ''' + + visited = [] # all visited nodes in a tree + stack = [action] # nodes waiting for inspection + + while stack: + vertex = stack.pop() + + if vertex in visited: + continue + + visited.append(vertex) + children = [] # list of tuples (index/key, child_value) + + if isinstance(vertex, collections.MutableMapping): + children = vertex.items() # list of (key, value) + elif isinstance(vertex, collections.MutableSequence): + children = list(enumerate(vertex)) # list of (index, value) + else: + log.warn("Unknown structure '%s' in YAML file, this shouldn't " + "happen: %s", type(vertex), vertex) + + for index, child_val in children: + if isinstance(child_val, basestring): + # leaf node and a string, replace variables + vertex[index] = _replace_vars(child_val, variables) + elif isinstance(child_val, collections.Iterable): + # traversable further down, mark for visit + stack.append(child_val) diff --git a/libtaskotron/yumrepoinfo.py b/libtaskotron/yumrepoinfo.py index 5e7ca36..f151f0a 100644 --- a/libtaskotron/yumrepoinfo.py +++ b/libtaskotron/yumrepoinfo.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- # Copyright 2009-2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''A wrapper object for yumrepoinfo.conf to access its information easily''' +from __future__ import absolute_import import ConfigParser import os diff --git a/readme.rst b/readme.rst index 7f00c6b..0d811af 100644 --- a/readme.rst +++ b/readme.rst @@ -18,7 +18,7 @@ For more information, please reference the online documentation: * `Development Version `_ - * `Latest Release `_ Installing a Development Environment @@ -85,13 +85,13 @@ Running a Task A relatively simple example task is `rpmlint `_. -To run that task against a koji build with envr `` for some arch `` +To run that task against a koji build with NEVR `` for some arch `` , do the following:: git clone https://bitbucket.org/fedoraqa/task-rpmlint.git - runtask -i -t koji_build -a task-rpmlint/rpmlint.yml + runtask -i -t koji_build -a task-rpmlint/rpmlint.yml -This will download the `` from koji into a temp directory under `/var/tmp/`, +This will download the `` from koji into a temp directory under `/var/tmp/`, run rpmlint on the downloaded rpms and print output in TAP format to stdout. Example:: @@ -124,3 +124,19 @@ parameter instead. If you write new tests, be sure to run this to see whether the code is sufficiently covered by your tests. + +Building Documentation +====================== + +Libtaskotron's documentation is written in `reStructuredText +`_ and built using `Sphinx +`_. + +The documentation is easy to build if you have followed the instructions to set +up a development environment. + +To actually build the documentation:: + + cd docs/ + make html + diff --git a/requirements.txt b/requirements.txt index e81ef9d..8d4e310 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,10 @@ py>=1.4.18 pyaml==13.07.1 pytest>=2.4.2 pytest-cov>=1.6 -jinja2>=2.7.2 -bayeux>=0.8 -yamlish>=0.11 -pytap13>=0.0.2 +bayeux>=0.9 +yamlish>=0.12 +pytap13>=0.1.0 resultsdb_api>=1.0.1 python-fedora>=0.3.33 bunch>=1.0.1 +Sphinx>=1.2.2 diff --git a/runtask.py b/runtask.py index dfa1d99..4ed9382 100644 --- a/runtask.py +++ b/runtask.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +from __future__ import absolute_import from libtaskotron import runner if __name__ == '__main__': diff --git a/testing/conftest.py b/testing/conftest.py index e6b90aa..bbd81cd 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,20 +1,7 @@ -# Copyright 2011, Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Author: Tim Flink +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''py.test configuration and plugins Read more at: http://pytest.org/latest/plugins.html#conftest-py-plugins''' diff --git a/testing/functest_config.py b/testing/functest_config.py index c066bc4..7e5e19b 100644 --- a/testing/functest_config.py +++ b/testing/functest_config.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Functional tests for libtaskotron/config.py''' diff --git a/testing/functest_python_directive.py b/testing/functest_python_directive.py index 9d17d2e..591eaa9 100644 --- a/testing/functest_python_directive.py +++ b/testing/functest_python_directive.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + import os import copy from libtaskotron.directives import python_directive diff --git a/testing/functest_yumrepoinfo.py b/testing/functest_yumrepoinfo.py index ea4c4cb..1a8d85b 100644 --- a/testing/functest_yumrepoinfo.py +++ b/testing/functest_yumrepoinfo.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Functional tests for libtaskotron/yumrepoinfo.py''' diff --git a/testing/some_loadable_test.py b/testing/some_loadable_test.py index 54fa8f7..6e6ca33 100644 --- a/testing/some_loadable_test.py +++ b/testing/some_loadable_test.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + class TaskClass(object): def my_task(self): return "I'm a task class method!" diff --git a/testing/test_bodhi_comment_directive.py b/testing/test_bodhi_comment_directive.py index c2b8006..4fe9f4c 100644 --- a/testing/test_bodhi_comment_directive.py +++ b/testing/test_bodhi_comment_directive.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + """Unit tests for libtaskotron/directives/resultsdb_directive.py""" import pytest @@ -24,11 +29,14 @@ class TestBodhiCommentReport(): Config.reporting_enabled = True Config.report_to_bodhi = True + self.ref_job_id = 123 + self.ref_log_url = '%s/builders/all/builds/%d/steps/runtask/logs/stdio'\ + % (Config.taskotron_master, self.ref_job_id) self.comment = { "author": Config.fas_username, "timestamp": "2014-01-31 10:11:12", - "text": "Taskotron: depcheck test PASSED on x86_64. Result log: http://example.com", + "text": "Taskotron: depcheck test PASSED on x86_64. Result log: %s" % self.ref_log_url, } self.update = { @@ -47,9 +55,7 @@ class TestBodhiCommentReport(): ) - #TODO: remove this ugly hack - it is here because of the bug in bayeux - tap = "TAP version 13\n1..1\n%s" % check.export_TAP(self.cd) - + tap = check.export_TAP(self.cd) self.ref_input = { 'results': tap, @@ -57,6 +63,7 @@ class TestBodhiCommentReport(): } self.ref_envdata = { 'checkname': 'depcheck', + 'jobid': 'all/%d' % self.ref_job_id, } def test_config_reporting_disabled(self): @@ -240,13 +247,15 @@ class TestBodhiCommentReport(): def test_post_testresult(self, prepare_for_bodhi_comment): """Checks whether _post_testresult maps arguments correctly to the bodhi comment call.""" + ref_comment = "Taskotron: depcheck test PASSED on noarch. "\ + "Result log: %s (results are informative only)" % self.ref_log_url self.stub_bodhi.client = Dingus('bodhi_api.bodhi') outcome = bodhi_comment_directive._post_testresult(self.stub_bodhi, update = "update_title", testname = "depcheck", result = "PASSED", - url = "http://example.com") + url = self.ref_log_url) # Select the first call of "comment" method. call = [call for call in self.stub_bodhi.client.calls() if call[0] == 'comment'][0] @@ -254,13 +263,15 @@ class TestBodhiCommentReport(): call_data = call[1] assert call_data[0] == "update_title" - assert call_data[1] == "Taskotron: depcheck test PASSED on noarch. Result log: http://example.com (results are informative only)" + assert call_data[1] == ref_comment assert call_data[2] == 0 assert call_data[3] == False def test_tap_mapping(self, prepare_for_bodhi_comment): """Checks whether TAP output is mapped correctly to the bodhi comment call.""" + ref_comment = "Taskotron: depcheck test PASSED on noarch. "\ + "Result log: %s (results are informative only)" % self.ref_log_url self.stub_bodhi.client = Dingus('bodhi_api.bodhi') self.directive.process(self.ref_input, self.ref_envdata) @@ -271,7 +282,7 @@ class TestBodhiCommentReport(): call_data = call[1] assert call_data[0] == "update_title" - assert call_data[1] == "Taskotron: depcheck test PASSED on noarch. Result log: http://example.com (results are informative only)" + assert call_data[1] == ref_comment assert call_data[2] == 0 assert call_data[3] == False diff --git a/testing/test_bodhi_directive.py b/testing/test_bodhi_directive.py index 66d46fc..1e26294 100644 --- a/testing/test_bodhi_directive.py +++ b/testing/test_bodhi_directive.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/directives/bodhi_directive.py''' @@ -27,7 +28,7 @@ class TestBodhiDownloads(): 'other_keys': {} } self.ref_input = {'action': 'download', - 'arch': ', '.join(self.ref_arch), + 'arch': self.ref_arch, 'bodhi_id': self.ref_bodhi_id } self.ref_envdata = {'workdir': self.ref_workdir} @@ -61,7 +62,7 @@ class TestBodhiDownloads(): '''Test download when all arches are demanded''' self.ref_arch = ['all'] self.ref_input = {'action': 'download', - 'arch': ', '.join(self.ref_arch), + 'arch': self.ref_arch, 'bodhi_id': self.ref_bodhi_id } @@ -77,13 +78,13 @@ class TestBodhiDownloads(): req_arches = map(lambda x: x[1][2], getrpms_calls) # checks whether all get_nvr_rpms calls demanded all arches - assert True == all(map(lambda x: set(x) == ref_arches, req_arches)) + assert all(map(lambda x: set(x) == ref_arches, req_arches)) def test_action_download_source(self): '''Test download of source packages''' self.ref_arch = ['src'] self.ref_input = {'action': 'download', - 'arch': ', '.join(self.ref_arch), + 'arch': self.ref_arch, 'bodhi_id': self.ref_bodhi_id } @@ -104,7 +105,7 @@ class TestBodhiDownloads(): '''Test download of multiple arches packages''' self.ref_arch = ['x86_64', 'noarch'] self.ref_input = {'action': 'download', - 'arch': ', '.join(self.ref_arch), + 'arch': self.ref_arch, 'bodhi_id': self.ref_bodhi_id } @@ -125,7 +126,7 @@ class TestBodhiDownloads(): '''Test whether noarch is automaticaly added''' self.ref_arch = ['i386'] self.ref_input = {'action': 'download', - 'arch': ', '.join(self.ref_arch), + 'arch': self.ref_arch, 'bodhi_id': self.ref_bodhi_id } @@ -136,7 +137,6 @@ class TestBodhiDownloads(): test_bodhi.process(self.ref_input, self.ref_envdata) - ref_arches = set(['i386', 'noarch']) getrpms_calls = stub_koji.calls() req_arches = map(lambda x: x[1][2], getrpms_calls) diff --git a/testing/test_bodhi_utils.py b/testing/test_bodhi_utils.py index 6a71ad2..67cfd36 100644 --- a/testing/test_bodhi_utils.py +++ b/testing/test_bodhi_utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/bodhi_utils.py''' diff --git a/testing/test_buildbot_utils.py b/testing/test_buildbot_utils.py new file mode 100644 index 0000000..331137e --- /dev/null +++ b/testing/test_buildbot_utils.py @@ -0,0 +1,62 @@ +from dingus import Dingus + +from libtaskotron import buildbot_utils + +class TestBuildbotUtils(object): + + def setup_method(self, method): + self.default_builder = 'default' + self.default_buildid = '-1' + + def test_valid_jobid_parse(self): + ref_builder = 'builder' + ref_buildid = '1234' + ref_jobid = '%s/%s' % (ref_builder, ref_buildid) + + test_builder, test_buildid = buildbot_utils.parse_jobid(ref_jobid) + + assert test_builder == ref_builder + assert test_buildid == ref_buildid + + def test_default_jobid_parse(self): + ref_jobid = '-1' + + test_builder, test_buildid = buildbot_utils.parse_jobid(ref_jobid) + + assert test_builder == self.default_builder + assert test_buildid == self.default_buildid + + def test_invalid_jobid_parse_small(self): + ref_jobid = 'crammalamma' + + test_builder, test_buildid = buildbot_utils.parse_jobid(ref_jobid) + + assert test_builder == self.default_builder + assert test_buildid == self.default_buildid + + def test_invalid_jobid_parse_long(self): + ref_jobid = 'crammalamma/swiss/cake/roll' + + test_builder, test_buildid = buildbot_utils.parse_jobid(ref_jobid) + + assert test_builder == self.default_builder + assert test_buildid == self.default_buildid + + def test_invalid_jobid_parse_baddelim(self): + ref_jobid = 'builder,1234' + + test_builder, test_buildid = buildbot_utils.parse_jobid(ref_jobid) + + assert test_builder == self.default_builder + assert test_buildid == self.default_buildid + + def test_get_urls(self): + ref_taskotron_master = 'http://taskotron-stg.fedoraproject.org/taskmaster/' + ref_task_stepname = 'runtask' + ref_builder = 'all' + ref_buildid = '1234' + ref_jobid = '%s/%s' % (ref_builder, ref_buildid) + job_url, log_url = buildbot_utils.get_urls(ref_jobid, ref_taskotron_master, ref_task_stepname) + + assert job_url == '%s/builders/%s/builds/%s' % (ref_taskotron_master, ref_builder, ref_buildid) + assert log_url == '%s/steps/%s/logs/stdio' % (job_url, ref_task_stepname) diff --git a/testing/test_check.py b/testing/test_check.py index 3622923..32191ba 100644 --- a/testing/test_check.py +++ b/testing/test_check.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/check.py''' @@ -268,7 +269,7 @@ xchat.x86_64: W: no-manual-page-for-binary xchat''' cd3 = check.CheckDetail(item='f20-updates', outcome='INFO', summary='2 stale updates', report_type=check.ReportType.YUM_REPOSITORY) - tap_output = check.export_TAP(self.cd, cd2, cd3) + tap_output = check.export_TAP([self.cd, cd2, cd3]) lines = tap_output.splitlines() assert lines[1].strip() == '1..3' @@ -419,3 +420,22 @@ ok 1 bar""" with pytest.raises(exc.TaskotronValueError): check.import_TAP(tap_output) + def test_invalid_tap_multi_header(self): + #simplified test data from phab/T205 + tap_output_multi_header = """ +TAP version 13 +1..1 +ok - $CHECKNAME for XXX +--- +item: XXX +... +TAP version 13 +1..1 +ok - $CHECKNAME for YYY +--- +item: YYY +... +""" + with pytest.raises(ValueError): + check.import_TAP(tap_output_multi_header) + diff --git a/testing/test_config.py b/testing/test_config.py index af722fc..a7c44e0 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/config.py''' diff --git a/testing/test_directives/test_directive.py b/testing/test_directives/test_directive.py index ecd3f72..a577632 100644 --- a/testing/test_directives/test_directive.py +++ b/testing/test_directives/test_directive.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + from libtaskotron.directives import BaseDirective class TestDirective(BaseDirective): diff --git a/testing/test_koji_directive.py b/testing/test_koji_directive.py index bc22a9b..d9f7e91 100644 --- a/testing/test_koji_directive.py +++ b/testing/test_koji_directive.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + from dingus import Dingus from libtaskotron.directives import koji_directive @@ -22,7 +27,7 @@ class TestKojiDirective(): 'arch': self.ref_arch}, {'name': self.ref_name, 'version': self.ref_version, 'release': self.ref_release, 'nvr': self.ref_nvr, - 'arch': 'src'}] + 'arch': ['src']}] self.ref_tagged = [{'nvr': self.ref_nvr}] @@ -38,8 +43,8 @@ class TestKojiDirective(): def test_parse_download_command(self): self.ref_input = {'action': 'download', - 'arch': ', '.join(self.ref_arch), - 'envr': self.ref_nvr} + 'arch': self.ref_arch, + 'koji_build': self.ref_nvr} stub_koji = Dingus(get_nvr_rpms__returns = self.ref_rpms) @@ -49,15 +54,17 @@ class TestKojiDirective(): getrpm_calls = stub_koji.calls() requested_nvr = getrpm_calls[0][1][0] + requested_src = getrpm_calls[0][2]['src'] assert len(getrpm_calls) == 1 assert requested_nvr == self.ref_nvr + assert not requested_src def test_parse_download_tag_command(self): self.ref_input = {'action': 'download_tag', - 'arch': ', '.join(self.ref_arch), - 'tag': self.ref_tag} + 'arch': self.ref_arch, + 'koji_tag': self.ref_tag} stub_koji = Dingus(get_tagged_rpms__returns=self.ref_rpms) @@ -75,8 +82,8 @@ class TestKojiDirective(): self.ref_arch = ['noarch', 'i386'] self.ref_input = {'action': 'download_tag', - 'arch': ', '.join(self.ref_arch), - 'tag': self.ref_tag} + 'arch': self.ref_arch, + 'koji_tag': self.ref_tag} stub_koji = Dingus('koji_utils') @@ -87,8 +94,8 @@ class TestKojiDirective(): def test_parse_single_arch(self): self.ref_input = {'action': 'download_tag', - 'arch': ', '.join(self.ref_arch), - 'tag': self.ref_tag} + 'arch': self.ref_arch, + 'koji_tag': self.ref_tag} stub_koji = Dingus('koji_utils') @@ -98,8 +105,8 @@ class TestKojiDirective(): assert test_helper.arches == self.ref_arch def test_add_noarch_download_command(self): - ref_input = {'action': 'download', 'arch': 'x86_64', - 'envr': self.ref_nvr} + ref_input = {'action': 'download', 'arch': ['x86_64'], + 'koji_build': self.ref_nvr} ref_envdata = {'workdir': '/var/tmp/foo'} stub_koji = Dingus() @@ -107,16 +114,15 @@ class TestKojiDirective(): test_helper.process(ref_input, ref_envdata) getrpm_calls = stub_koji.calls() - print getrpm_calls - requested_arches = getrpm_calls[0][1][2] + requested_arches = getrpm_calls[0][2]['arches'] assert len(getrpm_calls) == 1 assert 'x86_64' in requested_arches assert 'noarch' in requested_arches def test_add_noarch_download_tag_command(self): - ref_input = {'action': 'download_tag', 'arch': 'x86_64', - 'tag': self.ref_tag} + ref_input = {'action': 'download_tag', 'arch': ['x86_64'], + 'koji_tag': self.ref_tag} ref_envdata = {'workdir': '/var/tmp/foo'} stub_koji = Dingus(get_tagged_rpms__returns=self.ref_rpms) @@ -129,3 +135,20 @@ class TestKojiDirective(): assert len(getrpm_calls) == 1 assert 'x86_64' in requested_arches assert 'noarch' in requested_arches + + def test__download_command_src(self): + ref_input = {'action': 'download', 'arch': ['src'], + 'koji_build': self.ref_nvr} + ref_envdata = {'workdir': '/var/tmp/foo'} + + stub_koji = Dingus() + test_helper = koji_directive.KojiDirective(stub_koji) + test_helper.process(ref_input, ref_envdata) + + getrpm_calls = stub_koji.calls() + requested_arches = getrpm_calls[0][2]['arches'] + requested_src = getrpm_calls[0][2]['src'] + + assert len(getrpm_calls) == 1 + assert ['src'] == requested_arches + assert requested_src diff --git a/testing/test_koji_utils.py b/testing/test_koji_utils.py index 6ecab4f..ca73ed7 100644 --- a/testing/test_koji_utils.py +++ b/testing/test_koji_utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/koji_utils.py''' diff --git a/testing/test_python_directive.py b/testing/test_python_directive.py index 457806b..5e4f5f8 100644 --- a/testing/test_python_directive.py +++ b/testing/test_python_directive.py @@ -1,4 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + from libtaskotron.directives import python_directive +from libtaskotron.exceptions import TaskotronDirectiveError from dingus import Dingus import os import imp @@ -50,7 +56,7 @@ class TestExecutePyfile(object): ref_methodname = 'bar' ref_kwargs = {'baz': 'why', 'blah': 2} - stub_getattr = Dingus() + stub_getattr = Dingus(return_value=lambda baz, blah: "str") test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) @@ -65,15 +71,95 @@ class TestExecutePyfile(object): ref_methodname = 'bar' ref_kwargs = {'baz': 'why', 'blah': 2} - stub_getattr = Dingus() + stub_task = Dingus(return_value="str") + stub_getattr = Dingus(return_value=stub_task) + + test_directive = python_directive.PythonDirective() + monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) + + test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + + assert (stub_task.calls()[0].name, stub_task.calls()[0].args, + stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) + + +class TestPyfileOutputException(object): + + def test_task_doesnt_rise_on_unicode_str(self, monkeypatch): + ref_modulename = 'foo' + ref_methodname = 'bar' + ref_kwargs = {'baz': 'why', 'blah': 2} + + stub_task = Dingus(return_value=u"unicode") + stub_getattr = Dingus(return_value=stub_task) + + test_directive = python_directive.PythonDirective() + monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) + + test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + + assert (stub_task.calls()[0].name, stub_task.calls()[0].args, + stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) + + def test_task_doesnt_rise_on_empty_str(self, monkeypatch): + ref_modulename = 'foo' + ref_methodname = 'bar' + ref_kwargs = {'baz': 'why', 'blah': 2} + + stub_task = Dingus(return_value="") + stub_getattr = Dingus(return_value=stub_task) + + test_directive = python_directive.PythonDirective() + monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) + + test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + + assert (stub_task.calls()[0].name, stub_task.calls()[0].args, + stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) + + def test_task_doesnt_rise_on_none(self, monkeypatch): + ref_modulename = 'foo' + ref_methodname = 'bar' + ref_kwargs = {'baz': 'why', 'blah': 2} + + stub_task = Dingus(return_value=None) + stub_getattr = Dingus(return_value=stub_task) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) - assert (stub_getattr.calls()[1].name, stub_getattr.calls()[1].args, - stub_getattr.calls()[1].kwargs) == ('()', (), ref_kwargs) + assert (stub_task.calls()[0].name, stub_task.calls()[0].args, + stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) + + def test_task_rises_on_empty_list(self, monkeypatch): + ref_modulename = 'foo' + ref_methodname = 'bar' + ref_kwargs = {'baz': 'why', 'blah': 2} + + stub_task = Dingus(return_value=[]) + stub_getattr = Dingus(return_value=stub_task) + + test_directive = python_directive.PythonDirective() + monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) + + with pytest.raises(TaskotronDirectiveError): + test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + + def test_task_rises_on_wrong_type(self, monkeypatch): + ref_modulename = 'foo' + ref_methodname = 'bar' + ref_kwargs = {'baz': 'why', 'blah': 2} + + stub_task = Dingus(return_value={'han': 'shot', 'first': True}) + stub_getattr = Dingus(return_value=stub_task) + + test_directive = python_directive.PythonDirective() + monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) + + with pytest.raises(TaskotronDirectiveError): + test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) class TestProcess(object): @@ -141,4 +227,4 @@ class TestProcess(object): execute_call = self.stub_execute.calls[0] assert (execute_call.name, execute_call.args, execute_call.kwargs) == ( - '()', (self.stub_imported, self.ref_input['callable'], ref_kwargs), {}) \ No newline at end of file + '()', (self.stub_imported, self.ref_input['callable'], ref_kwargs), {}) diff --git a/testing/test_python_utils.py b/testing/test_python_utils.py index 20b5258..57f54d1 100644 --- a/testing/test_python_utils.py +++ b/testing/test_python_utils.py @@ -1,30 +1,77 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/python_utils.py''' +import pytest + from libtaskotron import python_utils +import libtaskotron.exceptions as exc + +class TestCollectionOf: + '''This basically tests `iterable` and `sequence`.''' + + def test_iterable(self): + assert python_utils.iterable([1, 2]) + assert python_utils.iterable(['a', 'b']) + assert python_utils.iterable(('foo',)) + assert python_utils.iterable({'foo', 'bar', 'baz'}) + assert python_utils.iterable(set()) + assert python_utils.iterable(()) + assert python_utils.iterable([u'foo', u'bar']) + assert python_utils.iterable({'a': 1, 'b': 2}) + + def test_not_iterable(self): + assert not python_utils.iterable('a') + assert not python_utils.iterable(u'a') + assert not python_utils.iterable(unicode('foo')) + assert not python_utils.iterable(3) + assert not python_utils.iterable(3.14) + assert not python_utils.iterable(None) + assert not python_utils.iterable(object()) + + def test_iterable_items(self): + assert python_utils.iterable([1, 2], int) + assert not python_utils.iterable([1, 2], float) + assert not python_utils.iterable([1, 2.2], float) + + assert python_utils.iterable(['a', 'b'], str) + assert python_utils.iterable(['a', 'b'], basestring) + assert not python_utils.iterable(['a', 'b'], unicode) + + assert python_utils.iterable([[],[]], list) + + # empty classes + X = type('X', (object,), {}) + Y = type('Y', (X,), {}) + assert python_utils.iterable((X(), X()), X) + assert python_utils.iterable((X(), Y()), X) + assert not python_utils.iterable((X(), Y()), Y) + + def test_raise(self): + with pytest.raises(exc.TaskotronValueError): + assert python_utils.iterable([1, 2], 1) + + with pytest.raises(exc.TaskotronValueError): + assert python_utils.iterable([1, 2], 'a') + + with pytest.raises(exc.TaskotronValueError): + assert python_utils.iterable([1, 2], []) + + X = type('X', (object,), {}) + with pytest.raises(exc.TaskotronValueError): + assert python_utils.iterable([1, 2], X()) + + def test_sequence(self): + assert python_utils.sequence([1, 2]) + assert python_utils.sequence(['a', 'b']) + assert python_utils.sequence(('foo',)) + assert python_utils.sequence(()) + assert python_utils.sequence([u'foo', u'bar']) -class TestPythonUtils: - - def test_listlike_true(self): - '''Test 'listlike' method when it returns True''' - assert python_utils.listlike([1, 2]) - assert python_utils.listlike(['a', 'b']) - assert python_utils.listlike(('foo',)) - assert python_utils.listlike({'foo', 'bar', 'baz'}) - assert python_utils.listlike(set()) - assert python_utils.listlike(()) - assert python_utils.listlike([u'foo', u'bar']) - assert python_utils.listlike({'a': 1, 'b': 2}) - - def test_listlike_false(self): - '''Test 'listlike' method when it returns False''' - assert not python_utils.listlike('a') - assert not python_utils.listlike(u'a') - assert not python_utils.listlike(unicode('foo')) - assert not python_utils.listlike(3) - assert not python_utils.listlike(3.14) - assert not python_utils.listlike(None) - assert not python_utils.listlike(object()) + def test_not_sequence(self): + assert not python_utils.sequence({'foo', 'bar', 'baz'}) + assert not python_utils.sequence(set()) + assert not python_utils.sequence({'a': 1, 'b': 2}) diff --git a/testing/test_resultsdb_directive.py b/testing/test_resultsdb_directive.py index fb0ad79..10bc329 100644 --- a/testing/test_resultsdb_directive.py +++ b/testing/test_resultsdb_directive.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + """Unit tests for libtaskotron/directives/resultsdb_directive.py""" import pytest @@ -181,52 +186,3 @@ class TestResultsdbReport(): # dingus doesn't record calls to exception_raiser assert len(rdb_calls) == 1 assert rdb_calls[0][0] == 'get_testcase' -class TestResultsdbJobidParsing(object): - - def setup_method(self, method): - self.default_builder = 'default' - self.default_buildid = '-1' - self.stub_rdb = Dingus('resultsdb_api') - self.test_rdb = resultsdb_directive.ResultsdbDirective(self.stub_rdb) - - def test_valid_jobid_parse(self): - ref_builder = 'builder' - ref_buildid = '1234' - ref_jobid = '%s/%s' % (ref_builder, ref_buildid) - - test_builder, test_buildid = self.test_rdb.parse_jobid(ref_jobid) - - assert test_builder == ref_builder - assert test_buildid == ref_buildid - - def test_default_jobid_parse(self): - ref_jobid = '-1' - - test_builder, test_buildid = self.test_rdb.parse_jobid(ref_jobid) - - assert test_builder == self.default_builder - assert test_buildid == self.default_buildid - - def test_invalid_jobid_parse_small(self): - ref_jobid = 'crammalamma' - - test_builder, test_buildid = self.test_rdb.parse_jobid(ref_jobid) - - assert test_builder == self.default_builder - assert test_buildid == self.default_buildid - - def test_invalid_jobid_parse_long(self): - ref_jobid = 'crammalamma/swiss/cake/roll' - - test_builder, test_buildid = self.test_rdb.parse_jobid(ref_jobid) - - assert test_builder == self.default_builder - assert test_buildid == self.default_buildid - - def test_invalid_jobid_parse_baddelim(self): - ref_jobid = 'builder,1234' - - test_builder, test_buildid = self.test_rdb.parse_jobid(ref_jobid) - - assert test_builder == self.default_builder - assert test_buildid == self.default_buildid diff --git a/testing/test_rpm_utils.py b/testing/test_rpm_utils.py index 7747628..1247ed0 100644 --- a/testing/test_rpm_utils.py +++ b/testing/test_rpm_utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/rpm_utils.py''' diff --git a/testing/test_runner.py b/testing/test_runner.py index 03f2e49..4026053 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + import pytest import os import sys @@ -10,7 +15,7 @@ from libtaskotron.exceptions import TaskotronYamlError class TestRunnerInputVerify(): def test_yamlrunner_valid_input(self): ref_cmd = '-i foo-1.2-3.fc99 -t koji_build -a x86_64 footask.yml' - ref_argdata = {'input': {'args': 'envr,arch'}} + ref_argdata = {'input': {'args': ['koji_build', 'arch']}} test_parser = runner.get_argparser() test_input_args = vars(test_parser.parse_args(ref_cmd.split())) @@ -99,8 +104,9 @@ class TestRunnerInputVerify(): class TestRunnerSetup(): def test_trivial_creation(self, tmpdir): - ref_taskdata = {'preparation': {'koji': 'download envr'}, - 'input': {'args': 'envr,arch'}, 'post': {'shell': 'clean'}, + ref_taskdata = {'preparation': {'koji': 'download nvr'}, + 'input': {'args': ['koji_build','arch']}, + 'post': {'shell': 'clean'}, 'execution': {'python': 'run_rpmlint.py'}, 'dependencies': ['rpmlint', 'libtaskbot']} ref_inputdata = {} @@ -210,7 +216,7 @@ class TestRunnerSingleAction(): ref_messagename = 'variable_messsage' ref_action = {'name': 'Dummy Action', 'dummy': {'result': 'pass', - 'msg': '{{ %s }}' % ref_messagename}, + 'msg': '${%s}' % ref_messagename}, 'export': '%s' % ref_exportname} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) @@ -262,7 +268,7 @@ class TestRunnerDoActions(): ref_messagename = 'variable_messsage' ref_action = {'name': 'Dummy Action', 'dummy': {'result': 'pass', - 'msg': '{{ %s }}' % ref_messagename}, + 'msg': '${%s}' % ref_messagename}, 'export': '%s' % ref_exportname} self.ref_taskdata = {'task': [ref_action, ref_action]} @@ -301,7 +307,7 @@ class TestRunnerRenderAction(): ref_message = 'This is a variable message' ref_messagename = 'variable_messsage' ref_action = {'name': 'test dummy action', 'dummy': - {'result': 'pass', 'msg': '{{ %s }}' % ref_messagename}} + {'result': 'pass', 'msg': '${%s}' % ref_messagename}} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) test_runner.working_data[ref_messagename] = ref_message @@ -310,6 +316,38 @@ class TestRunnerRenderAction(): assert test_rendered_action['dummy']['msg'] == ref_message + def test_runner_render_list_variable(self): + ref_var = ['foo', 'bar'] + ref_varname = 'testvar' + ref_action = {'name': 'test dummy action', 'dummy': + {'result': 'pass', 'msg': '${%s}' % ref_varname}} + + test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) + test_runner.working_data[ref_varname] = ref_var + + test_rendered_action = test_runner._render_action(ref_action) + + assert test_rendered_action['dummy']['msg'] == ref_var + + def test_runner_render_multi_variable(self): + ref_var1 = 'foo' + ref_var1name = 'testvar1' + ref_var2 = 'bar' + ref_var2name = 'testvar2' + ref_action = {'name': 'test dummy action', 'dummy': + {'result': 'pass', 'msg': '--foo ${%s} --bar ${%s}' % + (ref_var1name, ref_var2name)}} + + test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) + test_runner.working_data[ref_var1name] = ref_var1 + test_runner.working_data[ref_var2name] = ref_var2 + + test_rendered_action = test_runner._render_action(ref_action) + + assert (test_rendered_action['dummy']['msg'] == '--foo %s --bar %s' % + (ref_var1, ref_var2)) + + class TestRunnerProcessArgs(): def test_process_args_koji_build(self): @@ -319,8 +357,7 @@ class TestRunnerProcessArgs(): 'task': 'sometask.yml'} ref_args = copy.deepcopy(ref_input) - ref_args['envr'] = 'foo-1.2-3.fc99' - ref_args['arch'] = 'x86_64' + ref_args['koji_build'] = 'foo-1.2-3.fc99' test_args = runner.process_args(ref_input) @@ -334,7 +371,6 @@ class TestRunnerProcessArgs(): ref_args = copy.deepcopy(ref_input) ref_args['bodhi_id'] = 'foo-1.2-3.fc99' - ref_args['arch'] = 'x86_64' test_args = runner.process_args(ref_input) @@ -347,8 +383,7 @@ class TestRunnerProcessArgs(): 'task': 'sometask.yml'} ref_args = copy.deepcopy(ref_input) - ref_args['tag'] = 'dist-fc99-updates' - ref_args['arch'] = 'x86_64' + ref_args['koji_tag'] = 'dist-fc99-updates' test_args = runner.process_args(ref_input) @@ -361,8 +396,7 @@ class TestRunnerProcessArgs(): 'task': 'sometask.yml'} ref_args = copy.deepcopy(ref_input) - ref_args['tag'] = 'dist-fc99-updates' - ref_args['arch'] = 'x86_64,i386,noarch' + ref_args['koji_tag'] = 'dist-fc99-updates' test_args = runner.process_args(ref_input) @@ -372,7 +406,7 @@ class TestRunnerProcessArgs(): ref_input = { 'type': 'invalid', 'arch': None} ref_args = copy.deepcopy(ref_input) - ref_args['arch'] = 'noarch' + ref_args['arch'] = ['noarch'] test_args = runner.process_args(ref_input) assert test_args == ref_args diff --git a/testing/test_taskyaml.py b/testing/test_taskyaml.py index 9c3c956..60ca9ce 100644 --- a/testing/test_taskyaml.py +++ b/testing/test_taskyaml.py @@ -1,23 +1,33 @@ -import yaml +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +'''Unit tests for libtaskotron/taskyaml.py''' + +import copy +import yaml +import pytest from libtaskotron import taskyaml +import libtaskotron.exceptions as exc + -ref_filename = '/home/testuser/testtask.yml' ref_taskname = 'testtask' ref_taskdesc = 'this is a test task' ref_maintainer = 'testuser' ref_deps = ['libtaskotron', 'libpanacea'] -ref_input = {'args': 'arch, envr'} +ref_input = {'args': 'arch, koji_build'} ref_actions = [] -def create_yamldata(name=ref_taskname, desc=ref_taskdesc, maint= ref_maintainer, - deps=ref_deps, input=ref_input, actions=ref_actions): +def create_yamldata(name=ref_taskname, desc=ref_taskdesc, maint=ref_maintainer, + deps=ref_deps, input_=ref_input, actions=ref_actions): return {'name': name, 'desc': desc, 'maintainer': maint, 'deps': deps, - 'input': input, + 'input': input_, 'task': actions} @@ -58,3 +68,122 @@ class TestTaskYamlActionsNoVariables(object): test_task = taskyaml.parse_yaml(yaml.safe_dump(taskdata)) assert len(test_task['task']) == 3 + + +class TestReplaceVarsInAction(object): + + def setup_method(self, method): + self.action = { + 'name': 'run foo${foo}', + 'export': '${export}', + 'python': { + 'file': 'somefile.py', + 'args': ['arg1', '${number}', 'arg2', '${list}', '${dict}'], + 'cmdline': '-s ${foo} -i ${number}', + 'nest': { + 'deepnest': '${dict}', + 'null': '${null}', + } + } + } + self.vars = { + 'foo': 'bar', + 'export': 'a:b\nc:d\n-e', + 'number': 1, + 'list': [1, 'item'], + 'dict': {'key': 'val', 'pi': 3.14}, + 'null': None, + 'extra': 'juicy', + } + + def test_complex(self): + rend_action = copy.deepcopy(self.action) + rend_action['name'] = 'run foo%s' % self.vars['foo'] + rend_action['export'] = self.vars['export'] + rend_action['python']['args'] = ['arg1', self.vars['number'], 'arg2', + self.vars['list'], self.vars['dict']] + rend_action['python']['cmdline'] = '-s %s -i %s' % (self.vars['foo'], + self.vars['number']) + rend_action['python']['nest']['deepnest'] = self.vars['dict'] + rend_action['python']['nest']['null'] = self.vars['null'] + + taskyaml.replace_vars_in_action(self.action, self.vars) + assert self.action == rend_action + + def test_raise(self): + with pytest.raises(exc.TaskotronYamlError): + taskyaml.replace_vars_in_action(self.action, {}) + + del self.vars['number'] + with pytest.raises(exc.TaskotronYamlError): + taskyaml.replace_vars_in_action(self.action, self.vars) + + +class TestReplaceVars(object): + + def test_standadlone_var_str(self): + res = taskyaml._replace_vars('${foo}', {'foo': 'hedgehog'}) + assert res == 'hedgehog' + + def test_standalone_var_nonstr(self): + res = taskyaml._replace_vars('${foo}', {'foo': ['chicken', 'egg']}) + assert res == ['chicken', 'egg'] + + def test_standadlone_var_str_alternative(self): + res = taskyaml._replace_vars('$foo', {'foo': 'hedgehog'}) + assert res == 'hedgehog' + + def test_standalone_var_nonstr_alternative(self): + res = taskyaml._replace_vars('$foo', {'foo': ['chicken', 'egg']}) + assert res == ['chicken', 'egg'] + + def test_multivar(self): + res = taskyaml._replace_vars('a ${b} c${d}', {'b': 'bb', 'd': 'dd'}) + assert res == 'a bb cdd' + + def test_empty_space(self): + '''Some empty space around a single variable is like multi-var, i.e. + a string is always returned''' + res = taskyaml._replace_vars(' ${foo}', {'foo': 'bar'}) + assert res == ' bar' + + res = taskyaml._replace_vars(' ${foo}', {'foo': 42}) + assert res == ' 42' + + def test_recursive_content(self): + '''The content of a variable contains more variable names. But those + must not be expanded.''' + text = '${foo} ${bar}' + varz = {'foo': '${foo} ${bar}', + 'bar': '${foo} ${bar} ${baz}' + } + res = taskyaml._replace_vars(text, varz) + assert res == '%s %s' % (varz['foo'], varz['bar']) + + def test_empty(self): + res = taskyaml._replace_vars('', {'extra': 'var'}) + assert res == '' + + def test_same_var(self): + '''Only variables must get replaced, not same-sounding text''' + res = taskyaml._replace_vars('foo ${foo}', {'foo': 'bar'}) + assert res == 'foo bar' + + def test_combined_syntax(self): + '''Both $foo and ${foo} syntax must work, together with escaping''' + res = taskyaml._replace_vars('foo $foo ${foo} $$foo', {'foo': 'bar'}) + assert res == 'foo bar bar $foo' + + def test_raise_missing_var(self): + with pytest.raises(exc.TaskotronYamlError): + taskyaml._replace_vars('${foo}', {'bar': 'foo'}) + + def test_raise_missing_vars(self): + with pytest.raises(exc.TaskotronYamlError): + taskyaml._replace_vars('${foo} ${bar}', {'bar': 'foo'}) + + def test_raise_bad_escape(self): + '''Dollars must be doubled''' + with pytest.raises(exc.TaskotronYamlError): + taskyaml._replace_vars('${foo} $', {'foo': 'bar'}) + diff --git a/testing/test_yumrepoinfo.py b/testing/test_yumrepoinfo.py index 8cd4961..bb05e59 100644 --- a/testing/test_yumrepoinfo.py +++ b/testing/test_yumrepoinfo.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# Copyright 2014, Red Hat, Inc. -# License: GNU General Public License version 2 or later +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing '''Unit tests for libtaskotron/yumrepoinfo.py''' diff --git a/testing/test_yumrepoinfo_directive.py b/testing/test_yumrepoinfo_directive.py new file mode 100644 index 0000000..854c0f1 --- /dev/null +++ b/testing/test_yumrepoinfo_directive.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +import pytest +import StringIO +from dingus import Dingus + +from libtaskotron.directives import yumrepoinfo_directive +from libtaskotron.exceptions import TaskotronDirectiveError + +from libtaskotron import yumrepoinfo + + +TEST_CONF='''\ +[DEFAULT] +baseurl = http://download.fedoraproject.org/pub/fedora/linux +goldurl = %(baseurl)s/releases/%(path)s/Everything/%(arch)s/os +updatesurl = %(baseurl)s/updates/%(path)s/%(arch)s +rawhideurl = %(baseurl)s/%(path)s/%(arch)s/os +arches = i386, x86_64 +parent = +tag = %(__name__)s +supported = no + +[rawhide] +path = development/rawhide +url = %(rawhideurl)s +tag = f21 + +[f20] +url = %(goldurl)s +path = 20 +supported = yes + +[f20-updates] +url = %(updatesurl)s +path = 20 +parent = f20 +arches = x86_64 + +[f20-updates-testing] +url = %(updatesurl)s +path = testing/20 +parent = f20-updates +''' + + +class TestYumrepoinfoDirective(object): + + @classmethod + def setup_class(cls): + '''One-time class initialization''' + # create YumRepoInfo initialized with TEST_CONF + cls.ref_arch = ['x86_64'] + cls.repoinfo = yumrepoinfo.YumRepoInfo(filelist=[], arch=cls.ref_arch) + cls.repoinfo.parser.readfp(StringIO.StringIO(TEST_CONF)) + + def test_missing_kojitag(self): + directive = yumrepoinfo_directive.YumrepoinfoDirective(self.repoinfo) + ref_input = {"arch": ["x86_64"]} + + with pytest.raises(TaskotronDirectiveError): + directive.process(ref_input, None) + + def test_missing_arch(self): + directive = yumrepoinfo_directive.YumrepoinfoDirective(self.repoinfo) + ref_input = {"koji_tag":"rawhide"} + + with pytest.raises(TaskotronDirectiveError): + directive.process(ref_input, None) + + def test_pending(self): + directive = yumrepoinfo_directive.YumrepoinfoDirective(self.repoinfo) + ref_input = {"koji_tag": "f20-pending", "arch": ["x86_64"]} + + output = directive.process(ref_input, None) + + assert output == {"f20": "http://download.fedoraproject.org/pub/fedora/linux/releases/20/Everything/x86_64/os"} + + def test_rawhide(self): + directive = yumrepoinfo_directive.YumrepoinfoDirective(self.repoinfo) + ref_input = {"koji_tag": "rawhide", "arch": ["x86_64"]} + + output = directive.process(ref_input, None) + + assert output == {"rawhide": "http://download.fedoraproject.org/pub/fedora/linux/development/rawhide/x86_64/os"} + + def test_bad_kojitag(self): + directive = yumrepoinfo_directive.YumrepoinfoDirective(self.repoinfo) + ref_input = {"koji_tag": "my random tag"} + + with pytest.raises(TaskotronDirectiveError): + directive.process(ref_input, None) + + def test_repo_path(self): + + directive = yumrepoinfo_directive.YumrepoinfoDirective(self.repoinfo) + ref_input = {"koji_tag": "f20-updates", "arch": ["x86_64"]} + + output = directive.process(ref_input, None) + + assert output == { + "f20": "http://download.fedoraproject.org/pub/fedora/linux/releases/20/Everything/x86_64/os", + "f20-updates": "http://download.fedoraproject.org/pub/fedora/linux/updates/20/x86_64", + } + + def test_use_arch(self, monkeypatch): + """Make sure that the arch passed in as an arg is used to create the + yumrepoinfo object instead of falling back to the default system arch""" + ref_arch = 'i386' + + stub_getrepoinfo = Dingus(return_value=self.repoinfo) + monkeypatch.setattr(yumrepoinfo, 'get_yumrepoinfo', stub_getrepoinfo) + + # don't set the repoinfo object, we've stubbed out the code that would + # hit the filesystem, so it's not a risk here + directive = yumrepoinfo_directive.YumrepoinfoDirective() + ref_input = {"koji_tag": "f20-updates", "arch": [ref_arch]} + + directive.process(ref_input, None) + + # check the first arg of the first call to the stub object + assert stub_getrepoinfo.calls()[0][1][0] == ref_arch + +