From 24c29b66afc943d6987ff4a492394a4f73ebd482 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Aug 22 2014 08:53:35 +0000 Subject: Merge branch 'develop' into master for 0.3.6 release --- diff --git a/LICENSE b/LICENSE index 87f5983..39d9685 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,6 @@ The libtaskotron project is distributed under the GPLv3. Each source -file is licensed as GPLv2+, unless otherwise noted. +file is licensed as GPLv2+, unless otherwise noted here or inside the +file itself. Files licensed as GPLv3: docs/generate_directive_docs.py diff --git a/conf/taskotron.yaml.example b/conf/taskotron.yaml.example index 72dfec7..4bf019c 100644 --- a/conf/taskotron.yaml.example +++ b/conf/taskotron.yaml.example @@ -80,6 +80,20 @@ ## The location of temporary files for Taskotron #tmpdir: /var/tmp/taskotron +## ==== LOGGING section ==== +## This section contains configuration of logging + +## Configuration of logging level. Here can be configured which messages +## will be logged. You can specify different level for logging to standard +## output (option log_level_stream) and logging to file (log_level_file). +## Possible values can be found here: +## https://docs.python.org/2.7/library/logging.html#logging-levels +#log_level_stream: INFO +#log_level_file: DEBUG + +## If True, logging to file will be enabled. +## [default: True for production, False for development] +#log_file_enabled: True ## ==== SECRETS section ==== ## All login credentials and other secrets are here. If you add some secret diff --git a/conf/yumrepoinfo.conf.example b/conf/yumrepoinfo.conf.example index 2c985bf..2d73d39 100644 --- a/conf/yumrepoinfo.conf.example +++ b/conf/yumrepoinfo.conf.example @@ -22,21 +22,38 @@ parent = # koji tag defaults to section name tag = %(__name__)s -# true for "top" repos corresponding to currently supported Fedora releases -supported = no - +# release_status can be one of: obsolete, stable, branched or rawhide +# for non-top-parent repos this is an empty string +release_status = # Rawhide [rawhide] +url = %(rawhideurl)s path = development/rawhide +tag = f22 +release_status = rawhide + +# Fedora 21 +[f21] url = %(rawhideurl)s -tag = f21 +path = development/21 +release_status = branched + +[f21-updates] +url = %(updatesurl)s +path = 21 +parent = f21 + +[f21-updates-testing] +url = %(updatesurl)s +path = testing/21 +parent = f21-updates # Fedora 20 [f20] url = %(goldurl)s path = 20 -supported = yes +release_status = stable [f20-updates] url = %(updatesurl)s @@ -52,7 +69,7 @@ parent = f20-updates [f19] url = %(goldurl)s path = 19 -supported = yes +release_status = stable [f19-updates] url = %(updatesurl)s diff --git a/docs/source/conf.py b/docs/source/conf.py index b6c6dba..477ae09 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,9 +51,9 @@ copyright = u'2014, Fedora QA Devel' # built documents. # # The short X.Y version. -version = '0.3.3' +version = '0.3.6' # The full version, including alpha/beta/rc tags. -release = '0.3.3' +release = '0.3.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/index.rst b/docs/source/index.rst index 1628119..f720164 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ What is libtaskotron? Libtaskotron is one of the core components which make up the `Taskotron`_ system. Libtaskotron is responsible for running tasks written in -:ref:`taskotron-yaml-format`. +:ref:`taskotron-formula-format`. While libtaskotron was designed for use with `Fedora `_, the Fedora specific parts are isolated and the core should be usable with any @@ -23,6 +23,7 @@ Contents .. toctree:: :maxdepth: 1 + quickstart writingtasks taskyaml library @@ -75,6 +76,9 @@ Once the repository is enabled, install libtaskotron with:: sudo yum install libtaskotron +Note: The permissions on the log directory (``/var/log/taskotron``) are set to +``777``. If desired, change ``logdir`` in ``/etc/taskotron/taskotron.yaml`` +to a directory with more appropriate permissions. .. _running-tasks: diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst new file mode 100644 index 0000000..118435e --- /dev/null +++ b/docs/source/quickstart.rst @@ -0,0 +1,94 @@ +===================== +Taskotron Quick Start +===================== + + +Add repo +======== + +:: + + sudo wget http://copr-fe.cloud.fedoraproject.org/coprs/tflink/taskotron/repo/fedora-20-i386/tflink-taskotron-fedora-20-i386.repo -O /etc/yum.repos.d/tflink-taskotron-fedora-20-i386.repo + + +Install +======= + +:: + + sudo yum install libtaskotron + +`Detailed install instructions `_ + + +Run +=== + +:: + + runtask -i -t -a + +`More details on running tasks `_ + + +Formulae: General Syntax +======================== + +.. code-block:: yaml + + name: polyjuice potion + desc: potion that allows to take the form of someone else + maintainer: hermione + + input: + args: + - arg1 # supported args: + - arg2 # arch, koji_build, bodhi_id, koji_tag + + environment: + rpm: + - dependency1 # not checked in current version + - dependency2 # only for human reference + + actions: + - name: pick fluxweed on a full moon + directivename: # e.g. koji, bodhi, python, ... + arg1: value1 + arg2: ${some_variable} # e.g. input arg + export: firststep_output # export usable as input arg of next directive + + - name: next step of formula + nextdirectivename: + arg1: ${firststep_output} + +`Detailed instructions on writing tasks `_ + + +Recipes: Examples From Our Git +============================== + +Running Rpmlint +--------------- + +`Rpmlint `_ + +:: + + git clone https://bitbucket.org/fedoraqa/task-rpmlint.git + +:: + + runtask -i gourmet-0.16.0-2.fc20 -t koji_build -a x86_64 task-rpmlint/rpmlint.yml + + +Other Examples +-------------- + + +`Rpmbuild `_ + +`Example bodhi `_ + +`Example reporting `_ + +Check `our git `_ for more. diff --git a/docs/source/taskyaml.rst b/docs/source/taskyaml.rst index 540cf18..310ffcb 100644 --- a/docs/source/taskyaml.rst +++ b/docs/source/taskyaml.rst @@ -1,13 +1,14 @@ -.. _taskotron-yaml-format: +.. _taskotron-formula-format: -========================== -Taskotron Task YAML Format -========================== +============================= +Taskotron Task Formula Format +============================= -One of the core bits of Taskotron is the yaml file used to describe tasks to be -run. As Taskotron is still a very young project, some components of the task yaml -files are not yet used by the task runner but are useful to human readers of -the task. Those sections will be noted and those notes updated as things change. +One of the core bits of Taskotron is the formula yaml file used to describe +tasks to be run. As Taskotron is still a very young project, some components +of the task formulae are not yet used by the task runner but are useful to human +readers of the task. Those sections will be noted and those notes updated as +things change. The documentation here is a description of how things are implemented but are a little light on the practical creation of tasks. :ref:`writing-tasks-for-taskotron` @@ -22,7 +23,7 @@ Task Description Metadata -------- -Some metadata is required to be in the task yaml file: +Some metadata is required to be in the task formula: * task name * description diff --git a/docs/source/writingtasks.rst b/docs/source/writingtasks.rst index 9de6410..026041e 100644 --- a/docs/source/writingtasks.rst +++ b/docs/source/writingtasks.rst @@ -30,7 +30,7 @@ Examples ======== examplebodhi -------------- +------------ `examplebodhi task git repository `_ @@ -62,9 +62,9 @@ creating something new. To write a new task, you'll need some information: The taskotron runner works on specially formatted YAML files (see -:ref:`taskotron-yaml-format` for details) which describe the task that is to -be executed. In addition to the task YAML file, any other code or resources needed -for task execution should be accessible by the runner. +:ref:`taskotron-formula-format` for details) called formulae which describe +the task that is to be executed. In addition to the task formula, any other code +or resources needed for task execution should be accessible by the runner. A directory layout could look like:: @@ -75,7 +75,7 @@ A directory layout could look like:: .. note:: - Taskotron was designed to work with tasks stored in git repostitories and + Taskotron was designed to work with tasks stored in git repositories and while this is not strictly required for the local execution case, it is highly recommended that you limit tasks to one per directory. @@ -125,7 +125,7 @@ Let's have this new task do the following: .. code-block:: yaml - task: + actions: # we can use bodhi_id and arch as variables here because they're defined # for us by the runner which also verifies that all required input listed # above is present before executing the task @@ -153,7 +153,6 @@ Let's have this new task do the following: - name: report mytask results to resultsdb resultsdb: results: ${mytask_output} - checkname: 'mytask' .. this needs to be fixed, this output is doctored since it doesn't actually work. @@ -203,7 +202,7 @@ Our original task looked like: # this is a trivial example that doesn't do everything we want it to do, more # details will be added later in the example def run_mytask(rpmfiles): - for rpmfile in rpmfiles: + for rpmfile in rpmfiles['downloaded_rpms']: print "Running mytask on %s" % rpmfile To create valid TAP, we want to populate a :py:class:`libtaskotron.check.CheckDetail` @@ -224,8 +223,8 @@ with the data required for reporting. summary = 'mycheck %s for %s' % (result, bodhi_id) detail = check.CheckDetail(bodhi_id, check.ReportType.BODHI_UPDATE, result, summary) - for rpmfile in rpmfiles: - detail.store(rpmfile, False) + for rpmfile in rpmfiles['downloaded_rpms']: + detail.store(rpmfile, printout=False) return check.export_TAP(detail) @@ -246,6 +245,72 @@ Now if we run the task we get output that ends with:: type: bodhi_update ... +Working with CheckDetail Objects +-------------------------------- + +Check can have one of these outcomes (with increasing priority): ``PASSED``, +``INFO``, ``FAILED``, ``NEEDS_INSPECTION``, ``ABORTED`` and ``CRASHED``. These +are available as a list (where higher index means bigger priority) in +:py:attr:`CheckDetail.outcome_priority `. +You can set outcome during :py:class:`libtaskotron.check.CheckDetail` creation, by setting +:py:attr:`CheckDetail.outcome ` +directly or by using :py:meth:`libtaskotron.check.CheckDetail.update_outcome` - +it changes task outcome only if it has higher priority than current outcome. + +:py:meth:`libtaskotron.check.CheckDetail.store` is a convenience method if you +receive some output gradually and you want to store it and also print it at the +same time. You can also set check's output by setting :py:attr:`CheckDetail.output` +or in :py:class:`CheckDetail ` constructor. + +.. code-block:: python + + import os + import random + + from libtaskotron import check + + def random_choice(_): + return random.choice(['PASSED', 'FAILED', 'ABORTED']) + + def run_mytask(rpmfiles, bodhi_id): + """run through all passed in rpmfiles and emit TAP13. randomly report + failure or pass""" + + print "Running mytask on %s" % bodhi_id + + results = [(rpmfile, random_choice(rpmfile)) for rpmfile in rpmfiles['downloaded_rpms']] + + detail = check.CheckDetail(bodhi_id, check.ReportType.BODHI_UPDATE) + + for rpmfile, outcome in results: + # e.g. "xchat-1.2-3.fc20.x86_64.rpm: FAILED" + detail.store('%s: %s' % (os.path.basename(rpmfile), outcome)) + detail.update_outcome(outcome) + + detail.summary = "mytask %s for %s" % (detail.outcome, bodhi_id) + + return check.export_TAP(detail, checkname="mytask") + +Running this code as ``python runtask.py -i tzdata-2014f-1.fc20 -t bodhi_id -a x86_64 ../task-mytask/mytask.yml`` +ends with:: + + TAP version 13 + 1..1 + not ok - mytask for Bodhi update tzdata-2014f-1.fc20 # FAIL + --- + details: + output: |- + tzdata-2014f-1.fc20.noarch.rpm: ABORTED + tzdata-java-2014f-1.fc20.noarch.rpm: PASSED + item: tzdata-2014f-1.fc20 + outcome: ABORTED + summary: mytask ABORTED for tzdata-2014f-1.fc20 + type: bodhi_update + ... + +For more information about APIs and ``CheckDetail`` usage, see +:py:class:`CheckDetail class `. + .. note:: Reporting is disabled in the default configuration for Taskotron. While it is possible to set up a local resultsdb instance to check reporting, we want diff --git a/libtaskotron.spec b/libtaskotron.spec index 875fd2f..b0d01f1 100644 --- a/libtaskotron.spec +++ b/libtaskotron.spec @@ -3,7 +3,7 @@ %{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} Name: libtaskotron -Version: 0.3.3 +Version: 0.3.6 Release: 1%{?dist} Summary: Taskotron Support Library @@ -36,7 +36,7 @@ BuildRequires: python-bunch BuildRequires: python-dingus BuildRequires: python-urlgrabber BuildRequires: koji -BuildRequires: pytap13 >= 0.1.0 +BuildRequires: pytap13 >= 0.3.0 BuildRequires: python-bayeux BuildRequires: bodhi-client BuildRequires: resultsdb_api @@ -72,6 +72,9 @@ mkdir -p %{buildroot}%{_sysconfdir}/taskotron/ install conf/taskotron.yaml.example %{buildroot}%{_sysconfdir}/taskotron/taskotron.yaml install conf/yumrepoinfo.conf.example %{buildroot}%{_sysconfdir}/taskotron/yumrepoinfo.conf +# log dir +install -m 777 -d %{buildroot}/%{_localstatedir}/log/taskotron + %files %doc readme.rst LICENSE @@ -80,6 +83,8 @@ install conf/yumrepoinfo.conf.example %{buildroot}%{_sysconfdir}/taskotron/yumre %attr(755,root,root) %{_bindir}/runtask +%dir %attr(777, root, root) %{_localstatedir}/log/taskotron + %files -n libtaskotron-config %dir %{_sysconfdir}/taskotron %config(noreplace) %{_sysconfdir}/taskotron/taskotron.yaml @@ -88,6 +93,12 @@ install conf/yumrepoinfo.conf.example %{buildroot}%{_sysconfdir}/taskotron/yumre %changelog +* Fri Aug 22 2014 Martin Krizek - 0.3.6-1 +- Releasing libtaskotron 0.3.6 + +* Tue Jul 08 2014 Martin Krizek - 0.3.3-2 +- Add /var/log/taskotron directory + * Mon Jun 30 2014 Tim Flink - 0.3.3-1 - Changed distibution license to gpl3 - New user-facing docs diff --git a/libtaskotron/__init__.py b/libtaskotron/__init__.py index e0d57bc..cadb30c 100644 --- a/libtaskotron/__init__.py +++ b/libtaskotron/__init__.py @@ -4,4 +4,4 @@ # See the LICENSE file for more details on Licensing from __future__ import absolute_import -__version__ = '0.3.3' +__version__ = '0.3.6' diff --git a/libtaskotron/check.py b/libtaskotron/check.py index 0345ee2..8939a63 100644 --- a/libtaskotron/check.py +++ b/libtaskotron/check.py @@ -241,7 +241,7 @@ class ReportType(object): def export_TAP(check_details, checkname="$CHECKNAME"): '''Generate TAP output used for reporting to ResultsDB. - Note: You need to provide all your :class:`CheckDetail`s in a single pass + Note: You need to provide all your :class:`CheckDetail`\s in a single pass in order to generate a valid TAP output. You can't call this method several times and then simply join the outputs simply as strings. diff --git a/libtaskotron/config.py b/libtaskotron/config.py index 1b7ae6d..4810e9d 100644 --- a/libtaskotron/config.py +++ b/libtaskotron/config.py @@ -11,12 +11,11 @@ import yaml import collections import libtaskotron - -from . import exceptions as exc -from .logger import log -from .config_defaults import (Config, ProductionConfig, TestingConfig, - ProfileName) -from . import file_utils +from libtaskotron import exceptions as exc +from libtaskotron.logger import log +from libtaskotron.config_defaults import (Config, ProductionConfig, + TestingConfig, ProfileName) +from libtaskotron import file_utils CONF_DIRS = [ # local checkout dir first, then system wide dir @@ -91,6 +90,7 @@ def _load(): log.warn('No config file %s found in dirs: %s' % (CONF_FILE, CONF_DIRS)) return config + log.debug('Using config file: %s', filename) file_config = _load_file(filename) file_profile = file_config.get('profile', None) @@ -154,20 +154,18 @@ def _load_file(conf_file): # convert file path to file handle if needed if isinstance(conf_file, basestring): - log.debug('Trying to open configuration file: %s', conf_file) try: conf_file = open(conf_file) except IOError as e: - log.exception('Could not open configuration file: %s', conf_file) + log.exception('Could not open config file: %s', conf_file) raise exc.TaskotronConfigError(e) filename = (conf_file.name if hasattr(conf_file, 'name') else '') try: - log.debug('Trying to parse configuration file: %s', filename) conf_obj = yaml.safe_load(conf_file) except yaml.YAMLError as e: - log.exception('Could not parse configuration file: %s', filename) + log.exception('Could not parse config file: %s', filename) raise exc.TaskotronConfigError(e) # config file might be empty (all commented out), returning None. For diff --git a/libtaskotron/config_defaults.py b/libtaskotron/config_defaults.py index 5efe668..fc05abd 100644 --- a/libtaskotron/config_defaults.py +++ b/libtaskotron/config_defaults.py @@ -56,9 +56,14 @@ class Config(object): tmpdir = '/var/tmp/taskotron' #: logdir = '/var/log/taskotron' #: - main_log_name = 'taskotron.log' + log_name = 'taskotron.log' '''name of the main log file in :attr:`logdir`''' + log_level_stream = 'INFO' #: + log_level_file = 'DEBUG' #: + + log_file_enabled = False #: + fas_username = 'taskotron' #: fas_password = '' #: @@ -76,9 +81,20 @@ class ProductionConfig(Config): profile = ProfileName.PRODUCTION #: reports_enabled = True #: + log_level_stream = 'INFO' #: + log_level_file = 'DEBUG' #: + + log_file_enabled = True #: + class TestingConfig(Config): '''Configuration for testing suite profile. Inherits values from :class:`Config` and overrides some. Read Config documentation.''' profile = ProfileName.TESTING #: + + tmpdir = '/var/tmp/taskotron-test/tmp' #: + logdir = '/var/tmp/taskotron-test/log' #: + + log_level_stream = 'DEBUG' #: + log_level_file = 'DEBUG' #: diff --git a/libtaskotron/directives/bodhi_comment_directive.py b/libtaskotron/directives/bodhi_comment_directive.py index c456e1d..7a187b9 100644 --- a/libtaskotron/directives/bodhi_comment_directive.py +++ b/libtaskotron/directives/bodhi_comment_directive.py @@ -112,7 +112,7 @@ class BodhiCommentDirective(BaseDirective): Config = config.get_config() if not (Config.reporting_enabled and Config.report_to_bodhi): - log.info("Reporting to Bodhi is disabled.") + log.info("Reporting to Bodhi is disabled, not doing anything.") return diff --git a/libtaskotron/directives/koji_directive.py b/libtaskotron/directives/koji_directive.py index f3a1824..463c0fa 100644 --- a/libtaskotron/directives/koji_directive.py +++ b/libtaskotron/directives/koji_directive.py @@ -13,35 +13,39 @@ description: | a specific build, or you can download all RPMs from all builds belonging to a specific Koji tag. parameters: + action: + required: true + description: choose whether to download a single build or all builds + belonging to a Koji tag + type: str + choices: [tag, build] arch: required: true description: | a list of architectures for which to download RPMs for the requested - build/tag. In addition to architectures supported by Taskotron by default, - ``src`` is supported as well. + build/tag. If you want to download RPMs for all arches, use ``['all']``. Note: ``noarch`` RPMs are always automatically downloaded even when not - requested, unless ``src`` is the only architecture requested. + requested, unless ``arch=[]`` and ``src=True``. type: list of str - choices: [standard Taskotron architectures, src] - command: - required: true - description: choose whether to download a single build or all builds - belonging to a Koji tag - type: str - choices: [tag, build] + choices: [supported architectures, ['all']] koji_build: required: true description: | N(E)VR of a Koji build to download. Only required when - ``command="download"``. Example: ``xchat-2.8.8-21.fc20`` + ``action="download"``. Example: ``xchat-2.8.8-21.fc20`` type: str koji_tag: required: true description: | name of a Koji tag to download all builds from. Only required when - ``command="download_tag"``. Example: ``f20-updates-pending`` + ``action="download_tag"``. Example: ``f20-updates-pending`` type: str + src: + required: false + description: download also ``src`` RPM files + type: bool + default: False target_dir: required: false description: directory into which to download builds @@ -54,19 +58,24 @@ returns: | RPMs raises: | * :class:`.TaskotronRemoteError`: if downloading failed + * :class:`.TaskotronValueError`: if ``arch=[]`` and ``src=False``, therefore + there is nothing to download version_added: 0.4 """ EXAMPLES = """ -Rpmlint needs to download a specific build from Koji:: +Rpmlint needs to download a specific build from Koji, all architectures +including src.rpm:: - name: download rpms from koji koji: action: download koji_build: ${koji_build} - arch: ${arch} + arch: ['all'] + src: True -Depcheck needs to download all builds in a specific Koji tag:: +Depcheck needs to download all builds in a specific Koji tag for the current +architecture:: - name: download koji tag koji: @@ -77,10 +86,9 @@ Depcheck needs to download all builds in a specific Koji tag:: """ from libtaskotron.koji_utils import KojiClient -from libtaskotron.logger import log from libtaskotron.directives import BaseDirective -from libtaskotron.exceptions import TaskotronDirectiveError from libtaskotron import rpm_utils +import libtaskotron.exceptions as exc directive_class = 'KojiDirective' @@ -94,60 +102,55 @@ class KojiDirective(BaseDirective): self.koji = koji_session def process(self, input_data, env_data): + # process params valid_actions = ['download', 'download_tag'] - - output_data = {} - action = input_data['action'] - if not action in valid_actions: - raise TaskotronDirectiveError('%s is not a valid command for koji ' - 'helper' % action) - - if 'arch' not in input_data: - detected_args = ', '.join(input_data.keys()) - raise TaskotronDirectiveError( - "The koji directive requires 'arch' as an argument. Detected" - "arguments: %s" % detected_args) + raise exc.TaskotronDirectiveError('%s is not a valid action for koji ' + 'directive' % action) if not 'target_dir' in input_data: self.target_dir = env_data['workdir'] else: self.target_dir = input_data['target_dir'] - self.arches = input_data['arch'] + if 'arch' not in input_data: + detected_args = ', '.join(input_data.keys()) + raise exc.TaskotronDirectiveError( + "The koji directive requires 'arch' as an argument. Detected " + "arguments: %s" % detected_args) - if 'noarch' not in self.arches and not self.arches == ['src']: + self.arches = input_data['arch'] + if self.arches and ('noarch' not in self.arches): self.arches.append('noarch') + self.src = input_data.get('src', False) + + # download files + output_data = {} + if action == 'download': if 'koji_build' not in input_data: detected_args = ', '.join(input_data.keys()) - raise TaskotronDirectiveError( - "The koji directive requires 'koji_build' for the 'download'" + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_build' for the 'download' " "action. Detected arguments: %s" % detected_args) nvr = rpm_utils.rpmformat(input_data['koji_build'], 'nvr') - log.info("Getting RPMs for koji build %s (%s) and downloading to %s", - nvr, str(self.arches), self.target_dir) output_data['downloaded_rpms'] = self.koji.get_nvr_rpms(nvr, - self.target_dir, - arches=self.arches, - src=('src' in self.arches)) + self.target_dir, arches=self.arches, src=self.src) + elif action == 'download_tag': if 'koji_tag' not in input_data: detected_args = ', '.join(input_data.keys()) - raise TaskotronDirectiveError( - "The koji directive requires 'koji_tag' for the 'download_tag'" + raise exc.TaskotronDirectiveError( + "The koji directive requires 'koji_tag' for the 'download_tag' " "action. Detected arguments: %s" % detected_args) koji_tag = input_data['koji_tag'] - log.info("Getting koji builds for koji tag '%s', arches '%s' and " - "downloading to %s", koji_tag, str(self.arches), self.target_dir) output_data['downloaded_rpms'] = self.koji.get_tagged_rpms(koji_tag, - self.target_dir, - self.arches) + self.target_dir, arches=self.arches, src=self.src) return output_data diff --git a/libtaskotron/directives/mash_directive.py b/libtaskotron/directives/mash_directive.py index ec9384e..8c37f60 100644 --- a/libtaskotron/directives/mash_directive.py +++ b/libtaskotron/directives/mash_directive.py @@ -34,7 +34,8 @@ parameters: required: false description: absolute or relative path to an output directory, where repository should be created. If not specified, then ``rpmdir`` is used - as an output directory. + as an output directory. ``outdir`` will be created automatically if not + present. type: str default: None dodelta: @@ -79,24 +80,12 @@ from ConfigParser import RawConfigParser from libtaskotron.directives import BaseDirective from libtaskotron.logger import log from libtaskotron.exceptions import TaskotronDirectiveError +import libtaskotron.file_utils as file_utils directive_class = 'MashDirective' -class MashDirective(BaseDirective): - """This will create a YUM repository in ``outdir`` from from all RPM - packages in ``rpmdir``. If ``outdir`` is not specified, repo is created - in ``rpmdir``. It will call ``mash`` tool to perform this task. - - The directive returns actual path where repo was created. - - For the mash directive, we're expecting a yaml declaration of the form: - - mash: - rpmdir=/path/to/some/dir - dodelta=[True, False] - arch=["i386", "x86_64", "armhfp", "noarch"] - """ +class MashDirective(BaseDirective): def do_mash(self, rpmdir, dodelta, arch, outdir=None): """Set up a mash object with an ad-hoc config @@ -104,7 +93,9 @@ class MashDirective(BaseDirective): :param bool dodelta: create drpms during mash :param str arch: arch of the rpms :param str outdir: output directory where repo is created, - if not specified or None, rpmdir is used + if not specified or None, ``rpmdir`` is used. + ``outdir`` will be created automatically if + not present. :rtype str :returns: path to directory where repo was created """ @@ -147,7 +138,8 @@ class MashDirective(BaseDirective): for rpm in os.listdir(rpmdir): os.symlink(os.path.join(rpmdir, rpm), os.path.join(outdir, rpm)) else: - p = sub.Popen(['createrepo', rpmdir], stdout=sub.PIPE, stderr=sub.PIPE) + p = sub.Popen(['createrepo', rpmdir], stdout=sub.PIPE, + stderr=sub.PIPE) output, errors = p.communicate() @@ -176,4 +168,12 @@ class MashDirective(BaseDirective): outdir = input_data.get('outdir', None) arch = input_data['arch'] + # create outdir if it doesn't exist + if outdir: + try: + file_utils.makedirs(outdir) + except OSError, e: + log.exception("Can't create directory: %s", outdir) + raise TaskotronDirectiveError(e) + return self.do_mash(rpmdir, dodelta, arch, outdir) diff --git a/libtaskotron/directives/python_directive.py b/libtaskotron/directives/python_directive.py index 4dc6b0d..49bc6cc 100644 --- a/libtaskotron/directives/python_directive.py +++ b/libtaskotron/directives/python_directive.py @@ -116,9 +116,11 @@ C) callable instance reference:: import imp import os +import re from libtaskotron.directives import BaseDirective from libtaskotron.exceptions import TaskotronDirectiveError +from libtaskotron.logger import log directive_class = 'PythonDirective' @@ -147,6 +149,9 @@ class PythonDirective(BaseDirective): task_method = self._do_getattr(task_module, method_name) + module = re.sub('.py[co]?$', '', os.path.basename(task_module.__file__)) + log.info("Executing Python: %s.%s() with args %s", module, method_name, + kwargs) output = task_method(**kwargs) if output is not None and not isinstance(output, basestring): diff --git a/libtaskotron/directives/resultsdb_directive.py b/libtaskotron/directives/resultsdb_directive.py index a3fcdcf..5d22d89 100644 --- a/libtaskotron/directives/resultsdb_directive.py +++ b/libtaskotron/directives/resultsdb_directive.py @@ -159,15 +159,17 @@ class ResultsdbDirective(BaseDirective): # serves as validation of input results try: check_details = check.import_TAP(input_data['results']) + log.debug("TAP output is OK.") except TaskotronValueError as e: raise TaskotronDirectiveError("Failed to load 'results': %s" % e.message) conf = config.get_config() if not (conf.reporting_enabled and conf.report_to_resultsdb): - log.info("TAP is OK.") - log.info("Reporting to ResultsDB is disabled.") - log.info("Once enabled, the following would be reported:") - log.info('\n'.join([str(detail) for detail in check_details])) + log.info("Reporting to ResultsDB is disabled. Once enabled, the " + "following would get reported:\n%s" % + report_summary(input_data['results'], env_data['checkname'])) + log.info('Hint: Enabling debug output allows you to see unstripped ' + 'values during variable export.') return # for now, we're creating the resultsdb job at reporting time @@ -182,6 +184,7 @@ class ResultsdbDirective(BaseDirective): output = [] + log.info('Posting %s results to ResultsDB...' % len(check_details)) for detail in check_details: try: result = self.resultsdb.create_result( @@ -205,3 +208,30 @@ class ResultsdbDirective(BaseDirective): return output + +def report_summary(tap, checkname): + '''Create a pretty summary of what will/would be reported into ResultsDB. + The summary is based on TAP output, with the TAP header stripped out and + most of the check output stripped out. + + :param str tap: TAP-formatted results + :param str checkname: the name of the check + :return: summary in plain text + :rtype: str + :raise TaskotronValueError: if `tap` has invalid format + ''' + + check_details = check.import_TAP(tap) + summary = [] + + for detail in check_details: + # keep just the first and the last line of output + lines = ('\n'.join(detail.output)).splitlines() + if len(lines) > 3: + detail.output = [lines[0], '', lines[-1]] + out = check.export_TAP(detail, checkname=checkname) + # strip "TAP version 13\n1..1" (first two lines) + out = '\n'.join(out.splitlines()[2:]) + summary.append(out) + + return '\n'.join(summary) diff --git a/libtaskotron/file_utils.py b/libtaskotron/file_utils.py index 4f4badb..c9467a4 100644 --- a/libtaskotron/file_utils.py +++ b/libtaskotron/file_utils.py @@ -60,10 +60,10 @@ def download(url, dirname, filename=None, overwrite=False, grabber=None): '''Download a file. :param str url: file URL to download - :param str dirname directory path, if the directory does not exist, it gets - created (and all its parent directories). - :param str filename name of downloaded file, if not provided basename is - extracted from URL + :param str dirname: directory path, if the directory does not exist, it gets + created (and all its parent directories). + :param str filename: name of downloaded file, if not provided basename is + extracted from URL :param bool overwrite: if the destination file already exists, whether to overwrite or not. If ``False``, a simple check is performed whether the remote file is the same as the diff --git a/libtaskotron/koji_utils.py b/libtaskotron/koji_utils.py index 816f342..58026b2 100644 --- a/libtaskotron/koji_utils.py +++ b/libtaskotron/koji_utils.py @@ -35,125 +35,174 @@ class KojiClient(object): self.session = (koji_session or koji.ClientSession(config.get_config().koji_url)) - # contains a rpm_filename -> build info mapping used in rpm_to_build - self._rpm_to_build_cache = {} - - def rpm_to_build(self, rpm, prefetch=True): - '''Get koji build object for the rpm. - - :param str rpm: filename as either ``/my/path/nvr.a.rpm`` or ``nvr.a.rpm`` - :param bool prefetch: if set to True, get list of all rpms for the build - and store it for future use - this will speed up subsequent queries - for rpms belonging to already queried build. - :return: Koji buildinfo dictionary (as returned e.g. from :meth:`koji.getBuild`) - :rtype: :class:`bunch.Bunch` - :raise TaskotronRemoteError: if rpm or it's related build is not found + def rpms_to_build(self, rpms): + '''Get list of koji build objects for the rpms. Order of koji objects + in this list is the same as order of the respective rpm objects. + + :param rpms: list of filenames as either ``/my/path/nvr.a.rpm`` or + ``nvr.a.rpm`` + :type rpms: list of str + :return: list of Koji buildinfo dictionaries (as returned e.g. + from :meth:`koji.getBuild`) in the same respective order as in``rpms`` + :rtype: list + :raise TaskotronRemoteError: if rpm or it's related build is not + found ''' - # extract filename from the (possible) path - rpm = os.path.split(rpm)[-1] - - if rpm in self._rpm_to_build_cache: - return self._rpm_to_build_cache[rpm] - - rpminfo = self.session.getRPM(rpm) - if rpminfo is None: - raise exc.TaskotronRemoteError('RPM not found') - - buildinfo = self.session.getBuild(rpminfo['build_id']) - if buildinfo is None: - raise exc.TaskotronRemoteError('Build %r not found' % rpminfo['build_id']) - - if prefetch: - rpms = self.session.listRPMs(rpminfo['build_id']) - for r in rpms: - filename = "%s.%s.rpm" % (r['nvr'], r['arch']) - self._rpm_to_build_cache[filename] = buildinfo - else: - self._rpm_to_build_cache[rpm] = buildinfo - - return buildinfo - - - def nvr_to_urls(self, nvr, arches=None, debuginfo=False, src=True): + self.session.multicall = True + for rpm in rpms: + # extract filename from the (possible) path + rpm = os.path.split(rpm)[-1] + + self.session.getRPM(rpm) + + # the list will contain one element for each method added to the + # multicall, in the order it was added to the multicall + rpminfos = self.session.multiCall() + + # because it is probable that several rpms will come from the same build + # use set for builds + builds = set() + for i, rpminfo in enumerate(rpminfos): + # according to documentation, multiCall() returns list, where + # each element will be either a one-element list containing the + # result of the method call, or a dict containing "faultCode" and + # "faultString" keys, describing the error that occurred during the + # method call. + # + # getRPM() returns None if there is no RPM with the given ID + if isinstance(rpminfo, dict): + raise exc.TaskotronRemoteError('Problem with RPM %s: %d: %s' % ( + rpms[i], rpminfo["faultCode"], rpminfo["faultString"])) + elif rpminfo[0] is None: + raise exc.TaskotronRemoteError('RPM %s not found' % rpms[i]) + else: + builds.add(rpminfo[0]['build_id']) + + builds = list(builds) # so that we could use build order + self.session.multicall = True + for build in builds: + self.session.getBuild(build) + + buildinfos = self.session.multiCall() + + for i, buildinfo in enumerate(buildinfos): + # see ^ + if isinstance(buildinfo, dict): + raise exc.TaskotronRemoteError( + 'Problem with build %s: %d: %s' % ( + builds[i], buildinfo["faultCode"], + buildinfo["faultString"])) + elif buildinfo[0] is None: + raise exc.TaskotronRemoteError( + 'Build %s not found' % builds[i]) + + build_to_buildinfo = dict(zip(builds, buildinfos)) + result = [] + for rpminfo in rpminfos: + build_id = rpminfo[0]['build_id'] + result.append(build_to_buildinfo[build_id][0]) + + return result + + + def nvr_to_urls(self, nvr, arches=['all'], debuginfo=False, src=True): '''Get list of URLs for RPMs corresponding to a build. :param str nvr: build NVR - :param arches: restrict the arches of builds to provide URLs for. Note: - If basearch ``i386`` is in the list, ``i686`` arch is - automatically added (since that's the arch Koji API uses). + :param arches: restrict the arches of builds to provide URLs for. By + default, all architectures are considered. If you want to consider + just some selected arches, provide their names in a list. + + .. note:: If basearch ``i386`` is in the list, ``i686`` arch is + automatically added (since that's the arch Koji API uses). + :type arches: list of str + :param bool debuginfo: whether to provide URLs for debuginfo RPM files + or ignore them + :param bool src: whether to include a URL for the source RPM + :rtype: list of str + :raise TaskotronRemoteError: when the requested build doesn't exist + :raise TaskotronValueError: if ``arches=[]`` and ``src=False``, + therefore there is nothing to query for ''' log.info('Querying Koji for a list of RPMS for: %s', nvr) # add i686 arch if i386 is present in arches - if arches and 'i386' in arches and 'i686' not in arches: + if 'i386' in arches and 'i686' not in arches: arches.append('i686') + # if src is enabled, we need to add it so that it is included in the + # Koji query + if src and ('src' not in arches): + arches.append('src') + + # if "nothing" is requested, it's probably an error + if not arches: + raise exc.TaskotronValueError('Nothing to query for, `arches` was ' + 'set to an empty list and `src` was disabled. At least one of ' + 'them need to be non-empty.') + + # find the koji build info = self.session.getBuild(nvr) if info is None: raise exc.TaskotronRemoteError( - "No build information found for %s" % nvr) - - baseurl = '/'.join((config.get_config().pkg_url, info['package_name'], - info['version'], info['release'])) + "No such build found in Koji: %s" % nvr) - rpms = self.session.listRPMs(buildID=info['id'], arches=arches) + # list its RPM files + req_arches = None if 'all' in arches else arches + rpms = self.session.listRPMs(buildID=info['id'], arches=req_arches) if not debuginfo: rpms = [r for r in rpms if not r['name'].endswith('-debuginfo')] if not src: rpms = [r for r in rpms if not r['arch'] == 'src'] + # create URLs + baseurl = '/'.join((config.get_config().pkg_url, info['package_name'], + info['version'], info['release'])) urls = ['%s/%s' % (baseurl, koji.pathinfo.rpm(r)) for r in rpms] - if len(urls) == 0: - raise exc.TaskotronRemoteError('No RPMs found for %s' % nvr) return sorted(urls) - def get_nvr_rpms(self, nvr, rpm_dir, arches=None, debuginfo=False, + def get_nvr_rpms(self, nvr, dest, arches=['all'], debuginfo=False, src=False): '''Retrieve the RPMs associated with a build NVR into the specified - directory. + directory. - :param str nvr build: NVR - :param str rpm_dir: location where to store the RPMs + :param str nvr: build NVR + :param str dest: location where to store the RPMs + :param arches: see :meth:`nvr_to_urls` + :param debuginfo: see :meth:`nvr_to_urls` + :param src: see :meth:`nvr_to_urls` :return: list of local filenames of the grabbed RPMs :rtype: list of str - :raise TaskotronRemoteError: if issues with target directory + :raise TaskotronRemoteError: if the files can't be downloaded or saved + :raise TaskotronValueError: if ``arches=[]`` and ``src=False``, + therefore there is nothing to download ''' - try: - rpm_urls = self.nvr_to_urls(nvr, - arches=arches, - debuginfo=debuginfo, - src=src) - except exc.TaskotronRemoteError, e: - log.warn('Error while retrieving urls for %s (%s)' % (nvr, arches)) - log.warn(e) - return + rpm_urls = self.nvr_to_urls(nvr, arches, debuginfo, src) rpm_files = [] - log.info('Fetching %s RPMs for: %s', len(rpm_urls), nvr) + log.info('Fetching %s RPMs for: %s (into %s)', len(rpm_urls), nvr, dest) for url in rpm_urls: - log.debug(' Fetching %s', url) - rpm_file = file_utils.download(url, rpm_dir) + rpm_file = file_utils.download(url, dest) rpm_files.append(rpm_file) return rpm_files - def get_tagged_rpms(self, tag, dest, arches): + def get_tagged_rpms(self, tag, dest, arches=['all'], debuginfo=False, + src=False): '''Downloads all RPMs of all NVRs tagged by a specific Koji tag. + Note: This works basically the same as :meth:`get_nvr_rpms`, it just + downloads a lot of builds instead of a single one. For description of + all shared parameters and return values, please see that method. + :param str tag: Koji tag to be queried for available builds, e.g. ``f20-updates-pending`` - :param str dest: destination directory - :param arches: list of architectures - :type arches: list of str - :return: list of local filenames of the grabbed RPMs - :rtype: list of str - :raise TaskotronRemoteError: if issues with target directory ''' log.debug('Querying Koji for tag: %s' % tag) tag_data = self.session.listTagged(tag) @@ -164,7 +213,7 @@ class KojiClient(object): log.debug('Builds to be downloaded:\n %s', '\n '.join(nvrs)) for nvr in nvrs: - rpms.append(self.get_nvr_rpms(nvr, dest, arches)) + rpms.append(self.get_nvr_rpms(nvr, dest, arches, debuginfo, src)) return rpms diff --git a/libtaskotron/logger.py b/libtaskotron/logger.py index 0b99321..577bc25 100644 --- a/libtaskotron/logger.py +++ b/libtaskotron/logger.py @@ -5,14 +5,64 @@ from __future__ import absolute_import import sys +import os +import tempfile import logging import logging.handlers import traceback - +# you must not import libtaskotron.config here because of cyclic dependencies # http://docs.python.org/2/howto/logging.html#configuring-logging-for-a-library +#: the main logger for libtaskotron library, easily accessible from all our +#: modules log = logging.getLogger('libtaskotron') -log.addHandler(logging.NullHandler()) +log.setLevel(logging.DEBUG) # need to set level, otherwise root level is inherited +log.addHandler(logging.NullHandler()) # this is needed when running in library mode + +_fmt = '[%(name)s:%(filename)s:%(lineno)d] '\ + '%(asctime)s %(levelname)-7s %(message)s' +_datefmt = '%Y-%m-%d %H:%M:%S' +_formatter = logging.Formatter(fmt=_fmt, datefmt=_datefmt) + +#: our current stream handler sending logged messages to stderr +stream_handler = None + +#: our current syslog handler sending logged messages to syslog +syslog_handler = None + +#: our current file handler sending logged messages to file log +file_handler = None + + +def _create_handlers(syslog=False, filelog_path=None): + '''Create all handlers. This should be called before any method tries to + operate on handlers. Handlers are created only if they don't exist yet + (they are `None`), otherwise they are skipped. So this method can easily be + called multiple times. + + Note: Handlers can't be created during module import, because that breaks + stream capturing functionality when running through ``pytest``. + + :param bool syslog: create syslog handler. Syslog must be available. + :param str filelog_path: path to the log file where :data:`file_handler` + should emit messages. If this is ``None``, then + :data:`file_handler` is not created. + :raise socket.error: if syslog handler creation failed + ''' + global stream_handler, syslog_handler, file_handler + + if not stream_handler: + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(_formatter) + + if syslog and not syslog_handler: + syslog_handler = logging.handlers.SysLogHandler(address='/dev/log', + facility=logging.handlers.SysLogHandler.LOG_LOCAL4) + + if not file_handler and filelog_path: + file_handler = logging.handlers.RotatingFileHandler(filelog_path, + maxBytes=500000, backupCount=5) + file_handler.setFormatter(_formatter) def _log_excepthook(*exc_info): @@ -20,53 +70,135 @@ def _log_excepthook(*exc_info): log.critical(''.join(traceback.format_exception(*exc_info))) -def init(name='libtaskotron', level=logging.INFO, stream=True, syslog=False, - filelog=None, set_rootlogger=False): - """Setup a logger +def init_prior_config(level_stream=None): + '''Initialize Taskotron logging with default values which do not rely on + a config file. Only stream logging is enabled here. This is used before the + config file is loaded. After that a proper initialization should take place + through the :func:`init` method. - :param str name: name of the logger, 'libtaskotron' is default - :param level: level of logging - :type level: level definitions from :mod:`logging` - :param bool stream: enables logging to stderr - :param bool syslog: enables logging to syslog - :param str filelog: enables logging to a file, the value is the file name - :param bool set_rootlogger: adds output handlers to root logger - """ + Note: Since this touches the root logger, it should be called only when + Taskotron is run as the main program (through its runner), not when it is + used as a library. - # overwrite default except hook - sys.excepthook = _log_excepthook + :param int level_stream: message level of the stream logger. The level + definitions are in :mod:`logging`. If ``None``, the default level is + used (i.e. :data:`logging.NOTSET`). + ''' + _create_handlers() - logger = logging.getLogger(name) + if level_stream is not None: + stream_handler.setLevel(level_stream) rootlogger = logging.getLogger() - logger.setLevel(level) - logger.addHandler(logging.NullHandler()) + rootlogger.addHandler(stream_handler) + + sys.excepthook = _log_excepthook + + +def _set_level(handler, level, conf_name): + '''Set logging level to a handler. Fall back to config defaults if the level + name is invalid. + + :param handler: log handler to configure + :type handler: instance of :class:`logging.Handler` + :param level: level identification from :mod:`logging` + :type level: ``str`` or ``int`` + :param str conf_name: the name of the configuration option of the default + level for this handler. If ``level`` value is invalid, it will retrieve + the default value from the config file and set it instead. + ''' + from libtaskotron import config + try: + handler.setLevel(level) + except ValueError: + conf_defaults = config._load_defaults(config.get_config().profile) + default_level = getattr(conf_defaults, conf_name) + log.warning("Invalid logging level '%s' for '%s'. Resetting to default " + "value '%s'.", level, conf_name, default_level) + handler.setLevel(default_level) + + +def init(level_stream=None, level_file=None, + stream=True, syslog=False, filelog=None, + filelog_path=None): + """Initialize Taskotron logging. + + Note: Since this touches the root logger, it should be called only when + Taskotron is run as the main program (through its runner), not when it is + used as a library. + + :param int level_stream: level of stream logging as defined in + :mod:`logging`. If ``None``, a default level from config file is used. + :param int level_file: level of file logging as defined in + :mod:`logging`. If ``None``, a default level from config file is used. + :param bool stream: enable logging to process stream (stderr) + :param bool syslog: enable logging to syslog + :param bool filelog: enable logging to a file log. If ``None``, the value is + loaded from config file. + :param str filelog_path: path to the log file. If ``None``, the value is + loaded from config file. + """ - fmt = '[%(name)s:%(filename)s:%(lineno)d] '\ - '%(asctime)s %(levelname)-7s %(message)s' - datefmt = '%Y-%m-%d %H:%M:%S' - formatter = logging.Formatter(fmt=fmt, datefmt=datefmt) + # We import libtaskotron.config here because import from beginning + # of this module causes problems with cyclic dependencies + from libtaskotron import config + conf = config.get_config() + + if level_stream is None: + level_stream = conf.log_level_stream + if level_file is None: + level_file = conf.log_level_file + if filelog is None: + filelog = conf.log_file_enabled + if filelog_path is None: + filelog_path = os.path.join(conf.logdir, conf.log_name) + + _create_handlers() + rootlogger = logging.getLogger() + sys.excepthook = _log_excepthook if stream: - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - if set_rootlogger: - rootlogger.addHandler(stream_handler) - logger.debug("doing stream logging") + _set_level(stream_handler, level_stream, "log_level_stream") + rootlogger.addHandler(stream_handler) + log.debug("Stream logging enabled with level: %s", + logging.getLevelName(stream_handler.level)) + else: + rootlogger.removeHandler(stream_handler) if syslog: - syslog_handler = logging.handlers.SysLogHandler(address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_LOCAL4) - syslog_handler.setFormatter(formatter) - if set_rootlogger: - rootlogger.addHandler(syslog_handler) - logger.debug("doing syslog logging") + _create_handlers(syslog=True) + rootlogger.addHandler(syslog_handler) + log.debug("Syslog logging enabled with level: %s", + logging.getLevelName(syslog_handler.level)) + else: + rootlogger.removeHandler(syslog_handler) if filelog: - file_handler = logging.handlers.RotatingFileHandler(filelog, - maxBytes=500000, backupCount=5) - file_handler.setFormatter(formatter) - if set_rootlogger: - rootlogger.addHandler(file_handler) - logger.debug('doing file logging to %s', filelog) - - return logger + # try access + try: + f = open(filelog_path, 'a') + # since we have the file opened at the moment, put a separator + # into it, it will help to differentiate between different task runs + f.write('#'*120 + '\n') + f.close() + except IOError as e: + log.warning("Log file can't be opened for writing: %s\n %s", + filelog_path, e) + # we will use a temporary log file + filelog_path = tempfile.mkstemp(suffix='.log', prefix='taskotron-', + dir=conf.tmpdir)[1] + log.warning("Using a temporary log file instead: %s", filelog_path) + # update config options so they correspond to reality + conf.logdir = conf.tmpdir + conf.log_name = os.path.basename(filelog_path) + + global file_handler + rootlogger.removeHandler(file_handler) + file_handler = None + _create_handlers(filelog_path=filelog_path) + _set_level(file_handler, level_file, "log_level_file") + rootlogger.addHandler(file_handler) + log.debug('File logging enabled with level %s into: %s', + logging.getLevelName(file_handler.level), filelog_path) + else: + rootlogger.removeHandler(file_handler) + diff --git a/libtaskotron/runner.py b/libtaskotron/runner.py index ec0197c..b9944b8 100644 --- a/libtaskotron/runner.py +++ b/libtaskotron/runner.py @@ -12,7 +12,7 @@ import imp import copy import collections -from libtaskotron import taskyaml +from libtaskotron import taskformula from libtaskotron import logger from libtaskotron import python_utils from libtaskotron import config @@ -30,15 +30,22 @@ class Runner(object): self.envdata = argdata self.working_data = {} self.directives = {} - self.workdir = workdir or tempfile.mkdtemp(prefix="task-", - dir=config.get_config().tmpdir) + self.workdir = workdir def run(self): self._validate_input() + + if not self.workdir: # create temporary workdir if needed + self.workdir = tempfile.mkdtemp(prefix="task-", + dir=config.get_config().tmpdir) + log.debug("Current workdir: %s", self.workdir) self.envdata['workdir'] = self.workdir self.envdata['checkname'] = self.taskdata['name'] - log.debug("Current workdir: %s", self.workdir) + #override variable values + for var, val in self.envdata['override']: + log.debug("Overriding variable %s, new value: %s", var, val) + self.envdata[var] = eval(val, {}, {}) self.do_actions() @@ -63,8 +70,7 @@ class Runner(object): from :attr:`env_data` and :attr:`working_data`. See :meth:`do_actions` to see how an action looks like. - :param dict action: An action specification parsed from the task yaml - file + :param dict action: An action specification parsed from the task formula :return: a rendered action :rtype: dict ''' @@ -73,7 +79,7 @@ class Runner(object): variables = copy.deepcopy(self.envdata) variables.update(self.working_data) - taskyaml.replace_vars_in_action(rendered_action, variables) + taskformula.replace_vars_in_action(rendered_action, variables) return rendered_action @@ -88,8 +94,7 @@ class Runner(object): '''Execute a single action from the task. See :meth:`do_actions` to see how an action looks like. - :param dict action: An action specification parsed from the task yaml - file + :param dict action: An action specification parsed from the task formula ''' directive_name = self._extract_directive_from_action(action) @@ -108,10 +113,12 @@ class Runner(object): if 'export' in action: self.working_data[action['export']] = output + log.debug("Variable ${%s} was exported with value:\n%s" % + (action['export'], output)) def do_actions(self): '''Sequentially run all actions for a task. An 'action' is a single step - under the ``task:`` key. An example action looks like:: + under the ``actions:`` key. An example action looks like:: - name: download rpms from koji koji: @@ -119,11 +126,11 @@ class Runner(object): koji_build: $koji_build arch: $arch ''' - if 'task' not in self.taskdata or not self.taskdata['task']: + if 'actions' not in self.taskdata or not self.taskdata['actions']: raise TaskotronYamlError("At least one task should be specified" - " in input yaml file") + " in input formula") - for action in self.taskdata['task']: + for action in self.taskdata['actions']: self.do_single_action(action) def _validate_input(self): @@ -166,6 +173,16 @@ def get_argparser(): help="type of --item argument") parser.add_argument("-j", "--jobid", default="-1", help="optional job identifier used to render log urls") + parser.add_argument("-d", "--debug", action="store_true", + help="Enable debug output " + "(set logging level to 'DEBUG')") + parser.add_argument("--override", action="append", default=[], + metavar='VAR=VALUE', + help="override internal variable values used in runner " + "and the task formula. Value itself is evaluated " + "by eval(). This option can be used multiple times. " + "Example: --override \"workdir='/some/dir/'\"") + return parser @@ -186,26 +203,45 @@ def process_args(args): if args['arch'] is None: args['arch'] = ['noarch'] + # using list of tuples instead of dictionary so the args['override'] is + # still list at the end of process_args() - for better testability + override = [] + for var in args['override']: + name = var.split('=')[0] + value = var.split('=')[1] + override.append((name, value)) + args['override'] = override + return args def main(): + # Preliminary initialization of logging, so all messages before regular + # initialization can be logged to stream. + # FIXME: Remove this once this ticket is resolved: + # https://phab.qadevel.cloud.fedoraproject.org/T273 + logger.init_prior_config() + + # parse cmdline parser = get_argparser() args = parser.parse_args() + log.debug('Parsed arguments: %s', args) - logger.init(level=logging.DEBUG, set_rootlogger=True) + # full logging initialization + level_stream = logging.DEBUG if args.debug else None + logger.init(level_stream=level_stream) arg_data = process_args(vars(args)) arg_data['taskfile'] = args.task[0] - task_data = taskyaml.parse_yaml_from_file(arg_data['taskfile']) - + # parse task formula + task_data = taskformula.parse_yaml_from_file(arg_data['taskfile']) if not task_data: raise TaskotronYamlError('Input file should not be empty') + # run the task task_runner = Runner(task_data, arg_data) task_runner.run() - log.info('Check execution finished. Showing stored variables:') - for name, value in task_runner.working_data.items(): - log.info("${%s}:\n%s" % (name, value)) + # finalization + log.info('Check execution finished.') diff --git a/libtaskotron/taskformula.py b/libtaskotron/taskformula.py new file mode 100644 index 0000000..186ffbc --- /dev/null +++ b/libtaskotron/taskformula.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +'''Methods for operating with a task formula''' + +from __future__ import absolute_import +import collections +import string + +import yaml +from libtaskotron import exceptions as exc +from libtaskotron.logger import log + + +def parse_yaml_from_file(filename): + with open(filename, 'r') as datafile: + return parse_yaml(datafile.read()) + + +def parse_yaml(contents): + return yaml.safe_load(contents) + + +def _replace_vars(text, variables): + '''Go through ``text`` and replace all variables (in the form of what is + supported by :class:`string.Template`) with their values, as provided in + ``variables``. This is used for variable expansion in the task formula. + + :param str text: input text where to search for variables + :param dict variables: names (keys) and values of variables to replace + :return: if ``text`` contains just a single variable and nothing else, the + variable value is directly returned (i.e. with matching type, not + cast to ``str``). If ``text`` contains something else as well + (other variables or text), a string is returned. + :raise TaskotronYamlError: if ``text`` contains a variable that is not + present in ``variables``, or if the variable + syntax is incorrect + ''' + + try: + # try to find the first match + match = string.Template.pattern.search(text) + + if not match: + return text + + # There are 4 groups in the pattern: 1 - escaped, 2 - named, 3 - braced, + # 4 - invalid. Group 0 returns the whole match. + if match.group(0) == text and (match.group(2) or match.group(3)): + # We found a single variable and nothing more. We shouldn't return + # a string, but the exact value, so that we don't lose value type. + # This makes it possible to pass lists, dicts, etc as variables. + var_name = match.group(2) or match.group(3) + return variables[var_name] + + # Now it's clear there's also something else in `text` than just a + # single variable. We will replace all variables and return a string + # again. + output = string.Template(text).substitute(variables) + return output + + except KeyError as e: + raise exc.TaskotronYamlError("The task formula includes a variable, " + "but no value has been provided for it: %s" % e) + + except ValueError as e: + raise exc.TaskotronYamlError("The task formula includes an incorrect " + "variable definition. Dollar signs must be doubled if they " + "shouldn't be considered as a variable denotation.\n" + "Error: %s\n" + "Text: %s" % (e, text)) + + +def replace_vars_in_action(action, variables): + '''Find all variables that are leaves (in a tree sense) in an action. Leaves + are variables which can no longer be traversed, i.e. "primitive" types like + ``str`` or ``int``. Non-leaves are containers like ``dict`` or ``list``. + + For all leaves, call :func:`_replace_vars` and update their value with + the function's output. + + :param dict action: An action specification parsed from the task formula. + See :meth:`.Runner.do_actions` to see what an action + looks like. + :param dict variables: names (keys) and values of variables to replace + :raise TaskotronYamlError: if ``text`` contains a variable that is not + present in ``variables``, or if the variable + syntax is incorrect + ''' + + visited = [] # all visited nodes in a tree + stack = [action] # nodes waiting for inspection + + while stack: + vertex = stack.pop() + + if vertex in visited: + continue + + visited.append(vertex) + children = [] # list of tuples (index/key, child_value) + + if isinstance(vertex, collections.MutableMapping): + children = vertex.items() # list of (key, value) + elif isinstance(vertex, collections.MutableSequence): + children = list(enumerate(vertex)) # list of (index, value) + else: + log.warn("Unknown structure '%s' in YAML file, this shouldn't " + "happen: %s", type(vertex), vertex) + + for index, child_val in children: + if isinstance(child_val, basestring): + # leaf node and a string, replace variables + vertex[index] = _replace_vars(child_val, variables) + elif isinstance(child_val, collections.Iterable): + # traversable further down, mark for visit + stack.append(child_val) diff --git a/libtaskotron/taskyaml.py b/libtaskotron/taskyaml.py deleted file mode 100644 index eaf3a17..0000000 --- a/libtaskotron/taskyaml.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2009-2014, Red Hat, Inc. -# License: GPL-2.0+ -# See the LICENSE file for more details on Licensing - -'''Methods for operating with a task YAML file''' - -from __future__ import absolute_import -import collections -import string - -import yaml -from libtaskotron import exceptions as exc -from libtaskotron.logger import log - - -def parse_yaml_from_file(filename): - with open(filename, 'r') as datafile: - return parse_yaml(datafile.read()) - - -def parse_yaml(contents): - return yaml.safe_load(contents) - - -def _replace_vars(text, variables): - '''Go through ``text`` and replace all variables (in the form of what is - supported by :class:`string.Template`) with their values, as provided in - ``variables``. This is used for variable expansion in the task yaml file. - - :param str text: input text where to search for variables - :param dict variables: names (keys) and values of variables to replace - :return: if ``text`` contains just a single variable and nothing else, the - variable value is directly returned (i.e. with matching type, not - cast to ``str``). If ``text`` contains something else as well - (other variables or text), a string is returned. - :raise TaskotronYamlError: if ``text`` contains a variable that is not - present in ``variables``, or if the variable - syntax is incorrect - ''' - - try: - # try to find the first match - match = string.Template.pattern.search(text) - - if not match: - return text - - # There are 4 groups in the pattern: 1 - escaped, 2 - named, 3 - braced, - # 4 - invalid. Group 0 returns the whole match. - if match.group(0) == text and (match.group(2) or match.group(3)): - # We found a single variable and nothing more. We shouldn't return - # a string, but the exact value, so that we don't lose value type. - # This makes it possible to pass lists, dicts, etc as variables. - var_name = match.group(2) or match.group(3) - return variables[var_name] - - # Now it's clear there's also something else in `text` than just a - # single variable. We will replace all variables and return a string - # again. - output = string.Template(text).substitute(variables) - return output - - except KeyError as e: - raise exc.TaskotronYamlError("The task yaml file includes a variable, " - "but no value has been provided for it: %s" % e) - - except ValueError as e: - raise exc.TaskotronYamlError("The task yaml file includes an incorrect " - "variable definition. Dollar signs must be doubled if they " - "shouldn't be considered as a variable denotation.\n" - "Error: %s\n" - "Text: %s" % (e, text)) - - -def replace_vars_in_action(action, variables): - '''Find all variables that are leaves (in a tree sense) in an action. Leaves - are variables which can no longer be traversed, i.e. "primitive" types like - ``str`` or ``int``. Non-leaves are containers like ``dict`` or ``list``. - - For all leaves, call :func:`_replace_vars` and update their value with - the function's output. - - :param dict action: An action specification parsed from the task yaml file. - See :meth:`.Runner.do_actions` to see what an action - looks like. - :param dict variables: names (keys) and values of variables to replace - :raise TaskotronYamlError: if ``text`` contains a variable that is not - present in ``variables``, or if the variable - syntax is incorrect - ''' - - visited = [] # all visited nodes in a tree - stack = [action] # nodes waiting for inspection - - while stack: - vertex = stack.pop() - - if vertex in visited: - continue - - visited.append(vertex) - children = [] # list of tuples (index/key, child_value) - - if isinstance(vertex, collections.MutableMapping): - children = vertex.items() # list of (key, value) - elif isinstance(vertex, collections.MutableSequence): - children = list(enumerate(vertex)) # list of (index, value) - else: - log.warn("Unknown structure '%s' in YAML file, this shouldn't " - "happen: %s", type(vertex), vertex) - - for index, child_val in children: - if isinstance(child_val, basestring): - # leaf node and a string, replace variables - vertex[index] = _replace_vars(child_val, variables) - elif isinstance(child_val, collections.Iterable): - # traversable further down, mark for visit - stack.append(child_val) diff --git a/libtaskotron/yumrepoinfo.py b/libtaskotron/yumrepoinfo.py index f151f0a..d67805f 100644 --- a/libtaskotron/yumrepoinfo.py +++ b/libtaskotron/yumrepoinfo.py @@ -83,14 +83,31 @@ class YumRepoInfo(object): def repos(self): - '''Get the list of all known repository names.''' + '''Get the list of all known repository names. + + :rtype: list of str + ''' return self.parser.sections() def releases(self): - '''Get the list of stable (supported) Fedora releases.''' - return [r for r in self.repos() if self.parser.getboolean(r, - 'supported')] + '''Get the list of stable (supported) Fedora releases. + + :rtype: list of str + ''' + return [r for r in self.repos() if self.get(r, 'release_status').lower() + == "stable"] + + + def branched(self): + '''Get branched Fedora release (or None if it doesn't exist). + + :rtype: str or None + ''' + for r in self.repos(): + if self.get(r, 'release_status').lower() == 'branched': + return r + return None def arches(self, reponame): @@ -181,6 +198,18 @@ class YumRepoInfo(object): repo = parent + def release_status(self, reponame): + '''Return release status of specified repo. For non-top-parent repos, + return release_status of top parent repo. + + :param str reponame: repository name + :return: release status of specified repo, lowercased. One of: + `rawhide`, `branched`, `stable`, `obsolete`. + :rtype: str + ''' + return self.get(self.top_parent(reponame), 'release_status').lower() + + def _read(self): '''Read first available config file from the list of provided config files.''' diff --git a/requirements.txt b/requirements.txt index 8d4e310..a41dc48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ PyYAML==3.10 dingus==0.3.4 -py>=1.4.18 +py>=1.4.22 pyaml==13.07.1 pytest>=2.4.2 pytest-cov>=1.6 bayeux>=0.9 -yamlish>=0.12 -pytap13>=0.1.0 +yamlish>=0.14 +pytap13>=0.3.0 resultsdb_api>=1.0.1 python-fedora>=0.3.33 bunch>=1.0.1 diff --git a/testing/conftest.py b/testing/conftest.py index bbd81cd..f95a558 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -8,10 +8,10 @@ Read more at: http://pytest.org/latest/plugins.html#conftest-py-plugins''' import os import logging -import tempfile -import libtaskotron.config import libtaskotron.logger +import libtaskotron.config +import libtaskotron.config_defaults def pytest_addoption(parser): @@ -44,14 +44,11 @@ def pytest_configure(config): os.environ[libtaskotron.config.PROFILE_VAR] = \ libtaskotron.config.ProfileName.TESTING - # create temporary directories instead of using real system directories - testdir = tempfile.mkdtemp(prefix='taskotron-test-') - libtaskotron.config_defaults.Config.tmpdir = testdir + '/tmpdir' - libtaskotron.config_defaults.Config.logdir = testdir + '/logdir' - # enable debug logging for taskotron # This will allow us to use libtaskotron-logged output for failed tests. # You can also use "py.test -s" option to see all output even for passed # tests (of course, it's better to run the suite with just a single file # rather than the whole directory in this case) - libtaskotron.logger.init(level=logging.DEBUG) + libtaskotron.logger.init_prior_config() + # and this will show debug messages from many other libraries as well + logging.getLogger().setLevel(logging.NOTSET) diff --git a/testing/functest_config.py b/testing/functest_config.py index 7e5e19b..dade823 100644 --- a/testing/functest_config.py +++ b/testing/functest_config.py @@ -49,6 +49,10 @@ class TestConfig(object): # reset the singleton instance back config._config = None + def setup_method(self, method): + '''Run this before every test method execution start''' + # make sure singleton instance is empty + config._config = None def test_devel_profile_empty_config(self, tmpconffile): diff --git a/testing/functest_koji_utils.py b/testing/functest_koji_utils.py new file mode 100644 index 0000000..72d868d --- /dev/null +++ b/testing/functest_koji_utils.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +'''Functional tests for libtaskotron/koji_utils.py''' + +from libtaskotron import koji_utils + +from dingus import Dingus + + +class TestKojiClient(): + + def setup_method(self, method): + self.ref_nvr = 'foo-1.2-3.fc99' + self.ref_arch = 'noarch' + self.ref_name = 'foo' + self.ref_version = '1.2' + self.ref_release = '3.fc99' + self.ref_buildid = 123456 + + self.ref_build = {'package_name': self.ref_name, + 'version': self.ref_version, + 'release': self.ref_release, + 'id': self.ref_buildid} + + def test_handle_norpms_in_build(self, tmpdir): + """ This tests to make sure that missing rpms in a build are handled + gracefully during download so that execution isn't stopped when a build + is missing an rpm """ + + rpmdir = tmpdir.mkdir("rpmdownload") + stub_koji = Dingus(getBuild__returns = self.ref_build) + test_koji = koji_utils.KojiClient(stub_koji) + + test_koji.get_nvr_rpms(self.ref_nvr, str(rpmdir), arches=[self.ref_arch]) diff --git a/testing/functest_logger.py b/testing/functest_logger.py new file mode 100644 index 0000000..fb5d723 --- /dev/null +++ b/testing/functest_logger.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +import os +import logging +import pytest + +from libtaskotron import logger +from libtaskotron import config + + +class TestLogger(): + + def setup_method(self, method): + '''Run before every method''' + # remember the list of root handlers + self.root_handlers = logging.getLogger().handlers + # remember the level of the stream handler + self.stream_level = (logger.stream_handler.level if logger.stream_handler + else None) + + def teardown_method(self, method): + '''Run after every method''' + # reset the list of root handlers + logging.getLogger().handlers = self.root_handlers + # reset the stream handler level + if logger.stream_handler: + logger.stream_handler.level = self.stream_level + + + def test_logfile(self, tmpdir): + '''Messages should be logged to file when enabled''' + log_path = tmpdir.join('test.log').strpath + logger.init(filelog=True, filelog_path=log_path) + + msg = 'This should appear in the log file' + logger.log.debug(msg) + logger.file_handler.flush() + + with open(log_path) as log_file: + lines = log_file.readlines() + + assert msg in lines[-1] + + def msg_in_tmp_log(self, msg, bad_log_path): + '''A helper method to ensure a message has been logged to a temporary + log file and not into (somehow inaccessible) requested log file. The + temporary log file location is extracted from the config file. + :param str msg: message to search for in a log file + :param str bad_log_path: path to a requested log file which must not be + possible to open + :raise AssertError: if something is wrong + ''' + # the temporary file must be somewhere else + conf = config.get_config() + tmp_log_path = os.path.join(conf.logdir, conf.log_name) + assert tmp_log_path != bad_log_path + + # the inaccessible log file must not be possible to open + with pytest.raises(IOError): + open(bad_log_path) + + # the message must be in the temporary log file + with open(tmp_log_path) as tmp_log: + lines = tmp_log.readlines() + assert msg in lines[-1] + + def test_logfile_no_write(self, tmpdir): + '''If file log is not writeable, a temporary file should be used''' + log_file = tmpdir.join('test.log') + log_path = log_file.strpath + log_file.write('') + # make the file inaccessible for writing + os.chmod(log_path, 0) + + logger.init(filelog=True, filelog_path=log_path) + msg = 'This should be in a temporary log file' + logger.log.debug(msg) + logger.file_handler.flush() + + self.msg_in_tmp_log(msg, log_path) + + def test_logfile_missing_dir(self, tmpdir): + '''If file log is not writeable, a temporary file should be used''' + log_path = tmpdir.join('missing_dir', 'test.log').strpath + + logger.init(filelog=True, filelog_path=log_path) + msg = 'This should be in a temporary log file' + logger.log.debug(msg) + logger.file_handler.flush() + + self.msg_in_tmp_log(msg, log_path) diff --git a/testing/functest_mash_directive.py b/testing/functest_mash_directive.py index 6e625fa..f3be5d2 100644 --- a/testing/functest_mash_directive.py +++ b/testing/functest_mash_directive.py @@ -4,17 +4,15 @@ # See the LICENSE file for more details on Licensing from libtaskotron.directives import mash_directive -from libtaskotron.exceptions import TaskotronDirectiveError import os import shutil import pytest -from tempfile import mkdtemp -from rpmfluff import SimpleRpmBuild -from operator import and_ +import rpmfluff class TestMashDirective(object): - def setup_method(self, method): + @pytest.fixture(autouse=True) + def setup_method(self, tmpdir): NVRAs = [ ["wine", "0.1", "3", ["i686", "x86_64"]], ["foo", "0.2", "2", ["i686"]], @@ -23,7 +21,7 @@ class TestMashDirective(object): #store current dir startdir = os.getcwd() - workdir = mkdtemp() + workdir = tmpdir.strpath self.rpmdir = os.path.join(workdir, "rpmdir") print self.rpmdir @@ -31,7 +29,7 @@ class TestMashDirective(object): os.chdir(workdir) os.mkdir(self.rpmdir) - packages = [SimpleRpmBuild(*NVRA) for NVRA in NVRAs] + packages = [rpmfluff.SimpleRpmBuild(*NVRA) for NVRA in NVRAs] for package in packages: package.make() @@ -73,6 +71,21 @@ class TestMashDirective(object): assert self.rpms[0] in before and self.rpms[0] in after and \ self.rpms[1] in before and self.rpms[1] not in after + def test_outdir_is_created(self, tmpdir): + outdir_path = tmpdir.join('somewhere/along/the/highway').strpath + input_data = { + 'rpmdir': self.rpmdir, + 'outdir': outdir_path, + 'arch': ['i386'] + } + + directive = mash_directive.MashDirective() + ret_path = directive.process(input_data, {}) + + assert ret_path == outdir_path + assert os.path.isdir(ret_path) + + def test_repodata_in_rpmdir(self): input_data = { 'rpmdir': self.rpmdir, @@ -84,8 +97,8 @@ class TestMashDirective(object): assert 'repodata' in after - def test_repodata_in_outdir(self): - outdir = mkdtemp() + def test_repodata_in_outdir(self, tmpdir): + outdir = tmpdir.strpath input_data = { 'rpmdir': self.rpmdir, 'outdir': outdir, @@ -96,4 +109,3 @@ class TestMashDirective(object): after = os.listdir(self.rpmdir) assert 'repodata' not in after and 'repodata' in os.listdir(outdir) - diff --git a/testing/functest_runner.py b/testing/functest_runner.py new file mode 100644 index 0000000..3008236 --- /dev/null +++ b/testing/functest_runner.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +import sys +import pytest +import logging + +from libtaskotron import runner +from libtaskotron import logger + +class TestRunnerMain(): + '''Test main()''' + + def test_debug(self, tmpdir, monkeypatch, capfd): + '''Test whether --debug cmdline option works''' + fake_recipe_path = tmpdir.join('nonexistent.yml').strpath + fake_cmdline = ['runner.py', '--debug', fake_recipe_path] + monkeypatch.setattr(sys, 'argv', fake_cmdline) + + with pytest.raises(IOError): + # an error should be raised, because the yaml file does not exist + runner.main() + + # even though the error was raised, the debug level should already be set + assert logger.stream_handler.level == logging.DEBUG + + # and debug messages should be captured + msg = "testing debug logging with --debug cmdline option" + out, err = capfd.readouterr() + logger.log.debug(msg) + out, err = capfd.readouterr() + assert msg in err diff --git a/testing/test_directive_modules.py b/testing/test_directive_modules.py new file mode 100644 index 0000000..e292e7d --- /dev/null +++ b/testing/test_directive_modules.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + + +import pytest +import imp +import os +from glob import glob + +from libtaskotron import directives + + + +class TestDirectives(): + @classmethod + def setup_class(cls): + cls.directive_modules = [] + + # find all directive modules in libtaskotron.directives directory + for filepath in glob(os.path.join(os.path.dirname(directives.__file__), + '*.py')): + if 'directive' in os.path.basename(filepath): + real_name = os.path.basename(filepath)[:-len('.py')] + cls.directive_modules.append((real_name, filepath)) + + def test_directive_class_var_exists(self): + """Every directive module must contain a global variable named + `directive_class` that contains classname of directive class + """ + for module in self.directive_modules: + loaded = imp.load_source(*module) + class_name = getattr(loaded, 'directive_class') + instance = getattr(loaded, class_name)() + + assert class_name == instance.__class__.__name__ + + def test_directive_class_inherits_from_base(self): + for module in self.directive_modules: + loaded = imp.load_source(*module) + class_name = getattr(loaded, 'directive_class') + directive_class = getattr(loaded, class_name) + + assert issubclass(directive_class, directives.BaseDirective) + + def test_directive_class_process_method_exists(self): + for module in self.directive_modules: + loaded = imp.load_source(*module) + class_name = getattr(loaded, 'directive_class') + instance = getattr(loaded, class_name)() + + process_method = getattr(instance, "process", None) + + assert process_method is not None + assert hasattr(process_method, '__call__') \ No newline at end of file diff --git a/testing/test_koji_directive.py b/testing/test_koji_directive.py index d9f7e91..a847be1 100644 --- a/testing/test_koji_directive.py +++ b/testing/test_koji_directive.py @@ -130,14 +130,14 @@ class TestKojiDirective(): test_helper.process(ref_input, ref_envdata) getrpm_calls = stub_koji.calls() - requested_arches = getrpm_calls[0][1][2] + requested_arches = getrpm_calls[0][2]['arches'] assert len(getrpm_calls) == 1 assert 'x86_64' in requested_arches assert 'noarch' in requested_arches def test__download_command_src(self): - ref_input = {'action': 'download', 'arch': ['src'], + ref_input = {'action': 'download', 'arch': [], 'src': True, 'koji_build': self.ref_nvr} ref_envdata = {'workdir': '/var/tmp/foo'} @@ -150,5 +150,5 @@ class TestKojiDirective(): requested_src = getrpm_calls[0][2]['src'] assert len(getrpm_calls) == 1 - assert ['src'] == requested_arches + assert [] == requested_arches assert requested_src diff --git a/testing/test_koji_utils.py b/testing/test_koji_utils.py index 35e3650..3779d51 100644 --- a/testing/test_koji_utils.py +++ b/testing/test_koji_utils.py @@ -12,6 +12,21 @@ from libtaskotron import koji_utils from libtaskotron import exceptions as exc from libtaskotron import config + +# http://stackoverflow.com/questions/3190706/nonlocal-keyword-in-python-2-x +def create_multicall(first, second): + first_call = {"value": True} + + def multicall(): + if first_call["value"]: + first_call["value"] = False + return first + else: + return second + + return multicall + + class TestKojiClient(): def setup_method(self, method): self.ref_nvr = 'foo-1.2-3.fc99' @@ -34,36 +49,56 @@ class TestKojiClient(): 'release': self.ref_release, 'nvr': self.ref_nvr, 'arch': 'src', 'build_id': self.ref_buildid, }] - def test_rpm_to_build(self): - stub_koji = Dingus(getBuild__returns = self.ref_build, - listRPMs__returns = self.ref_rpms, - getRPM__returns = self.ref_rpms[0]) + def test_rpms_to_build(self): + stub_koji = Dingus(getBuild__returns=None, + getRPM__returns=None, + multiCall=create_multicall( + [[self.ref_rpms[0]], [self.ref_rpms[1]]], + [[self.ref_build], [self.ref_build]])) test_koji = koji_utils.KojiClient(stub_koji) - outcome = test_koji.rpm_to_build(self.ref_filename) + outcome = test_koji.rpms_to_build([self.ref_filename,self.ref_filename]) - assert outcome is self.ref_build + assert outcome == [self.ref_build, self.ref_build] + # because two rpms come from same build, it gets called twice for each + # rpm, once for build assert len(stub_koji.calls) == 3 - # test prefetch - outcome = test_koji.rpm_to_build(self.ref_filename) - assert len(stub_koji.calls) == 3 - def test_rpm_to_build_exceptions(self): - stub_koji = Dingus(getRPM__returns = None) + def test_rpms_to_build_exceptions(self): + stub_koji = Dingus(getRPM__returns=None, + multiCall__returns=[{"faultCode": -1, + "faultString": "failed"}]) test_koji = koji_utils.KojiClient(stub_koji) with pytest.raises(exc.TaskotronRemoteError): - test_koji.rpm_to_build(self.ref_filename) + test_koji.rpms_to_build([self.ref_filename]) + + stub_koji = Dingus(getBuild__returns=None, + getRPM__returns=None, + multiCall=create_multicall([[self.ref_rpms[0]]], [ + {"faultCode": -1, "faultString": "failed"}])) + test_koji = koji_utils.KojiClient(stub_koji) + with pytest.raises(exc.TaskotronRemoteError): + test_koji.rpms_to_build([self.ref_filename]) - stub_koji = Dingus(getBuild__returns = None, - listRPMs__returns = self.ref_rpms, - getRPM__returns = self.ref_rpms[0]) + stub_koji = Dingus(getBuild__returns=None, + getRPM__returns=None, + multiCall__returns=[[None]]) test_koji = koji_utils.KojiClient(stub_koji) with pytest.raises(exc.TaskotronRemoteError): - test_koji.rpm_to_build(self.ref_filename) + test_koji.rpms_to_build([self.ref_filename]) + + stub_koji = Dingus(getBuild__returns=None, + getRPM__returns=None, + multiCall=create_multicall([[self.ref_rpms[0]]], + [[None]])) + + test_koji = koji_utils.KojiClient(stub_koji) + with pytest.raises(exc.TaskotronRemoteError): + test_koji.rpms_to_build([self.ref_filename]) def test_get_noarch_rpmurls_from_nvr(self): @@ -105,26 +140,46 @@ class TestKojiClient(): assert 'i686' in requested_arches assert 'i386' in requested_arches - def should_throw_exception_norpms(self): + def should_not_throw_exception_norpms(self): + '''It's possible to have no RPMs (for the given arch) in a build''' stub_koji = Dingus(getBuild__returns = self.ref_build) + test_koji = koji_utils.KojiClient(stub_koji) + test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch]) + + def test_nvr_to_urls_src_filtering(self): + stub_koji = Dingus(getBuild__returns=self.ref_build, + listRPMs__returns=self.ref_rpms) test_koji = koji_utils.KojiClient(stub_koji) - with pytest.raises(exc.TaskotronRemoteError): - test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch]) + # arch and src enabled + urls = test_koji.nvr_to_urls(self.ref_nvr, arches=[self.ref_arch]) + assert len(urls) == 2 + assert 'src' in urls[0] or 'src' in urls[1] - def test_handle_norpms_in_build(self, tmpdir): - """ This tests to make sure that missing rpms in a build are handled - gracefully during download so that execution isn't stopped when a build - is missing an rpm """ + # arch enabled, src disabled + urls = test_koji.nvr_to_urls(self.ref_nvr, arches=[self.ref_arch], + src=False) + assert len(urls) == 1 + assert 'src' not in urls[0] - rpmdir = tmpdir.mkdir("rpmdownload") + def test_nvr_to_urls_only_src(self): + stub_koji = Dingus(getBuild__returns=self.ref_build) + test_koji = koji_utils.KojiClient(stub_koji) - stub_koji = Dingus(getBuild__returns = self.ref_build) + # arch disabled, src enabled + test_koji.nvr_to_urls(self.ref_nvr, arches=[], src=True) + listrpm_calls = stub_koji.calls('listRPMs') + requested_arches = listrpm_calls[0][2]['arches'] + assert len(requested_arches) == 1 + assert requested_arches[0] == 'src' + def should_raise_no_arch_no_src(self): + stub_koji = Dingus() test_koji = koji_utils.KojiClient(stub_koji) - test_koji.get_nvr_rpms(self.ref_nvr, str(rpmdir), arches = [self.ref_arch]) + with pytest.raises(exc.TaskotronValueError): + test_koji.nvr_to_urls(self.ref_nvr, arches=[], src=False) class TestGetENVR(object): diff --git a/testing/test_logger.py b/testing/test_logger.py new file mode 100644 index 0000000..df0d003 --- /dev/null +++ b/testing/test_logger.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +import logging + +from libtaskotron import logger +from libtaskotron import config + +class TestLogger(): + + def setup_method(self, method): + '''Run before every method''' + # remember the list of root handlers + self.root_handlers = logging.getLogger().handlers + # remember the level of the stream handler + self.stream_level = (logger.stream_handler.level if logger.stream_handler + else None) + + def teardown_method(self, method): + '''Run after every method''' + # reset the list of root handlers + logging.getLogger().handlers = self.root_handlers + # reset the stream handler level + if logger.stream_handler: + logger.stream_handler.level = self.stream_level + + + def test_logging_messages(self, capfd): + '''Basic message logging functionality''' + msg = 'Quo vadis?' + logger.log.info(msg) + out, err = capfd.readouterr() + assert not out + assert msg in err + + def test_levels(self, monkeypatch, capfd): + '''Messages with lower level than set should be ignored, others included''' + monkeypatch.setattr(logger.log, 'level', logging.WARN) + + # ignore this message + msg = 'Quo vadis?' + logger.log.info(msg) + out, err = capfd.readouterr() + assert not out + assert not err + + # include this message + msg = 'Whither goest thou?' + logger.log.warn(msg) + out, err = capfd.readouterr() + assert not out + assert msg in err + + def test_no_syslog_filelog(self): + '''Syslog and filelog should be disabled under testing profile''' + logger.init() + root = logging.getLogger() + + msg = ('If this failed, it means we need either set syslog=False and ' + 'filelog=False as defaults in logger.init(), or we need to ' + 'introduce new variables in Config to make sure syslog and ' + 'filelog are disabled during the test suite.') + + if logger.syslog_handler is not None: + assert logger.syslog_handler not in root.handlers, msg + + if logger.file_handler is not None: + assert logger.file_handler not in root.handlers, msg + + def test_override_level(self, monkeypatch): + '''A log level from config file should be correctly applied''' + conf = config.get_config() + monkeypatch.setattr(conf, 'log_level_stream', 'CRITICAL') + logger.init() + assert logging.getLevelName(logger.stream_handler.level) == 'CRITICAL' + + def test_invalid_level(self, monkeypatch): + '''Invalid log level in config file should be reverted to default''' + conf = config.get_config() + default_level = conf.log_level_stream + + monkeypatch.setattr(conf, 'log_level_stream', 'INVALID') + logger.init() + assert logging.getLevelName(logger.stream_handler.level) != 'INVALID' + assert logging.getLevelName(logger.stream_handler.level) == default_level diff --git a/testing/test_python_directive.py b/testing/test_python_directive.py index 18c2a44..6a6b784 100644 --- a/testing/test_python_directive.py +++ b/testing/test_python_directive.py @@ -57,14 +57,15 @@ class TestExecutePyfile(object): ref_kwargs = {'baz': 'why', 'blah': 2} stub_getattr = Dingus(return_value=lambda baz, blah: "str") + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) assert (stub_getattr.calls()[0].name, stub_getattr.calls()[0].args) == ( - '()', (ref_modulename, ref_methodname)) + '()', (stub_module, ref_methodname)) def test_task_method_is_executed(self, monkeypatch): ref_modulename = 'foo' @@ -73,11 +74,12 @@ class TestExecutePyfile(object): stub_task = Dingus(return_value="str") stub_getattr = Dingus(return_value=stub_task) + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) assert (stub_task.calls()[0].name, stub_task.calls()[0].args, stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) @@ -92,11 +94,12 @@ class TestPyfileOutputException(object): stub_task = Dingus(return_value=u"unicode") stub_getattr = Dingus(return_value=stub_task) + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) assert (stub_task.calls()[0].name, stub_task.calls()[0].args, stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) @@ -108,11 +111,12 @@ class TestPyfileOutputException(object): stub_task = Dingus(return_value="") stub_getattr = Dingus(return_value=stub_task) + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) assert (stub_task.calls()[0].name, stub_task.calls()[0].args, stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) @@ -124,11 +128,12 @@ class TestPyfileOutputException(object): stub_task = Dingus(return_value=None) stub_getattr = Dingus(return_value=stub_task) + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) assert (stub_task.calls()[0].name, stub_task.calls()[0].args, stub_task.calls()[0].kwargs) == ('()', (), ref_kwargs) @@ -140,12 +145,13 @@ class TestPyfileOutputException(object): stub_task = Dingus(return_value=[]) stub_getattr = Dingus(return_value=stub_task) + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) with pytest.raises(TaskotronDirectiveError): - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) def test_task_rises_on_wrong_type(self, monkeypatch): ref_modulename = 'foo' @@ -154,12 +160,13 @@ class TestPyfileOutputException(object): stub_task = Dingus(return_value={'han': 'shot', 'first': True}) stub_getattr = Dingus(return_value=stub_task) + stub_module = Dingus(__file__=ref_modulename) test_directive = python_directive.PythonDirective() monkeypatch.setattr(test_directive, '_do_getattr', stub_getattr) with pytest.raises(TaskotronDirectiveError): - test_directive.execute(ref_modulename, ref_methodname, ref_kwargs) + test_directive.execute(stub_module, ref_methodname, ref_kwargs) class TestProcess(object): diff --git a/testing/test_runner.py b/testing/test_runner.py index 51ef3c1..df8ae43 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -7,6 +7,7 @@ import pytest import os import sys import copy +from dingus import Dingus from libtaskotron import runner from libtaskotron import exceptions as exc @@ -225,7 +226,7 @@ class TestRunnerDoActions(): def test_runner_do_single_dummy_pass(self): ref_action = {'name': 'Dummy Action', 'dummy': {'result': 'pass'}} - self.ref_taskdata = {'task': [ref_action]} + self.ref_taskdata = {'actions': [ref_action]} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) @@ -234,7 +235,7 @@ class TestRunnerDoActions(): def test_runner_do_single_dummy_fail(self): ref_action = {'name': 'Dummy Action', 'dummy': {'result': 'fail'}} - self.ref_taskdata = {'task': [ref_action]} + self.ref_taskdata = {'actions': [ref_action]} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) @@ -244,7 +245,7 @@ class TestRunnerDoActions(): def test_runner_do_multiple_dummy_fail(self): ref_action = {'name': 'Dummy Action', 'dummy': {'result': 'fail'}} - self.ref_taskdata = {'task': [ref_action, ref_action]} + self.ref_taskdata = {'actions': [ref_action, ref_action]} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) @@ -260,7 +261,7 @@ class TestRunnerDoActions(): 'msg': '${%s}' % ref_messagename}, 'export': '%s' % ref_exportname} - self.ref_taskdata = {'task': [ref_action, ref_action]} + self.ref_taskdata = {'actions': [ref_action, ref_action]} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) test_runner.working_data[ref_messagename] = ref_message @@ -270,7 +271,7 @@ class TestRunnerDoActions(): assert test_runner.working_data[ref_exportname] == ref_message def test_runner_fails_missing_action(self): - self.ref_taskdata = {'task': None} + self.ref_taskdata = {'actions': None} test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata) @@ -343,7 +344,8 @@ class TestRunnerProcessArgs(): ref_input = { 'arch': ['x86_64'], 'item': 'foo-1.2-3.fc99', 'type': 'koji_build', - 'task': 'sometask.yml'} + 'task': 'sometask.yml', + 'override': []} ref_args = copy.deepcopy(ref_input) ref_args['koji_build'] = 'foo-1.2-3.fc99' @@ -356,7 +358,8 @@ class TestRunnerProcessArgs(): ref_input = { 'arch': ['x86_64'], 'item': 'foo-1.2-3.fc99', 'type': 'bodhi_id', - 'task': 'sometask.yml'} + 'task': 'sometask.yml', + 'override': []} ref_args = copy.deepcopy(ref_input) ref_args['bodhi_id'] = 'foo-1.2-3.fc99' @@ -369,7 +372,8 @@ class TestRunnerProcessArgs(): ref_input = { 'arch': ['x86_64'], 'item': 'dist-fc99-updates', 'type': 'koji_tag', - 'task': 'sometask.yml'} + 'task': 'sometask.yml', + 'override': []} ref_args = copy.deepcopy(ref_input) ref_args['koji_tag'] = 'dist-fc99-updates' @@ -382,7 +386,8 @@ class TestRunnerProcessArgs(): ref_input = { 'arch': ['x86_64', 'i386', 'noarch'], 'item': 'dist-fc99-updates', 'type': 'koji_tag', - 'task': 'sometask.yml'} + 'task': 'sometask.yml', + 'override': []} ref_args = copy.deepcopy(ref_input) ref_args['koji_tag'] = 'dist-fc99-updates' @@ -393,10 +398,41 @@ class TestRunnerProcessArgs(): def test_process_args_no_arch(self): ref_input = { 'type': 'invalid', - 'arch': None} + 'arch': None, + 'override': []} ref_args = copy.deepcopy(ref_input) ref_args['arch'] = ['noarch'] test_args = runner.process_args(ref_input) assert test_args == ref_args + +class TestRunnerCheckOverride(): + + def test_override_existing(self): + ref_input = { 'arch': ['x86_64'], + 'type': 'koji_build', + 'item': 'foo-1.2-3.fc99', + 'override': ["workdir='/some/dir/'"]} + + ref_inputdata = runner.process_args(ref_input) + + test_runner = runner.Runner(Dingus(), ref_inputdata) + test_runner.do_actions = lambda: None + test_runner.run() + + assert test_runner.envdata['workdir'] == "/some/dir/" + + def test_override_nonexisting(self): + ref_input = { 'arch': ['x86_64'], + 'type': 'koji_build', + 'item': 'foo-1.2-3.fc99', + 'override': ["friendship='is magic'"]} + + ref_inputdata = runner.process_args(ref_input) + + test_runner = runner.Runner(Dingus(), ref_inputdata) + test_runner.do_actions = lambda: None + test_runner.run() + + assert test_runner.envdata['friendship'] == "is magic" diff --git a/testing/test_taskformula.py b/testing/test_taskformula.py new file mode 100644 index 0000000..19dc5e3 --- /dev/null +++ b/testing/test_taskformula.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# Copyright 2009-2014, Red Hat, Inc. +# License: GPL-2.0+ +# See the LICENSE file for more details on Licensing + +'''Unit tests for libtaskotron/taskformula.py''' + +import copy + +import yaml +import pytest +from libtaskotron import taskformula +import libtaskotron.exceptions as exc + + +ref_taskname = 'testtask' +ref_taskdesc = 'this is a test task' +ref_maintainer = 'testuser' +ref_deps = ['libtaskotron', 'libpanacea'] +ref_input = {'args': 'arch, koji_build'} +ref_actions = [] + + +def create_yamldata(name=ref_taskname, desc=ref_taskdesc, maint=ref_maintainer, + deps=ref_deps, input_=ref_input, actions=ref_actions): + return {'name': name, + 'desc': desc, + 'maintainer': maint, + 'deps': deps, + 'input': input_, + 'actions': actions} + + +class TestTaskYamlMetadata(object): + def setup_method(self, method): + self.taskdata = create_yamldata() + self.test_task = taskformula.parse_yaml(yaml.safe_dump(self.taskdata)) + + def test_taskyaml_taskname(self): + assert self.test_task['name'] == self.taskdata['name'] + + def test_taskyaml_taskdesc(self): + assert self.test_task['desc'] == self.taskdata['desc'] + + def test_taskyaml_taskmaintainer(self): + assert self.test_task['maintainer'] == self.taskdata['maintainer'] + + def test_taskyaml_taskdeps(self): + assert self.test_task['deps'] == self.taskdata['deps'] + + def test_taskyaml_taskinput(self): + assert self.test_task['input'] == self.taskdata['input'] + + +class TestTaskYamlActionsNoVariables(object): + def test_taskyaml_simple_action(self): + ref_action = {'name': 'testaction', 'action': {'key': 'value'}} + taskdata = create_yamldata(actions=[ref_action]) + + test_task = taskformula.parse_yaml(yaml.safe_dump(taskdata)) + + assert test_task['actions'][0] == ref_action + + def test_taskyaml_multiple_actions(self): + ref_action = {'name': 'testaction', 'action': {'key': 'value'}} + taskdata = create_yamldata(actions=[ref_action, ref_action, ref_action]) + + test_task = taskformula.parse_yaml(yaml.safe_dump(taskdata)) + + assert len(test_task['actions']) == 3 + + +class TestReplaceVarsInAction(object): + + def setup_method(self, method): + self.action = { + 'name': 'run foo${foo}', + 'export': '${export}', + 'python': { + 'file': 'somefile.py', + 'args': ['arg1', '${number}', 'arg2', '${list}', '${dict}'], + 'cmdline': '-s ${foo} -i ${number}', + 'nest': { + 'deepnest': '${dict}', + 'null': '${null}', + } + } + } + self.vars = { + 'foo': 'bar', + 'export': 'a:b\nc:d\n-e', + 'number': 1, + 'list': [1, 'item'], + 'dict': {'key': 'val', 'pi': 3.14}, + 'null': None, + 'extra': 'juicy', + } + + def test_complex(self): + rend_action = copy.deepcopy(self.action) + rend_action['name'] = 'run foo%s' % self.vars['foo'] + rend_action['export'] = self.vars['export'] + rend_action['python']['args'] = ['arg1', self.vars['number'], 'arg2', + self.vars['list'], self.vars['dict']] + rend_action['python']['cmdline'] = '-s %s -i %s' % (self.vars['foo'], + self.vars['number']) + rend_action['python']['nest']['deepnest'] = self.vars['dict'] + rend_action['python']['nest']['null'] = self.vars['null'] + + taskformula.replace_vars_in_action(self.action, self.vars) + assert self.action == rend_action + + def test_raise(self): + with pytest.raises(exc.TaskotronYamlError): + taskformula.replace_vars_in_action(self.action, {}) + + del self.vars['number'] + with pytest.raises(exc.TaskotronYamlError): + taskformula.replace_vars_in_action(self.action, self.vars) + + +class TestReplaceVars(object): + + def test_standadlone_var_str(self): + res = taskformula._replace_vars('${foo}', {'foo': 'hedgehog'}) + assert res == 'hedgehog' + + def test_standalone_var_nonstr(self): + res = taskformula._replace_vars('${foo}', {'foo': ['chicken', 'egg']}) + assert res == ['chicken', 'egg'] + + def test_standadlone_var_str_alternative(self): + res = taskformula._replace_vars('$foo', {'foo': 'hedgehog'}) + assert res == 'hedgehog' + + def test_standalone_var_nonstr_alternative(self): + res = taskformula._replace_vars('$foo', {'foo': ['chicken', 'egg']}) + assert res == ['chicken', 'egg'] + + def test_multivar(self): + res = taskformula._replace_vars('a ${b} c${d}', {'b': 'bb', 'd': 'dd'}) + assert res == 'a bb cdd' + + def test_empty_space(self): + '''Some empty space around a single variable is like multi-var, i.e. + a string is always returned''' + res = taskformula._replace_vars(' ${foo}', {'foo': 'bar'}) + assert res == ' bar' + + res = taskformula._replace_vars(' ${foo}', {'foo': 42}) + assert res == ' 42' + + def test_recursive_content(self): + '''The content of a variable contains more variable names. But those + must not be expanded.''' + text = '${foo} ${bar}' + varz = {'foo': '${foo} ${bar}', + 'bar': '${foo} ${bar} ${baz}' + } + res = taskformula._replace_vars(text, varz) + assert res == '%s %s' % (varz['foo'], varz['bar']) + + def test_empty(self): + res = taskformula._replace_vars('', {'extra': 'var'}) + assert res == '' + + def test_same_var(self): + '''Only variables must get replaced, not same-sounding text''' + res = taskformula._replace_vars('foo ${foo}', {'foo': 'bar'}) + assert res == 'foo bar' + + def test_combined_syntax(self): + '''Both $foo and ${foo} syntax must work, together with escaping''' + res = taskformula._replace_vars('foo $foo ${foo} $$foo', {'foo': 'bar'}) + assert res == 'foo bar bar $foo' + + def test_raise_missing_var(self): + with pytest.raises(exc.TaskotronYamlError): + taskformula._replace_vars('${foo}', {'bar': 'foo'}) + + def test_raise_missing_vars(self): + with pytest.raises(exc.TaskotronYamlError): + taskformula._replace_vars('${foo} ${bar}', {'bar': 'foo'}) + + def test_raise_bad_escape(self): + '''Dollars must be doubled''' + with pytest.raises(exc.TaskotronYamlError): + taskformula._replace_vars('${foo} $', {'foo': 'bar'}) + diff --git a/testing/test_taskyaml.py b/testing/test_taskyaml.py deleted file mode 100644 index 60ca9ce..0000000 --- a/testing/test_taskyaml.py +++ /dev/null @@ -1,189 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2009-2014, Red Hat, Inc. -# License: GPL-2.0+ -# See the LICENSE file for more details on Licensing - -'''Unit tests for libtaskotron/taskyaml.py''' - -import copy - -import yaml -import pytest -from libtaskotron import taskyaml -import libtaskotron.exceptions as exc - - -ref_taskname = 'testtask' -ref_taskdesc = 'this is a test task' -ref_maintainer = 'testuser' -ref_deps = ['libtaskotron', 'libpanacea'] -ref_input = {'args': 'arch, koji_build'} -ref_actions = [] - - -def create_yamldata(name=ref_taskname, desc=ref_taskdesc, maint=ref_maintainer, - deps=ref_deps, input_=ref_input, actions=ref_actions): - return {'name': name, - 'desc': desc, - 'maintainer': maint, - 'deps': deps, - 'input': input_, - 'task': actions} - - -class TestTaskYamlMetadata(object): - def setup_method(self, method): - self.taskdata = create_yamldata() - self.test_task = taskyaml.parse_yaml(yaml.safe_dump(self.taskdata)) - - def test_taskyaml_taskname(self): - assert self.test_task['name'] == self.taskdata['name'] - - def test_taskyaml_taskdesc(self): - assert self.test_task['desc'] == self.taskdata['desc'] - - def test_taskyaml_taskmaintainer(self): - assert self.test_task['maintainer'] == self.taskdata['maintainer'] - - def test_taskyaml_taskdeps(self): - assert self.test_task['deps'] == self.taskdata['deps'] - - def test_taskyaml_taskinput(self): - assert self.test_task['input'] == self.taskdata['input'] - - -class TestTaskYamlActionsNoVariables(object): - def test_taskyaml_simple_action(self): - ref_action = {'name': 'testaction', 'action': {'key': 'value'}} - taskdata = create_yamldata(actions=[ref_action]) - - test_task = taskyaml.parse_yaml(yaml.safe_dump(taskdata)) - - assert test_task['task'][0] == ref_action - - def test_taskyaml_multiple_actions(self): - ref_action = {'name': 'testaction', 'action': {'key': 'value'}} - taskdata = create_yamldata(actions=[ref_action, ref_action, ref_action]) - - test_task = taskyaml.parse_yaml(yaml.safe_dump(taskdata)) - - assert len(test_task['task']) == 3 - - -class TestReplaceVarsInAction(object): - - def setup_method(self, method): - self.action = { - 'name': 'run foo${foo}', - 'export': '${export}', - 'python': { - 'file': 'somefile.py', - 'args': ['arg1', '${number}', 'arg2', '${list}', '${dict}'], - 'cmdline': '-s ${foo} -i ${number}', - 'nest': { - 'deepnest': '${dict}', - 'null': '${null}', - } - } - } - self.vars = { - 'foo': 'bar', - 'export': 'a:b\nc:d\n-e', - 'number': 1, - 'list': [1, 'item'], - 'dict': {'key': 'val', 'pi': 3.14}, - 'null': None, - 'extra': 'juicy', - } - - def test_complex(self): - rend_action = copy.deepcopy(self.action) - rend_action['name'] = 'run foo%s' % self.vars['foo'] - rend_action['export'] = self.vars['export'] - rend_action['python']['args'] = ['arg1', self.vars['number'], 'arg2', - self.vars['list'], self.vars['dict']] - rend_action['python']['cmdline'] = '-s %s -i %s' % (self.vars['foo'], - self.vars['number']) - rend_action['python']['nest']['deepnest'] = self.vars['dict'] - rend_action['python']['nest']['null'] = self.vars['null'] - - taskyaml.replace_vars_in_action(self.action, self.vars) - assert self.action == rend_action - - def test_raise(self): - with pytest.raises(exc.TaskotronYamlError): - taskyaml.replace_vars_in_action(self.action, {}) - - del self.vars['number'] - with pytest.raises(exc.TaskotronYamlError): - taskyaml.replace_vars_in_action(self.action, self.vars) - - -class TestReplaceVars(object): - - def test_standadlone_var_str(self): - res = taskyaml._replace_vars('${foo}', {'foo': 'hedgehog'}) - assert res == 'hedgehog' - - def test_standalone_var_nonstr(self): - res = taskyaml._replace_vars('${foo}', {'foo': ['chicken', 'egg']}) - assert res == ['chicken', 'egg'] - - def test_standadlone_var_str_alternative(self): - res = taskyaml._replace_vars('$foo', {'foo': 'hedgehog'}) - assert res == 'hedgehog' - - def test_standalone_var_nonstr_alternative(self): - res = taskyaml._replace_vars('$foo', {'foo': ['chicken', 'egg']}) - assert res == ['chicken', 'egg'] - - def test_multivar(self): - res = taskyaml._replace_vars('a ${b} c${d}', {'b': 'bb', 'd': 'dd'}) - assert res == 'a bb cdd' - - def test_empty_space(self): - '''Some empty space around a single variable is like multi-var, i.e. - a string is always returned''' - res = taskyaml._replace_vars(' ${foo}', {'foo': 'bar'}) - assert res == ' bar' - - res = taskyaml._replace_vars(' ${foo}', {'foo': 42}) - assert res == ' 42' - - def test_recursive_content(self): - '''The content of a variable contains more variable names. But those - must not be expanded.''' - text = '${foo} ${bar}' - varz = {'foo': '${foo} ${bar}', - 'bar': '${foo} ${bar} ${baz}' - } - res = taskyaml._replace_vars(text, varz) - assert res == '%s %s' % (varz['foo'], varz['bar']) - - def test_empty(self): - res = taskyaml._replace_vars('', {'extra': 'var'}) - assert res == '' - - def test_same_var(self): - '''Only variables must get replaced, not same-sounding text''' - res = taskyaml._replace_vars('foo ${foo}', {'foo': 'bar'}) - assert res == 'foo bar' - - def test_combined_syntax(self): - '''Both $foo and ${foo} syntax must work, together with escaping''' - res = taskyaml._replace_vars('foo $foo ${foo} $$foo', {'foo': 'bar'}) - assert res == 'foo bar bar $foo' - - def test_raise_missing_var(self): - with pytest.raises(exc.TaskotronYamlError): - taskyaml._replace_vars('${foo}', {'bar': 'foo'}) - - def test_raise_missing_vars(self): - with pytest.raises(exc.TaskotronYamlError): - taskyaml._replace_vars('${foo} ${bar}', {'bar': 'foo'}) - - def test_raise_bad_escape(self): - '''Dollars must be doubled''' - with pytest.raises(exc.TaskotronYamlError): - taskyaml._replace_vars('${foo} $', {'foo': 'bar'}) - diff --git a/testing/test_yumrepoinfo.py b/testing/test_yumrepoinfo.py index bb05e59..2ad1793 100644 --- a/testing/test_yumrepoinfo.py +++ b/testing/test_yumrepoinfo.py @@ -22,17 +22,23 @@ rawhideurl = %(baseurl)s/%(path)s/%(arch)s/os arches = i386, x86_64 parent = tag = %(__name__)s -supported = no +release_status = [rawhide] path = development/rawhide url = %(rawhideurl)s -tag = f21 +tag = f22 +release_status = rawhide + +[f21] +url = %(rawhideurl)s +path = development/21 +release_status = Branched [f20] url = %(goldurl)s path = 20 -supported = yes +release_status = STABLE [f20-updates] url = %(updatesurl)s @@ -44,6 +50,11 @@ arches = x86_64 url = %(updatesurl)s path = testing/20 parent = f20-updates + +[f15] +url = not_really_an_url +path = 15 +release_status = obsolete ''' @@ -92,13 +103,17 @@ class TestYumRepoInfo(object): def test_repos(self): '''repos() should return all known repo names''' - assert self.repoinfo.repos() == ['rawhide', 'f20', 'f20-updates', - 'f20-updates-testing'] + assert self.repoinfo.repos() == ['rawhide', 'f21', 'f20', 'f20-updates', + 'f20-updates-testing', 'f15'] def test_releases(self): '''releases() should return only stable releases repo names''' assert self.repoinfo.releases() == ['f20'] + def test_branched(self): + '''branched() should return branched release repo name''' + assert self.repoinfo.branched() == 'f21' + def test_arches(self): '''arches() should return what's defined in a section''' assert self.repoinfo.arches('f20') == ['i386', 'x86_64'] @@ -107,7 +122,7 @@ class TestYumRepoInfo(object): def test_get(self): '''get() simply returns a key''' - assert self.repoinfo.get('rawhide', 'supported') == 'no' + assert self.repoinfo.get('rawhide', 'release_status') == 'rawhide' assert self.repoinfo.get('DEFAULT', 'parent') == '' def test_get_raise(self): @@ -122,7 +137,7 @@ class TestYumRepoInfo(object): '''repo() returns a repo dict''' rawhide = self.repoinfo.repo('rawhide') assert rawhide['name'] == 'rawhide' - assert rawhide['tag'] == 'f21' + assert rawhide['tag'] == 'f22' assert set(rawhide.keys()).issuperset( set(('arches', 'parent', 'tag', 'url', 'path', 'name')) ) @@ -132,8 +147,8 @@ class TestYumRepoInfo(object): f20up = self.repoinfo.repo_by_tag('f20-updates') assert f20up['name'] == f20up['tag'] == 'f20-updates' - f21 = self.repoinfo.repo_by_tag('f21') - assert f21['tag'] == 'f21' + f21 = self.repoinfo.repo_by_tag('f22') + assert f21['tag'] == 'f22' assert f21['name'] == 'rawhide' assert self.repoinfo.repo_by_tag('foobar') is None @@ -158,6 +173,12 @@ parent = repo1 with pytest.raises(exc.TaskotronConfigError): repoinfo.top_parent('repo1') + def test_repo_release_status(self): + assert self.repoinfo.release_status('f21') == 'branched' + assert self.repoinfo.release_status('f20') == 'stable' + assert self.repoinfo.release_status('f20-updates-testing') == 'stable' + assert self.repoinfo.release_status('f15') == 'obsolete' + def test_get_yumrepoinfo(self, clean_singleton): '''get_yumrepoinfo must work as a singleton'''