From 5389bf71469ff22245b89e26f08254429b1c9c36 Mon Sep 17 00:00:00 2001 From: Kamil Páral Date: Feb 10 2017 12:53:19 +0000 Subject: fix deps, improve develop and test workflow * Fix dependencies in requirements.txt: we need `fedmsg[consumers]`, not just `fedmsg`, and add some missing ones * Remove deps from `setup.py`, because it was not up-to-date anyway and there's little sense to keep deps defined there as well, as long as we don't publish this on PyPI * Make all Makefile targets phony (defined at the very same place, so that it's not forgotten next time we add a target) * Print test coverage when the test suite is run * Use `virtualenv --ignore-installed` to make sure all the deps possible are installed into virtualenv, regardless of system libs. * Extend instructions how to set up a development environment * Add testing instructions * Use the same test approach in spec as we do in development Differential Revision: https://phab.qa.fedoraproject.org/D1123 --- diff --git a/Makefile b/Makefile index f739e88..a5ae30a 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,6 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -.PHONY: test test-ci pylint pep8 docs clean virtualenv - # general variables VENV=test_env SRC=taskotron-trigger @@ -34,23 +32,33 @@ GITBRANCH:=$(shell git rev-parse --abbrev-ref HEAD) TARGETDIST:=fc24 BUILDTARGET=fedora-24-x86_64 +.PHONY: test test: $(VENV) - sh -c "set -e; . $(VENV)/bin/activate; TEST='true' py.test --cov $(MODULENAME); deactivate" + sh -c "set -e; . $(VENV)/bin/activate; \ + TEST='true' py.test --cov-report=term-missing --cov $(MODULENAME); \ + deactivate" +.PHONY: test-ci test-ci: $(VENV) - sh -c "set -e; . $(VENV)/bin/activate; TEST='true' py.test --cov-report xml --cov $(MODULENAME); deactivate" + sh -c "set -e; . $(VENV)/bin/activate; \ + TEST='true' py.test --cov-report=xml --cov $(MODULENAME); deactivate" +.PHONY: pylint pylint: pylint -f parseable $(SRC) | tee pylint.out +.PHONY: pep8 pep8: pep8 $(SRC)/*.py $(SRC)/*/*.py | tee pep8.out +.PHONY: ci ci: test-ci pylint pep8 +.PHONY: docs docs: sphinx-build -b html -d docs/_build/doctrees docs/source docs/_build/html +.PHONY: clean clean: rm -rf dist rm -rf taskotron-trigger.egg-info @@ -58,30 +66,40 @@ clean: rm -f pep8.out rm -f pylint.out +.PHONY: archive archive: $(SRC)-$(VERSION).tar.gz +.PHONY: $(SRC)-$(VERSION).tar.gz $(SRC)-$(VERSION).tar.gz: git archive $(GITBRANCH) --prefix=$(SRC)-$(VERSION)/ | gzip -c9 > $@ +.PHONY: mocksrpm mocksrpm: archive mock -r $(BUILDTARGET) --buildsrpm --spec $(SPECFILE) --sources . cp /var/lib/mock/$(BUILDTARGET)/result/$(NVR).$(TARGETDIST).src.rpm . +.PHONY: mockbuild mockbuild: mocksrpm mock -r $(BUILDTARGET) --no-clean --rebuild $(NVR).$(TARGETDIST).src.rpm cp /var/lib/mock/$(BUILDTARGET)/result/$(NVR).$(TARGETDIST).noarch.rpm . +#.PHONY: kojibuild #kojibuild: mocksrpm # koji build --scratch dist-6E-epel-testing-candidate $(NVR).$(TARGETDIST).src.rpm +.PHONY: nvr nvr: @echo $(NVR) +.PHONY: cleanvenv cleanvenv: rm -rf $(VENV) +.PHONY: virtualenv virtualenv: $(VENV) +.PHONY: $(VENV) $(VENV): - virtualenv --distribute --system-site-packages $(VENV) - sh -c "set -e; . $(VENV)/bin/activate; pip install --force-reinstall -r requirements.txt; deactivate" + virtualenv --system-site-packages $(VENV) + sh -c "set -e; . $(VENV)/bin/activate; \ + pip install --ignore-installed -r requirements.txt; deactivate" diff --git a/README.rst b/README.rst index a21a3a2..a657db3 100644 --- a/README.rst +++ b/README.rst @@ -4,21 +4,23 @@ Taskotron Job Trigger This is very simple code that listens for fedmsg messages of specific types and triggers jobs in taskotron in buildbot as needed. +Project page: https://pagure.io/taskotron/taskotron-trigger + Installation ------------ Install the rpm dependencies on the system (only works on Fedora for now):: - yum install fedmsg fedmsg-hub + $ dnf install fedmsg-hub git koji Parts of the trigger do need to be installed on the local system:: - python setup.py install + $ python setup.py install Updating Production Logging Configuration ----------------------------------------- -In /etc/fedmsg.d/logging.py, add the following handler:: +In ``/etc/fedmsg.d/logging.py``, add the following handler:: filelog={ "class": "logging.handlers.RotatingFileHandler", @@ -45,32 +47,72 @@ Running the Trigger Once the fedmsg component has been installed, run the following command in the working directory with triggerjob.py:: - fedmsg-hub + $ fedmsg-hub Setting Up a Dev Environment ---------------------------- -Install the following packages if they aren't already installed: - - * koji +Install the RPM dependencies mentioned in the Installation section. Then also +install the following:: - * fedmsg-hub + $ sudo dnf install python2-virtualenv python-pip Create a virtualenv for the trigger, including system site-packages to make sure that we get koji:: - virtualenv --system-site-packages env_trigger/ + $ virtualenv --system-site-packages env_trigger/ + $ source env_trigger/bin/activate Install requirements:: - source env_trigger/bin/activate - pip install -r requirements.txt -Install fedmsg and taskotron-trigger into the virtualenv after activating (so -that it picks up the trigger in the virtualenv's site-packages):: + $ pip install --ignore-installed -r requirements.txt + +Install taskotron-trigger in a development mode:: + + $ pip install -e . + +Create a local configuration file:: + + $ cp conf/trigger.cfg.example conf/trigger.cfg + +Edit ``conf/trigger.cfg`` and set: - pip install -U --force-reinstall fedmsg - pip install -e . + * ``job_logging = True`` + * ``joblog_file`` and ``git_cache_dir`` to a path where you have write access + * ``rules_template`` to a file with your trigger rules, you can use + ``conf/trigger_rules.yml.example`` as a starting point + * ``critpath_filepath`` to a critpath definition, you can use + ``docker_data/critpath_whitelist.example`` as a starting point Once this is all done, you can run fedmsg-hub and it will pick up the local -configuration without modifying the local system. +configuration without modifying the local system:: + + $ fedmsg-hub + +If you use the default trigger rules, you should see similar messages to +confirm they have been enabled:: + + [ fedmsg INFO] enabled by config - jobtriggers.dist_git_commit_msg:DistGitCommitJobConsumer + [ fedmsg INFO] No backlog handling. status: None, url: None + [ fedmsg INFO] enabled by config - jobtriggers.koji_build_msg:KojiBuildCompletedJobConsumer + [ fedmsg INFO] No backlog handling. status: None, url: None + [ fedmsg INFO] enabled by config - jobtriggers.compose_complete_msg:ComposeCompletedJobConsumer + [ fedmsg INFO] No backlog handling. status: None, url: None + [ fedmsg INFO] enabled by config - jobtriggers.koji_tag_msg:KojiTagChangedJobConsumer + [ fedmsg INFO] No backlog handling. status: None, url: None + +After a short time of waiting you should see new events being detected, e.g.:: + + [ fedmsg INFO] triggering rpmlint for kig-16.12.2-1.fc26 on x86_64 + [ fedmsg INFO] StreamRunner received: + item: kig-16.12.2-1.fc26 + item_type: koji_build + taskname: rpmlint + arch: x86_64 + +Running the test suite +---------------------- + +From inside the virtualenv run:: + $ py.test diff --git a/requirements.txt b/requirements.txt index d480074..76b97ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,21 @@ +# This is a list of pypi packages to be installed into virtualenv. Alternatively, +# you can install these as RPMs instead of pypi packages. See the dependecies +# with: +# $ rpmspec -q --requires taskotron-trigger.spec +# $ rpmspec -q --buildrequires taskotron-trigger.spec + +# This list is in the same order as the requirements in the specfile. + +fedmsg[consumers] +mongoquery +requests +Twisted +PyYAML == 3.11 + +# Test suite requirements + dingus == 0.3.4 -mock==1.3.0 -moksha.hub == 1.4.6 +mock == 1.3.0 munch == 2.0.2 -psutil == 3.2.1 -Pygments == 2.1.3 pytest == 2.9.2 pytest-cov == 2.2.1 -PyYAML == 3.11 -mongoquery diff --git a/setup.py b/setup.py index 4a86e3a..88495c6 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,6 @@ setup( }, include_package_data=True, install_requires=[ - 'fedmsg', - 'requests', ], cmdclass={'test': PyTest} ) diff --git a/taskotron-trigger.spec b/taskotron-trigger.spec index 2c158f0..8d0a4ba 100644 --- a/taskotron-trigger.spec +++ b/taskotron-trigger.spec @@ -1,7 +1,7 @@ Name: taskotron-trigger # NOTE: if you update version, *make sure* to also update `setup.py` Version: 0.4.4 -Release: 1%{?dist} +Release: 2%{?dist} Summary: Triggering Taskotron jobs via fedmsg License: GPLv2+ @@ -20,12 +20,13 @@ Requires: python-twisted Requires: PyYAML BuildRequires: fedmsg BuildRequires: koji -BuildRequires: pytest BuildRequires: python2-devel BuildRequires: python-dingus BuildRequires: python-mock BuildRequires: python-mongoquery BuildRequires: python-munch +BuildRequires: python2-pytest +BuildRequires: python-pytest-cov BuildRequires: python-setuptools BuildRequires: PyYAML @@ -36,7 +37,7 @@ Triggering Taskotron jobs via fedmsg. %setup -q %check -py.test testing +py.test rm -f %{buildroot}%{_sysconfdir}/fedmsg.d/*.py{c,o} %build @@ -75,6 +76,11 @@ install -d %{buildroot}%{_sharedstatedir}/taskotron-trigger/ %config(noreplace) %{_sysconfdir}/logrotate.d/taskotron-trigger %changelog +* Fri Feb 10 2017 Kamil Páral - 0.4.4-2 +- add python-pytest-cov dependency +- use the same test approach we use in development (just run py.test and + inherit everything from tox.ini) + * Mon Jan 23 2017 Martin Krizek - 0.4.4-1 - Fix CLI to work with the trigger rules (D1081) diff --git a/tox.ini b/tox.ini index 41bd383..43a1000 100644 --- a/tox.ini +++ b/tox.ini @@ -11,4 +11,4 @@ max-line-length=99 minversion=2.0 python_functions=test should python_files=test_* functest_* -addopts=--functional testing/ +addopts=--functional testing/ --cov-report=term-missing --cov jobtriggers