PR#28: Skip tickets for specific composes. - releng/compose-tracker

releng / compose-tracker

#28 Skip tickets for specific composes.

Closed 4 years ago by mohanboddu. Opened 4 years ago by cverna.

releng/ cverna/compose-tracker fix_15 into master

Skip tickets for specific composes.

Clement Verna • 4 years ago

f0c9d25

Dockerfile

file modified

+5 -15

		`@@ -1,4 +1,4 @@`
		`- FROM registry.fedoraproject.org/fedora:30`
		`+ FROM registry.fedoraproject.org/fedora:31`

		`# set PYTHONUNBUFFERED env var to non-empty string so that our`
		`# periods with no newline get printed immediately to the screen`
		`@@ -13,31 +13,21 @@`
		`RUN mkdir /work`
		`WORKDIR /work`

		`+`
		`+ ADD compose-tracker.toml /work/my_config.toml`
		`# Copy the fedora config for fedora-messaging and also generate a random UUID`
		`# https://fedora-messaging.readthedocs.io/en/latest/fedora-broker.html#getting-connected`
		`# Note this will mean that if there is more than one container running`
		`# using this image they will be reading from the same queue. Generally`
		`# I expect this to only be running in one place.`
		`- RUN sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /etc/fedora-messaging/fedora.toml > /work/my_config.toml`
		`-`
		`- # Set the Application Name`
		`- RUN sed -i 's\|Example Application\|Compose Tracker: https://pagure.io/releng/compose-tracker\|' /work/my_config.toml`
		`+ RUN sed -i "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /work/my_config.toml`

		`# Lower log levels to WARNING level`
		`#RUN sed -i 's/INFO/WARNING/' /work/my_config.toml`

		`- # Set the log config of compose_tracker`
		`- RUN sed -i 's/\[log_config\]/\[log_config\]\nlevel = "DEBUG"/' /work/my_config.toml`
		`-`
		`- # Set the format for the log messages`
		`- RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml`
		`-`
		`- # We only care about pungi.compose.status.change messages`
		`- RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"]/' /work/my_config.toml`
		`-`
		`# Put compose-tracker into a location that can be imported`
		`ADD compose_tracker.py /usr/lib/python3.7/site-packages/`

		`# Call fedora-messaging CLI and tell it to use the ComposeTracker`
		`# class from the compose-tracker module.`
		`- CMD fedora-messaging --conf /work/my_config.toml consume --callback=compose_tracker:Consumer`
		`+ CMD fedora-messaging --conf /work/my_config.toml consume`

README.md

file modified

+24 -2

		`@@ -1,6 +1,6 @@`
		`# compose-tracker`

		`- Source code that parses pungi logs and opens issues against`
		`+ Source code that parses pungi logs and opens issues against`
		`https://pagure.io/releng/failed-composes`

		`# Deploying in Fedora`
		`@@ -33,7 +33,7 @@`
		`oc new-build --strategy=docker https://pagure.io/releng/compose-tracker --to compose-tracker-img`
		```

		`- Export pagure token to use as an env var and then use`
		`+ Export pagure token to use as an env var and then use`
		`[kedge](https://github.com/kedgeproject/kedge)`
		`to get up and running in openshift:`

		`@@ -41,3 +41,25 @@`
		`export PAGURE_TOKEN=<pagure_token>`
		`kedge apply -f kedge.yaml`
		```
		`+`
		`+ # Development Environment`
		`+`
		`+ To run compose_tracker's tests on your local machine you need to create a virtual environment`
		`+`
		+ ```
		`+ $ python -m venv .venv`
		`+ $ source .venv/bin/activate`
		`+ (.venv) $`
		+ ```
		`+`
		`+ Then you can install the dependencies using the requirements.txt file`
		`+`
		+ ```
		`+ (.venv) $ pip install -r requirements.txt`
		+ ```
		`+`
		`+ Finally you can run the tests`
		`+`
		+ ```
		`+ (.venv) $ py.test test_consumer.py -v`
		+ ```

compose-tracker.toml

file added

+78

		`@@ -0,0 +1,78 @@`
		`+ # A basic configuration for Fedora's message broker`
		`+ #`
		`+ # This file is in the TOML format.`
		`+ amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"`
		`+ callback = "compose_tracker:Consumer"`
		`+`
		`+ [tls]`
		`+ ca_cert = "/etc/fedora-messaging/cacert.pem"`
		`+ keyfile = "/etc/fedora-messaging/fedora-key.pem"`
		`+ certfile = "/etc/fedora-messaging/fedora-cert.pem"`
		`+`
		`+ [client_properties]`
		`+ app = "compose_tracker"`
		`+ app_url = "https://pagure.io/releng/compose-tracker"`
		`+`
		`+ [exchanges."amq.topic"]`
		`+ type = "topic"`
		`+ durable = true`
		`+ auto_delete = false`
		`+ arguments = {}`
		`+`
		`+ # Queue names must be in the normal UUID format: run "uuidgen" and use the`
		`+ # output as your queue name. If your queue is not exclusive, anyone can connect`
		`+ # and consume from it, causing you to miss messages, so do not share your queue`
		`+ # name. Any queues that are not auto-deleted on disconnect are garbage-collected`
		`+ # after approximately one hour.`
		`+ #`
		`+ # If you require a stronger guarantee about delivery, please talk to Fedora's`
		`+ # Infrastructure team.`
		`+ [queues.00000000-0000-0000-0000-000000000000]`
		`+ durable = false`
		`+ auto_delete = true`
		`+ exclusive = true`
		`+ arguments = {}`
		`+`
		`+ [[bindings]]`
		`+ queue = "00000000-0000-0000-0000-000000000000"`
		`+ exchange = "amq.topic"`
		`+ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"]`
		`+`
		`+ [consumer_config]`
		`+ composes_to_skip = ["IoT"]`
		`+`
		`+ [qos]`
		`+ prefetch_size = 0`
		`+ prefetch_count = 25`
		`+`
		`+ [log_config]`
		`+ level = "DEBUG"`
		`+ version = 1`
		`+ disable_existing_loggers = true`
		`+`
		`+ [log_config.formatters.simple]`
		`+ format = "%(asctime)s %(levelname)s %(name)s - %(message)s"`
		`+`
		`+ [log_config.handlers.console]`
		`+ class = "logging.StreamHandler"`
		`+ formatter = "simple"`
		`+ stream = "ext://sys.stdout"`
		`+`
		`+ [log_config.loggers.fedora_messaging]`
		`+ level = "INFO"`
		`+ propagate = false`
		`+ handlers = ["console"]`
		`+`
		`+ [log_config.loggers.twisted]`
		`+ level = "INFO"`
		`+ propagate = false`
		`+ handlers = ["console"]`
		`+`
		`+ [log_config.loggers.pika]`
		`+ level = "WARNING"`
		`+ propagate = false`
		`+ handlers = ["console"]`
		`+`
		`+ [log_config.root]`
		`+ level = "ERROR"`
		`+ handlers = ["console"]`

compose_tracker.py

file modified

+13 -8

		`@@ -6,17 +6,16 @@`
		`# modify it under the terms of the GNU Lesser General Public`
		`# License as published by the Free Software Foundation; either`
		`# version 2 of the License, or (at your option) any later version.`
		`- #`
		`+ #`
		`# This library is distributed in the hope that it will be useful,`
		`# but WITHOUT ANY WARRANTY; without even the implied warranty of`
		`# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU`
		`# Lesser General Public License for more details.`
		`- #`
		`+ #`
		`# You should have received a copy of the GNU Lesser General Public`
		`# License along with this library. If not, see <http://www.gnu.org/licenses/>.`

		`import datetime`
		`- import fedora_messaging.api`
		`import json`
		`import logging`
		`import os`
		`@@ -25,9 +24,12 @@`
		`import sys`
		`import traceback`

		`+ import fedora_messaging.api`
		`+ import fedora_messaging.config`
		`from ogr import PagureService`

		`- # Set local logging`
		`+`
		`+ # Set local logging`
		`logger = logging.getLogger(__name__)`
		`logger.setLevel(logging.INFO)`

		`@@ -42,7 +44,8 @@`

		`# We are processing the org.fedoraproject.prod.pungi.compose.status.change topic`
		`# https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000`
		`- EXAMPLE_MESSAGE_BODY = json.loads("""`
		`+ EXAMPLE_MESSAGE_BODY = json.loads(`
		`+ """`
		`{`
		`"status": "DOOMED",`
		`"release_type": "ga",`
		`@@ -85,6 +88,7 @@`

		`# Used for printing out a value when the day has changed`
		`self.date = datetime.date.today()`
		`+ self.config = fedora_messaging.config.conf["consumer_config"]`

		`def get_supporting_text(self, lines):`
		`""" given a log file line determine if it has a koji task ID in it`
		`@@ -177,9 +181,10 @@`

		`# It was requested that we not file issues for IoT composes`
		`# https://pagure.io/releng/failed-composes/issue/39#comment-591054`
		`- if 'IoT' in msg['compose_id']:`
		`- logger.info("Skipping filing issues for IoT composes")`
		`- return`
		`+ for compose in self.config["composes_to_skip"]:`
		`+ if compose in msg["compose_id"]:`
		`+ logger.info(f"Skipping filing issues for {compose} composes")`
		`+ return`

		`# variable to hold description for issue`
		`content = "[pungi.global.log](%s)\n\n" % logfileurl`

requirements.txt

file modified

+7 -4

		`@@ -1,4 +1,7 @@`
		`- requests==2.21.0`
		`- fedora_messaging>=1.7.1`
		`- libpagure>=0.22`
		`- PyYAML==5.1.2`
		`+ requests`
		`+ fedora_messaging`
		`+ ogr`
		`+ PyYAML`
		`+ # Development`
		`+ pytest`
		`+ pytest-mock`

test_consumer.py

file added

+98

		`@@ -0,0 +1,98 @@`
		`+ import copy`
		`+ import json`
		`+`
		`+ import pytest`
		`+ import fedora_messaging.api`
		`+ import fedora_messaging.config`
		`+`
		`+ from compose_tracker import Consumer`
		`+`
		`+`
		`+ EXAMPLE_MESSAGE_BODY = json.loads(`
		`+ """`
		`+ {`
		`+ "status": "DOOMED",`
		`+ "release_type": "ga",`
		`+ "compose_label": null,`
		`+ "compose_respin": 0,`
		`+ "compose_date": "20190619",`
		`+ "release_version": "Rawhide",`
		`+ "location": "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20190619.n.0/compose",`
		`+ "compose_type": "nightly",`
		`+ "release_is_layered": false,`
		`+ "release_name": "Fedora",`
		`+ "release_short": "Fedora",`
		`+ "compose_id": "Fedora-Rawhide-20190619.n.0"`
		`+ }`
		`+ """`
		`+ )`
		`+`
		`+`
		`+ def test_consumer_message_process(mocker, caplog):`
		`+ "Test that we can successfully process a message"`
		`+ mocker.patch("compose_tracker.PagureService")`
		`+ mocker.patch("compose_tracker.requests")`
		`+ mocker.patch(`
		`+ "compose_tracker.fedora_messaging.config.conf",`
		`+ fedora_messaging.config.conf.load_config("compose-tracker.toml"),`
		`+ )`
		`+ con = Consumer()`
		`+ msg = fedora_messaging.api.Message(`
		`+ topic="org.fedoraproject.prod.pungi.compose.status.change",`
		`+ body=EXAMPLE_MESSAGE_BODY,`
		`+ )`
		`+ con.process(msg)`
		`+ assert "Fedora-Rawhide-20190619.n.0 DOOMED" in caplog.text`
		`+`
		`+`
		`+ def test_consumer_settings_ignore_compose(mocker, caplog):`
		`+ "Test that we ignore the composes specified in the settings"`
		`+ mocker.patch("compose_tracker.PagureService")`
		`+ mocker.patch("compose_tracker.requests")`
		`+ mocker.patch(`
		`+ "compose_tracker.fedora_messaging.config.conf",`
		`+ fedora_messaging.config.conf.load_config("compose-tracker.toml"),`
		`+ )`
		`+ con = Consumer()`
		`+ body = copy.copy(EXAMPLE_MESSAGE_BODY)`
		`+ body["compose_id"] = "Fedora-IoT-20190619.n.0"`
		`+ msg = fedora_messaging.api.Message(`
		`+ topic="org.fedoraproject.prod.pungi.compose.status.change", body=body,`
		`+ )`
		`+ con.process(msg)`
		`+ assert "Skipping filing issues for IoT composes" in caplog.text`
		`+`
		`+`
		`+ def test_consumer_settings_ignore_compose_2_values(mocker, caplog):`
		`+ "Test that we ignore the composes specified in the settings with 2 values"`
		`+ mocker.patch("compose_tracker.PagureService")`
		`+ mocker.patch("compose_tracker.requests")`
		`+ mocker.patch(`
		`+ "compose_tracker.fedora_messaging.config.conf",`
		`+ {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"]}},`
		`+ )`
		`+ con = Consumer()`
		`+ msg = fedora_messaging.api.Message(`
		`+ topic="org.fedoraproject.prod.pungi.compose.status.change",`
		`+ body=EXAMPLE_MESSAGE_BODY,`
		`+ )`
		`+ con.process(msg)`
		`+ assert "Skipping filing issues for Rawhide composes" in caplog.text`
		`+`
		`+`
		`+ def test_consumer_ignore_good_composes(mocker, caplog):`
		`+ "Test that we ignore the composes that finished correctly"`
		`+ mocker.patch("compose_tracker.PagureService")`
		`+ mocker.patch("compose_tracker.requests")`
		`+ mocker.patch(`
		`+ "compose_tracker.fedora_messaging.config.conf",`
		`+ fedora_messaging.config.conf.load_config("compose-tracker.toml"),`
		`+ )`
		`+ con = Consumer()`
		`+ body = copy.copy(EXAMPLE_MESSAGE_BODY)`
		`+ body["status"] = "FINISHED"`
		`+ msg = fedora_messaging.api.Message(`
		`+ topic="org.fedoraproject.prod.pungi.compose.status.change", body=body,`
		`+ )`
		`+ con.process(msg)`
		`+ assert "Fedora-Rawhide-20190619.n.0" not in caplog.text`

cverna commented 4 years ago

This commit uses the fedora-messaging configuration file to configure a
list of composes to skip.
This commit also add unit tests in order to test that feature.
A brief description on how to run the tests was also added to the README file.

Signed-off-by: Clement Verna cverna@tutanota.com