From f0c9d2581afd03b41dfc304365890e5484843beb Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Feb 02 2020 20:22:31 +0000 Subject: Skip tickets for specific composes. This commit uses the fedora-messaging configuration file to configure a list of composes to skip. This commit also add unit tests in order to test that feature. A brief description on how to run the tests was also added to the README file. Signed-off-by: Clement Verna --- diff --git a/Dockerfile b/Dockerfile index 22ec84f..14d87ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:30 +FROM registry.fedoraproject.org/fedora:31 # set PYTHONUNBUFFERED env var to non-empty string so that our # periods with no newline get printed immediately to the screen @@ -13,31 +13,21 @@ RUN dnf -y install python3-ogr fedora-messaging && dnf clean all RUN mkdir /work WORKDIR /work + +ADD compose-tracker.toml /work/my_config.toml # Copy the fedora config for fedora-messaging and also generate a random UUID # https://fedora-messaging.readthedocs.io/en/latest/fedora-broker.html#getting-connected # Note this will mean that if there is more than one container running # using this image they will be reading from the same queue. Generally # I expect this to only be running in one place. -RUN sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /etc/fedora-messaging/fedora.toml > /work/my_config.toml - -# Set the Application Name -RUN sed -i 's|Example Application|Compose Tracker: https://pagure.io/releng/compose-tracker|' /work/my_config.toml +RUN sed -i "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /work/my_config.toml # Lower log levels to WARNING level #RUN sed -i 's/INFO/WARNING/' /work/my_config.toml -# Set the log config of compose_tracker -RUN sed -i 's/\[log_config\]/\[log_config\]\nlevel = "DEBUG"/' /work/my_config.toml - -# Set the format for the log messages -RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml - -# We only care about pungi.compose.status.change messages -RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"]/' /work/my_config.toml - # Put compose-tracker into a location that can be imported ADD compose_tracker.py /usr/lib/python3.7/site-packages/ # Call fedora-messaging CLI and tell it to use the ComposeTracker # class from the compose-tracker module. -CMD fedora-messaging --conf /work/my_config.toml consume --callback=compose_tracker:Consumer +CMD fedora-messaging --conf /work/my_config.toml consume diff --git a/README.md b/README.md index 3c32906..3135d4c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # compose-tracker -Source code that parses pungi logs and opens issues against +Source code that parses pungi logs and opens issues against https://pagure.io/releng/failed-composes # Deploying in Fedora @@ -33,7 +33,7 @@ oc new-project compose-tracker oc new-build --strategy=docker https://pagure.io/releng/compose-tracker --to compose-tracker-img ``` -Export pagure token to use as an env var and then use +Export pagure token to use as an env var and then use [kedge](https://github.com/kedgeproject/kedge) to get up and running in openshift: @@ -41,3 +41,25 @@ to get up and running in openshift: export PAGURE_TOKEN= kedge apply -f kedge.yaml ``` + +# Development Environment + +To run compose_tracker's tests on your local machine you need to create a virtual environment + +``` +$ python -m venv .venv +$ source .venv/bin/activate +(.venv) $ +``` + +Then you can install the dependencies using the requirements.txt file + +``` +(.venv) $ pip install -r requirements.txt +``` + +Finally you can run the tests + +``` +(.venv) $ py.test test_consumer.py -v +``` diff --git a/compose-tracker.toml b/compose-tracker.toml new file mode 100644 index 0000000..1f8b4ab --- /dev/null +++ b/compose-tracker.toml @@ -0,0 +1,78 @@ +# A basic configuration for Fedora's message broker +# +# This file is in the TOML format. +amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" +callback = "compose_tracker:Consumer" + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/fedora-key.pem" +certfile = "/etc/fedora-messaging/fedora-cert.pem" + +[client_properties] +app = "compose_tracker" +app_url = "https://pagure.io/releng/compose-tracker" + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +# Queue names *must* be in the normal UUID format: run "uuidgen" and use the +# output as your queue name. If your queue is not exclusive, anyone can connect +# and consume from it, causing you to miss messages, so do not share your queue +# name. Any queues that are not auto-deleted on disconnect are garbage-collected +# after approximately one hour. +# +# If you require a stronger guarantee about delivery, please talk to Fedora's +# Infrastructure team. +[queues.00000000-0000-0000-0000-000000000000] +durable = false +auto_delete = true +exclusive = true +arguments = {} + +[[bindings]] +queue = "00000000-0000-0000-0000-000000000000" +exchange = "amq.topic" +routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"] + +[consumer_config] +composes_to_skip = ["IoT"] + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +level = "DEBUG" +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "%(asctime)s %(levelname)s %(name)s - %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"] diff --git a/compose_tracker.py b/compose_tracker.py index af6cccd..a354526 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -6,17 +6,16 @@ # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. -# +# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. -# +# # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . import datetime -import fedora_messaging.api import json import logging import os @@ -25,9 +24,12 @@ import requests import sys import traceback +import fedora_messaging.api +import fedora_messaging.config from ogr import PagureService -# Set local logging + +# Set local logging logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -42,7 +44,8 @@ KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID=' # We are processing the org.fedoraproject.prod.pungi.compose.status.change topic # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000 -EXAMPLE_MESSAGE_BODY = json.loads(""" +EXAMPLE_MESSAGE_BODY = json.loads( + """ { "status": "DOOMED", "release_type": "ga", @@ -85,6 +88,7 @@ class Consumer(object): # Used for printing out a value when the day has changed self.date = datetime.date.today() + self.config = fedora_messaging.config.conf["consumer_config"] def get_supporting_text(self, lines): """ given a log file line determine if it has a koji task ID in it @@ -177,9 +181,10 @@ class Consumer(object): # It was requested that we not file issues for IoT composes # https://pagure.io/releng/failed-composes/issue/39#comment-591054 - if 'IoT' in msg['compose_id']: - logger.info("Skipping filing issues for IoT composes") - return + for compose in self.config["composes_to_skip"]: + if compose in msg["compose_id"]: + logger.info(f"Skipping filing issues for {compose} composes") + return # variable to hold description for issue content = "[pungi.global.log](%s)\n\n" % logfileurl diff --git a/requirements.txt b/requirements.txt index be59f7b..f42c790 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ -requests==2.21.0 -fedora_messaging>=1.7.1 -libpagure>=0.22 -PyYAML==5.1.2 +requests +fedora_messaging +ogr +PyYAML +# Development +pytest +pytest-mock diff --git a/test_consumer.py b/test_consumer.py new file mode 100644 index 0000000..445f0d7 --- /dev/null +++ b/test_consumer.py @@ -0,0 +1,98 @@ +import copy +import json + +import pytest +import fedora_messaging.api +import fedora_messaging.config + +from compose_tracker import Consumer + + +EXAMPLE_MESSAGE_BODY = json.loads( + """ +{ + "status": "DOOMED", + "release_type": "ga", + "compose_label": null, + "compose_respin": 0, + "compose_date": "20190619", + "release_version": "Rawhide", + "location": "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20190619.n.0/compose", + "compose_type": "nightly", + "release_is_layered": false, + "release_name": "Fedora", + "release_short": "Fedora", + "compose_id": "Fedora-Rawhide-20190619.n.0" +} +""" +) + + +def test_consumer_message_process(mocker, caplog): + "Test that we can successfully process a message" + mocker.patch("compose_tracker.PagureService") + mocker.patch("compose_tracker.requests") + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + fedora_messaging.config.conf.load_config("compose-tracker.toml"), + ) + con = Consumer() + msg = fedora_messaging.api.Message( + topic="org.fedoraproject.prod.pungi.compose.status.change", + body=EXAMPLE_MESSAGE_BODY, + ) + con.process(msg) + assert "Fedora-Rawhide-20190619.n.0 DOOMED" in caplog.text + + +def test_consumer_settings_ignore_compose(mocker, caplog): + "Test that we ignore the composes specified in the settings" + mocker.patch("compose_tracker.PagureService") + mocker.patch("compose_tracker.requests") + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + fedora_messaging.config.conf.load_config("compose-tracker.toml"), + ) + con = Consumer() + body = copy.copy(EXAMPLE_MESSAGE_BODY) + body["compose_id"] = "Fedora-IoT-20190619.n.0" + msg = fedora_messaging.api.Message( + topic="org.fedoraproject.prod.pungi.compose.status.change", body=body, + ) + con.process(msg) + assert "Skipping filing issues for IoT composes" in caplog.text + + +def test_consumer_settings_ignore_compose_2_values(mocker, caplog): + "Test that we ignore the composes specified in the settings with 2 values" + mocker.patch("compose_tracker.PagureService") + mocker.patch("compose_tracker.requests") + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"]}}, + ) + con = Consumer() + msg = fedora_messaging.api.Message( + topic="org.fedoraproject.prod.pungi.compose.status.change", + body=EXAMPLE_MESSAGE_BODY, + ) + con.process(msg) + assert "Skipping filing issues for Rawhide composes" in caplog.text + + +def test_consumer_ignore_good_composes(mocker, caplog): + "Test that we ignore the composes that finished correctly" + mocker.patch("compose_tracker.PagureService") + mocker.patch("compose_tracker.requests") + mocker.patch( + "compose_tracker.fedora_messaging.config.conf", + fedora_messaging.config.conf.load_config("compose-tracker.toml"), + ) + con = Consumer() + body = copy.copy(EXAMPLE_MESSAGE_BODY) + body["status"] = "FINISHED" + msg = fedora_messaging.api.Message( + topic="org.fedoraproject.prod.pungi.compose.status.change", body=body, + ) + con.process(msg) + assert "Fedora-Rawhide-20190619.n.0" not in caplog.text