#11 Port resultsdb-listener to fedora-messaging, tox and python3
Merged 4 years ago by pingou. Opened 4 years ago by pingou.
pingou/ci-resultsdb-listener fedora-messaging  into  master

file modified
+4
@@ -1,1 +1,5 @@ 

  .tox/

+ config.toml

+ dev_config.toml

+ *.pyc

+ *.swp

file modified
+38 -2
@@ -1,5 +1,41 @@ 

  ci-resultsdb-listener

  =====================

  

- Listens to fedmsg messages sent by the Fedora Atomic CI pipeline running in

- ci.centos.org and uploads them to a specified resultsdb.

+ Listens to fedora-messaging messages sent by the Fedora CI pipeline running

+ in ci.centos.org and uploads them to a specified resultsdb.

+ 

+ 

+ Get it running

+ --------------

+ 

+ * In another terminal, setup and get running a resultsdb instance.

+   Instructions can be found at: https://pagure.io/taskotron/resultsdb/

+ 

+ * Install the dependencies:

+ ::

+ 

+    sudo dnf install python3-fedora-messaging

+ 

+ * Adjust the fedora-messaging example configuration with your own uuid

+ ::

+ 

+   sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" conf/config.toml.dev-example > dev_config.toml

+ 

+ * Run the consumer:

+ ::

+ 

+   PYTHONPATH=. fedora-messaging --conf dev_config.toml consume

+ 

+ 

+ Run the tests

+ -------------

+ 

+ * Install tox:

+ ::

+ 

+   sudo dnf install python3-tox

+ 

+ * Run the tests:

+ ::

+ 

+   tox .

@@ -0,0 +1,85 @@ 

+ # A sample configuration for fedora-messaging. This file is in the TOML format.

+ # For complete details on all configuration options, see the documentation

+ # https://fedora-messaging.readthedocs.io/en/latest/configuration.html.

+ 

+ amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"

+ 

+ publish_exchange = "amq.topic"

+ 

+ callback = "resultsdb_listener.consumer:consume"

+ 

+ [exchanges."amq.topic"]

+ type = "topic"

+ durable = true

+ auto_delete = false

+ arguments = {}

+ 

+ # Queue names *must* be in the normal UUID format: run "uuidgen" and use the

+ # output as your queue name. If your queue is not exclusive, anyone can connect

+ # and consume from it, causing you to miss messages, so do not share your queue

+ # name. Any queues that are not auto-deleted on disconnect are garbage-collected

+ # after approximately one hour.

+ [queues.00000000-0000-0000-0000-000000000000]

+ durable = false

+ auto_delete = true

+ exclusive = true

+ arguments = {}

+ 

+ # Note the double brackets below.

+ # To add another binding, add another [[bindings]] section.

+ [[bindings]]

+ queue = "00000000-0000-0000-0000-000000000000"

+ exchange = "amq.topic"

+ routing_keys = [

+     'org.centos.prod.ci.pipeline.complete',

+     'org.centos.prod.ci.pipeline.compose.complete',

+     'org.centos.prod.ci.pipeline.compose.test.integration.complete',

+     'org.centos.prod.ci.pipeline.image.complete',

+     'org.centos.prod.ci.pipeline.image.test.smoke.complete',

+     'org.centos.prod.ci.pipeline.package.complete',

+     'org.centos.prod.ci.pipeline.package.ignore',

+     'org.centos.prod.ci.pipeline.allpackages-build.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.image.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.package.test.functional.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.package.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.package.ignored',

+ ]

+ 

+ [tls]

+ ca_cert = "/etc/fedora-messaging/cacert.pem"

+ keyfile = "/etc/fedora-messaging/fedora-key.pem"

+ certfile = "/etc/fedora-messaging/fedora-cert.pem"

+ 

+ [client_properties]

+ app = "resultsdb_listener"

+ # Some suggested extra fields:

+ # URL of the project that provides this consumer

+ app_url = "https://pagure.io/ci-resultsdb-listener/"

+ # Contact emails for the maintainer(s) of the consumer - in case the

+ # broker admin needs to contact them, for e.g.

+ app_contacts_email = ["pingou@fedoraproject.org"]

+ 

+ [qos]

+ prefetch_size = 0

+ prefetch_count = 25

+ 

+ [log_config]

+ version = 1

+ disable_existing_loggers = true

+ 

+ [log_config.formatters.simple]

+ format = "[%(name)s %(levelname)s] %(message)s"

+ 

+ [log_config.handlers.console]

+ class = "logging.StreamHandler"

+ formatter = "simple"

+ stream = "ext://sys.stdout"

+ 

+ [log_config.loggers.fedora_messaging]

+ level = "INFO"

+ propagate = false

+ handlers = ["console"]

+ 

+ [log_config.root]

+ level = "WARNING"

+ handlers = ["console"]

@@ -0,0 +1,80 @@ 

+ # A sample configuration for fedora-messaging. This file is in the TOML format.

+ # For complete details on all configuration options, see the documentation

+ # https://fedora-messaging.readthedocs.io/en/latest/configuration.html.

+ 

+ amqp_url = "amqps://"

+ 

+ publish_exchange = "amq.topic"

+ 

+ callback = "resultsdb_listener.consumer:consume"

+ 

+ [exchanges."amq.topic"]

+ type = "topic"

+ durable = true

+ auto_delete = false

+ arguments = {}

+ 

+ [queues.resultsdb_listener]

+ durable = true

+ auto_delete = false

+ exclusive = false # so you can have more than one consumer/worker

+ arguments = {}

+ 

+ # Note the double brackets below.

+ # To add another binding, add another [[bindings]] section.

+ [[bindings]]

+ queue = "resultsdb_listener"

+ exchange = "amq.topic"

+ routing_keys = [

+     'org.centos.prod.ci.pipeline.complete',

+     'org.centos.prod.ci.pipeline.compose.complete',

+     'org.centos.prod.ci.pipeline.compose.test.integration.complete',

+     'org.centos.prod.ci.pipeline.image.complete',

+     'org.centos.prod.ci.pipeline.image.test.smoke.complete',

+     'org.centos.prod.ci.pipeline.package.complete',

+     'org.centos.prod.ci.pipeline.package.ignore',

+     'org.centos.prod.ci.pipeline.allpackages-build.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.image.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.package.test.functional.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.package.complete',

+     'org.centos.prod.ci.pipeline.allpackages-build.package.ignored',

+ ]

+ 

+ [tls]

+ ca_cert = "/etc/fedora-messaging/cacert.pem"

+ keyfile = "/etc/fedora-messaging/fedora-key.pem"

+ certfile = "/etc/fedora-messaging/fedora-cert.pem"

+ 

+ [client_properties]

+ app = "resultsdb_listener"

+ # Some suggested extra fields:

+ # URL of the project that provides this consumer

+ app_url = "https://pagure.io/ci-resultsdb-listener/"

+ # Contact emails for the maintainer(s) of the consumer - in case the

+ # broker admin needs to contact them, for e.g.

+ app_contacts_email = ["pingou@fedoraproject.org"]

+ 

+ [qos]

+ prefetch_size = 0

+ prefetch_count = 25

+ 

+ [log_config]

+ version = 1

+ disable_existing_loggers = true

+ 

+ [log_config.formatters.simple]

+ format = "[%(name)s %(levelname)s] %(message)s"

+ 

+ [log_config.handlers.console]

+ class = "logging.StreamHandler"

+ formatter = "simple"

+ stream = "ext://sys.stdout"

+ 

+ [log_config.loggers.fedora_messaging]

+ level = "INFO"

+ propagate = false

+ handlers = ["console"]

+ 

+ [log_config.root]

+ level = "WARNING"

+ handlers = ["console"]

@@ -1,6 +1,3 @@ 

- [general]

- datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw

- 

  [sender]

  sender_type = StreamSender

  resultsdb_url = http://127.0.0.1:5001/api/v2.0/

@@ -1,3 +0,0 @@ 

- config = {

-     'resultsdb_listener.enabled': True,

- }

@@ -1,9 +1,9 @@ 

  %global modname resultsdb_listener

  

  Name:               python-ci-resultsdb-listener

- Version:            0.3

+ Version:            1.0

  Release:            1%{?dist}

- Summary:            A fedmsg consumer that sents Atomic CI pipeline messages to resultsdb

+ Summary:            A fedora-messaging consumer uploading results to resultsdb

  

  Group:              Development/Libraries

  License:            GPLv2+
@@ -12,15 +12,17 @@ 

  

  BuildArch:          noarch

  

- BuildRequires:      python2-devel

- BuildRequires:      python-setuptools

- BuildRequires:      fedmsg

+ BuildRequires:      python3-devel

+ BuildRequires:      python3-setuptools

+ BuildRequires:      python3-fedora-messaging

  

- Requires:           fedmsg

+ Requires:           python3-fedora-messaging

+ Requires:           python3-resultsdb_api

  

  %description

- This application listens for messages sent by the Atomic CI pipeline running

- in ci.centos.org and uploads the messages it sees to resultsdb.

+ This application listens for ci messages sent by ci.centos.org for Fedora

+ packages in the fedora-messaging message bus and upload their results to

+ resultsdb.

  

  %prep

  %setup -q -n %{modname}-%{version}
@@ -29,14 +31,10 @@ 

  rm -rf %{modname}.egg-info

  

  %build

- %{__python2} setup.py build

+ %{__python3} setup.py build

  

  %install

- %{__python2} setup.py install -O1 --skip-build --root=%{buildroot}

- 

- # Install the fedmsg config file

- mkdir -p %{buildroot}%{_sysconfdir}/fedmsg.d/

- cp -p fedmsg.d/ci-resultsdb-listener.py %{buildroot}%{_sysconfdir}/fedmsg.d/ci-resultsdb-listener.py

+ %{__python3} setup.py install -O1 --skip-build --root=%{buildroot}

  

  # Install the application config file

  mkdir -p %{buildroot}%{_sysconfdir}/ci-resultsdb-listener/
@@ -45,11 +43,10 @@ 

  %files

  %doc README.rst

  %license LICENSE

- %config(noreplace) %{_sysconfdir}/fedmsg.d/ci-resultsdb-listener.py*

  %config(noreplace) %{_sysconfdir}/ci-resultsdb-listener/listener.cfg

  

- %{python2_sitelib}/%{modname}/*

- %{python2_sitelib}/%{modname}-%{version}*

+ %{python3_sitelib}/%{modname}/*

+ %{python3_sitelib}/%{modname}-%{version}*

  

  

  %changelog

file added
+2
@@ -0,0 +1,2 @@ 

+ fedora-messaging

+ resultsdb_api

@@ -1,45 +0,0 @@ 

- from . import senders

- from . import config

- 

- class CIProcessor(object):

- 

-     def __init__(self, log, sender=None):

-         if sender is None:

-             self.sender = senders.get_sender(config.sender_type)

-         else:

-             self.sender = sender

- 

-         self.sender = self.sender()

- 

-     def process(self, msg):

-         """ Extract most of the overlapping information and let the

-         sub-class complement it if they want.

-         """

- 

-         data = {

-             "msg_id": msg['msg_id'],

-             "build_id": msg['msg']['build_id'],

-             "build_url": msg['msg']['build_url'],

-             "branch":  msg['msg']['branch'],

-             "ref": msg['msg']['ref'],

-             "rev":  msg['msg']['rev'],

-             "repo": msg['msg']['repo'],

-             "namespace": msg['msg']['namespace'],

-             "username": msg['msg']['username'],

-             "status": msg['msg']['status'],

-         }

- 

-         if 'topic' in msg['msg']:

-             data['topic'] = msg['msg']['topic']

-         elif 'ci_topic' in msg['msg']:

-             data['topic'] = msg['msg']['ci_topic']

- 

-         for key in ['package_url', 'compose_url', 'nvr', 'original_spec_nvr']:

-             if key in msg['msg']:

-                 data[key] = msg['msg'][key]

- 

-         return data

- 

- 

-     def send(self, data):

-         self.sender.send(data)

file modified
+6 -10
@@ -1,21 +1,17 @@ 

- import ConfigParser

+ import configparser

  import io

  

  defaults = """

- [general]

- datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw

- 

  [sender]

  sender_type = StreamSender

  resultsdb_url = http://127.0.0.1:5001/api/v2.0/

  

  """

  

- config = ConfigParser.ConfigParser()

- config.readfp(io.BytesIO(defaults))

- config.read(['/etc/ci-resultsdb-listener/listener.cfg', './conf/listener.cfg'])

+ config = configparser.ConfigParser()

+ config.readfp(io.StringIO(defaults))

+ config.read(["/etc/ci-resultsdb-listener/listener.cfg", "./conf/listener.cfg"])

  

  

- datagrepper_url = config.get('general', 'datagrepper_url')

- sender_type = config.get('sender', 'sender_type')

- resultsdb_url = config.get('sender', 'resultsdb_url')

+ sender_type = config.get("sender", "sender_type")

+ resultsdb_url = config.get("sender", "resultsdb_url")

@@ -0,0 +1,79 @@ 

+ import logging

+ 

+ from fedora_messaging import config as fm_config

+ 

+ from . import config

+ from . import exceptions as exc

+ from . import senders

+ 

+ 

+ fm_config.conf.setup_logging()

+ _log = logging.getLogger(__name__)

+ 

+ 

+ def message_to_payload(message):

+     """ Extract most of the overlapping information and let the

+     sub-class complement it if they want.

+     """

+ 

+     data = {

+         "msg_id": message.id,

+         "build_id": message.body["build_id"],

+         "build_url": message.body["build_url"],

+         "branch": message.body["branch"],

+         "ref": message.body["ref"],

+         "rev": message.body["rev"],

+         "repo": message.body["repo"],

+         "namespace": message.body["namespace"],

+         "username": message.body["username"],

+         "status": message.body["status"],

+     }

+ 

+     data["topic"] = message.topic

+ 

+     for key in ["package_url", "compose_url", "nvr", "original_spec_nvr"]:

+         if key in message.body:

+             data[key] = message.body[key]

+ 

+     return data

+ 

+ 

+ class Consumer:

+     """All resultsdb-listener messages are received by this class's

+     __call__() method.

+     """

+ 

+     def __init__(self):

+         """ Initialize the consumer, sets up processor.

+         """

+         try:

+             processor = senders.get_sender(config.sender_type)

+             self.processor = processor()

+         except exc.ListenerMsgError as e:

+             _log.error(e)

+             raise

+ 

+     def __call__(self, message):

+         """ React on the messages seen on fedora-messaging's bus whose topics

+         are listed/defined in fedora-messaging's configuration file.

+ 

+         In this case, the function will convert the message into a payload that

+         will be uploaded to resultsdb.

+ 

+         There is not need to take any special precaution for duplicated

+         messages, we can just upload them twice if that happens.

+         """

+ 

+         _log.info(

+             "PipelineConsumerFedoraMessaging: Got a message %r: %r",

+             message.topic,

+             message.id,

+         )

+ 

+         try:

+             data = message_to_payload(message)

+             self.processor.send(data)

+         except exc.ResultsdbListenerException as e:

+             _log.error(e)

+ 

+         _log.info("PipelineConsumerFedoraMessaging: Done")

@@ -1,4 +1,3 @@ 

- 

  class ResultsdbListenerException(Exception):

      pass

  
@@ -9,4 +8,3 @@ 

  

  class ListenerMsgError(ResultsdbListenerException):

      pass

- 

@@ -1,52 +0,0 @@ 

- import logging

- 

- import fedmsg.consumers

- 

- from . import ci_processor

- from . import exceptions as exc

- 

- 

- _log = logging.getLogger(__name__)

- 

- 

- class PipelineConsumer(fedmsg.consumers.FedmsgConsumer):

-     topic = [

-         'org.centos.prod.ci.pipeline.complete',

-         'org.centos.prod.ci.pipeline.compose.complete',

-         'org.centos.prod.ci.pipeline.compose.test.integration.complete',

-         'org.centos.prod.ci.pipeline.image.complete',

-         'org.centos.prod.ci.pipeline.image.test.smoke.complete',

-         'org.centos.prod.ci.pipeline.package.complete',

-         'org.centos.prod.ci.pipeline.package.ignore',

-         'org.centos.prod.ci.pipeline.allpackages-build.complete',

-         'org.centos.prod.ci.pipeline.allpackages-build.image.complete',

-         'org.centos.prod.ci.pipeline.allpackages-build.package.test.functional.complete',

-         'org.centos.prod.ci.pipeline.allpackages-build.package.complete',

-         'org.centos.prod.ci.pipeline.allpackages-build.package.ignored',

-     ]

- 

-     config_key = 'resultsdb_listener.enabled'

- 

-     def __init__(self, *args, **kw):

-         super(PipelineConsumer, self).__init__(*args, **kw)

-         self.processor = ci_processor.CIProcessor(self.log)

- 

-     def consume(self, message):

-         msg = message['body']

-         topic = None

-         if 'topic' in msg:

-             topic = msg['topic']

-         elif 'ci_topic' in msg:

-             topic = msg['ci_topic']

- 

-         _log.info(

-             "PipelineConsumer: Got a message %r: %r",

-             topic, msg['msg_id'])

- 

-         try:

-             data = self.processor.process(msg)

-             self.processor.send(data)

-         except exc.ListenerMsgError, e:

-             self.log.debug(e)

- 

-         _log.info("PipelineConsumer: Done")

file modified
+22 -17
@@ -1,4 +1,3 @@ 

- 

  """

  Basically, this should replace the runner functionality from trigger so that

  we can more easily debug what's going on without sending stuff out
@@ -14,52 +13,56 @@ 

      def send(self, result):

          raise NotImplementedError

  

- # these don't overlap so well but the best mapping seems to be SUCCESS == PASSED and anything else

- # is FAILED. I don't see a place in here where INFO or NEEDS_INSPECTION are appropriate unless I'm

- # misunderstanding what the outcomes from CI mean

  

- CI_OUTCOMES = ('SUCCESS', 'FAILURE','ABORTED')

- RESULTSDB_OUTCOMES = ('PASSED', 'INFO', 'FAILED', 'NEEDS_INSPECTION')

+ # these don't overlap so well but the best mapping seems to be

+ # SUCCESS == PASSED and anything else is FAILED. I don't see a place in here

+ # where INFO or NEEDS_INSPECTION are appropriate unless I'm misunderstanding

+ # what the outcomes from CI mean

+ 

+ CI_OUTCOMES = ("SUCCESS", "FAILURE", "ABORTED")

+ RESULTSDB_OUTCOMES = ("PASSED", "INFO", "FAILED", "NEEDS_INSPECTION")

+ 

  

  class ResultsdbSender(BaseSender):

      def __init__(self, rdb_connection=None):

          if rdb_connection is None:

-             self.rdb_connection = resultsdb_api.ResultsDBapi(config.resultsdb_url)

+             self.rdb_connection = resultsdb_api.ResultsDBapi(

+                 config.resultsdb_url

+             )

          else:

              self.rdb_connection = rdb_connection

  

      def _check_testcase_exists(self, testcase):

          try:

              self.rdb_connection.get_testcase(testcase)

-         except resultsdb_api.ResultsDBapiException as e:

+         except resultsdb_api.ResultsDBapiException:

              raise exc.SenderError("Testcase was not found")

  

      def _create_testcase(self, testcase):

          self.rdb_connection.create_testcase(testcase)

  

      def _translate_outcome(self, outcome):

-         if outcome == 'SUCCESS':

-             return 'PASSED'

+         if outcome == "SUCCESS":

+             return "PASSED"

          else:

-             return 'FAILED'

- 

+             return "FAILED"

  

      # topic is the testcase

      # add all data as kv pairs

      def send(self, result):

-         testcase_name = result['topic']

+         testcase_name = result["topic"]

  

          # make sure that the testcase exists, if not, create it

          try:

              self._check_testcase_exists(testcase_name)

-         except exc.SenderError as e:

+         except exc.SenderError:

              self._create_testcase(testcase_name)

  

          # translate the outcome from CI to resultsdb-speak

-         rdb_outcome = self._translate_outcome(result['status'])

+         rdb_outcome = self._translate_outcome(result["status"])

  

          # construct the data that is to be sent

-         ref_url = result['build_url']

+         ref_url = result["build_url"]

  

          # send the result to resultsdb

          self.rdb_connection.create_result(
@@ -68,11 +71,13 @@ 

              group=None,

              note=None,

              ref_url=ref_url,

-             **result)

+             **result

+         )

  

  

  class StreamSender(BaseSender):

      def send(self, result):

+         print("StreamSender\n------------")

          print(result)

  

  

file modified
+16 -17
@@ -3,32 +3,31 @@ 

  

  class PyTest(Command):

      user_options = []

+ 

      def initialize_options(self):

          pass

+ 

      def finalize_options(self):

          pass

+ 

      def run(self):

          import subprocess

-         errno = subprocess.call(['py.test', 'testing'])

+ 

+         errno = subprocess.call(["py.test", "testing"])

          raise SystemExit(errno)

  

  

  setup(

-     name='resultsdb_listener',

-     version='0.3',

-     description='Listens for ci completion fedmsgs, translates to resultsdb',

-     author='Tim Flink',

-     author_email='tflink@fedoraproject.org',

-     license='GPLv2+',

-     url='https://pagure.io/resultsdb_listener',

-     packages=['resultsdb_listener'],

-     entry_points={

-         'moksha.consumer': [

-             'pipelineconsumer = resultsdb_listener.pipeline_consumer:PipelineConsumer',

-         ]

-     },

+     name="resultsdb_listener",

+     version="1.0",

+     description="Listens for ci messages in fedora-messaging and upload "

+     "them to resultsdb",

+     author="Tim Flink",

+     author_email="tflink@fedoraproject.org",

+     license="GPLv2+",

+     url="https://pagure.io/resultsdb_listener",

+     packages=["resultsdb_listener"],

      include_package_data=True,

-     install_requires=[

-     ],

-     cmdclass={'test': PyTest}

+     install_requires=[],

+     cmdclass={"test": PyTest},

  )

file modified
+11 -6
@@ -16,8 +16,8 @@ 

  #

  # Author: Tim Flink <tflink@redhat.com>

  

- '''py.test configuration and plugins

- Read more at: http://pytest.org/latest/plugins.html#conftest-py-plugins'''

+ """py.test configuration and plugins

+ Read more at: http://pytest.org/latest/plugins.html#conftest-py-plugins"""

  

  

  def pytest_addoption(parser):
@@ -26,16 +26,21 @@ 

      should be detected and run

      """

  

-     parser.addoption('-F', '--functional', action='store_true', default=False,

-                      help='Add functional tests')

+     parser.addoption(

+         "-F",

+         "--functional",

+         action="store_true",

+         default=False,

+         help="Add functional tests",

+     )

  

  

  def pytest_ignore_collect(path, config):

      """Prevents collection of any files named functest* to speed up non

          integration tests"""

-     if path.fnmatch('*functest*'):

+     if path.fnmatch("*functest*"):

          try:

-             is_functional = config.getvalue('functional')

+             is_functional = config.getvalue("functional")

          except KeyError:

              return True

  

file removed
-4
@@ -1,4 +0,0 @@ 

- [pytest]

- minversion = 2.0

- python_functions=test should

- python_files=test_* functest_*

file modified
+2 -2
@@ -1,10 +1,10 @@ 

- import mock

- 

  from resultsdb_listener import senders

  

+ 

  class StubSender(senders.BaseSender):

      def send(self, result):

          self.last_result = result

  

+ 

  class TestCIListener(object):

      pass

@@ -1,47 +1,97 @@ 

- from resultsdb_listener import pipeline_compose_complete

- 

- example_message = {

-   "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934", 

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.compose.complete",   

-   'msg': { 

-         "CI_TYPE": "custom", 

+ import logging

+ from unittest import mock

+ 

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ from resultsdb_listener import exceptions as exc

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.compose.complete",

+     body={

+         "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-ostree-compose",

          "topic": "org.centos.prod.ci.pipeline.compose.complete",

-         "build_id": "79", 

-         "build_url": "https://ci.centos.org/job/cci-pipeline-ostree-compose/79/",

-         "compose_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/f26/ostree",

+         "build_id": "79",

+         "build_url": "https://ci.centos.org/job/"

+         "cci-pipeline-ostree-compose/79/",

+         "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/f26/ostree",

          "branch": "f26",

          "ref": "fedora/f26/x86_64/atomic-host",

-         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f", 

-         "repo": "perl", 

-         "namespace": "rpms", 

-         "username": "fedora-atomic", 

+         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+         "repo": "perl",

+         "namespace": "rpms",

+         "username": "fedora-atomic",

          "test_guidance": "<comma-separated-list-of-test-suites-to-run>",

          "message-content": "",

-         "status": "SUCCESS"

-   }

- }

+         "status": "SUCCESS",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelineComposeComplete(object):

-     def test_exists(self):

-         pipeline_compose_complete.PipelineComposeCompleteProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_compose_complete.PipelineComposeCompleteProcessor(None)

-         test_data = test_listener.process(example_message)

- 

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['compose_url'] == example_message['msg']['compose_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

+ 

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "79",

+                 "build_url": "https://ci.centos.org/job/"

+                 "cci-pipeline-ostree-compose/79/",

+                 "branch": "f26",

+                 "ref": "fedora/f26/x86_64/atomic-host",

+                 "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+                 "repo": "perl",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "SUCCESS",

+                 "topic": "org.centos.prod.ci.pipeline.compose.complete",

+                 "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+                 "fedora-atomic/f26/ostree",

+             }

+         )

+ 

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_fail_to_parse_example_message(self, senders, caplog):

+         caplog.set_level(logging.DEBUG)

+         senders.get_sender.side_effect = exc.ListenerMsgError("boo!")

+ 

+         try:

+             consumer.Consumer()

+         except exc.ListenerMsgError:

+             pass

+ 

+         warn_higher_records = [

+             r for r in caplog.records if r.levelno >= logging.WARNING

+         ]

+         assert len(warn_higher_records) == 1

+         log_record = warn_higher_records[0]

+         assert log_record.levelno == logging.ERROR

+         assert log_record.message == "boo!"

+ 

+     def test_fail_to_send_example_message(self, caplog):

+         caplog.set_level(logging.DEBUG)

+ 

+         sender = mock.MagicMock()

+         sender.send.side_effect = exc.SenderError("boo!")

+ 

+         cons = consumer.Consumer()

+         cons.processor = sender

+         cons(example_message)

+ 

+         warn_higher_records = [

+             r for r in caplog.records if r.levelno >= logging.WARNING

+         ]

+         assert len(warn_higher_records) == 1

+         log_record = warn_higher_records[0]

+         assert log_record.levelno == logging.ERROR

+         assert log_record.message == "boo!"

@@ -1,47 +1,60 @@ 

- from resultsdb_listener import pipeline_compose_test_integration_complete

+ from unittest import mock

  

- example_message = {

-   "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934", 

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.test.integration.complete",   

-   'msg': { 

-         "CI_TYPE": "custom", 

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.test.integration.complete",

+     body={

+         "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-ostree-boot-sanity",

          "topic": "org.centos.prod.ci.pipeline.test.integration.complete",

-         "build_id": "79", 

-         "build_url": "https://ci.centos.org/job/ci-pipeline-ostree-boot-sanity/79/",

-         "compose_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/f26/ostree",

+         "build_id": "79",

+         "build_url": "https://ci.centos.org/job/"

+         "ci-pipeline-ostree-boot-sanity/79/",

+         "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/f26/ostree",

          "branch": "f26",

          "ref": "fedora/f26/x86_64/atomic-host",

-         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f", 

-         "repo": "perl", 

-         "namespace": "rpms", 

-         "username": "fedora-atomic", 

+         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+         "repo": "perl",

+         "namespace": "rpms",

+         "username": "fedora-atomic",

          "test_guidance": "<comma-separated-list-of-test-suites-to-run>",

          "message-content": "",

-         "status": "SUCCESS"

-   }

- }

+         "status": "SUCCESS",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelineComposeTestIntegrationComplete(object):

-     def test_exists(self):

-         pipeline_compose_test_integration_complete.PipelineComposeTestIntegrationCompleteProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_compose_test_integration_complete.PipelineComposeTestIntegrationCompleteProcessor(None)

-         test_data = test_listener.process(example_message)

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

  

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['compose_url'] == example_message['msg']['compose_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "79",

+                 "build_url": "https://ci.centos.org/job/"

+                 "ci-pipeline-ostree-boot-sanity/79/",

+                 "branch": "f26",

+                 "ref": "fedora/f26/x86_64/atomic-host",

+                 "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+                 "repo": "perl",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "SUCCESS",

+                 "topic": "org.centos.prod.ci.pipeline.test.integration"

+                 ".complete",

+                 "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+                 "fedora-atomic/f26/ostree",

+             }

+         )

@@ -1,51 +1,65 @@ 

- from resultsdb_listener import pipeline_image_complete

+ from unittest import mock

  

- example_message = {

- "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934",

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.image.complete",

-   'msg': { 

-         "CI_TYPE": "custom", 

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.image.complete",

+     body={

+         "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-ostree-image-compose",

          "topic": "org.centos.prod.ci.pipeline.image.complete",

-         "build_id": "82", 

-         "build_url": "https://ci.centos.org/job/ci-pipeline-ostree-image-compose/82/",

-         "image_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/f26/images/fedora-atomic-26.85-c2dcff5e8d4637a.qcow2",

+         "build_id": "82",

+         "build_url": "https://ci.centos.org/job/"

+         "ci-pipeline-ostree-image-compose/82/",

+         "image_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/f26/images/fedora-atomic-26.85-c2dcff5e8d4637a.qcow2",

          "image_name": "fedora-atomic-26.85-c2dcff5e8d4637a.qcow2",

          "type": "qcow2",

-         "compose_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/f26/ostree",

-         "compose_rev": "c2dcff5e8d4637a6cb19dd0f3e867b48b3d3b6fa0528dd2d64de23169f9221df",

+         "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/f26/ostree",

+         "compose_rev": "c2dcff5e8d4637a6cb19dd0f3e867b48b3d3b6fa0528dd2d6"

+         "4de23169f9221df",

          "branch": "f26",

          "ref": "fedora/f26/x86_64/atomic-host",

-         "rev": "N/A", 

-         "repo": "N/A", 

-         "namespace": "rpms", 

-         "username": "fedora-atomic", 

+         "rev": "N/A",

+         "repo": "N/A",

+         "namespace": "rpms",

+         "username": "fedora-atomic",

          "test_guidance": "<comma-separated-list-of-test-suites-to-run>",

          "message-content": "",

-         "status": "<SUCCESS/FAILURE/ABORTED/ABORTED>"

-   }

- }

+         "status": "<SUCCESS/FAILURE/ABORTED/ABORTED>",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelineComposeComplete(object):

-     def test_exists(self):

-         pipeline_image_complete.PipelineImageCompleteProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_image_complete.PipelineImageCompleteProcessor(None)

-         test_data = test_listener.process(example_message)

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

  

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['compose_url'] == example_message['msg']['compose_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "82",

+                 "build_url": "https://ci.centos.org/job/"

+                 "ci-pipeline-ostree-image-compose/82/",

+                 "branch": "f26",

+                 "ref": "fedora/f26/x86_64/atomic-host",

+                 "rev": "N/A",

+                 "repo": "N/A",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "<SUCCESS/FAILURE/ABORTED/ABORTED>",

+                 "topic": "org.centos.prod.ci.pipeline.image.complete",

+                 "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+                 "fedora-atomic/f26/ostree",

+             }

+         )

@@ -1,51 +1,66 @@ 

- from resultsdb_listener import pipeline_image_test_smoke_complete

+ from unittest import mock

  

- example_message = {

-   "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934",

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.image.test.smoke.complete",

-   'msg': {

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.image.test.smoke.complete",

+     body={

          "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-ostree-image-boot-sanity",

          "topic": "org.centos.prod.ci.pipeline.image.test.smoke.complete",

-         "build_id": "82", 

-         "build_url": "https://ci.centos.org/job/cci-pipeline-ostree-image-boot-sanity/82/",

-         "image_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/f26/images/fedora-atomic-26.85-c2dcff5e8d4637a.qcow2",

+         "build_id": "82",

+         "build_url": "https://ci.centos.org/job/"

+         "cci-pipeline-ostree-image-boot-sanity/82/",

+         "image_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/f26/images/fedora-atomic-26.85-c2dcff5e8d4637a.qcow2",

          "image_name": "fedora-atomic-26.85-c2dcff5e8d4637a.qcow2",

          "type": "qcow2",

-         "compose_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/f26/ostree",

-         "compose_rev": "c2dcff5e8d4637a6cb19dd0f3e867b48b3d3b6fa0528dd2d64de23169f9221df",

+         "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/f26/ostree",

+         "compose_rev": "c2dcff5e8d4637a6cb19dd0f3e867b48b3d3b6fa05"

+         "28dd2d64de23169f9221df",

          "branch": "f26",

          "ref": "fedora/f26/x86_64/atomic-host",

          "rev": "N/A",

          "repo": "N/A",

-         "namespace": "rpms", 

+         "namespace": "rpms",

          "username": "fedora-atomic",

          "test_guidance": "<comma-separated-list-of-test-suites-to-run>",

          "message-content": "",

-         "status": "<SUCCESS/FAILURE/ABORTED/ABORTED>"

-   }

- }

+         "status": "<SUCCESS/FAILURE/ABORTED/ABORTED>",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelineImageTestSmokeComplete(object):

-     def test_exists(self):

-         pipeline_image_test_smoke_complete.PipelineImageTestSmokeCompleteProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_image_test_smoke_complete.PipelineImageTestSmokeCompleteProcessor(None)

-         test_data = test_listener.process(example_message)

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

  

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['compose_url'] == example_message['msg']['compose_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "82",

+                 "build_url": "https://ci.centos.org/job/"

+                 "cci-pipeline-ostree-image-boot-sanity/82/",

+                 "branch": "f26",

+                 "ref": "fedora/f26/x86_64/atomic-host",

+                 "rev": "N/A",

+                 "repo": "N/A",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "<SUCCESS/FAILURE/ABORTED/ABORTED>",

+                 "topic": "org.centos.prod.ci.pipeline.image.test."

+                 "smoke.complete",

+                 "compose_url": "http://artifacts.ci.centos.org/artifacts/"

+                 "fedora-atomic/f26/ostree",

+             }

+         )

@@ -1,47 +1,60 @@ 

- from resultsdb_listener import pipeline_package_complete

+ from unittest import mock

  

- example_message = { 

-   "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934", 

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.package.complete",

-   'msg': { 

-         "CI_TYPE": "custom", 

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.package.complete",

+     body={

+         "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-duffy-allocate-new",

          "topic": "org.centos.prod.ci.pipeline.package.complete",

-         "build_id": "79", 

-         "build_url": "https://ci.centos.org/job/ci-pipeline-duffy-allocate-new/79/",

-         "package_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/rawhide/repo/perl_repo/perl-5.26.0-392.fc27.x86_64.rpm",

+         "build_id": "79",

+         "build_url": "https://ci.centos.org/job/"

+         "ci-pipeline-duffy-allocate-new/79/",

+         "package_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/rawhide/repo/perl_repo/perl-5.26.0-392.fc27.x86_64.rpm",

          "branch": "rawhide",

          "ref": "fedora/rawhide/x86_64/atomic-host",

-         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f", 

-         "repo": "perl", 

-         "namespace": "rpms", 

-         "username": "fedora-atomic", 

+         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+         "repo": "perl",

+         "namespace": "rpms",

+         "username": "fedora-atomic",

          "test_guidance": "''",

          "message-content": "",

-         "status": "<SUCCESS/FAILURE/ABORTED>"

-   }

- }

+         "status": "<SUCCESS/FAILURE/ABORTED>",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelinePackageComplete(object):

-     def test_exists(self):

-         pipeline_package_complete.PipelinePackageCompleteProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_package_complete.PipelinePackageCompleteProcessor(None)

-         test_data = test_listener.process(example_message)

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

  

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['package_url'] == example_message['msg']['package_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "79",

+                 "build_url": "https://ci.centos.org/job/"

+                 "ci-pipeline-duffy-allocate-new/79/",

+                 "branch": "rawhide",

+                 "ref": "fedora/rawhide/x86_64/atomic-host",

+                 "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+                 "repo": "perl",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "<SUCCESS/FAILURE/ABORTED>",

+                 "topic": "org.centos.prod.ci.pipeline.package.complete",

+                 "package_url": "http://artifacts.ci.centos.org/artifacts/"

+                 "fedora-atomic/rawhide/repo/perl_repo/"

+                 "perl-5.26.0-392.fc27.x86_64.rpm",

+             }

+         )

@@ -1,45 +1,54 @@ 

- from resultsdb_listener import pipeline_package_ignore

+ from unittest import mock

  

- example_message = { 

-   "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934", 

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.package.ignore",   

-   'msg': { 

-         "CI_TYPE": "custom", 

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.package.ignore",

+     body={

+         "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-rpmbuild",

          "topic": "org.centos.prod.ci.pipeline.package.ignore",

-         "build_id": "79", 

+         "build_id": "79",

          "build_url": "https://ci.centos.org/job/ci-pipeline-rpmbuild/79/",

          "branch": "rawhide",

          "ref": "fedora/rawhide/x86_64/atomic-host",

-         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f", 

-         "repo": "perl", 

-         "namespace": "rpms", 

-         "username": "fedora-atomic", 

+         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+         "repo": "perl",

+         "namespace": "rpms",

+         "username": "fedora-atomic",

          "test_guidance": "<comma-separated-list-of-test-suites-to-run>",

          "message-content": "",

-         "status": "SUCCESS"

-   }

- }

+         "status": "SUCCESS",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelinePackageIgnore(object):

-     def test_exists(self):

-         pipeline_package_ignore.PipelinePackageIgnoreProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_package_ignore.PipelinePackageIgnoreProcessor(None)

-         test_data = test_listener.process(example_message)

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

  

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "79",

+                 "build_url": "https://ci.centos.org/job/"

+                 "ci-pipeline-rpmbuild/79/",

+                 "branch": "rawhide",

+                 "ref": "fedora/rawhide/x86_64/atomic-host",

+                 "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+                 "repo": "perl",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "SUCCESS",

+                 "topic": "org.centos.prod.ci.pipeline.package.ignore",

+             }

+         )

@@ -1,47 +1,61 @@ 

- from resultsdb_listener import pipeline_package_complete

+ from unittest import mock

  

- example_message = {

-   "i": 1,

-   "timestamp": 1496382015, 

-   "msg_id": "2017-f2b46ca1-1b07-4539-8d86-f7d3cb1b5934", 

-   "crypto": "x509", 

-   "topic": "org.centos.prod.ci.pipeline.test.functional.complete",   

-   'msg': { 

-         "CI_TYPE": "custom", 

+ from fedora_messaging.api import Message

+ 

+ from resultsdb_listener import consumer

+ 

+ 

+ example_message = Message(

+     topic="org.centos.prod.ci.pipeline.test.functional.complete",

+     body={

+         "CI_TYPE": "custom",

          "CI_NAME": "ci-pipeline-atomic-host-tests",

          "topic": "org.centos.prod.ci.pipeline.test.functional.complete",

-         "build_id": "79", 

-         "build_url": "https://ci.centos.org/job/ci-pipeline-atomic-host-tests/79/",

-         "package_url": "http://artifacts.ci.centos.org/artifacts/fedora-atomic/rawhide/repo/perl_repo/perl-5.26.0-392.fc27.x86_64.rpm",

+         "build_id": "79",

+         "build_url": "https://ci.centos.org/job/"

+         "ci-pipeline-atomic-host-tests/79/",

+         "package_url": "http://artifacts.ci.centos.org/artifacts/"

+         "fedora-atomic/rawhide/repo/perl_repo/perl-5.26.0-392.fc27.x86_64.rpm",

          "branch": "rawhide",

          "ref": "fedora/rawhide/x86_64/atomic-host",

-         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f", 

-         "repo": "perl", 

-         "namespace": "rpms", 

-         "username": "fedora-atomic", 

+         "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+         "repo": "perl",

+         "namespace": "rpms",

+         "username": "fedora-atomic",

          "test_guidance": "<comma-separated-list-of-test-suites-to-run>",

          "message-content": "",

-         "status": "SUCCESS"

-   }

- }

+         "status": "SUCCESS",

+     },

+ )

+ example_message.id = "random_uuid"

  

  

  class TestPipelinePackageComplete(object):

-     def test_exists(self):

-         pipeline_package_complete.PipelinePackageCompleteProcessor(None)

- 

-     def test_parse_example_message(self):

-         test_listener = pipeline_package_complete.PipelinePackageCompleteProcessor(None)

-         test_data = test_listener.process(example_message)

+     @mock.patch("resultsdb_listener.consumer.senders")

+     def test_parse_example_message(self, senders):

+         processor = mock.MagicMock()

+         senders.get_sender.return_value = processor

+         cons = consumer.Consumer()

+         cons(example_message)

  

-         assert test_data['msg_id'] == example_message['msg_id']

-         assert test_data['topic'] == example_message['msg']['topic']

-         assert test_data['build_id'] == example_message['msg']['build_id']

-         assert test_data['build_url'] == example_message['msg']['build_url']

-         assert test_data['package_url'] == example_message['msg']['package_url']

-         assert test_data['branch'] == example_message['msg']['branch']

-         assert test_data['ref'] == example_message['msg']['ref']

-         assert test_data['rev'] == example_message['msg']['rev']

-         assert test_data['repo'] == example_message['msg']['repo']

-         assert test_data['namespace'] == example_message['msg']['namespace']

-         assert test_data['username'] == example_message['msg']['username']

+         assert processor().send.call_count == 1

+         processor().send.assert_called_with(

+             {

+                 "msg_id": "random_uuid",

+                 "build_id": "79",

+                 "build_url": "https://ci.centos.org/job/"

+                 "ci-pipeline-atomic-host-tests/79/",

+                 "branch": "rawhide",

+                 "ref": "fedora/rawhide/x86_64/atomic-host",

+                 "rev": "0546fa18041a8ca223f4f441dc1868fc81ddce0f",

+                 "repo": "perl",

+                 "namespace": "rpms",

+                 "username": "fedora-atomic",

+                 "status": "SUCCESS",

+                 "topic": "org.centos.prod.ci.pipeline.test.functional."

+                 "complete",

+                 "package_url": "http://artifacts.ci.centos.org/artifacts/"

+                 "fedora-atomic/rawhide/repo/perl_repo/perl-5.26.0-392."

+                 "fc27.x86_64.rpm",

+             }

+         )

file modified
+23 -13
@@ -1,18 +1,21 @@ 

- import mock

+ from unittest import mock

  import pytest

  import resultsdb_api

  

  from resultsdb_listener import senders

  from resultsdb_listener import exceptions as exc

  

+ 

  class TestResultsdbSender(object):

      def setup_method(self, method):

-         self.ref_testcase = 'some.important.test'

-         self.ref_status = 'SUCCESS'

-         self.ref_url = 'http://127.0.0.1/result/test'

-         self.ref_result = {'topic': self.ref_testcase,

-                            'status': self.ref_status,

-                            'build_url': self.ref_url}

+         self.ref_testcase = "some.important.test"

+         self.ref_status = "SUCCESS"

+         self.ref_url = "http://127.0.0.1/result/test"

+         self.ref_result = {

+             "topic": self.ref_testcase,

+             "status": self.ref_status,

+             "build_url": self.ref_url,

+         }

  

      def test_sender_testcase_exists(self):

          stub_rdb = mock.MagicMock()
@@ -25,7 +28,11 @@ 

  

      def test_sender_testcase_exists_failure(self):

          stub_rdb = mock.MagicMock()

-         stub_rdb.get_testcase = mock.Mock(side_effect=resultsdb_api.ResultsDBapiException("testcase not found"))

+         stub_rdb.get_testcase = mock.Mock(

+             side_effect=resultsdb_api.ResultsDBapiException(

+                 "testcase not found"

+             )

+         )

  

          test_sender = senders.ResultsdbSender(stub_rdb)

  
@@ -43,7 +50,11 @@ 

  

      def test_sender_testcase_exists_failure_create(self):

          stub_rdb = mock.MagicMock()

-         stub_rdb.get_testcase = mock.Mock(side_effect=resultsdb_api.ResultsDBapiException("testcase not found"))

+         stub_rdb.get_testcase = mock.Mock(

+             side_effect=resultsdb_api.ResultsDBapiException(

+                 "testcase not found"

+             )

+         )

  

          test_sender = senders.ResultsdbSender(stub_rdb)

  
@@ -57,7 +68,6 @@ 

  

          test_sender = senders.ResultsdbSender(stub_rdb)

  

-         assert test_sender._translate_outcome('SUCCESS') == 'PASSED'

-         assert test_sender._translate_outcome('FAILURE') == 'FAILED'

-         assert test_sender._translate_outcome('ABORTED') == 'FAILED'

- 

+         assert test_sender._translate_outcome("SUCCESS") == "PASSED"

+         assert test_sender._translate_outcome("FAILURE") == "FAILED"

+         assert test_sender._translate_outcome("ABORTED") == "FAILED"

file modified
+27 -6
@@ -1,14 +1,35 @@ 

  # This is a common file where different test suites/linters can be configured.

- # Phabricator uses this file when running `arc unit` or `arc lint`.

+ [tox]

+ envlist = py36,py37,lint

+ #,py38 does not work atm

+ # If the user is missing an interpreter, don't fail

+ skip_missing_interpreters = True

+ 

+ [testenv]

+ deps =

+     -rrequirements.txt

+     pytest

+     pytest-cov

+ passenv = HOME

+ commands =

+     pytest tests/

+ 

+ 

+ [testenv:lint]

+ deps =

+     flake8

+     black

+ commands =

+     flake8 .

+     black -l 79 --check . tests/

+ 

  

  [flake8]

- max-line-length=99

+ max-line-length=80

  

  [pep8]

- max-line-length=99

+ max-line-length=80

  

  [pytest]

  minversion=2.0

- python_functions=test should

- python_files=test_* functest_*

- addopts=--functional testing/ --cov-report=term-missing --cov resultsdb_listener

+ addopts= --cov-report=term-missing --cov resultsdb_listener

no initial comment

2 new commits added

  • Update the spec file for python3 and the new code
  • Move the configuration files into the conf folder
4 years ago

rebased onto f933a2bc6a5eaf43a12ccd7b0269aaa7f57cb843

4 years ago

rebased onto 1751324

4 years ago

This PR can be reviewed commits by commits :)

If you do this, a new instance of the sender will be created for each incoming message. Looking at that code I see that it initializes a DB connection on instanciation, so it may be a good idea to instanciate it only once. To do that, you can switch to a class-based callback, like what Bodhi does for example : https://github.com/fedora-infra/bodhi/blob/develop/bodhi/server/consumers/__init__.py

Please document here how you handle duplicate messages (they may happen with Fedora Messaging in case of network errors or crashes, for exemple). If duplicate messages are not a problem (because it will just add a duplicate entry in the DB and we decide we don't care), then please say so. But if there's a chance that duplicate messages could cause a crash (duplicate index on DB insert for exemple) or other issues, then we should try to protect against it.
There are multiple ways to do that, but let's discuss it in a more appropriate place like an issue entry.

I'll double check but I believe we do not have to handle duplicate message (worst case we log twice that something passed/failed, which shouldn't affect any resulting decision).

Good idea, I'll look at that :)

I've refactor but note that it is not initializing a DB connection but a requests session in fact.
It is calling https://pagure.io/taskotron/resultsdb_api/blob/develop/f/resultsdb_api.py#_115

I've adjusted my CLI script from #7 and I was able to import twice the same message from fedora-ci (2019-f0e2ba1b-7ec8-47b6-b6ef-18cd762c5780 to be precise). So no issue with duplicate messages :)

1 new commit added

  • Move the fedora-messaging consumer to be a class-based callback
4 years ago

13 new commits added

  • Move the fedora-messaging consumer to be a class-based callback
  • Update the spec file for python3 and the new code
  • Move the configuration files into the conf folder
  • Run the tests with tox, document this in the README and fixes all linting issues
  • Fixup README
  • Adjust the unit-tests to test the new consumer
  • Project wide black run
  • Expend on the files ignored in .gitignore
  • Flake8 fixes
  • Make a little clearer that the following output is coming from this sender
  • Drop the datagrepper_url from the config, it's not used anywhere
  • Document in the README how to get the project running
  • Drop fedmsg for fedora-messaging
4 years ago

You can also just write cons(example_message).

Same here, you can just write cons(example_message).

Same here, you can also just write cons(example_message).

Same here, you can also just write cons(example_message)

13 new commits added

  • Move the fedora-messaging consumer to be a class-based callback
  • Update the spec file for python3 and the new code
  • Move the configuration files into the conf folder
  • Run the tests with tox, document this in the README and fixes all linting issues
  • Fixup README
  • Adjust the unit-tests to test the new consumer
  • Project wide black run
  • Expend on the files ignored in .gitignore
  • Flake8 fixes
  • Make a little clearer that the following output is coming from this sender
  • Drop the datagrepper_url from the config, it's not used anywhere
  • Document in the README how to get the project running
  • Drop fedmsg for fedora-messaging
4 years ago

LGTM, thanks!
I usually think it's OK to leave the max-line-length at 99 and not 80, I've found that sometimes 80 makes you cut lines very inconveniently. But that's a matter of personal preference.

I usually think it's OK to leave the max-line-length at 99 and not 80, I've found that sometimes 80 makes you cut lines very inconveniently. But that's a matter of personal preference.

I code with two windows side by side, so 80 chars basically fits while 99 don't, so I look like an old-timer and stick to 80 chars ;-)

Thanks for the review! :)

Pull-Request has been merged by pingou

4 years ago