#12 Add a script uploading a message from datagrepper into resultsdb
Opened 4 years ago by pingou. Modified 4 years ago
pingou/ci-resultsdb-listener resulstdb_cli  into  master

@@ -47,7 +47,7 @@ 

  

  %{python3_sitelib}/%{modname}/*

  %{python3_sitelib}/%{modname}-%{version}*

- 

+ %{_bindir}/resultsdb-listener-cli

  

  %changelog

  * Fri Mar 01 2019 Pierre-Yves Chibon <pingou@pingoured.fr> - 0.3-1

@@ -0,0 +1,128 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ """

+  (c) 2019 - Copyright Red Hat Inc

+ 

+  Authors:

+    Pierre-Yves Chibon <pingou@pingoured.fr>

+ 

+ """

+ 

+ import argparse

+ import logging

+ 

+ import fedora_messaging.api

+ import requests

+ from requests.packages.urllib3.util import retry

+ 

+ from resultsdb_listener.consumer import Consumer

+ 

+ 

+ _log = logging.getLogger(__name__)

+ 

+ DG_URL = "https://apps.fedoraproject.org/datagrepper/"

+ 

+ 

+ def _parser_load_dg(subparser):

+     """ Set up the CLI argument parser for the load-dg action. """

+     local_parser = subparser.add_parser(

+         "load-dg",

+         help="Upload to resultsdb a message retrieved from datagrepper",

+     )

+     local_parser.add_argument(

+         "msg_ids",

+         nargs="+",

+         help="List of msg-id to retrieve from datagrepper and load",

+     )

+     local_parser.add_argument(

+         "--dg-url", dest="dg_url", help="Datagrepper URL to query"

+     )

+     local_parser.set_defaults(func=do_load_dg)

+ 

+ 

+ def parse_arguments():

+     """ Set-up the argument parsing. """

+     parser = argparse.ArgumentParser(

+         description="A CLI for ci-resultsdb-listener"

+     )

+ 

+     parser.add_argument(

+         "--debug",

+         default=False,

+         action="store_true",

+         help="Increase the verbosity of the information displayed",

+     )

+ 

+     subparser = parser.add_subparsers(title="actions")

+ 

+     # load-dg

+     _parser_load_dg(subparser)

+ 

+     return parser.parse_args()

+ 

+ 

+ def do_load_dg(args):

+     """ Load messages from datagrepper.

+ 

+     :arg args: the argparse object returned by ``parse_arguments()``.

+ 

+     """

+     _log.debug("dg-url:         %s", args.dg_url)

+     _log.debug("msg-ids:        %s", args.msg_ids)

+     base_url = args.dg_url or DG_URL

+ 

+     requests_session = requests.Session()

+     timeout = (20, 20)  # Connect/Read timeouts

+     retries = 3

+     retry_conf = retry.Retry(

+         total=retries, connect=retries, read=retries, backoff_factor=1

+     )

+     retry_conf.BACKOFF_MAX = 5

+     requests_session.mount(

+         "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)

+     )

+ 

+     processor = Consumer()

+ 

+     base_url = "%s/id?id=" % base_url.rstrip("/")

+     for msg_id in args.msg_ids:

+         url = base_url + msg_id

+         _log.debug("Calling %s", url)

+         req = requests_session.get(url, timeout=timeout)

+         data = req.json()

+         _log.debug("Processing: %s", data)

+         message = fedora_messaging.api.Message(

+             topic=data["topic"], body=data["msg"]

+         )

+         processor.__call__(message)

+         _log.debug("Done")

+ 

+ 

+ def main():

+     """ Main entry point for the CLI tool. """

+     # Parse the arguments

+     args = parse_arguments()

+ 

+     logging.basicConfig()

+     if args.debug:

+         _log.setLevel(logging.DEBUG)

+ 

+     # Act based on the arguments given

+     return_code = 0

+     try:

+         args.func(args)

+     except KeyboardInterrupt:

+         print("\nInterrupted by user.")

+         return_code = 1

+     except Exception as err:

+         print("Error: {0}".format(err))

+         logging.exception("Generic error catched:")

+         return_code = 2

+ 

+     return return_code

+ 

+ 

+ if __name__ == "__main__":

+     import sys

+ 

+     sys.exit(main())

file modified
+5 -2
@@ -28,6 +28,9 @@ 

      url="https://pagure.io/resultsdb_listener",

      packages=["resultsdb_listener"],

      include_package_data=True,

-     install_requires=[],

-     cmdclass={"test": PyTest},

+     install_requires=['fedora_messaging', 'requests', 'resultsdb_api'],

+     cmdclass={'test': PyTest},

+     console_scripts: [

+         'resultsdb-listener-cli = resultsdb_listener.cli:main',

+     ],

  )

This uses the code used when consuming messages from the bus thus
allowing to test it in real conditions.

Signed-off-by: Pierre-Yves Chibon pingou@pingoured.fr

rebased onto 8ead2bad4edfcb48a7a61aa98225de6012f15d41

4 years ago

rebased onto ecf58ac

4 years ago