#7 Add script to upload a message from datagrepper into resultsdb
Closed 4 years ago by pingou. Opened 6 years ago by pingou.
pingou/ci-resultsdb-listener dg_upload  into  master

file modified
+1
@@ -1,1 +1,2 @@ 

  .tox/

+ tests/.cache/

@@ -3,7 +3,7 @@ 

  

  class CIProcessor(object):

  

-     def __init__(self, log, sender=None):

+     def __init__(self, sender=None):

          if sender is None:

              self.sender = senders.get_sender(config.sender_type)

          else:

@@ -0,0 +1,115 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ """

+  (c) 2017 - Copyright Red Hat Inc

2018?

It was last changed in 2017 :)

+ 

+  Authors:

+    Pierre-Yves Chibon <pingou@pingoured.fr>

+ 

+ """

+ 

+ import argparse

+ import logging

+ 

+ import requests

+ from requests.packages.urllib3.util import retry

+ 

+ from resultsdb_listener import ci_processor

+ 

+ 

+ _log = logging.getLogger(__name__)

+ 

+ DG_URL = 'https://apps.fedoraproject.org/datagrepper/'

+ 

+ 

+ def _parser_load_dg(subparser):

+     """ Set up the CLI argument parser for the load-dg action. """

+     local_parser = subparser.add_parser(

+         'load-dg',

+         help='Upload to resultsdb a message retrieved from datagrepper')

+     local_parser.add_argument(

+         'msg_ids', nargs='+',

+         help="List of msg-id to retrieve from datagrepper and load")

+     local_parser.add_argument(

+         '--dg-url', dest='dg_url', help="Datagrepper URL to query")

+     local_parser.set_defaults(func=do_load_dg)

+ 

+ 

+ def parse_arguments():

+     """ Set-up the argument parsing. """

+     parser = argparse.ArgumentParser(

+         description='A CLI for ci-resultsdb-listener')

+ 

+     parser.add_argument(

+         '--debug', default=False, action='store_true',

+         help='Increase the verbosity of the information displayed')

+ 

+     subparser = parser.add_subparsers(title='actions')

+ 

+     # load-dg

+     _parser_load_dg(subparser)

+ 

+     return parser.parse_args()

+ 

+ 

+ def do_load_dg(args):

+     """ Load messages from datagrepper.

+ 

+     :arg args: the argparse object returned by ``parse_arguments()``.

+ 

+     """

+     _log.debug('dg-url:         %s', args.dg_url)

+     _log.debug('msg-ids:        %s', args.msg_ids)

+     base_url = args.dg_url or DG_URL

+ 

+     requests_session = requests.Session()

+     timeout = (20, 20)  # Connect/Read timeouts

+     retries = 3

+     retry_conf = retry.Retry(

+         total=retries, connect=retries, read=retries, backoff_factor=1)

+     retry_conf.BACKOFF_MAX = 5

+     requests_session.mount(

+         'http://', requests.adapters.HTTPAdapter(max_retries=retry_conf))

Do we want to do any plaintext requests?

Likely not, I just reused existing code here, I can drop these two lines

+     requests_session.mount(

+         'https://', requests.adapters.HTTPAdapter(max_retries=retry_conf))

+ 

+     processor = ci_processor.CIProcessor()

+ 

+     base_url = "%s/id?id=" % base_url.rstrip('/')

+     for msg_id in args.msg_ids:

+         url = base_url + msg_id

+         _log.debug('Calling %s', url)

+         req = requests_session.get(url, timeout=timeout)

+         data = req.json()

+         _log.debug('Processing: %s', data)

+         processor.send(processor.process(data))

+         _log.debug('Done')

+ 

+ 

+ def main():

+     """ Main entry point for the CLI tool. """

+     # Parse the arguments

+     args = parse_arguments()

+ 

+     logging.basicConfig()

+     if args.debug:

+         _log.setLevel(logging.DEBUG)

+ 

+     # Act based on the arguments given

+     return_code = 0

+     try:

+         args.func(args)

+     except KeyboardInterrupt:

+         print("\nInterrupted by user.")

+         return_code = 1

+     except Exception as err:

+         print('Error: {0}'.format(err))

+         logging.exception("Generic error catched:")

+         return_code = 2

+ 

+     return return_code

+ 

+ 

+ if __name__ == '__main__':

+     import sys

+     sys.exit(main())

@@ -29,7 +29,7 @@ 

  

      def __init__(self, *args, **kw):

          super(PipelineConsumer, self).__init__(*args, **kw)

-         self.processor = ci_processor.CIProcessor(self.log)

+         self.processor = ci_processor.CIProcessor()

  

      def consume(self, message):

          msg = message['body']

file modified
+6
@@ -25,10 +25,16 @@ 

      entry_points={

          'moksha.consumer': [

              'pipelineconsumer = resultsdb_listener.pipeline_consumer:PipelineConsumer',

+         ],

+         'console_scripts': [

+             'resultsdb-listener-cli = resultsdb_listener.cli:main',

          ]

      },

      include_package_data=True,

      install_requires=[

+         'fedmsg',

+         'requests',

+         'resultsdb_api',

      ],

      cmdclass={'test': PyTest}

  )

@@ -1,1 +0,0 @@ 

- {} 

\ No newline at end of file

no initial comment

It was last changed in 2017 :)

Do we want to do any plaintext requests?

rebased onto f617e8afa8e1de1167e3b8d05cdcaf734487fa63

5 years ago

rebased onto 078fe800f8aefad75943f24ca7762a692019c2cd

5 years ago

Likely not, I just reused existing code here, I can drop these two lines

rebased onto 4b728abaf3eeb25d814d94084056e5be4ef6d18e

5 years ago

rebased onto 01afeb3

4 years ago

I'm going to close this PR.

https://pagure.io/ci-resultsdb-listener/pull-request/12 updates this script to the new code

Pull-Request has been closed by pingou

4 years ago