From 5c7979bf953849760c4ffe41d4f039bec9724f7b Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jun 19 2019 19:21:57 +0000 Subject: [PATCH 1/2] compose-tracker: adapt to change in message body See https://github.com/fedora-infra/fedmsg-migration-tools/issues/20 --- diff --git a/compose_tracker.py b/compose_tracker.py index 3da8648..c024f4c 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -39,7 +39,6 @@ KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID=' # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000 ########import json ########msg = json.loads("""{ -######## "msg": { ######## "status": "DOOMED", ######## "release_type": "ga", ######## "compose_label": null, @@ -52,7 +51,7 @@ KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID=' ######## "release_name": "Fedora-Modular", ######## "release_short": "Fedora-Modular", ######## "compose_id": "Fedora-Modular-Bikeshed-20180215.n.0" -######## }} +######## } ########""" ########) @@ -94,7 +93,7 @@ class Consumer(object): # Grab the raw message body and the status from that msg = message.body - status = msg['msg']['status'] + status = msg['status'] # Print out a log statement if the day has changed today = datetime.date.today() @@ -116,8 +115,8 @@ class Consumer(object): # We have a compose that either failed or had missing artifacts # create a new issue. - title = msg['msg']['compose_id'] + ' ' + status - logfileurl = msg['msg']['location'] + '/../logs/global/pungi.global.log' + title = msg['compose_id'] + ' ' + status + logfileurl = msg['location'] + '/../logs/global/pungi.global.log' logger.info("%s\t%s" % (title, logfileurl)) # variable to hold description for issue @@ -154,7 +153,7 @@ class Consumer(object): logger.debug(content) # pull only part of the compose ID for the tag to set - tag = re.search('(.*)-\d{8}', msg['msg']['compose_id']).group(1) + tag = re.search('(.*)-\d{8}', msg['compose_id']).group(1) #TODO figure out how to set tag on an issue # Should be able to do this now https://pagure.io/libpagure/issue/31 From 9bb672b9855fee2379e56889feb3dd90b229173f Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Jun 19 2019 20:12:24 +0000 Subject: [PATCH 2/2] compose-tracker: add main for local testing/manual runs Allows for local testing or also manually triggering a run. You can manually trigger a run by passing in a URL to a datagrepper message like: compose_tracker.py 'https://apps.fedoraproject.org/datagrepper/id?id=2019-05494a6e-2619-4290-96d0-07d8804c3f2a&is_raw=true&size=extra-large' If you want an actual pagure issue to be opened then set PAGURE_TOKEN first: export PAGURE_TOKEN=abcd compose_tracker.py 'https://apps.fedoraproject.org/datagrepper/id?id=2019-05494a6e-2619-4290-96d0-07d8804c3f2a&is_raw=true&size=extra-large' --- diff --git a/compose_tracker.py b/compose_tracker.py index c024f4c..f72cd6d 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -16,12 +16,15 @@ # License along with this library. If not, see . import datetime -import fedora_messaging +import fedora_messaging.api +import json +import logging import os import re import requests +import sys + from libpagure import Pagure -import logging # Set local logging logger = logging.getLogger(__name__) @@ -37,23 +40,22 @@ KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID=' # We are processing the org.fedoraproject.prod.pungi.compose.status.change topic # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000 -########import json -########msg = json.loads("""{ -######## "status": "DOOMED", -######## "release_type": "ga", -######## "compose_label": null, -######## "compose_respin": 0, -######## "compose_date": "20180215", -######## "release_version": "Bikeshed", -######## "location": "http://kojipkgs.fedoraproject.org/compose/Fedora-Modular-Bikeshed-20180215.n.0/compose", -######## "compose_type": "nightly", -######## "release_is_layered": false, -######## "release_name": "Fedora-Modular", -######## "release_short": "Fedora-Modular", -######## "compose_id": "Fedora-Modular-Bikeshed-20180215.n.0" -######## } -########""" -########) +EXAMPLE_MESSAGE_BODY = json.loads(""" +{ + "status": "DOOMED", + "release_type": "ga", + "compose_label": null, + "compose_respin": 0, + "compose_date": "20190619", + "release_version": "Rawhide", + "location": "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20190619.n.0/compose", + "compose_type": "nightly", + "release_is_layered": false, + "release_name": "Fedora", + "release_short": "Fedora", + "compose_id": "Fedora-Rawhide-20190619.n.0" +} +""") class Consumer(object): def __init__(self): @@ -159,3 +161,35 @@ class Consumer(object): if self.token: self.pg.create_issue(title=title, content=content) + +# The code in this file is expected to be run through fedora messaging +# However, you can run the script directly for testing purposes. The +# below code allows us to do that and also fake feeding data to the +# call by updating the json text below. +if __name__ == '__main__': + sh = logging.StreamHandler() + sh.setFormatter(logging.Formatter('%(asctime)s %(message)s')) + logger.addHandler(sh) + + # Allow a user to pass in a url to a datagrepper page and we'll + # parse that and pass it into the Consumer. + if len(sys.argv) == 1: + # no args, just use example message body + body = EXAMPLE_MESSAGE_BODY + else: + # User passed in a url like: + # https://apps.fedoraproject.org/datagrepper/id?id=2019-ec728a49-57a7-4e19-9f5b-c8ac09920d36&is_raw=true&size=extra-large + url = sys.argv[1] + logger.info(f'Attempting to retrieve data from {url}') + r = requests.get(url) + data = json.loads(r.text) + logger.debug('Retrieved JSON data:') + logger.debug(json.dumps(data, indent=4, sort_keys=True)) + body = data['msg'] + + # Create a Message and then call the Consumer() + m = fedora_messaging.api.Message( + topic = 'org.fedoraproject.prod.pungi.compose.status.change', + body = body) + c = Consumer() + c.__call__(m)