From 3ceac217bc90a9d29bb8d7c6156e80e02985fca7 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Nov 13 2019 23:09:04 +0000 Subject: Add a consumer for updating AMI link page This page won't be properly populated when the event is first created, as that happens as soon as the compose is complete but the AMIs are published over the next hour or two. We do not get a signal when *all* of the AMIs are published, unfortunately, so all we can do is regenerate the page *each time* a relevant AMI is published. That's what this consumer does: whenever an 'AMI published' message is received, it checks for a validation event related to that compose, and if there is one, it updates its AMI page. Signed-off-by: Adam Williamson --- diff --git a/README.md b/README.md index e45a11d..1c09346 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ taking over my job or something. As long as relval is also installed, relvalconsumer will also run the image size check on the newly-created event, whenever it creates an event. +The project also includes another consumer, RelvalAMIConsumer, which updates +a wiki page containing information on available EC2 AMIs for a validation +event whenever a relevant 'AMI published' message is received from fedimg +(the tool that publishes Fedora AMI images to EC2). + ## Requirements Python libraries: @@ -60,7 +65,8 @@ You will also typically need to do: A sample configuration file which is set up to listen for real compose events on the production fedora-messaging bus is provided as `relvalconsumer.toml`. -To use this, you must at least change the queue name from +A sample file for the AMI consumer is provided as `relvalamiconsumer.toml`. +To use these, you must at least change the queue name from `00000000-0000-0000-0000-000000000000` to a unique and private string; the official recommendation is to use a UUID generated by uuidgen. Then place the file in `/etc/fedora-messaging`. @@ -70,11 +76,17 @@ You can then enable and start the consumer as a systemd service: sudo systemctl enable fm-consumer@relvalconsumer sudo systemctl start fm-consumer@relvalconsumer +For the AMI consumer: + + sudo systemctl enable fm-consumer@relvalamiconsumer + sudo systemctl start fm-consumer@relvalamiconsumer + ## Test and production modes A configuration setting, `relval_prod`, decides whether the consumer operates in 'production' or 'test' mode. This is set in the consumer configuration -file's `consumer_config` section. In test mode: +file's `consumer_config` section. The corresponding setting for the AMI +consumer is `relvalami_prod`. In test mode: * The consumer creates events on the staging wiki (not production wiki) * The announcement email is logged rather than being mailed out diff --git a/relvalamiconsumer.toml b/relvalamiconsumer.toml new file mode 100644 index 0000000..3e3eae0 --- /dev/null +++ b/relvalamiconsumer.toml @@ -0,0 +1,93 @@ +# sample fedora-messaging consumer configuration file for +# relvalconsumer. Note you must change the UUID here to something +# unique before using this. +# +# This file is in the TOML format. +amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" +callback = "relvalconsumer:RelvalAMIConsumer" + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/fedora-key.pem" +certfile = "/etc/fedora-messaging/fedora-cert.pem" + +[client_properties] +app = "Fedora release validation event AMI page updater (relvalconsumer)" +app_url = "https://pagure.io/fedora-qa/relvalconsumer" +app_contacts_email = ["qa-devel@lists.fedoraproject.org"] + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +# Queue names *must* be in the normal UUID format: run "uuidgen" and use the +# output as your queue name. If your queue is not exclusive, anyone can connect +# and consume from it, causing you to miss messages, so do not share your queue +# name. Any queues that are not auto-deleted on disconnect are garbage-collected +# after approximately one hour. +# +# If you require a stronger guarantee about delivery, please talk to Fedora's +# Infrastructure team. +[queues.00000000-0000-0000-0000-000000000000] +durable = false +auto_delete = true +exclusive = true +arguments = {} + +[[bindings]] +queue = "00000000-0000-0000-0000-000000000000" +exchange = "amq.topic" +routing_keys = ["org.fedoraproject.prod.fedimg.image.publish"] +# need this to receive messages from ZMQ->AMQP bridge +[[bindings]] +queue = "00000000-0000-0000-0000-000000000000" +exchange = "zmq.topic" +routing_keys = ["org.fedoraproject.prod.fedimg.image.publish"] + +[consumer_config] +# The production instance has this set true. There should be only one +# production instance anywhere in the world at any one time; unless +# you are very sure you are maintaining it, leave this at false. +relvalami_prod = false + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.RelvalAMIConsumer] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"] diff --git a/relvalconsumer.py b/relvalconsumer.py index 8881b7f..fd96c58 100644 --- a/relvalconsumer.py +++ b/relvalconsumer.py @@ -37,6 +37,42 @@ from urllib.error import (URLError, HTTPError) __version__ = "2.0.1" +class RelvalAMIConsumer(object): + """A fedora-messaging consumer that updates the AMI info page for + a release validation test event when a relevant AMI is published. + """ + def __init__(self): + self.relvalami_prod = fedora_messaging.config.conf["consumer_config"]["relvalami_prod"] + self.logger = logging.getLogger(self.__class__.__name__) + + def __call__(self, message): + """Consumer incoming message. relvalami_prod' indicates prod + or test mode. Message should be a fedimg.image.publish. + """ + if self.relvalami_prod: + serv = 'fedoraproject.org' + else: + serv = 'stg.fedoraproject.org' + site = wikitcms.wiki.Wiki(serv, '/w/') + site.login() + cid = message.body.get('compose') + try: + event = site.get_validation_event(cid=cid) + except fedfind.exceptions.UnsupportedComposeError: + # this is a common, expected case, not worth logging at info + self.logger.debug("compose %s not supported by fedfind", cid) + raise fedora_messaging.exceptions.Drop + except ValueError: + self.logger.info("Could not determine event for compose %s", cid) + raise fedora_messaging.exceptions.Drop + self.logger.info("Working on AMI %s for compose %s", message.body['extra']['id'], cid) + if event.result_pages: + # this is a proxy for 'event exists for this compose'; if + # so, we'll just re-create the AMI page from scratch, we + # don't have the capability to smartly add a single AMI + event.ami_page.write(createonly=None) + + class RelvalConsumer(object): """A fedora-messaging consumer that creates release validation test events for new composes when appropriate. diff --git a/test_relvalconsumer.py b/test_relvalconsumer.py index 6755206..1fe2ce0 100644 --- a/test_relvalconsumer.py +++ b/test_relvalconsumer.py @@ -103,24 +103,28 @@ mock.patch('wikitcms.wiki.Wiki.login', autospec=True).start() # so does site init mock.patch('mwclient.client.Site.site_init', autospec=True).start() -# initialize two test consumers with different configs +# initialize two pairs of test consumers with different configs PRODCONF = { 'consumer_config': { 'relval_prod': True, + 'relvalami_prod': True, 'relval_bugzilla': True, } } TESTCONF = { 'consumer_config': { 'relval_prod': False, + 'relvalami_prod': False, 'relval_bugzilla': False, } } with mock.patch.dict('fedora_messaging.config.conf', PRODCONF): CONSUMER = relvalconsumer.RelvalConsumer() + AMICONSUMER = relvalconsumer.RelvalAMIConsumer() with mock.patch.dict('fedora_messaging.config.conf', TESTCONF): TESTCONSUMER = relvalconsumer.RelvalConsumer() + TESTAMICONSUMER = relvalconsumer.RelvalAMIConsumer() # These are all fake IDs for various composes, in the order we'll test them. RAWHIDE1 = 'Fedora-Rawhide-20160601.n.0' @@ -459,3 +463,37 @@ class TestRelvalConsumerPostBranch: # FIXME we should check we used stg wiki, but it's a bit tricky # with these mocks fakesubproc.reset_mock() + +# get_current_release requires a network trip +@mock.patch('fedfind.helpers.get_current_release', _fakegetcurr1) +# don't actually write anything +@mock.patch('wikitcms.page.AMIPage.write', autospec=True) +class TestRelvalAMIConsumer: + """All tests for the AMI consumer.""" + amimsg = Message( + topic='org.fedoraproject.prod.fedimg.image.publish', + body={ + 'compose': 'Fedora-Rawhide-20191109.n.0', + 'extra': {'id': 'ami-0f4c7283a18abbc53'}, + } + ) + + # make consumer believe event 'exists' (real return value would be + # a list of Page instances, but meh) + @mock.patch('wikitcms.event.ValidationEvent.result_pages', return_value=[1]) + def test_ami_exists(self, fakeres, fakewrite): + """We should write the AMI page when the event exists.""" + AMICONSUMER(self.amimsg) + assert fakewrite.call_count == 1 + fakewrite.reset_mock() + TESTAMICONSUMER(self.amimsg) + assert fakewrite.call_count == 1 + + # make consumer believe event does not 'exist' + @mock.patch('wikitcms.event.ValidationEvent.result_pages', []) + def test_ami_not_exists(self, fakewrite): + """We shouldn't write the AMI page when the event doesn't + exist. + """ + AMICONSUMER(self.amimsg) + assert fakewrite.call_count == 0