From 56f9c3226ab6bd5a19a3a9c19607fe5293bc3d88 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Apr 22 2018 12:04:26 +0000 Subject: Ondemand task scheduling based on testcase --- diff --git a/.gitignore b/.gitignore index 28f1a4c..b6dac6a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ # pytest cache /testing/.cache +.pytest_cache # virtualenv /env_trigger/ diff --git a/conf/trigger_rules.yml.example b/conf/trigger_rules.yml.example index c11b09f..43515b5 100644 --- a/conf/trigger_rules.yml.example +++ b/conf/trigger_rules.yml.example @@ -63,3 +63,8 @@ message_type: GitHubPullRequestOpened do: - {tasks: [task-mtf]} +- when: + message_type: TaskotronResultMissing + testcase: {$regex: '/^dist.rpmgrill(\..+)*/'} + do: + - {tasks: [rpmgrill], arches: ${arches}} diff --git a/fedmsg.d/taskotron-trigger.py b/fedmsg.d/taskotron-trigger.py index 8c2afd0..6d2f77a 100644 --- a/fedmsg.d/taskotron-trigger.py +++ b/fedmsg.d/taskotron-trigger.py @@ -6,4 +6,5 @@ config = { 'taskotron.cloudcomposecompletedconsumer.enabled': True, 'taskotron.modulebuilddonejobconsumer.enabled': True, 'taskotron.githubpropenedjobconsumer.enabled': True, + 'taskotron.taskotronresultmissingjobconsumer.enabled': True, } diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py index 5cfc1c1..389026d 100644 --- a/jobtriggers/jobtrigger.py +++ b/jobtriggers/jobtrigger.py @@ -92,6 +92,9 @@ class JobTrigger(object): else: self.log.error("Rule has invalid `do` section: %r", rule) + if 'arches' in item: + task['arches'] = item['arches'] + tasks.append(task) return tasks diff --git a/jobtriggers/mongoquery_string_template.py b/jobtriggers/mongoquery_string_template.py index dc7a6d5..f23b17e 100644 --- a/jobtriggers/mongoquery_string_template.py +++ b/jobtriggers/mongoquery_string_template.py @@ -79,6 +79,12 @@ class MongoTemplate(string.Template): val = mapping[named] # We use this idiom instead of str() because the latter will # fail if val is a Unicode containing non-ASCII characters. + # FIXME - list, tuple and dict value handling hotfixed and not properly tested + if isinstance(val, list) or isinstance(val, tuple): + return "[%s]" % ",".join(val) + if isinstance(val, dict): + items = ["%s: %s" % (k, v) for k, v in val.iteritems()] + return "{%s}" % ",".join(items) return '%s' % (val,) if mo.group('escaped') is not None: return self.delimiter diff --git a/jobtriggers/taskotron_result_missing_msg.py b/jobtriggers/taskotron_result_missing_msg.py new file mode 100644 index 0000000..0cd2c55 --- /dev/null +++ b/jobtriggers/taskotron_result_missing_msg.py @@ -0,0 +1,45 @@ +import fedmsg.consumers + +from . import config +from .jobtrigger import JobTrigger +from . import exceptions as exc + +MESSAGE_TYPE = 'TaskotronResultMissing' + +class TaskotronResultMissingJobTrigger(JobTrigger): + def process(self, msg): + item = msg['msg']['item'] + item_type = msg['msg']['item_type'] + testcase = msg['msg']['testcase'] + if 'arch' in msg['msg']: + arches = [msg['msg']['arch']] + else: + arches = config.valid_arches + + data = { + "_msg": msg, + "message_type": MESSAGE_TYPE, + "item": item, + "item_type": item_type, + "testcase": testcase, + "arches": arches, + } + return data + + +class TasktoronResultMissingJobConsumer(fedmsg.consumers.FedmsgConsumer): + topic = "org.fedoraproject.%s.taskotron.result_missing" % config.deployment_type + config_key = 'taskotron.taskotronresultmissingjobconsumer.enabled' + + def __init__(self, *args, **kw): + super(TasktoronResultMissingJobConsumer, self).__init__(*args, **kw) + self.trigger = TaskotronResultMissingJobTrigger(self.log) + + def consume(self, message): + msg = fedmsg.encoding.loads(message.body) + try: + data = self.trigger.process(msg) + self.trigger.do_trigger(data) + except exc.TriggerMsgError, e: + self.log.debug(e) + return diff --git a/requirements.txt b/requirements.txt index 7d87b34..e1aa418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ fedmsg[consumers] mongoquery requests Twisted +koji PyYAML >= 3.11 # Test suite requirements diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py index 9fa5921..6b3ed5f 100644 --- a/testing/test_jobtrigger.py +++ b/testing/test_jobtrigger.py @@ -81,6 +81,17 @@ class TestJobtrigger(): assert tasks[0]['item_type'] == self.ref_message_data['item_type'] assert tasks[0]['arches'] == self.ref_default_arches + + def test__tasks_from_rule_with_arches(self): + rule = {'do': [{'tasks': self.ref_tasks, 'arches': ['bogusarch']}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert len(tasks) == 1 + assert tasks[0]['tasks'] == self.ref_tasks + assert tasks[0]['item'] == self.ref_message_data['item'] + assert tasks[0]['item_type'] == self.ref_message_data['item_type'] + assert tasks[0]['arches'] == ['bogusarch'] + + def test__tasks_from_rule_discover(self): ref_repo = 'http://bogus.repo/firefox.git' rule = {'do': [{'discover': {'repo': ref_repo}}]} @@ -140,6 +151,34 @@ class TestJobtrigger(): assert runner_calls[3][1] == ( self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1]) + def test_do_trigger_runtasks_with_arches(self, monkeypatch): + ref_arches = ['arch_foo', 'arch_bar'] + mock__load_rules = mock.Mock(return_value=[ + { + 'do': [{'tasks': self.ref_tasks, 'arches': ref_arches}], + 'when': {'message_type': 'RunTasks'} + }, + ]) + monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules) + + message_data = copy.deepcopy(self.ref_message_data) + message_data['message_type'] = 'RunTasks' + + self.helper.do_trigger(message_data) + + runner_calls = self.helper.runner.trigger_job.calls() + + assert len(runner_calls) == 4 + + assert runner_calls[0][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0]) + assert runner_calls[1][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1]) + assert runner_calls[2][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0]) + assert runner_calls[3][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1]) + def test_do_trigger_discover(self, monkeypatch): mock__load_rules = mock.Mock(return_value=self.ref_rules) monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules) diff --git a/testing/test_taskotron_result_missing_trigger.py b/testing/test_taskotron_result_missing_trigger.py new file mode 100644 index 0000000..6902fa2 --- /dev/null +++ b/testing/test_taskotron_result_missing_trigger.py @@ -0,0 +1,68 @@ +import pytest +from dingus import Dingus +from munch import Munch +from copy import deepcopy +import fedmsg.encoding +import fedmsg.consumers + +from jobtriggers import taskotron_result_missing_msg + + +@pytest.mark.usefixtures('prepare') +class TestTaskotronResultMissingJobConsumer(): + + @pytest.fixture + def prepare(self, monkeypatch): + self.ref_item = 'ref_item' + self.ref_item_type = 'ref_item_type' + self.ref_testcase = 'ref.testcase.subcase' + self.ref_arch = 'ref_arch' + self.ref_task = 'ref_task' + self.ref_validarches = ['i386', 'x86_64'] + + self._create_msg( + self.ref_item, self.ref_item_type, self.ref_testcase, self.ref_arch) + + self.ref_data = { + "_msg": {}, + "message_type": "TaskotronResultMissing", + "item": self.ref_item, + "item_type": self.ref_item_type, + "testcase": self.ref_testcase, + "arches": [self.ref_arch], + } + + stub_hub = Munch(config=Munch(get=0)) + self.helper = taskotron_result_missing_msg.TasktoronResultMissingJobConsumer(stub_hub) + + self.helper.trigger.runner = Dingus() + taskotron_result_missing_msg.config.trigger_rules_template = """--- +- do: + - {tasks: [ref_task], arches: ${arches}} + when: {message_type: TaskotronResultMissing, testcase: {$regex: '/^ref.testcase(\..+)*/'}} +""" + + taskotron_result_missing_msg.config.valid_arches = self.ref_validarches + taskotron_result_missing_msg.config.job_logging = False + + def _create_msg(self, ref_item, ref_item_type, ref_testcase, ref_arch): + self.ref_message = Munch(body='{"i": 1,\ + "msg": {"item": "%s",\ + "item_type": "%s",\ + "testcase": "%s",\ + "arch": "%s"},\ + "timestamp": 1359603469.21164,\ + "topic": "org.fedoraproject.prod.taskotron.result_missing",\ + "username": "apache"}' % + (ref_item, ref_item_type, ref_testcase, ref_arch)) + + def test_consume(self): + self.helper.consume(self.ref_message) + + runner_calls = self.helper.trigger.runner.trigger_job.calls() + + assert len(runner_calls) == 1 + + assert runner_calls[0][1] == (self.ref_item, self.ref_item_type, + self.ref_task, self.ref_arch) +