From d01478c45175bdc605dee66e80fe20b5ad924145 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Apr 25 2018 11:03:48 +0000 Subject: Make fixed-architecture job scheduling possible --- diff --git a/conf/trigger.cfg.example b/conf/trigger.cfg.example index 553245d..355f95e 100644 --- a/conf/trigger.cfg.example +++ b/conf/trigger.cfg.example @@ -3,6 +3,7 @@ url = http://localhost:8080/buildbot/change_hook [trigger] valid_arches = x86_64 +default_run_arch = noarch datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw execdb_server = http://localhost:5003 runner_type = StreamRunner diff --git a/jobtriggers/config.py b/jobtriggers/config.py index 91bbc69..6694405 100644 --- a/jobtriggers/config.py +++ b/jobtriggers/config.py @@ -7,6 +7,7 @@ url = http://localhost:8080/buildbot/change_hook [trigger] valid_arches = x86_64 +default_run_arch = noarch datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw execdb_server = http://localhost:5003 runner_type = StreamRunner @@ -29,6 +30,7 @@ config.read(['/etc/taskotron/trigger.cfg', './conf/trigger.cfg']) buildbot_url = config.get('buildbot', 'url') valid_arches = [e.strip() for e in config.get('trigger', 'valid_arches').split(',') if e] +default_run_arch = config.get('trigger', 'default_run_arch') datagrepper_url = config.get('trigger', 'datagrepper_url') execdb_server = config.get('trigger', 'execdb_server') runner_type = config.get('trigger', 'runner_type') diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py index 389026d..161b922 100644 --- a/jobtriggers/jobtrigger.py +++ b/jobtriggers/jobtrigger.py @@ -26,23 +26,24 @@ class JobTrigger(object): self.log = log self.runner = runners.get_runner(config.runner_type) - def trigger_tasks(self, item, item_type, tasks, arches, **kwargs): + def trigger_jobs(self, item, item_type, tasks, arches, arch_dependent, **kwargs): for task, arch in itertools.product(tasks, arches): - self.log.info('triggering %s for %s on %s', task, item, arch) + run_arch = arch if arch_dependent else config.default_run_arch # FIXME make 'noarch' configurable + self.log.info('triggering %s for %s on %s (run on %s)', task, item, arch, run_arch) if config.job_logging: try: - utils.log_job(config.joblog_file, item, item_type, task, arch) + utils.log_job(config.joblog_file, item, item_type, task, arch, run_arch) except IOError, e: self.log.exception(e) - output = self.runner.trigger_job(item, item_type, task, arch, **kwargs) + output = self.runner.trigger_job(item, item_type, task, arch, run_arch, **kwargs) for line in output: self.log.info(line) def _tasks_from_rule(self, rule, message_data): """Takes a rule-dict as input, and produces list of dicts representing arguments required - by the trigger_tasks method (i.e. item, item_type, tasks, arches) plus any additional data + by the trigger_jobs method (i.e. item, item_type, tasks, arches) plus any additional data provided in the rule. """ @@ -52,6 +53,7 @@ class JobTrigger(object): 'item': message_data['item'], 'item_type': message_data['item_type'], 'arches': config.valid_arches, + 'arch_dependent': False, 'tasks': [] } except KeyError, e: @@ -95,6 +97,9 @@ class JobTrigger(object): if 'arches' in item: task['arches'] = item['arches'] + if item.get('arch_dependent', False): + task['arch_dependent'] = True + tasks.append(task) return tasks @@ -133,7 +138,7 @@ class JobTrigger(object): for task in tasks: try: - self.trigger_tasks(**task) + self.trigger_jobs(**task) except exc.TriggerMsgError, e: self.log.debug(e) except exc.TriggerError, e: diff --git a/jobtriggers/runners.py b/jobtriggers/runners.py index 85ee02a..ea36b40 100644 --- a/jobtriggers/runners.py +++ b/jobtriggers/runners.py @@ -13,22 +13,24 @@ class BaseRunner(object): 'item': , 'item_type': , 'taskname': , - 'arch': + 'arch': , + 'run_arch': } """ - def trigger_job(self, item, item_type, taskname, arch, **kwargs): + def trigger_job(self, item, item_type, taskname, arch, run_arch, **kwargs): raise NotImplementedError class StreamRunner(BaseRunner): - def trigger_job(self, item, item_type, taskname, arch, **kwargs): + def trigger_job(self, item, item_type, taskname, arch, run_arch, **kwargs): output = ["StreamRunner received:", "item: %s" % item, "item_type: %s" % item_type, "taskname: %s" % taskname, - "arch: %s" % arch] + "arch: %s" % arch, + "run_arch: %s" % run_arch] output += ["%s: %s" % (k, w) for k, w in kwargs.iteritems()] yield '\n'.join(output) @@ -36,12 +38,13 @@ class StreamRunner(BaseRunner): class BuildbotRunner(BaseRunner): - def trigger_job(self, item, item_type, taskname, arch, **kwargs): - output = ["Buildbot received:", + def trigger_job(self, item, item_type, taskname, arch, run_arch, **kwargs): + output = ["BuildbotRunner received:", "item: %s" % item, "item_type: %s" % item_type, "taskname: %s" % taskname, - "arch: %s" % arch] + "arch: %s" % arch, + "run_arch: %s" % run_arch] yield '\n'.join(output) @@ -50,6 +53,7 @@ class BuildbotRunner(BaseRunner): 'item_type': item_type, 'taskname': taskname, 'arch': arch, + 'run_arch': run_arch, } properties.update(kwargs) @@ -82,7 +86,7 @@ class BuildbotRunner(BaseRunner): r = requests.post(config.buildbot_url, data={'author': 'taskotron', 'project': project, - 'category': arch, + 'category': run_arch, 'repository': '', 'comments': 'build request from taskotron-trigger', 'properties': json.dumps(properties)}) diff --git a/jobtriggers/utils.py b/jobtriggers/utils.py index d2e73a3..921ea45 100644 --- a/jobtriggers/utils.py +++ b/jobtriggers/utils.py @@ -13,12 +13,12 @@ from jobtriggers import config from jobtriggers import exceptions as exc -def log_job(logname, item, item_type, task, arch): +def log_job(logname, item, item_type, task, arch, arch_dependent): now = datetime.datetime.now() with open(logname, 'a') as csvfile: writer = csv.writer(csvfile, delimiter=';') - writer.writerow([now, item, item_type, task, arch]) + writer.writerow([now, item, item_type, task, arch, arch_dependent]) def parse_yaml_from_file(filename): diff --git a/testing/test_dist_git_trigger.py b/testing/test_dist_git_trigger.py index 04e45e1..328bc0a 100644 --- a/testing/test_dist_git_trigger.py +++ b/testing/test_dist_git_trigger.py @@ -46,6 +46,7 @@ class TestDistGitCommitJobConsumer(): monkeypatch.setattr(utils, 'parse_yaml_from_file', mock_get_critpath) dist_git_commit_msg.config.valid_arches = ['x86_64'] + dist_git_commit_msg.config.default_run_arch = 'noarch' def _create_msg(self): return Munch(body={ @@ -113,4 +114,5 @@ class TestDistGitCommitJobConsumer(): 'dist_git_commit', 'speclint', 'x86_64', + 'noarch', ) diff --git a/testing/test_github_pr_opened_trigger.py b/testing/test_github_pr_opened_trigger.py index 0bdfdaa..0871ed5 100644 --- a/testing/test_github_pr_opened_trigger.py +++ b/testing/test_github_pr_opened_trigger.py @@ -50,6 +50,7 @@ class TestKojiTagChangedJobConsumer(): github_pull_request_opened.config.valid_arches = self.ref_validarches github_pull_request_opened.config.job_logging = False + github_pull_request_opened.config.default_run_arch = 'noarch' def _create_msg(self, ref_pr_url, ref_repo_name): return Munch(body='{"i": 1,\ @@ -70,17 +71,17 @@ class TestKojiTagChangedJobConsumer(): assert len(runner_calls) == 6 assert runner_calls[0][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') assert runner_calls[4][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[5][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') def test_trigger_data(self): trigger = github_pull_request_opened.GithubPullRequestOpenedJobTrigger(Dingus()) diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py index 6b3ed5f..7be4791 100644 --- a/testing/test_jobtrigger.py +++ b/testing/test_jobtrigger.py @@ -49,6 +49,7 @@ class TestJobtrigger(): ] jobtrigger.config.valid_arches = self.ref_default_arches + jobtrigger.config.default_run_arch = 'noarch' # short-circuit the logic to check if a repo exists stub_repoexist = mock.Mock(return_value=True) @@ -80,6 +81,7 @@ class TestJobtrigger(): assert tasks[0]['item'] == self.ref_message_data['item'] assert tasks[0]['item_type'] == self.ref_message_data['item_type'] assert tasks[0]['arches'] == self.ref_default_arches + assert tasks[0]['arch_dependent'] == False def test__tasks_from_rule_with_arches(self): @@ -90,6 +92,37 @@ class TestJobtrigger(): assert tasks[0]['item'] == self.ref_message_data['item'] assert tasks[0]['item_type'] == self.ref_message_data['item_type'] assert tasks[0]['arches'] == ['bogusarch'] + assert tasks[0]['arch_dependent'] == False + + def test__tasks_from_rule_with_arch_dependent(self): + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': True}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert len(tasks) == 1 + assert tasks[0]['tasks'] == self.ref_tasks + assert tasks[0]['item'] == self.ref_message_data['item'] + assert tasks[0]['item_type'] == self.ref_message_data['item_type'] + assert tasks[0]['arches'] == self.ref_default_arches + assert tasks[0]['arch_dependent'] == True + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': 'yes'}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == True + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': 'no'}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == True + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': False}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == False + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': 0}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == False + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': ''}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == False def test__tasks_from_rule_discover(self): @@ -143,13 +176,13 @@ class TestJobtrigger(): assert len(runner_calls) == 4 assert runner_calls[0][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[0], 'noarch') assert runner_calls[1][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[1], 'noarch') assert runner_calls[2][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[0], 'noarch') assert runner_calls[3][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1], 'noarch') def test_do_trigger_runtasks_with_arches(self, monkeypatch): ref_arches = ['arch_foo', 'arch_bar'] @@ -171,13 +204,13 @@ class TestJobtrigger(): assert len(runner_calls) == 4 assert runner_calls[0][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0], 'noarch') assert runner_calls[1][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1], 'noarch') assert runner_calls[2][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0], 'noarch') assert runner_calls[3][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1], 'noarch') def test_do_trigger_discover(self, monkeypatch): mock__load_rules = mock.Mock(return_value=self.ref_rules) @@ -194,10 +227,10 @@ class TestJobtrigger(): assert runner_calls[0][1] == ( self.ref_item, self.ref_item_type, self.ref_discovered_tasks[0], - self.ref_default_arches[0]) + self.ref_default_arches[0], 'noarch') assert runner_calls[1][1] == ( self.ref_item, self.ref_item_type, self.ref_discovered_tasks[0], - self.ref_default_arches[1]) + self.ref_default_arches[1], 'noarch') def test_do_trigger_disabled(self, monkeypatch): mock__load_rules = mock.Mock(return_value=self.ref_rules) diff --git a/testing/test_koji_build_trigger.py b/testing/test_koji_build_trigger.py index a8ed649..fd2dd45 100644 --- a/testing/test_koji_build_trigger.py +++ b/testing/test_koji_build_trigger.py @@ -55,6 +55,7 @@ class TestKojiBuildCompletedJobConsumer(): """ % (self.ref_tasks, self.message_type, self.ref_tasks_docker, self.message_type) koji_build_msg.config.valid_arches = self.ref_validarches + koji_build_msg.config.default_run_arch = 'noarch' koji_build_msg.config.job_logging = False mock_discover_tasks = mock.Mock(return_value=[]) @@ -101,13 +102,13 @@ class TestKojiBuildCompletedJobConsumer(): assert len(runner_calls) == 4 assert runner_calls[0][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') def test_consume_docker(self): # skip test when item is docker image @@ -124,17 +125,17 @@ class TestKojiBuildCompletedJobConsumer(): assert len(runner_calls) == 6 assert runner_calls[0][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') assert runner_calls[4][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks_docker[0], self.ref_validarches[0]) + self.ref_tasks_docker[0], self.ref_validarches[0], 'noarch') assert runner_calls[5][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks_docker[0], self.ref_validarches[1]) + self.ref_tasks_docker[0], self.ref_validarches[1], 'noarch') def test_consume_distgit(self, monkeypatch): ref_distgit_task = 'abicheck' @@ -148,17 +149,17 @@ class TestKojiBuildCompletedJobConsumer(): assert len(runner_calls) == 6 assert runner_calls[0][1] == (self.ref_nvr, self.item_type, - ref_distgit_task, self.ref_validarches[0]) + ref_distgit_task, self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_nvr, self.item_type, - ref_distgit_task, self.ref_validarches[1]) + ref_distgit_task, self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[4][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[5][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') def test_consume_not_primary_instance(self): ref_instance = 'ppc' diff --git a/testing/test_koji_tag_trigger.py b/testing/test_koji_tag_trigger.py index e0bddf3..dd6aab9 100644 --- a/testing/test_koji_tag_trigger.py +++ b/testing/test_koji_tag_trigger.py @@ -49,6 +49,7 @@ class TestKojiTagChangedJobConsumer(): """ koji_tag_msg.config.valid_arches = self.ref_validarches + koji_tag_msg.config.default_run_arch = 'noarch' koji_tag_msg.config.job_logging = False def _create_msg(self, ref_instance, ref_name, ref_release, ref_tag, ref_version): @@ -90,13 +91,13 @@ class TestKojiTagChangedJobConsumer(): assert len(runner_calls) == 4 assert runner_calls[0][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[1], 'noarch') assert self.helper.queued_tags == set() assert self.helper.queued_data == dict() @@ -135,9 +136,9 @@ class TestKojiTagChangedJobConsumer(): assert len(runner_calls) == 2 assert runner_calls[0][1] == ( - ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0]) + ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == ( - ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1]) + ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert self.helper.queued_tags == set() diff --git a/testing/test_module_build_trigger.py b/testing/test_module_build_trigger.py index 34b00e0..ac7c620 100644 --- a/testing/test_module_build_trigger.py +++ b/testing/test_module_build_trigger.py @@ -31,6 +31,7 @@ class TestModuleBuildDoneJobConsumer(): module_build_msg.config.job_logging = False module_build_msg.config.valid_arches = ['x86_64'] + module_build_msg.config.default_run_arch = 'noarch' def _create_msg(self, name, stream, version, state_name='done'): return Munch(body={ @@ -117,6 +118,7 @@ class TestModuleBuildDoneJobConsumer(): module_build_msg.ITEM_TYPE, self.ref_task, 'x86_64', + 'noarch', ) def test_consume_not_done(self): diff --git a/testing/test_pagure_commit_trigger.py b/testing/test_pagure_commit_trigger.py index 33bc68c..b97de46 100644 --- a/testing/test_pagure_commit_trigger.py +++ b/testing/test_pagure_commit_trigger.py @@ -43,6 +43,7 @@ class TestPagureGitReceivedJobConsumer(): monkeypatch.setattr(utils, 'clone_repo', mock_clone_repo) pagure_git_received.config.valid_arches = ['x86_64'] + pagure_git_received.config.default_run_arch = 'noarch' def _create_msg(self): return Munch(body={ @@ -115,4 +116,5 @@ class TestPagureGitReceivedJobConsumer(): 'git_commit', 'speclint', 'x86_64', + 'noarch', ) diff --git a/testing/test_taskotron_result_missing_trigger.py b/testing/test_taskotron_result_missing_trigger.py index 6461e66..ddf42f4 100644 --- a/testing/test_taskotron_result_missing_trigger.py +++ b/testing/test_taskotron_result_missing_trigger.py @@ -44,6 +44,7 @@ class TestTaskotronResultMissingJobConsumer(): taskotron_result_missing_msg.config.valid_arches = self.ref_validarches taskotron_result_missing_msg.config.job_logging = False + taskotron_result_missing_msg.config.default_run_arch = 'noarch' def _create_msg(self, ref_item, ref_item_type, ref_testcase, ref_arch): self.ref_message = Munch(body='{"i": 1,\ @@ -64,5 +65,5 @@ class TestTaskotronResultMissingJobConsumer(): assert len(runner_calls) == 1 assert runner_calls[0][1] == (self.ref_item, self.ref_item_type, - self.ref_task, self.ref_arch) + self.ref_task, self.ref_arch, 'noarch')