From ae1038ae4d50143343607c766d7088d9ab722f86 Mon Sep 17 00:00:00 2001 From: Bruno Goncalves Date: Apr 29 2018 21:22:30 +0000 Subject: using class remove verify_commit method fixed miss call --- diff --git a/fedora-ci-monitor/fedora_ci_monitor.py b/fedora-ci-monitor/fedora_ci_monitor.py index 609cd5b..2a3a8ad 100644 --- a/fedora-ci-monitor/fedora_ci_monitor.py +++ b/fedora-ci-monitor/fedora_ci_monitor.py @@ -12,16 +12,13 @@ import yaml requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) """ -This scripts checks if Fedora CI pipeline executes correctly when -there is commit to dist-git and branch has tests (/tests/tests.yml) +This scripts checks if Fedora CI pipeline executes correctly and branch has tests (/tests/tests.yml) """ -# Check datagrepper messages from the last 24 hours -DELTA = os.getenv("DELTA", 24*3600) JENKINS_URL = "https://jenkins-continuous-infra.apps.ci.centos.org" - DATAGREPPER_URL = "https://apps.fedoraproject.org/datagrepper/raw" + GIT_COMMIT_TOPIC = "org.fedoraproject.prod.git.receive" NEW_PR_TOPIC = "org.fedoraproject.prod.pagure.pull-request.new" NEW_PR_COMMENT_TOPIC = "org.fedoraproject.prod.pagure.pull-request.comment.added" @@ -51,14 +48,25 @@ BUILD_PIPELINE_TESTS_RUNNING_TOPIC = "org.centos.prod.ci.pipeline.allpackages-bu BUILD_PIPELINE_TESTS_COMPLETE_TOPIC = "org.centos.prod.ci.pipeline.allpackages-build.package.test.functional.complete" BUILD_PIPELINE_COMPLETE_TOPIC = "org.centos.prod.ci.pipeline.allpackages-build.complete" +VALID_PIPELINE_TOPICS = [PR_PIPELINE_PKG_QUEUED_TOPIC, PR_PIPELINE_PKG_RUNNING_TOPIC, PR_PIPELINE_PKG_COMPLETE_TOPIC, + PR_PIPELINE_IMG_QUEUED_TOPIC, PR_PIPELINE_IMG_RUNNING_TOPIC, PR_PIPELINE_IMG_COMPLETE_TOPIC, + PR_PIPELINE_TESTS_QUEUED_TOPIC, PR_PIPELINE_TESTS_RUNNING_TOPIC, PR_PIPELINE_TESTS_COMPLETE_TOPIC, + PR_PIPELINE_COMPLETE_TOPIC, PR_PIPELINE_PKG_IGNORED_TOPIC, + BUILD_PIPELINE_PKG_QUEUED_TOPIC, BUILD_PIPELINE_PKG_RUNNING_TOPIC, BUILD_PIPELINE_PKG_COMPLETE_TOPIC, + BUILD_PIPELINE_IMG_QUEUED_TOPIC, BUILD_PIPELINE_IMG_RUNNING_TOPIC, BUILD_PIPELINE_IMG_COMPLETE_TOPIC, + BUILD_PIPELINE_TESTS_QUEUED_TOPIC, BUILD_PIPELINE_TESTS_RUNNING_TOPIC, BUILD_PIPELINE_TESTS_COMPLETE_TOPIC, + BUILD_PIPELINE_COMPLETE_TOPIC, BUILD_PIPELINE_PKG_IGNORED_TOPIC] + +PIPELINES = {"pr": "fedora-%s-pr-pipeline", + "kojibuild": "fedora-%s-build-pipeline" } + + PASS = 0 INFRA_FAILURE = 1 TEST_FAILURE = 2 SKIP = 3 RUNNING = 4 -PIPELINES = {"pr": "fedora-%s-pr-pipeline", - "kojibuild": "fedora-%s-build-pipeline" } def _query_url(url): try: @@ -71,116 +79,6 @@ def _query_url(url): return None return resp.text -def _query_datagrepper(topic, delta=None): - if not delta: - delta = DELTA - page=1 - pages=9999 - data = [] - - while page <= pages: - url = "%s?topic=%s&delta=%s&page=%s" % (DATAGREPPER_URL, topic, delta, page) - result = _query_url("%s&page=%s" % (url, page)) - if not result: - return None - - jresult = json.loads(result) - data.extend(jresult['raw_messages']) - - pages = int(jresult['pages']) - page += 1 - - return data - -def get_recent_commits(delta=None, namespace="rpms"): - """ - Check datagrepper for recent commits - """ - if not delta: - delta = DELTA - commits = [] - page=1 - pages=9999 - - print("INFO: Getting commits for the last %s seconds" % delta) - data =_query_datagrepper(GIT_COMMIT_TOPIC, delta) - if not data: - return None - - for commit in data: - # Skip commits that are not from namespace - if namespace and commit['msg']['commit']['namespace'] != namespace: - continue - # Skip commits on forks - if commit['msg']['commit']['path'].startswith("/srv/git/repositories/forks"): - continue - commits.append(commit['msg']) - - return commits - -def get_recent_prs(delta=None, namespace="rpms"): - """ - Check datagrepper for recent Pull Requests - """ - if not delta: - delta = DELTA - pull_requests = [] - page=1 - pages=9999 - - print("INFO: Getting PRs from the last %s seconds" % delta) - topics = [NEW_PR_TOPIC, NEW_PR_COMMENT_TOPIC] - data = [] - for topic in topics: - msgs =_query_datagrepper(topic, delta) - if msgs: - data.extend(msgs) - if not data: - print("PASS: Got 0 PR messages") - return None - - for pullrequest in data: - # Skip PRs that are not from namespace - pr_info = pullrequest['msg']['pullrequest'] - if namespace and pr_info['project']['namespace'] != namespace: - continue - # Skip commits on forks - # if pr_info['path'].startswith("/srv/git/repositories/forks"): - # continue - pull_requests.append(pullrequest['msg']) - - print("PASS: Got %s PR messages" % len(pull_requests)) - return pull_requests - -def get_recent_builds(delta=None, namespace="rpms"): - """ - Check datagrepper for recent Pull Requests - """ - if not delta: - delta = DELTA - builds = [] - page=1 - pages=9999 - - print("INFO: Getting Koji builds from the last %s seconds" % delta) - data =_query_datagrepper(KOJIBUILD_TOPIC, delta) - if not data: - print("PASS: Got 0 Koji builds messages") - return None - - for build in data: - if build['msg']['new'] != 1: - continue - # Skip compose builds, not the best, but quicker than koji taskinfo - if build['msg']['owner'] == "releng": - continue - # Does not seem to be a valid package build - if not build['msg']['request']: - continue - builds.append(build['msg']) - - print("PASS: Got %s Koji builds messages" % len(builds)) - return builds def check_tests(project, branch="master"): """ @@ -223,7 +121,7 @@ def has_jenkins_pipeline(pipeline_type, branch): return True -def get_jenkins_build_info(branch, build_id): +def get_jenkins_build_info(pipeline_type, branch, build_id): if branch == "master": branch = "rawhide" @@ -260,7 +158,7 @@ def get_jenkins_build(project, branch, commit_id): for build in builds: message_url = ("%s/view/all/job/%s/%s/artifact/messages/message-audit.json" % - (JENKINS_URL, pipeline, build['number'])) + (JENKINS_URL, pipeline, build['number'])) result = _query_url(message_url) if not result: return None @@ -277,507 +175,520 @@ def get_jenkins_build(project, branch, commit_id): return None +class Monitor: + + def __init__(self): + # Check datagrepper messages from the last 24 hours + self.delta = os.getenv("DELTA", 24*3600) + # By default we wait for running builds on pipeline to complete + self.wait_complete = True + self.queried_topics = {} + self.pipeline_steps = {} + + + self.pipeline_steps["pr"] = [{"topic" : PR_PIPELINE_PKG_QUEUED_TOPIC, "timeout" : 2, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_PKG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_PKG_COMPLETE_TOPIC, "timeout" : 120, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_IMG_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_IMG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_IMG_COMPLETE_TOPIC, "timeout" : 60, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_TESTS_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_TESTS_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : PR_PIPELINE_TESTS_COMPLETE_TOPIC, "timeout" : 5*60, "no_msg" : INFRA_FAILURE, "failure" : TEST_FAILURE}, + {"topic" : PR_PIPELINE_COMPLETE_TOPIC, "timeout" : 5, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}] + + + self.pipeline_steps["build"] = [{"topic" : BUILD_PIPELINE_PKG_QUEUED_TOPIC, "timeout" : 2, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_PKG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_PKG_COMPLETE_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_IMG_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_IMG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_IMG_COMPLETE_TOPIC, "timeout" : 60, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_TESTS_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_TESTS_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}, + {"topic" : BUILD_PIPELINE_TESTS_COMPLETE_TOPIC, "timeout" : 5*60, "no_msg" : INFRA_FAILURE, "failure" : TEST_FAILURE}, + {"topic" : BUILD_PIPELINE_COMPLETE_TOPIC, "timeout" : 5, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE}] + + def set_wait_complete(self, value): + self.wait_complete = value + + def _query_datagrepper(self, topic): + page=1 + pages=9999 + data = [] + + if topic in self.queried_topics and not self.wait_complete: + # In this case we do not need to update the data from the topic + # We are processing many messages and we want the messages from the beging, + # otherwise some topics not be in specific delta any more + return self.queried_topics[topic] + + while page <= pages: + url = "%s?topic=%s&delta=%s&page=%s" % (DATAGREPPER_URL, topic, self.delta, page) + result = _query_url("%s&page=%s" % (url, page)) + if not result: + return None -def get_commit_topic(project, branch, commit_id, topic, timeout, delta=None): - """ - Check datagrepper for specific topic related to the commit - """ - if not delta: - delta = DELTA + jresult = json.loads(result) + data.extend(jresult['raw_messages']) + + pages = int(jresult['pages']) + page += 1 - count = 0 - topic_msg = None - while not topic_msg: - if count > timeout: + self.queried_topics[topic] = data + return data + + def query_all_topics(self): + print("INFO: Querying topics from all pipelines...") + for topic in VALID_PIPELINE_TOPICS: + self._query_datagrepper(topic) + print("INFO: All topics queried") + + def get_recent_commits(self, namespace="rpms"): + """ + Check datagrepper for recent commits + """ + commits = [] + + print("INFO: Getting commits for the last %s seconds" % self.delta) + data =_query_datagrepper(GIT_COMMIT_TOPIC) + if not data: + return None + + for commit in data: + # Skip commits that are not from namespace + if namespace and commit['msg']['commit']['namespace'] != namespace: + continue + # Skip commits on forks + if commit['msg']['commit']['path'].startswith("/srv/git/repositories/forks"): + continue + commits.append(commit['msg']) + + return commits + + def get_recent_prs(self, namespace="rpms"): + """ + Check datagrepper for recent Pull Requests + """ + pull_requests = [] + + print("INFO: Getting PRs from the last %s seconds" % self.delta) + topics = [NEW_PR_TOPIC, NEW_PR_COMMENT_TOPIC] + data = [] + for topic in topics: + msgs = self._query_datagrepper(topic) + if msgs: + data.extend(msgs) + if not data: + print("PASS: Got 0 PR messages") return None - count += 1 - # print("INFO: Searching topic %s for %s %s %s (%s / %s) delta %s" % (topic, project, branch, commit_id, count, timeout, delta)) - data =_query_datagrepper(topic, delta) + for pullrequest in data: + # Skip PRs that are not from namespace + pr_info = pullrequest['msg']['pullrequest'] + if namespace and pr_info['project']['namespace'] != namespace: + continue + # Skip commits on forks + # if pr_info['path'].startswith("/srv/git/repositories/forks"): + # continue + pull_requests.append(pullrequest['msg']) + + print("PASS: Got %s PR messages" % len(pull_requests)) + return pull_requests + + def get_recent_builds(self, namespace="rpms"): + """ + Check datagrepper for recent Pull Requests + """ + + builds = [] + + print("INFO: Getting Koji builds from the last %s seconds" % self.delta) + data = self._query_datagrepper(KOJIBUILD_TOPIC) if not data: + print("PASS: Got 0 Koji builds messages") + return None + + for build in data: + if build['msg']['new'] != 1: + continue + # Skip compose builds, not the best, but quicker than koji taskinfo + if build['msg']['owner'] == "releng": + continue + # Does not seem to be a valid package build + if not build['msg']['request']: + continue + builds.append(build['msg']) + + print("PASS: Got %s Koji builds messages" % len(builds)) + return builds + + + + def get_commit_topic(self, project, branch, commit_id, topic, timeout): + """ + Check datagrepper for specific topic related to the commit + """ + count = 0 + topic_msg = None + while not topic_msg: + if count > timeout: + return None + + count += 1 + # print("INFO: Searching topic %s for %s %s %s (%s / %s) delta %s" % (topic, project, branch, commit_id, count, timeout)) + data = self._query_datagrepper(topic) + if not data: + # Check if pipeline completed withtout sending expected topic message + if topic != PIPELINE_COMPLETE_TOPIC: + if get_commit_topic(project, branch, commit_id, PIPELINE_COMPLETE_TOPIC, 0): + print("FAIL: pileline completed, but topic %s was never sent" % topic) + return None + if count <= timeout: + time.sleep(60) + continue + + for info in data: + msg = info['msg'] + msg_repo = msg['repo'] + msg_branch = msg['branch'] + msg_rev = msg['rev'] + if msg_repo == project and msg_branch == branch and msg_rev == commit_id: + topic_msg = info + return info + # Check if pipeline completed withtout sending expected topic message if topic != PIPELINE_COMPLETE_TOPIC: - if get_commit_topic(project, branch, commit_id, PIPELINE_COMPLETE_TOPIC, 0, delta): + if get_commit_topic(project, branch, commit_id, PIPELINE_COMPLETE_TOPIC, 0): print("FAIL: pileline completed, but topic %s was never sent" % topic) return None + if count <= timeout: time.sleep(60) - continue - for info in data: - msg = info['msg'] - msg_repo = msg['repo'] - msg_branch = msg['branch'] - msg_rev = msg['rev'] - if msg_repo == project and msg_branch == branch and msg_rev == commit_id: - topic_msg = info - return info + return None - # Check if pipeline completed withtout sending expected topic message - if topic != PIPELINE_COMPLETE_TOPIC: - if get_commit_topic(project, branch, commit_id, PIPELINE_COMPLETE_TOPIC, 0, delta): - print("FAIL: pileline completed, but topic %s was never sent" % topic) + def get_pr_topic(self, project, branch, pr_id, topic, timeout): + """ + Check datagrepper for specific topic related to the PR + """ + count = 0 + topic_msg = None + while not topic_msg: + if count > timeout: return None - if count <= timeout: - time.sleep(60) - - return None - -def get_pr_topic(project, branch, pr_id, topic, timeout, delta=None): - """ - Check datagrepper for specific topic related to the PR - """ - if not delta: - delta = DELTA - - count = 0 - topic_msg = None - while not topic_msg: - if count > timeout: - return None + count += 1 + # print("INFO: Searching topic %s for %s %s %s (%s / %s)" % (topic, project, branch, pr_id, count, timeout)) + data = self._query_datagrepper(topic) + if not data: + # Check if pipeline completed without sending expected topic message + if topic != PR_PIPELINE_COMPLETE_TOPIC: + if self.get_pr_topic(project, branch, pr_id, PR_PIPELINE_COMPLETE_TOPIC, 0): + print("FAIL: pileline completed, but topic %s was never sent" % topic) + return None + if count <= timeout: + time.sleep(60) + continue + for info in data: + msg = info['msg'] + msg_repo = msg['repo'] + msg_branch = msg['branch'] + msg_rev = msg['rev'].replace("PR-", "") + if msg_repo == project and msg_branch == branch and msg_rev == pr_id: + topic_msg = info + return info - count += 1 - # print("INFO: Searching topic %s for %s %s %s (%s / %s) delta %s" % (topic, project, branch, pr_id, count, timeout, delta)) - data =_query_datagrepper(topic, delta) - if not data: # Check if pipeline completed without sending expected topic message if topic != PR_PIPELINE_COMPLETE_TOPIC: - if get_pr_topic(project, branch, pr_id, PR_PIPELINE_COMPLETE_TOPIC, 0, delta): + if self.get_pr_topic(project, branch, pr_id, PR_PIPELINE_COMPLETE_TOPIC, 0): print("FAIL: pileline completed, but topic %s was never sent" % topic) return None + if count <= timeout: time.sleep(60) - continue - for info in data: - msg = info['msg'] - msg_repo = msg['repo'] - msg_branch = msg['branch'] - msg_rev = msg['rev'].replace("PR-", "") - if msg_repo == project and msg_branch == branch and msg_rev == pr_id: - topic_msg = info - return info - - # Check if pipeline completed without sending expected topic message - if topic != PR_PIPELINE_COMPLETE_TOPIC: - if get_pr_topic(project, branch, pr_id, PR_PIPELINE_COMPLETE_TOPIC, 0, delta): - print("FAIL: pileline completed, but topic %s was never sent" % topic) - return None - if count <= timeout: - time.sleep(60) + return None - return None + def get_build_topic(self, project, branch, task_id, topic, timeout): + """ + Check datagrepper for specific topic related to the koji build + """ + count = 0 + topic_msg = None + while not topic_msg: + if count > timeout: + return None -def get_build_topic(project, branch, task_id, topic, timeout, delta=None): - """ - Check datagrepper for specific topic related to the koji build - """ - if not delta: - delta = DELTA + count += 1 + # print("INFO: Searching topic %s for %s %s %s (%s / %s)" % (topic, project, branch, commit_id, count, timeout)) + data = self._query_datagrepper(topic) + if not data: + # Check if pipeline completed withtout sending expected topic message + if topic != BUILD_PIPELINE_COMPLETE_TOPIC: + if self.get_build_topic(project, branch, task_id, BUILD_PIPELINE_COMPLETE_TOPIC, 0): + print("FAIL: pileline completed, but topic %s was never sent" % topic) + return None + if count <= timeout: + time.sleep(60) + continue - count = 0 - topic_msg = None - while not topic_msg: - if count > timeout: - return None + for info in data: + msg = info['msg'] + msg_repo = msg['repo'] + msg_branch = msg['branch'] + msg_id = msg['rev'].replace("kojitask-", "") + if msg_repo == project and msg_branch == branch and msg_id == task_id: + topic_msg = info + return info - count += 1 - # print("INFO: Searching topic %s for %s %s %s (%s / %s) delta %s" % (topic, project, branch, commit_id, count, timeout, delta)) - data =_query_datagrepper(topic, delta) - if not data: - # Check if pipeline completed withtout sending expected topic message + # Check if pipeline completed without sending expected topic message if topic != BUILD_PIPELINE_COMPLETE_TOPIC: - if get_build_topic(project, branch, task_id, BUILD_PIPELINE_COMPLETE_TOPIC, 0, delta): + if self.get_build_topic(project, branch, task_id, BUILD_PIPELINE_COMPLETE_TOPIC, 0): print("FAIL: pileline completed, but topic %s was never sent" % topic) return None + if count <= timeout: time.sleep(60) - continue - - for info in data: - msg = info['msg'] - msg_repo = msg['repo'] - msg_branch = msg['branch'] - msg_id = msg['rev'].replace("kojitask-", "") - if msg_repo == project and msg_branch == branch and msg_id == task_id: - topic_msg = info - return info - - # Check if pipeline completed without sending expected topic message - if topic != BUILD_PIPELINE_COMPLETE_TOPIC: - if get_build_topic(project, branch, task_id, BUILD_PIPELINE_COMPLETE_TOPIC, 0, delta): - print("FAIL: pileline completed, but topic %s was never sent" % topic) - return None - - if count <= timeout: - time.sleep(60) - - return None - -def verify_commit(project, branch, commit_id): - """ - Check if commit run properly on CI - """ - commit_result = {"project" : project, "branch" : branch, "commit" : commit_id, "status" : None} - if not check_tests(project, branch): - print("SKIP: %s - %s does not contains tests" % (project, branch)) - commit_result["status"] = SKIP - return commit_result - - if not has_jenkins_pipeline(branch): - print("SKIP: %s there is no pipeline for commits on branch %s" % (project, branch)) - commit_result["status"] = SKIP - return commit_result - - print("INFO: Checking pipeline for commit %s %s %s" % (project, branch, commit_id)) - - pkg_queued_step = {"topic" : PIPELINE_PKG_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - pkg_running_step = {"topic" : PIPELINE_PKG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - pkg_complete_step = {"topic" : PIPELINE_PKG_COMPLETE_TOPIC, "timeout" : 120, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_queued_step = {"topic" : PIPELINE_IMG_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_running_step = {"topic" : PIPELINE_IMG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_complete_step = {"topic" : PIPELINE_IMG_COMPLETE_TOPIC, "timeout" : 60, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_queued_step = {"topic" : PIPELINE_TESTS_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_running_step = {"topic" : PIPELINE_TESTS_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_complete_step = {"topic" : PIPELINE_TESTS_COMPLETE_TOPIC, "timeout" : 5*60, "no_msg" : INFRA_FAILURE, "failure" : TEST_FAILURE} - pipeline_complete_step = {"topic" : PIPELINE_COMPLETE_TOPIC, "timeout" : 5, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - - pipeline_steps = [pkg_queued_step, pkg_running_step, pkg_complete_step, img_queued_step, img_running_step, img_complete_step, - tests_queued_step, tests_running_step, tests_complete_step, pipeline_complete_step] - - step_results = [] - - pipeline_failed = False - topic_jenkins_build = None - for step in pipeline_steps: - topic = step['topic'] - timeout = step['timeout'] - if pipeline_failed and topic != PIPELINE_COMPLETE_TOPIC: - # step will not execute if previous step failed - step_results.append({'step': topic, 'status': SKIP}) - continue - topic_msg = get_commit_topic(project, branch, commit_id, topic, timeout) - if not topic_msg: - pipeline_failed = True - print("FAIL: Could not find topic %s" % topic) - step_results.append({'step': topic, 'status': step['no_msg']}) - continue - if topic_msg['msg']['status'] != "SUCCESS" and topic != PIPELINE_TESTS_COMPLETE_TOPIC: - pipeline_failed = True - print("FAIL: %s" % topic) - step_results.append({'step': topic, 'status': step['failure']}) - continue - if topic_msg['msg']['status'] != "SUCCESS" and topic == PIPELINE_TESTS_COMPLETE_TOPIC: - pipeline_failed = True - print("FAIL: %s" % topic) - if topic_msg['msg']['status'] == "UNSTABLE": - step_results.append({'step': topic, 'status': TEST_FAILURE}) - else: - step_results.append({'step': topic, 'status': INFRA_FAILURE}) - continue - print("PASS: %s" % topic) - step_results.append({'step': topic, 'status': PASS}) - - # At this point Jenkins pipeline should have the build - if topic == PIPELINE_PKG_RUNNING_TOPIC: - topic_jenkins_build = int(topic_msg['msg']['build_id']) - topic_jenkins_build_url = topic_msg['msg']['build_url'] - - - if topic_jenkins_build: - # Wait some time for jenkins build be completed - build_info = get_jenkins_build_info(branch, topic_jenkins_build) - count = 5 - while build_info['building'] and count > 0: - build_info = get_jenkins_build_info(branch, topic_jenkins_build) - time.sleep(60) - count -= 1 - - if count == 0: - print("FAIL: Jenkins build did not finish: %s" % build_info) - status = INFRA_FAILURE - step_results.append({'step': "Jenkins build complete", 'status': INFRA_FAILURE}) - else: - step_results.append({'step': "Jenkins build complete", 'status': PASS}) - print("INFO: Jenkins build URL: %s" % topic_jenkins_build_url) - commit_result["jenkins_build_url"] = topic_jenkins_build_url - else: - print("FAIL: Could not find Jenkins build") - step_results.append({'step': "Find Jenkins build", 'status': INFRA_FAILURE}) - - commit_result["steps"] = step_results - commit_result["status"] = PASS - for result in step_results: - # Set the status of first failure - if result['status'] == INFRA_FAILURE or result['status'] == TEST_FAILURE: - commit_result["status"] = result['status'] - break - return commit_result + return None -def verify_pull_request(project, branch, pr_id, wait_complete=True): - """ - Check if PR ran properly on CI - """ - pr_result = {"project" : project, "branch" : branch, "pr_id" : pr_id, "status" : None, "pipeline": "pullrequest"} - if not has_jenkins_pipeline("pr", branch): - topic_msg = get_pr_topic(project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC, 2) - if not topic_msg: - print("FAIL: %s %s %s Could not find topic %s" % (project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC)) - pr_result["status"] = INFRA_FAILURE + def verify_pull_request(self, project, branch, pr_id): + """ + Check if PR ran properly on CI + """ + pr_result = {"project" : project, "branch" : branch, "pr_id" : pr_id, "status" : None, "pipeline": "pullrequest"} + if not has_jenkins_pipeline("pr", branch): + topic_msg = self.get_pr_topic(project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC, 2) + if not topic_msg: + print("FAIL: %s %s %s Could not find topic %s" % (project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC)) + pr_result["status"] = INFRA_FAILURE + return pr_result + if "build_url" in topic_msg['msg']: + pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] + print("SKIP: %s - %s does not contains tests" % (project, branch)) + print("SKIP: %s there is no pipeline for pull request on branch %s" % (project, branch)) + pr_result["status"] = SKIP return pr_result - if "build_url" in topic_msg['msg']: - pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] - print("SKIP: %s - %s does not contains tests" % (project, branch)) - print("SKIP: %s there is no pipeline for pull request on branch %s" % (project, branch)) - pr_result["status"] = SKIP - return pr_result - if not check_tests(project, branch): - topic_msg = get_pr_topic(project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC, 2) - if not topic_msg: - print("FAIL: %s %s %s Could not find topic %s" % (project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC)) - pr_result["status"] = INFRA_FAILURE + if not check_tests(project, branch): + topic_msg = self.get_pr_topic(project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC, 2) + if not topic_msg: + print("FAIL: %s %s %s Could not find topic %s" % (project, branch, pr_id, PR_PIPELINE_PKG_IGNORED_TOPIC)) + pr_result["status"] = INFRA_FAILURE + return pr_result + if "build_url" in topic_msg['msg']: + pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] + print("SKIP: %s - %s does not contains tests" % (project, branch)) + pr_result["status"] = SKIP return pr_result - if "build_url" in topic_msg['msg']: - pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] - print("SKIP: %s - %s does not contains tests" % (project, branch)) - pr_result["status"] = SKIP - return pr_result - print("INFO: Checking pipeline for PR %s %s %s" % (project, branch, pr_id)) + print("INFO: Checking pipeline for PR %s %s %s" % (project, branch, pr_id)) + + if not self.wait_complete: + # in case the build still running we skip all other topics check + topic_msg = self.get_pr_topic(project, branch, pr_id, PR_PIPELINE_PKG_QUEUED_TOPIC, 2) + if not topic_msg: + print("FAIL: %s %s %s Could not find topic %s" % (project, branch, pr_id, PR_PIPELINE_PKG_QUEUED_TOPIC)) + pr_result["status"] = INFRA_FAILURE + return pr_result + if "build_url" in topic_msg['msg']: + pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] + topic_msg = self.get_pr_topic(project, branch, pr_id, PR_PIPELINE_COMPLETE_TOPIC, 0) + if not topic_msg: + print("SKIP: %s - %s - %s still running" % (project, branch, pr_id)) + pr_result["status"] = RUNNING + return pr_result + + + step_results = [] + + pipeline_failed = False + topic_jenkins_build = None + for step in self.pipeline_steps['pr']: + topic = step['topic'] + timeout = step['timeout'] + if pipeline_failed and topic != PR_PIPELINE_COMPLETE_TOPIC: + # step will not execute if previous step failed + step_results.append({'step': topic, 'status': SKIP}) + continue + topic_msg = self.get_pr_topic(project, branch, pr_id, topic, timeout) + if not topic_msg: + pipeline_failed = True + print("FAIL: Could not find topic %s" % topic) + step_results.append({'step': topic, 'status': step['no_msg']}) + continue + if topic_msg['msg']['status'] != "SUCCESS" and topic != PR_PIPELINE_TESTS_COMPLETE_TOPIC: + pipeline_failed = True + print("FAIL: %s" % topic) + step_results.append({'step': topic, 'status': step['failure']}) + continue + if topic_msg['msg']['status'] != "SUCCESS" and topic == PR_PIPELINE_TESTS_COMPLETE_TOPIC: + pipeline_failed = True + print("FAIL: %s" % topic) + if topic_msg['msg']['status'] == "UNSTABLE": + step_results.append({'step': topic, 'status': TEST_FAILURE}) + else: + step_results.append({'step': topic, 'status': INFRA_FAILURE}) + continue + print("PASS: %s" % topic) + step_results.append({'step': topic, 'status': PASS}) + + # At this point Jenkins pipeline should have the build + if topic == PR_PIPELINE_PKG_RUNNING_TOPIC: + topic_jenkins_build = int(topic_msg['msg']['build_id']) + topic_jenkins_build_url = topic_msg['msg']['build_url'] - if not wait_complete: - # in case the build still running we skip all other topics check - topic_msg = get_pr_topic(project, branch, pr_id, PR_PIPELINE_PKG_QUEUED_TOPIC, 2) - if not topic_msg: - print("FAIL: %s %s %s Could not find topic %s" % (project, branch, pr_id, PR_PIPELINE_PKG_QUEUED_TOPIC)) - pr_result["status"] = INFRA_FAILURE - return pr_result - if "build_url" in topic_msg['msg']: - pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] - topic_msg = get_pr_topic(project, branch, pr_id, PR_PIPELINE_COMPLETE_TOPIC, 0) - if not topic_msg: - print("SKIP: %s - %s - %s still running" % (project, branch, pr_id)) - pr_result["status"] = RUNNING - return pr_result - pkg_queued_step = {"topic" : PR_PIPELINE_PKG_QUEUED_TOPIC, "timeout" : 2, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - pkg_running_step = {"topic" : PR_PIPELINE_PKG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - pkg_complete_step = {"topic" : PR_PIPELINE_PKG_COMPLETE_TOPIC, "timeout" : 120, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_queued_step = {"topic" : PR_PIPELINE_IMG_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_running_step = {"topic" : PR_PIPELINE_IMG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_complete_step = {"topic" : PR_PIPELINE_IMG_COMPLETE_TOPIC, "timeout" : 60, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_queued_step = {"topic" : PR_PIPELINE_TESTS_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_running_step = {"topic" : PR_PIPELINE_TESTS_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_complete_step = {"topic" : PR_PIPELINE_TESTS_COMPLETE_TOPIC, "timeout" : 5*60, "no_msg" : INFRA_FAILURE, "failure" : TEST_FAILURE} - pipeline_complete_step = {"topic" : PR_PIPELINE_COMPLETE_TOPIC, "timeout" : 5, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - - pipeline_steps = [pkg_queued_step, pkg_running_step, pkg_complete_step, img_queued_step, img_running_step, img_complete_step, - tests_queued_step, tests_running_step, tests_complete_step, pipeline_complete_step] - - step_results = [] - - pipeline_failed = False - topic_jenkins_build = None - for step in pipeline_steps: - topic = step['topic'] - timeout = step['timeout'] - if pipeline_failed and topic != PR_PIPELINE_COMPLETE_TOPIC: - # step will not execute if previous step failed - step_results.append({'step': topic, 'status': SKIP}) - continue - topic_msg = get_pr_topic(project, branch, pr_id, topic, timeout) - if not topic_msg: - pipeline_failed = True - print("FAIL: Could not find topic %s" % topic) - step_results.append({'step': topic, 'status': step['no_msg']}) - continue - if topic_msg['msg']['status'] != "SUCCESS" and topic != PR_PIPELINE_TESTS_COMPLETE_TOPIC: - pipeline_failed = True - print("FAIL: %s" % topic) - step_results.append({'step': topic, 'status': step['failure']}) - continue - if topic_msg['msg']['status'] != "SUCCESS" and topic == PR_PIPELINE_TESTS_COMPLETE_TOPIC: - pipeline_failed = True - print("FAIL: %s" % topic) - if topic_msg['msg']['status'] == "UNSTABLE": - step_results.append({'step': topic, 'status': TEST_FAILURE}) + if topic_jenkins_build: + # Wait some time for jenkins build be completed + build_info = get_jenkins_build_info("pr", branch, topic_jenkins_build) + count = 5 + while build_info['building'] and count > 0: + build_info = get_jenkins_build_info("pr", branch, topic_jenkins_build) + time.sleep(60) + count -= 1 + + if count == 0: + print("FAIL: Jenkins build did not finish: %s" % build_info) + status = INFRA_FAILURE + step_results.append({'step': "Jenkins build complete", 'status': INFRA_FAILURE}) else: - step_results.append({'step': topic, 'status': INFRA_FAILURE}) - continue - print("PASS: %s" % topic) - step_results.append({'step': topic, 'status': PASS}) - - # At this point Jenkins pipeline should have the build - if topic == PR_PIPELINE_PKG_RUNNING_TOPIC: - topic_jenkins_build = int(topic_msg['msg']['build_id']) - topic_jenkins_build_url = topic_msg['msg']['build_url'] - - - if topic_jenkins_build: - # Wait some time for jenkins build be completed - build_info = get_jenkins_build_info(branch, topic_jenkins_build) - count = 5 - while build_info['building'] and count > 0: - build_info = get_jenkins_build_info(branch, topic_jenkins_build) - time.sleep(60) - count -= 1 - - if count == 0: - print("FAIL: Jenkins build did not finish: %s" % build_info) - status = INFRA_FAILURE - step_results.append({'step': "Jenkins build complete", 'status': INFRA_FAILURE}) + step_results.append({'step': "Jenkins build complete", 'status': PASS}) + print("INFO: Jenkins build URL: %s" % topic_jenkins_build_url) + pr_result["jenkins_build_url"] = topic_jenkins_build_url else: - step_results.append({'step': "Jenkins build complete", 'status': PASS}) - print("INFO: Jenkins build URL: %s" % topic_jenkins_build_url) - pr_result["jenkins_build_url"] = topic_jenkins_build_url - else: - print("FAIL: Could not find Jenkins build") - step_results.append({'step': "Find Jenkins build", 'status': INFRA_FAILURE}) - - pr_result["steps"] = step_results - pr_result["status"] = PASS - for result in step_results: - # Set the status of first failure - if result['status'] == INFRA_FAILURE or result['status'] == TEST_FAILURE: - pr_result["status"] = result['status'] - break + print("FAIL: Could not find Jenkins build") + step_results.append({'step': "Find Jenkins build", 'status': INFRA_FAILURE}) - return pr_result + pr_result["steps"] = step_results + pr_result["status"] = PASS + for result in step_results: + # Set the status of first failure + if result['status'] == INFRA_FAILURE or result['status'] == TEST_FAILURE: + pr_result["status"] = result['status'] + break -def verify_kojibuild(project, branch, task_id, wait_complete=True): - """ - Check if Kojibuild ran properly on CI - """ - if branch.lower() == "rawhide": - branch = "master" + return pr_result - build_result = {"project" : project, "branch" : branch, "task_id" : task_id, "status" : None, "pipeline": "kojibuild"} - if not has_jenkins_pipeline("kojibuild", branch): - topic_msg = get_build_topic(project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC, 2) - if not topic_msg: - print("FAIL: %s %s %s Could not find topic %s" % (project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC)) - build_result["status"] = INFRA_FAILURE + def verify_kojibuild(self, project, branch, task_id): + """ + Check if Kojibuild ran properly on CI + """ + if branch.lower() == "rawhide": + branch = "master" + + build_result = {"project" : project, "branch" : branch, "task_id" : task_id, "status" : None, "pipeline": "kojibuild"} + if not has_jenkins_pipeline("kojibuild", branch): + topic_msg = self.get_build_topic(project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC, 2) + if not topic_msg: + print("FAIL: %s %s %s Could not find topic %s" % (project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC)) + build_result["status"] = INFRA_FAILURE + return build_result + if "build_url" in topic_msg['msg']: + build_result["jenkins_build_url"] = topic_msg['msg']['build_url'] + print("SKIP: %s there is no pipeline for koji build on branch %s" % (project, branch)) + build_result["status"] = SKIP return build_result - if "build_url" in topic_msg['msg']: - build_result["jenkins_build_url"] = topic_msg['msg']['build_url'] - print("SKIP: %s there is no pipeline for koji build on branch %s" % (project, branch)) - build_result["status"] = SKIP - return build_result - if not check_tests(project, branch): - topic_msg = get_build_topic(project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC, 2) - if not topic_msg: - print("FAIL: %s %s %s Could not find topic %s" % (project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC)) - build_result["status"] = INFRA_FAILURE + if not check_tests(project, branch): + topic_msg = self.get_build_topic(project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC, 2) + if not topic_msg: + print("FAIL: %s %s %s Could not find topic %s" % (project, branch, task_id, BUILD_PIPELINE_PKG_IGNORED_TOPIC)) + build_result["status"] = INFRA_FAILURE + return build_result + if "build_url" in topic_msg['msg']: + build_result["jenkins_build_url"] = topic_msg['msg']['build_url'] + print("SKIP: %s - %s does not contains tests" % (project, branch)) + build_result["status"] = SKIP return build_result - if "build_url" in topic_msg['msg']: - build_result["jenkins_build_url"] = topic_msg['msg']['build_url'] - print("SKIP: %s - %s does not contains tests" % (project, branch)) - build_result["status"] = SKIP - return build_result - print("INFO: Checking pipeline for BUILD %s %s %s" % (project, branch, task_id)) + print("INFO: Checking pipeline for BUILD %s %s %s" % (project, branch, task_id)) + + if not self.wait_complete: + # in case the build still running we skip all other topics check + topic_msg = get_pr_topic(project, branch, task_id, BUILD_PIPELINE_PKG_QUEUED_TOPIC, 2) + if not topic_msg: + print("FAIL: %s %s %s Could not find topic %s" % (project, branch, task_id, BUILD_PIPELINE_PKG_QUEUED_TOPIC)) + build_result["status"] = INFRA_FAILURE + return build_result + if "build_url" in topic_msg['msg']: + pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] + topic_msg = get_pr_topic(project, branch, build_id, BUILD_PIPELINE_COMPLETE_TOPIC, 0) + if not topic_msg: + print("SKIP: %s - %s - %s still running" % (project, branch, build_id)) + pr_result["status"] = RUNNING + return pr_result + + + step_results = [] + + pipeline_failed = False + topic_jenkins_build = None + for step in self.pipeline_steps['build']: + topic = step['topic'] + timeout = step['timeout'] + if pipeline_failed and topic != BUILD_PIPELINE_COMPLETE_TOPIC: + # step will not execute if previous step failed + step_results.append({'step': topic, 'status': SKIP}) + continue + topic_msg = self.get_build_topic(project, branch, task_id, topic, timeout) + if not topic_msg: + pipeline_failed = True + print("FAIL: Could not find topic %s" % topic) + step_results.append({'step': topic, 'status': step['no_msg']}) + continue + if topic_msg['msg']['status'] != "SUCCESS" and topic != BUILD_PIPELINE_TESTS_COMPLETE_TOPIC: + pipeline_failed = True + print("FAIL: %s" % topic) + step_results.append({'step': topic, 'status': step['failure']}) + continue + if topic_msg['msg']['status'] != "SUCCESS" and topic == BUILD_PIPELINE_TESTS_COMPLETE_TOPIC: + pipeline_failed = True + print("FAIL: %s" % topic) + if topic_msg['msg']['status'] == "UNSTABLE": + step_results.append({'step': topic, 'status': TEST_FAILURE}) + else: + step_results.append({'step': topic, 'status': INFRA_FAILURE}) + continue + print("PASS: %s" % topic) + step_results.append({'step': topic, 'status': PASS}) + + if "build_id" in topic_msg['msg']: + topic_jenkins_build = int(topic_msg['msg']['build_id']) + topic_jenkins_build_url = topic_msg['msg']['build_url'] - if not wait_complete: - # in case the build still running we skip all other topics check - topic_msg = get_pr_topic(project, branch, task_id, BUILD_PIPELINE_PKG_QUEUED_TOPIC, 2) - if not topic_msg: - print("FAIL: %s %s %s Could not find topic %s" % (project, branch, task_id, BUILD_PIPELINE_PKG_QUEUED_TOPIC)) - build_result["status"] = INFRA_FAILURE - return build_result - if "build_url" in topic_msg['msg']: - pr_result["jenkins_build_url"] = topic_msg['msg']['build_url'] - topic_msg = get_pr_topic(project, branch, build_id, BUILD_PIPELINE_COMPLETE_TOPIC, 0) - if not topic_msg: - print("SKIP: %s - %s - %s still running" % (project, branch, build_id)) - pr_result["status"] = RUNNING - return pr_result - pkg_queued_step = {"topic" : BUILD_PIPELINE_PKG_QUEUED_TOPIC, "timeout" : 2, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - pkg_running_step = {"topic" : BUILD_PIPELINE_PKG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - pkg_complete_step = {"topic" : BUILD_PIPELINE_PKG_COMPLETE_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_queued_step = {"topic" : BUILD_PIPELINE_IMG_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_running_step = {"topic" : BUILD_PIPELINE_IMG_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - img_complete_step = {"topic" : BUILD_PIPELINE_IMG_COMPLETE_TOPIC, "timeout" : 60, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_queued_step = {"topic" : BUILD_PIPELINE_TESTS_QUEUED_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_running_step = {"topic" : BUILD_PIPELINE_TESTS_RUNNING_TOPIC, "timeout" : 10, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - tests_complete_step = {"topic" : BUILD_PIPELINE_TESTS_COMPLETE_TOPIC, "timeout" : 5*60, "no_msg" : INFRA_FAILURE, "failure" : TEST_FAILURE} - pipeline_complete_step = {"topic" : BUILD_PIPELINE_COMPLETE_TOPIC, "timeout" : 5, "no_msg" : INFRA_FAILURE, "failure" : INFRA_FAILURE} - - pipeline_steps = [pkg_queued_step, pkg_running_step, pkg_complete_step, img_queued_step, img_running_step, img_complete_step, - tests_queued_step, tests_running_step, tests_complete_step, pipeline_complete_step] - - step_results = [] - - pipeline_failed = False - topic_jenkins_build = None - for step in pipeline_steps: - topic = step['topic'] - timeout = step['timeout'] - if pipeline_failed and topic != BUILD_PIPELINE_COMPLETE_TOPIC: - # step will not execute if previous step failed - step_results.append({'step': topic, 'status': SKIP}) - continue - topic_msg = get_build_topic(project, branch, task_id, topic, timeout) - if not topic_msg: - pipeline_failed = True - print("FAIL: Could not find topic %s" % topic) - step_results.append({'step': topic, 'status': step['no_msg']}) - continue - if topic_msg['msg']['status'] != "SUCCESS" and topic != BUILD_PIPELINE_TESTS_COMPLETE_TOPIC: - pipeline_failed = True - print("FAIL: %s" % topic) - step_results.append({'step': topic, 'status': step['failure']}) - continue - if topic_msg['msg']['status'] != "SUCCESS" and topic == BUILD_PIPELINE_TESTS_COMPLETE_TOPIC: - pipeline_failed = True - print("FAIL: %s" % topic) - if topic_msg['msg']['status'] == "UNSTABLE": - step_results.append({'step': topic, 'status': TEST_FAILURE}) + if topic_jenkins_build: + # Wait some time for jenkins build be completed + build_info = get_jenkins_build_info("kojibuild", branch, topic_jenkins_build) + count = 5 + while build_info['building'] and count > 0: + build_info = get_jenkins_build_info("kojibuild", branch, topic_jenkins_build) + time.sleep(60) + count -= 1 + + if count == 0: + print("FAIL: Jenkins build did not finish: %s" % build_info) + status = INFRA_FAILURE + step_results.append({'step': "Jenkins build complete", 'status': INFRA_FAILURE}) else: - step_results.append({'step': topic, 'status': INFRA_FAILURE}) - continue - print("PASS: %s" % topic) - step_results.append({'step': topic, 'status': PASS}) - - if "build_id" in topic_msg['msg']: - topic_jenkins_build = int(topic_msg['msg']['build_id']) - topic_jenkins_build_url = topic_msg['msg']['build_url'] - - - if topic_jenkins_build: - # Wait some time for jenkins build be completed - build_info = get_jenkins_build_info(branch, topic_jenkins_build) - count = 5 - while build_info['building'] and count > 0: - build_info = get_jenkins_build_info(branch, topic_jenkins_build) - time.sleep(60) - count -= 1 - - if count == 0: - print("FAIL: Jenkins build did not finish: %s" % build_info) - status = INFRA_FAILURE - step_results.append({'step': "Jenkins build complete", 'status': INFRA_FAILURE}) + step_results.append({'step': "Jenkins build complete", 'status': PASS}) + print("INFO: Jenkins build URL: %s" % topic_jenkins_build_url) + build_result["jenkins_build_url"] = topic_jenkins_build_url else: - step_results.append({'step': "Jenkins build complete", 'status': PASS}) - print("INFO: Jenkins build URL: %s" % topic_jenkins_build_url) - build_result["jenkins_build_url"] = topic_jenkins_build_url - else: - print("FAIL: Could not find Jenkins build") - step_results.append({'step': "Find Jenkins build", 'status': INFRA_FAILURE}) - - build_result["steps"] = step_results - build_result["status"] = PASS - for result in step_results: - # Set the status of first failure - if result['status'] == INFRA_FAILURE or result['status'] == TEST_FAILURE: - build_result["status"] = result['status'] - break + print("FAIL: Could not find Jenkins build") + step_results.append({'step': "Find Jenkins build", 'status': INFRA_FAILURE}) - return build_result + build_result["steps"] = step_results + build_result["status"] = PASS + for result in step_results: + # Set the status of first failure + if result['status'] == INFRA_FAILURE or result['status'] == TEST_FAILURE: + build_result["status"] = result['status'] + break + + return build_result if __name__ == "__main__": @@ -787,8 +698,7 @@ if __name__ == "__main__": start_time = int(time.time()) - # By default we wait for running builds on pipeline - wait_complete = True + monitor = Monitor() ci_message = os.getenv("CI_MESSAGE", None) if ci_message: @@ -797,17 +707,18 @@ if __name__ == "__main__": else: if not args.pipeline: messages = [] - prs = get_recent_prs() + prs = monitor.get_recent_prs() if prs: messages.extend(prs) - builds = get_recent_builds() + builds = monitor.get_recent_builds() if builds: messages.extend(builds) elif args.pipeline == "pr": - messages = get_recent_prs() + messages = monitor.get_recent_prs() elif args.pipeline == "kojibuild": - messages = get_recent_builds() - wait_complete = False + messages = monitor.get_recent_builds() + monitor.set_wait_complete(False) + monitor.query_all_topics() if not messages: sys.exit(SKIP) @@ -826,7 +737,7 @@ if __name__ == "__main__": if message['pullrequest']['comments'] and "citest" not in message['pullrequest']['comments'][-1]['comment']: print("SKIP: %s %s %s - Comment added to Pull request is not for rebuild" % (project, branch, pr_id)) continue - pr_result = verify_pull_request(project, branch, pr_id, wait_complete) + pr_result = monitor.verify_pull_request(project, branch, pr_id) result_log["results"].append(pr_result) elif 'build_id' in message: project = message['name'] @@ -843,7 +754,7 @@ if __name__ == "__main__": result_log["results"].append(build_result) continue branch = re.sub("-.*", "", build_tag) - build_result = verify_kojibuild(project, branch, task_id, wait_complete) + build_result = monitor.verify_kojibuild(project, branch, task_id) result_log["results"].append(build_result) else: print("FAIL: Does not support ci_message: %s" % message) @@ -852,7 +763,7 @@ if __name__ == "__main__": finish_time = int(time.time()) result_log["start_time"] = start_time result_log["finish_time"] = finish_time - result_log["delta"] = DELTA + result_log["delta"] = monitor.delta with open('result.json', 'w') as resultfile: json.dump(result_log, resultfile, indent=4, sort_keys=True, separators=(',', ': '))