From 51a7631a33f3da499c45719d5d8efb24d87713e2 Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Feb 05 2020 15:40:04 +0000 Subject: Fixed multi-build, so that it conforms to the refactor, removed unused imports. --- diff --git a/monitor_gating.py b/monitor_gating.py index 2c30613..78a45b6 100644 --- a/monitor_gating.py +++ b/monitor_gating.py @@ -27,24 +27,14 @@ import logging import os import sys import tempfile -import time import toml -import requests -from utils import( - MonitoringException, - MonitoringUtils, - run_command, -) +from utils import MonitoringUtils _log = logging.getLogger(__name__) -class MonitoringException(Exception): - """The base class for all exceptions raised by this script.""" - - def get_arguments(args): """ Parse and return the CLI arguments. """ diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 02e96f1..5ff0769 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -27,42 +27,12 @@ import logging import os import sys import tempfile -import time import toml -import requests - -from utils import( - MonitoringException, - print_user, - run_command, - clone_repo, - add_remote, - switch_branch, - bump_release, - commit_changes, - chain_build_packages, - push_changes, - pull_changes, - open_pullrequest, - get_nevr, - build_package, - get_build_tags, - get_update_id, - lookup_results_datagrepper, - lookup_ci_resultsdb, - waive_update, - get_pr_flag, - merge_pr, - finalize, - create_update, -) - -_log = logging.getLogger(__name__) +from utils import MonitoringUtils -class MonitoringException(Exception): - """The base class for all exceptions raised by this script.""" +_log = logging.getLogger(__name__) def get_arguments(args): @@ -98,47 +68,6 @@ def get_arguments(args): return parser.parse_args(args) -def create_side_tag(command, folder): - """ Create a side-tag to build packages in it. """ - info_log = f"Creating the side-tag" - print_user(info_log) - command = [command, "request-side-tag"] - side_tag_name = None - try: - output = run_command(command, cwd=folder) - lines = output.decode("utf-8").split("\n") - _, side_tag_name, _ = lines[0].split("'") - info_log = f"Created side-tag {side_tag_name}" - print_user(info_log, success=True) - except (MonitoringException, Exception) as err: - print(err) - print_user(info_log, success=False) - return side_tag_name - - -def clone_and_bump(folder, nevrs, conf, name, target=None, new_side_tag=False): - """Clone the repo, bump the release, commit and push.""" - namespace = conf["namespace"] - branch = conf["branch"] - - clone_repo(conf["fedpkg"], namespace, name, folder=folder) - gitfolder = os.path.join(folder, name) - switch_branch(conf["fedpkg"], branch, folder=gitfolder) - side_tag_name = None - # Create a side-tag - if new_side_tag: - side_tag_name = create_side_tag(conf["fedpkg"], folder=gitfolder) - target = side_tag_name - bump_release(name, folder=gitfolder) - commit_changes("Bump release", folder=gitfolder) - nevr = get_nevr(conf["fedpkg"], folder=gitfolder) - nevrs[name] = nevr - # Push to the main repo - push_changes(gitfolder, "origin", branch) - print(f" Upcoming build : {nevr}") - return (nevrs, target) - - def main(args): """ Main method used by this script. """ start = datetime.datetime.utcnow() @@ -146,21 +75,20 @@ def main(args): args = get_arguments(args) conf = toml.load(args.conf) - - fas_username = conf["fas_username"] + utils = MonitoringUtils() with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: print(f"Working in {folder}\n") nevrs = {} # Bump the release on both packages: - nevrs, side_tag_name = clone_and_bump( + nevrs, side_tag_name = utils.clone_and_bump( folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True) - nevrs, _ = clone_and_bump( + nevrs, _ = utils.clone_and_bump( folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name) # Chain-build the packages - chain_build_packages( + utils.chain_build_packages( conf["fedpkg"], packages=conf["name_multi_1"], folder=os.path.join(folder, conf["name_multi_2"]), @@ -168,10 +96,10 @@ def main(args): ) if args.conflict: - clone_to_build(folder, nevrs, conf, name, target=None) + utils.clone_to_build(folder, nevrs, conf, conf["name_multi_1"], target=None) # Create the update - create_update( + utils.create_update( conf["bodhi-cli"], side_tag_name, prod=conf["_env"] == "prod", @@ -179,25 +107,25 @@ def main(args): password=conf.get("bodhi-password"), from_tag=True, ) - updateid = get_update_id( + updateid = utils.get_update_id( nevrs[list(nevrs.keys())[0]], conf["bodhi"], ) print(f" Update created : {updateid}") # Check the tag of the build - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevrs[conf["name_multi_1"]], expected_ends=["signing-pending", "testing-pending"], ) if not updateid: - finalize(start) + utils.finalize(start) return # Check that bodhi notified the pipeline it can run - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="bodhi to CI", topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." @@ -213,7 +141,7 @@ def main(args): nevr_names.append(nevr) # Check that the CI pipeline is running - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (running)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", @@ -221,7 +149,7 @@ def main(args): start=start_dg, ) # Check at the CI pipeline has completed - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (complete)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", @@ -230,19 +158,19 @@ def main(args): ) # Check the tag of the build - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) # Check that the CI results made it to resultsdb - lookup_ci_resultsdb( + utils.lookup_ci_resultsdb( nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] ) # Check that resultsdb announced the new results - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="resultsdb", topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", @@ -251,7 +179,7 @@ def main(args): ) # Check that greenwave reacted to resultsdb's new results - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="greenwave", topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", @@ -260,7 +188,7 @@ def main(args): ) # Check the tag of the build -- build is blocked but should be signed - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], @@ -268,7 +196,7 @@ def main(args): if not args.no_waive: nevr = nevrs[list(nevrs.keys())[0]] - waive_update( + utils.waive_update( conf["bodhi-cli"], updateid, prod=conf["_env"] == "prod", @@ -277,7 +205,7 @@ def main(args): ) # Check that waiverdb announced the new waiver - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="waiverdb", topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", @@ -285,7 +213,7 @@ def main(args): ) # Check that greenwave reacted to the new waiver - lookup_results_datagrepper( + utils.lookup_results_datagrepper( base_url=conf["datagrepper"], name="greenwave", topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", @@ -293,13 +221,13 @@ def main(args): ) # Check the tag of the build -- build was waived, let is through - get_build_tags( + utils.get_build_tags( conf.get("koji_hub"), nevr, expected_ends=["f32"], ) - finalize(start) + utils.finalize(start) if __name__ == "__main__": diff --git a/utils.py b/utils.py index afd629c..13645e5 100644 --- a/utils.py +++ b/utils.py @@ -10,10 +10,8 @@ import datetime import logging import os import subprocess -import tempfile import time -import toml import requests _log = logging.getLogger(__name__) @@ -128,7 +126,7 @@ class MonitoringUtils: def pull_changes(self, folder, target, branch): """ Pull all changes using git. """ - info_log = f"Pushing changes" + info_log = f"Pulling changes" self.print_user(info_log) try: cmd = ["git", "pull", "--rebase", target, branch] @@ -377,6 +375,13 @@ class MonitoringUtils: end_url = url end_url += f"&page={page}" data = requests.get(end_url).json() + if "raw_messages" not in data: + nomsg_log = f"No messages in data-grepper on {end_url} " + if 'error' in data: + nomsg_log += data['error'] + self.print_user(nomsg_log) + break + for message in data["raw_messages"]: # Old message format from the CI pipeline @@ -644,6 +649,46 @@ class MonitoringUtils: self.logs.append(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") print(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") + def create_side_tag(self, command, folder): + """ Create a side-tag to build packages in it. """ + info_log = f"Creating the side-tag" + self.print_user(info_log) + command = [command, "request-side-tag"] + side_tag_name = None + try: + output = run_command(command, cwd=folder) + lines = output.decode("utf-8").split("\n") + _, side_tag_name, _ = lines[0].split("'") + info_log = f"Created side-tag {side_tag_name}" + self.print_user(info_log, success=True) + except (MonitoringException, Exception) as err: + print(err) + self.print_user(info_log, success=False) + return side_tag_name + + + def clone_and_bump(self, folder, nevrs, conf, name, target=None, new_side_tag=False): + """Clone the repo, bump the release, commit and push.""" + namespace = conf["namespace"] + branch = conf["branch"] + + self.clone_repo(conf["fedpkg"], namespace, name, folder=folder) + gitfolder = os.path.join(folder, name) + self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) + side_tag_name = None + # Create a side-tag + if new_side_tag: + side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) + target = side_tag_name + self.bump_release(name, folder=gitfolder) + self.commit_changes("Bump release", folder=gitfolder) + nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) + nevrs[name] = nevr + # Push to the main repo + self.push_changes(gitfolder, "origin", branch) + print(f" Upcoming build : {nevr}") + return (nevrs, target) + def run_command(command, cwd=None): """ Run the specified command in a specific working directory if one