From 7452a477a2327c52966777f18ec01671939bba59 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Mar 25 2020 15:03:58 +0000 Subject: Improvements to maintainer pings Signed-off-by: Mohan Boddu --- diff --git a/compose-tracker.toml b/compose-tracker.toml index 1f8b4ab..759ffae 100644 --- a/compose-tracker.toml +++ b/compose-tracker.toml @@ -40,6 +40,7 @@ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"] [consumer_config] composes_to_skip = ["IoT"] +ks_repo = "https://pagure.io/fedora-kickstarts" [qos] prefetch_size = 0 diff --git a/compose_tracker.py b/compose_tracker.py index 1582656..1fe42ee 100755 --- a/compose_tracker.py +++ b/compose_tracker.py @@ -25,6 +25,7 @@ import requests import sys import traceback import toml +import itertools import fedora_messaging.api import fedora_messaging.config @@ -142,33 +143,31 @@ class Consumer(object): logger.info("Failed to retrieve maintainers info from ks repo.") return None maintainers_info = toml.loads(req.text) - arches = ['armhfp', 'aarch64', 'ppc64le', 'i686', 'x86_64', 's390x'] - variants = list(maintainers_info.keys()) - for variant in variants: - if variant.lower() in line.lower(): - subvariants = maintainers_info.get(variant) - for subvariant in subvariants: - if subvariant.lower() in line.lower(): - if 'arch *' in line.lower(): - maintainers_arches = [arch for arch in arches if arch in subvariants.get(subvariant).keys()] - if maintainers_arches: - maintainers = set() - for maintainers_arch in maintainers_arches: - maintainers.update(maintainers_info.get(variant).get(subvariant).get(maintainers_arch).get('fas')) - return [maintainers, variant, subvariant] - else: - maintainers = maintainers_info.get(variant).get(subvariant).get('fas') - return [maintainers, variant, subvariant] - else: - maintainers_arches = [arch for arch in arches if arch in subvariants.get(subvariant).keys()] - if maintainers_arches: - for maintainers_arch in maintainers_arches: - if maintainers_arch.lower() in line.lower(): - maintainers = maintainers_info.get(variant).get(subvariant).get(maintainers_arch).get('fas') - return [maintainers, variant, subvariant] - else: - maintainers = maintainers_info.get(variant).get(subvariant).get('fas') - return [maintainers, variant, subvariant] + try: + matching_lists = re.findall(r'variant (\w+), arch (\S+), subvariant (\w+)', line) + if matching_lists: + for ml in matching_lists: + # For all arches + if ml[1] == '*': + # Get the list of all maintainers for all arches for that variant and sub variant + maintainers_list = list(maintainers_info.get(ml[0].lower()).get(ml[2].lower()).values()) + # Get the list of all fas usernames from the above list + maintainer_fas_list = [maintainer.get('fas') for maintainer in maintainers_list] + # Since there can be repetitions of fas usernames, create a set of usernames + maintainers = set(itertools.chain(*maintainer_fas_list)) + # return the list of maintainers, variant and sub variant + return maintainers, ml[0], ml[2] + else: + # Get the list of all maintainers for the variant, sub variant and arch + maintainers_list = maintainers_info.get(ml[0].lower()).get(ml[2].lower()).get(ml[1].lower()) + # Get a set of fas usernames from the above list + maintainers = set(maintainers_list.get('fas')) + # return the list of maintainers, variant and sub variant + return maintainers, ml[0], ml[2] + except Exception: + logger.info("No maintainer info found, skipping") + return None + def __call__(self, message: fedora_messaging.api.Message): # Catch any exceptions and don't raise them further because @@ -260,28 +259,29 @@ class Consumer(object): # Find the maintainers in kickstarts repo under # mainatiners.toml file and ping them in the ticket # Find the branch in kickstarts repo - ks_branch = None - if 'rawhide'.lower() == msg['release_version'].lower(): - ks_branch = 'master' - elif 'epel' not in msg['release_name'].lower(): - ks_branch = 'f' + msg['release_version'] - ks_url = 'https://pagure.io/fedora-kickstarts/raw/' + ks_branch + '/f/maintainers.toml' + if self.config["ks_repo"]: + if 'rawhide' == msg['release_version'].lower(): + ks_branch = 'master' + elif 'epel' not in msg['release_name'].lower(): + ks_branch = 'f' + msg['release_version'] + ks_url = self.config["ks_repo"] + '/raw/' + ks_branch + '/f/maintainers.toml' kojitaskline, text = self.get_supporting_text(lines[x-1:]) content+=f'{text}\n' content+= "```\n%s\n%s\n```\n\n" % \ (line, kojitaskline or nextline) - # Ping the maintainers - pinging_list = self.get_maintainers(line, ks_url) - if not pinging_list: - logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") - else: - ping_line = 'Variant: {}, subvariant: {} task failed. Pinging maintainers: '.format(pinging_list[1], pinging_list[2]) - for maintainer in pinging_list[0]: - ping_line += '@{} '.format(maintainer) - content += ping_line + '\n' - logger.info(ping_line) + if self.config["ks_repo"]: + # Ping the maintainers + maintainers, variant, sub_variant = self.get_maintainers(line, ks_url) or (None, None, None) + if not maintainers: + logger.info("No maintainers info available. The ticket will be filed without pinging any maintainers") + else: + ping_line = f'Variant: {variant}, subvariant: {sub_variant} task failed. Pinging maintainers: ' + for maintainer in maintainers: + ping_line += f'@{maintainer} ' + content += ping_line + '\n' + logger.info(ping_line) # If this is the Compose run failed line, then add it # to the description too diff --git a/test_consumer.py b/test_consumer.py index fe6eef9..25b86f5 100644 --- a/test_consumer.py +++ b/test_consumer.py @@ -55,11 +55,13 @@ EXAMPLE_PUNGI_LOG_PING_MAINTAINERS = """2020-02-04 05:59:24 [INFO ] [BEGIN] - """ EXAMPLE_KS_MAINTAINERS = """ -[spins] -[spins.soas] [spins.soas.x86_64] fas = [ "pbrobinson" ] maintainers = [ "Peter Robinson" ] + +[spins.xfce.x86_64] +fas = [ "nonamedotc", "kevin", "maxamillion" ] +maintainers = [ "Mukundan Ragavan", "Kevin Fenzi", "Adam Miller" ] """ @@ -88,7 +90,7 @@ def test_consumer_settings_ignore_compose(mocker, caplog): mocker.patch("compose_tracker.requests") mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) con = Consumer() body = copy.copy(EXAMPLE_MESSAGE_BODY) @@ -106,7 +108,7 @@ def test_consumer_settings_ignore_compose_2_values(mocker, caplog): mocker.patch("compose_tracker.requests") mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"]}}, + {"consumer_config": {"composes_to_skip": ["IoT", "Rawhide"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) con = Consumer() msg = fedora_messaging.api.Message( @@ -171,7 +173,7 @@ def test_consumer_logfile_parsing_failures(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -201,7 +203,7 @@ def test_adding_labels(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) req = mocker.patch("compose_tracker.requests.get", autospec=True) text_mock = mocker.MagicMock() @@ -229,7 +231,7 @@ def test_consumer_maintainer_pings(mocker, caplog): mocker.patch( "compose_tracker.fedora_messaging.config.conf", - {"consumer_config": {"composes_to_skip": ["IoT"]}}, + {"consumer_config": {"composes_to_skip": ["IoT"], "ks_repo": "https://pagure.io/fedora-kickstarts"}}, ) # mock the pungi.global.log file req = mocker.patch("compose_tracker.requests.get", autospec=True) @@ -245,4 +247,6 @@ def test_consumer_maintainer_pings(mocker, caplog): topic="org.fedoraproject.prod.pungi.compose.status.change", body=EXAMPLE_MESSAGE_BODY, ) con.process(msg) - assert "Variant: spins, subvariant: soas task failed. Pinging maintainers: @pbrobinson" in caplog.text \ No newline at end of file + assert "Variant: Spins, subvariant: SoaS task failed. Pinging maintainers: @pbrobinson" in caplog.text + # Cannot assert particular maintainers since the set order is random + assert "Variant: Spins, subvariant: xfce task failed. Pinging maintainers" in caplog.text and "@kevin" in caplog.text and '@nonamedotc' in caplog.text and '@maxamillion' in caplog.text \ No newline at end of file