From e10907bd1ba96d95352038d73bc0c6715f5ad578 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 19 2020 15:55:36 +0000 Subject: Introduce the clean_retired_packages toddler This toddler ensures retired packages are properly orphaned and have no (co)maintainers set to properly reflect in the dist-git's UI that these packages are not maintained by anyone. Fixes https://pagure.io/fedora-infrastructure/issue/8600 (which contains all the references to how we came to do this, including link to the FESCo ticket approving this). Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/plugins/test_clean_retired_packages.py b/tests/plugins/test_clean_retired_packages.py new file mode 100644 index 0000000..016323b --- /dev/null +++ b/tests/plugins/test_clean_retired_packages.py @@ -0,0 +1,219 @@ +from unittest.mock import call, MagicMock, patch + +import fedora_messaging.api +import pytest + +import toddlers.plugins.clean_retired_packages + + +class TestCleanRetiredPackagesToddler: + + toddler_cls = toddlers.plugins.clean_retired_packages.CleanRetiredPackages + + def test_accepts_topic_invalid(self, toddler): + assert toddler.accepts_topic("foo.bar") is False + + @pytest.mark.parametrize( + "topic", + [ + "org.fedoraproject.*.toddlers.trigger.clean_retired_packages", + "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages", + "org.fedoraproject.stg.toddlers.trigger.clean_retired_packages", + ], + ) + def test_accepts_topic_valid(self, topic, toddler): + assert toddler.accepts_topic(topic) + + def test_process_config_missing_pdc_url(self, toddler): + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages" + msg.body = {"foo": "bar"} + + with pytest.raises( + Exception, + match=r"Invalid toddler configuration, no `pdc_active_branches` defined", + ): + assert toddler.process(config={}, message=msg) is None + + def test_process_config_missing_distgit_url(self, toddler): + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages" + msg.body = {"foo": "bar"} + + config = { + "pdc_active_branches": "https://pdc.fedoraproject.org/extras/active_branches.json" + } + + with pytest.raises( + Exception, + match=r"Invalid toddler configuration, no `dist_git_url` defined", + ): + assert toddler.process(config=config, message=msg) is None + + def test_process_config_missing_distgit_token(self, toddler): + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages" + msg.body = {"foo": "bar"} + + config = { + "pdc_active_branches": "https://pdc.fedoraproject.org/extras/active_branches.json", + "dist_git_url": "https://src.fedoraproject.org/", + } + + with pytest.raises( + Exception, + match=r"Invalid toddler configuration, no `dist_git_token` defined", + ): + assert toddler.process(config=config, message=msg) is None + + @patch("toddlers.plugins.clean_retired_packages.send_email") + def test_process(self, send_email, toddler, caplog): + pdc_branches = MagicMock() + pdc_branches.json.return_value = { + "flatpak": { + "bluez-gnome": [["master", False], ["epel8", False], ["epel7", False]], + }, + "rpm": { + "trojan": [["master", True], ["f33", True], ["f32", True]], + "pigment": [["master", False], ["epel8", True], ["epel7", True]], + "bluez-gnome": [["master", False], ["epel8", False], ["epel7", False]], + "guake": [["master", False], ["epel8", False], ["epel7", False]], + }, + } + distgit_info_bluez_gnome = MagicMock() + distgit_info_bluez_gnome.json.return_value = { + "user": {"name": "pingou"}, + "access_groups": { + "admin": [], + "collaborator": [], + "commit": ["infra-sig"], + "ticket": [], + }, + "access_users": { + "admin": [], + "collaborator": [], + "commit": ["ohaessler"], + "owner": ["pingou"], + "ticket": [], + }, + } + distgit_info_guake = MagicMock() + distgit_info_guake.json.return_value = { + "user": {"name": "orphan"}, + "access_groups": { + "admin": [], + "collaborator": [], + "commit": [], + "ticket": [], + }, + "access_users": { + "admin": ["orphan"], + "collaborator": [], + "commit": [], + "owner": ["pingou"], + "ticket": [], + }, + } + + toddler.requests_session.get.side_effect = ( + pdc_branches, + distgit_info_bluez_gnome, + distgit_info_guake, + ) + toddler.requests_session.patch.return_value = MagicMock(ok=False) + toddler.requests_session.post.return_value = MagicMock(ok=False) + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages" + msg.body = {"foo": "bar"} + + config = { + "pdc_active_branches": "https://pdc.fedoraproject.org/extras/active_branches.json", + "dist_git_url": "https://src.fedoraproject.org/", + "dist_git_token": "foobar", + "admin_email": "admin@fp.o", + "mail_server": "mail_server", + } + + assert toddler.process(config=config, message=msg) is None + send_email.assert_called_with( + to_addresses=["admin@fp.o"], + from_address="admin@fp.o", + subject="Toddlers cleaned up some retired packages", + content="Dear Admin,\n\n" + "The clean_retired_packages toddler just ran and adjusted some retired packages\n" + "for recording purposes, here is what it did:\n" + "- Orphaning rpms/bluez-gnome from pingou\n" + "- Removing access on rpms/bluez-gnome to ohaessler\n" + "- Removing access on rpms/bluez-gnome to pingou\n" + "- Removing access on rpms/bluez-gnome to @infra-sig\n" + "- Removing access on rpms/guake to pingou\n\n" + "In case someone asks, you can come back to this email but there isn't really\n" + "anything else to do with it.\n\n" + "Have a wonderful day and see you (maybe?) at the next run!\n\n", + mail_server="mail_server", + ) + toddler.requests_session.patch.assert_called() + toddler.requests_session.patch.assert_has_calls( + calls=[ + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome", + data={"main_admin": "orphan", "retain_access": False}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/git/modifyacls", + data={"user_type": "user", "name": "ohaessler"}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/git/modifyacls", + data={"user_type": "user", "name": "pingou"}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/git/modifyacls", + data={"user_type": "group", "name": "infra-sig"}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/guake/git/modifyacls", + data={"user_type": "user", "name": "pingou"}, + headers={"Authorization": "token foobar"}, + ), + ] + ) + toddler.requests_session.post.assert_called() + toddler.requests_session.post.assert_has_calls( + calls=[ + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/watchers/update", + data={"status": -1, "watcher": "pingou"}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/_dg/bzoverrides/rpms/bluez-gnome", + data={"fedora_assignee": None, "epel_assignee": None}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/watchers/update", + data={"status": -1, "watcher": "ohaessler"}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/watchers/update", + data={"status": -1, "watcher": "pingou"}, + headers={"Authorization": "token foobar"}, + ), + call( + "https://src.fedoraproject.org/api/0/rpms/guake/watchers/update", + data={"status": -1, "watcher": "pingou"}, + headers={"Authorization": "token foobar"}, + ), + ] + ) diff --git a/toddlers.toml.example b/toddlers.toml.example index 37327e0..793f3cb 100644 --- a/toddlers.toml.example +++ b/toddlers.toml.example @@ -128,6 +128,9 @@ mbs_url = "https://mbs.fedoraproject.org/module-build-service/2/module-builds/" [consumer_config.check_email_overrides] email_overrides_url = "https://pagure.io/fedora-infra/ansible/raw/master/f/roles/openshift-apps/distgit-bugzilla-sync/templates/email_overrides.toml" +[consumer_config.clean_retired_packages] +pdc_active_branches = "https://pdc.fedoraproject.org/extras/active_branches.json" + [qos] prefetch_size = 0 diff --git a/toddlers/plugins/clean_retired_packages.py b/toddlers/plugins/clean_retired_packages.py new file mode 100644 index 0000000..1fedec7 --- /dev/null +++ b/toddlers/plugins/clean_retired_packages.py @@ -0,0 +1,240 @@ +""" +This script queries PDC for all the packages in Fedora, it then go through +each of them and determines if they are inactive on all branches or not. +If they are, it will then query dist-git to retrieve all of its current +maintainers, if they do not have ``orphan`` as main user they will be orphaned +and all co-maintainers will be removed. + +Authors: Pierre-Yves Chibon + +""" + +import collections +import logging + +from toddlers.base import ToddlerBase +from toddlers.utils.notify import send_email +from toddlers.utils.requests import make_session + + +_log = logging.getLogger(__name__) + + +pdc_namespace_to_dist_git = { + "rpm": "rpms", + # "container": "container", + # "flatpak": "flatpaks", + # "module": "modules", +} + + +class CleanRetiredPackages(ToddlerBase): + """ Listens to messages sent by playtime (which lives in toddlers) and check + if the `email_overrides.toml` file stored in ansible is up to date. + """ + + name = "clean_retired_packages" + + amqp_topics = [ + "org.fedoraproject.*.toddlers.trigger.clean_retired_packages", + ] + + def __init__(self): + self.requests_session = make_session() + self.dist_git_token = None + self.dist_git_url = None + self.distgit_api_base_url = None + + def accepts_topic(self, topic): + """Returns a boolean whether this toddler is interested in messages + from this specific topic. + """ + return topic.startswith("org.fedoraproject.") and topic.endswith( + "toddlers.trigger.clean_retired_packages" + ) + + def process(self, config, message, dry_run=False): + """Process a given message.""" + + pdc_active_branches = config.get("pdc_active_branches") or None + if not pdc_active_branches: + raise Exception( + "Invalid toddler configuration, no `pdc_active_branches` defined" + ) + + dist_git_url = config.get("dist_git_url") or None + if not dist_git_url: + raise Exception("Invalid toddler configuration, no `dist_git_url` defined") + self.dist_git_url = dist_git_url.rstrip("/") + self.distgit_api_base_url = f"{self.dist_git_url}/api/0" + + self.dist_git_token = config.get("dist_git_token") or None + if not self.dist_git_token: + raise Exception( + "Invalid toddler configuration, no `dist_git_token` defined" + ) + + _log.info("Querying PDC") + branch_info = self.requests_session.get(pdc_active_branches).json() + + _log.info("Retrieving list of package retired from PDC") + fully_retired = collections.defaultdict(list) + for namespace in sorted(branch_info): + for package in sorted(branch_info[namespace]): + if True not in [el[1] for el in branch_info[namespace][package]]: + fully_retired[namespace].append(package) + + for namespace in fully_retired: + _log.info( + "%s fully retired packages found in %s", + len(fully_retired[namespace]), + namespace, + ) + + logs = [] + + for namespace in fully_retired: + # for namespace in ["rpm"]: + if namespace not in pdc_namespace_to_dist_git: + _log.info( + "Ignoring namespace: %s - not mapped from PDC to dist-git", + namespace, + ) + continue + + _log.info("Processing: %s", namespace) + for package in fully_retired[namespace]: + # _log.info(" %s/%s", namespace, package) + + ns = pdc_namespace_to_dist_git[namespace] + + url = f"{self.distgit_api_base_url}{ns}/{package}" + data = self.requests_session.get(url).json() + + if data["user"]["name"] != "orphan": + log = f"Orphaning {ns}/{package} from {data['user']['name']}" + print(log) + logs.append(log) + self.orphan_package(ns, package, current_poc=data["user"]["name"]) + + packagers = set() + for lvl in data["access_users"]: + packagers.update([u for u in data["access_users"][lvl]]) + + for packager in sorted(packagers): + if packager == "orphan": + continue + log = f"Removing access on {ns}/{package} to {packager}" + print(log) + logs.append(log) + self.remove_access(ns, package, packager, "user") + + groups = set() + for lvl in data["access_groups"]: + groups.update([u for u in data["access_groups"][lvl]]) + + for group in sorted(groups): + log = f"Removing access on {ns}/{package} to @{group}" + print(log) + logs.append(log) + self.remove_access(ns, package, group, "group") + + if logs: + logs_text = "\n- ".join(logs) + message = f"""Dear Admin, + +The clean_retired_packages toddler just ran and adjusted some retired packages +for recording purposes, here is what it did: +- {logs_text} + +In case someone asks, you can come back to this email but there isn't really +anything else to do with it. + +Have a wonderful day and see you (maybe?) at the next run! + +""" + send_email( + to_addresses=[config["admin_email"]], + from_address=config["admin_email"], + subject="Toddlers cleaned up some retired packages", + content=message, + mail_server=config["mail_server"], + ) + + def orphan_package(self, namespace, name, current_poc): + """ Give the specified project on dist_git to the ``orphan`` user. + """ + + # Orphan the package + url = f"{self.distgit_api_base_url}/{namespace}/{name}" + headers = {"Authorization": f"token {self.dist_git_token}"} + data = {"main_admin": "orphan", "retain_access": False} + + req = self.requests_session.patch(url, data=data, headers=headers) + if not req.ok: + _log.debug("**** REQUEST FAILED") + _log.debug(" - Orphan package") + _log.debug(req.url) + _log.debug(data) + _log.debug(headers) + _log.debug(req.text) + + # Reset the watching status + url = f"{self.distgit_api_base_url}/{namespace}/{name}/watchers/update" + data = {"status": -1, "watcher": current_poc} + + req = self.requests_session.post(url, data=data, headers=headers) + if not req.ok: + _log.debug("**** REQUEST FAILED") + _log.debug(" - Unwatch package") + _log.debug(req.url) + _log.debug(data) + _log.debug(headers) + _log.debug(req.text) + + # Reset the bugzilla overrides + url = f"{self.dist_git_url}/_dg/bzoverrides/{namespace}/{name}" + data = {"fedora_assignee": None, "epel_assignee": None} + + req = self.requests_session.post(url, data=data, headers=headers) + if not req.ok: + _log.debug("**** REQUEST FAILED") + _log.debug(" - Reset bugzilla overrides") + _log.debug(req.url) + _log.debug(data) + _log.debug(headers) + _log.debug(req.text) + + def remove_access(self, namespace, name, username, usertype): + """ Remove the ACL of the specified user/group on the specified project. """ + + # Remove user/group from the package + url = f"{self.distgit_api_base_url}/{namespace}/{name}/git/modifyacls" + headers = {"Authorization": f"token {self.dist_git_token}"} + data = { + "user_type": usertype, + "name": username, + } + + req = self.requests_session.patch(url, data=data, headers=headers) + if not req.ok: + _log.debug("**** REQUEST FAILED") + _log.debug(" - Orphan package") + _log.debug(req.url) + _log.debug(data) + _log.debug(headers) + _log.debug(req.text) + + if usertype == "user": + # Reset the watching status + url = f"{self.distgit_api_base_url}/{namespace}/{name}/watchers/update" + data = {"status": -1, "watcher": username} + + req = self.requests_session.post(url, data=data, headers=headers) + if not req.ok: + _log.debug("**** REQUEST FAILED") + _log.debug(" - Unwatch package") + _log.debug(req.url) + _log.debug(data) + _log.debug(headers) + _log.debug(req.text)