PR#35: Introduce the clean_retired_packages toddler - fedora-infra/toddlers

fedora-infra / toddlers

#35 Introduce the clean_retired_packages toddler

Merged 4 years ago by pingou. Opened 4 years ago by pingou.

clean_retired_packages into master

Use the orphan endpoint of dist-git to orphan the package

Pierre-Yves Chibon • 4 years ago

4a3409f

Introduce the clean_retired_packages toddler

Pierre-Yves Chibon • 4 years ago

e10907b

tests/plugins/test_clean_retired_packages.py

file added

+217

		`@@ -0,0 +1,217 @@`
		`+ from unittest.mock import call, MagicMock, patch`
		`+`
		`+ import fedora_messaging.api`
		`+ import pytest`
		`+`
		`+ import toddlers.plugins.clean_retired_packages`
		`+`
		`+`
		`+ class TestCleanRetiredPackagesToddler:`
		`+`
		`+ toddler_cls = toddlers.plugins.clean_retired_packages.CleanRetiredPackages`
		`+`
		`+ def test_accepts_topic_invalid(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is False`
		`+`
		`+ @pytest.mark.parametrize(`
		`+ "topic",`
		`+ [`
		`+ "org.fedoraproject.*.toddlers.trigger.clean_retired_packages",`
		`+ "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages",`
		`+ "org.fedoraproject.stg.toddlers.trigger.clean_retired_packages",`
		`+ ],`
		`+ )`
		`+ def test_accepts_topic_valid(self, topic, toddler):`
		`+ assert toddler.accepts_topic(topic)`
		`+`
		`+ def test_process_config_missing_pdc_url(self, toddler):`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages"`
		`+ msg.body = {"foo": "bar"}`
		`+`
		`+ with pytest.raises(`
		`+ Exception,`
		+ match=r"Invalid toddler configuration, no `pdc_active_branches` defined",
		`+ ):`
		`+ assert toddler.process(config={}, message=msg) is None`
		`+`
		`+ def test_process_config_missing_distgit_url(self, toddler):`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages"`
		`+ msg.body = {"foo": "bar"}`
		`+`
		`+ config = {`
		`+ "pdc_active_branches": "https://pdc.fedoraproject.org/extras/active_branches.json"`
		`+ }`
		`+`
		`+ with pytest.raises(`
		`+ Exception,`
		+ match=r"Invalid toddler configuration, no `dist_git_url` defined",
		`+ ):`
		`+ assert toddler.process(config=config, message=msg) is None`
		`+`
		`+ def test_process_config_missing_distgit_token(self, toddler):`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages"`
		`+ msg.body = {"foo": "bar"}`
		`+`
		`+ config = {`
		`+ "pdc_active_branches": "https://pdc.fedoraproject.org/extras/active_branches.json",`
		`+ "dist_git_url": "https://src.fedoraproject.org/",`
		`+ }`
		`+`
		`+ with pytest.raises(`
		`+ Exception,`
		+ match=r"Invalid toddler configuration, no `dist_git_token` defined",
		`+ ):`
		`+ assert toddler.process(config=config, message=msg) is None`
		`+`
		`+ @patch("toddlers.plugins.clean_retired_packages.send_email")`
		`+ def test_process(self, send_email, toddler, caplog):`
		`+ pdc_branches = MagicMock()`
		`+ pdc_branches.json.return_value = {`
		`+ "flatpak": {`
		`+ "bluez-gnome": [["master", False], ["epel8", False], ["epel7", False]],`
		`+ },`
		`+ "rpm": {`
		`+ "trojan": [["master", True], ["f33", True], ["f32", True]],`
		`+ "pigment": [["master", False], ["epel8", True], ["epel7", True]],`
		`+ "bluez-gnome": [["master", False], ["epel8", False], ["epel7", False]],`
		`+ "guake": [["master", False], ["epel8", False], ["epel7", False]],`
		`+ },`
		`+ }`
		`+ distgit_info_bluez_gnome = MagicMock()`
		`+ distgit_info_bluez_gnome.json.return_value = {`
		`+ "user": {"name": "pingou"},`
		`+ "access_groups": {`
		`+ "admin": [],`
		`+ "collaborator": [],`
		`+ "commit": ["infra-sig"],`
		`+ "ticket": [],`
		`+ },`
		`+ "access_users": {`
		`+ "admin": [],`
		`+ "collaborator": [],`
		`+ "commit": ["ohaessler"],`
		`+ "owner": ["pingou"],`
		`+ "ticket": [],`
		`+ },`
		`+ }`
		`+ distgit_info_guake = MagicMock()`
		`+ distgit_info_guake.json.return_value = {`
		`+ "user": {"name": "orphan"},`
		`+ "access_groups": {`
		`+ "admin": [],`
		`+ "collaborator": [],`
		`+ "commit": [],`
		`+ "ticket": [],`
		`+ },`
		`+ "access_users": {`
		`+ "admin": ["orphan"],`
		`+ "collaborator": [],`
		`+ "commit": [],`
		`+ "owner": ["pingou"],`
		`+ "ticket": [],`
		`+ },`
		`+ }`
		`+`
		`+ toddler.requests_session.get.side_effect = (`
		`+ pdc_branches,`
		`+ distgit_info_bluez_gnome,`
		`+ distgit_info_guake,`
		`+ )`
		`+ toddler.requests_session.patch.return_value = MagicMock(ok=False)`
		`+ toddler.requests_session.post.return_value = MagicMock(ok=False)`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.toddlers.trigger.clean_retired_packages"`
		`+ msg.body = {"foo": "bar"}`
		`+`
		`+ config = {`
		`+ "pdc_active_branches": "https://pdc.fedoraproject.org/extras/active_branches.json",`
		`+ "dist_git_url": "https://src.fedoraproject.org/",`
		`+ "dist_git_token": "foobar",`
		`+ "admin_email": "admin@fp.o",`
		`+ "mail_server": "mail_server",`
		`+ }`
		`+`
		`+ assert toddler.process(config=config, message=msg) is None`
		`+ send_email.assert_called_with(`
		`+ to_addresses=["admin@fp.o"],`
		`+ from_address="admin@fp.o",`
		`+ subject="Toddlers cleaned up some retired packages",`
		`+ content="Dear Admin,\n\n"`
		`+ "The clean_retired_packages toddler just ran and adjusted some retired packages\n"`
		`+ "for recording purposes, here is what it did:\n"`
		`+ "- Orphaning rpms/bluez-gnome from pingou\n"`
		`+ "- Removing access on rpms/bluez-gnome to ohaessler\n"`
		`+ "- Removing access on rpms/bluez-gnome to pingou\n"`
		`+ "- Removing access on rpms/bluez-gnome to @infra-sig\n"`
		`+ "- Removing access on rpms/guake to pingou\n\n"`
		`+ "In case someone asks, you can come back to this email but there isn't really\n"`
		`+ "anything else to do with it.\n\n"`
		`+ "Have a wonderful day and see you (maybe?) at the next run!\n\n",`
		`+ mail_server="mail_server",`
		`+ )`
		`+ toddler.requests_session.patch.assert_called()`
		`+ toddler.requests_session.patch.assert_has_calls(`
		`+ calls=[`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/git/modifyacls",`
		`+ data={"user_type": "user", "name": "ohaessler"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/git/modifyacls",`
		`+ data={"user_type": "user", "name": "pingou"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/git/modifyacls",`
		`+ data={"user_type": "group", "name": "infra-sig"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/guake/git/modifyacls",`
		`+ data={"user_type": "user", "name": "pingou"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ ]`
		`+ )`
		`+ toddler.requests_session.post.assert_called()`
		`+ toddler.requests_session.post.assert_has_calls(`
		`+ calls=[`
		`+ call(`
		`+ "https://src.fedoraproject.org/_dg/orphan/rpms/bluez-gnome",`
		`+ data={`
		`+ "orphan_reason": "other",`
		`+ "orphan_reason_info": "Package was retired before it was orphaned",`
		`+ },`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/_dg/bzoverrides/rpms/bluez-gnome",`
		`+ data={"fedora_assignee": None, "epel_assignee": None},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/watchers/update",`
		`+ data={"status": -1, "watcher": "ohaessler"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/bluez-gnome/watchers/update",`
		`+ data={"status": -1, "watcher": "pingou"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ call(`
		`+ "https://src.fedoraproject.org/api/0/rpms/guake/watchers/update",`
		`+ data={"status": -1, "watcher": "pingou"},`
		`+ headers={"Authorization": "token foobar"},`
		`+ ),`
		`+ ]`
		`+ )`

toddlers.toml.example

file modified

		`@@ -128,6 +128,9 @@`
		`[consumer_config.check_email_overrides]`
		`email_overrides_url = "https://pagure.io/fedora-infra/ansible/raw/master/f/roles/openshift-apps/distgit-bugzilla-sync/templates/email_overrides.toml"`

		`+ [consumer_config.clean_retired_packages]`
		`+ pdc_active_branches = "https://pdc.fedoraproject.org/extras/active_branches.json"`
		`+`

		`[qos]`
		`prefetch_size = 0`

toddlers/plugins/clean_retired_packages.py

file added

+230

		`@@ -0,0 +1,230 @@`
		`+ """`
		`+ This script queries PDC for all the packages in Fedora, it then go through`
		`+ each of them and determines if they are inactive on all branches or not.`
		`+ If they are, it will then query dist-git to retrieve all of its current`
		+ maintainers, if they do not have ``orphan`` as main user they will be orphaned
		`+ and all co-maintainers will be removed.`
		`+`
		`+ Authors: Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+ import collections`
		`+ import logging`
		`+`
		`+ from toddlers.base import ToddlerBase`
		`+ from toddlers.utils.notify import send_email`
		`+ from toddlers.utils.requests import make_session`
		`+`
		`+`
		`+ _log = logging.getLogger(__name__)`
		`+`
		`+`
		`+ pdc_namespace_to_dist_git = {`
		`+ "rpm": "rpms",`
		`+ # "container": "container",`
		`+ # "flatpak": "flatpaks",`
		`+ # "module": "modules",`
		`+ }`
		`+`
		`+`
		`+ class CleanRetiredPackages(ToddlerBase):`
		`+ """ Listens to messages sent by playtime (which lives in toddlers) and check`
		+ if the `email_overrides.toml` file stored in ansible is up to date.
		`+ """`
		`+`
		`+ name = "clean_retired_packages"`
		`+`
		`+ amqp_topics = [`
		`+ "org.fedoraproject.*.toddlers.trigger.clean_retired_packages",`
		`+ ]`
		`+`
		`+ def __init__(self):`
		`+ self.requests_session = make_session()`
		`+ self.dist_git_token = None`
		`+ self.dist_git_url = None`
		`+ self.distgit_api_base_url = None`
		`+`
		`+ def accepts_topic(self, topic):`
		`+ """Returns a boolean whether this toddler is interested in messages`
		`+ from this specific topic.`
		`+ """`
		`+ return topic.startswith("org.fedoraproject.") and topic.endswith(`
		`+ "toddlers.trigger.clean_retired_packages"`
		`+ )`
		`+`
		`+ def process(self, config, message, dry_run=False):`
		`+ """Process a given message."""`
		`+`
		`+ pdc_active_branches = config.get("pdc_active_branches") or None`
		`+ if not pdc_active_branches:`
		`+ raise Exception(`
		+ "Invalid toddler configuration, no `pdc_active_branches` defined"
		`+ )`
		`+`
		`+ dist_git_url = config.get("dist_git_url") or None`
		`+ if not dist_git_url:`
		+ raise Exception("Invalid toddler configuration, no `dist_git_url` defined")
		`+ self.dist_git_url = dist_git_url.rstrip("/")`
		`+ self.distgit_api_base_url = f"{self.dist_git_url}/api/0"`
		`+`
		`+ self.dist_git_token = config.get("dist_git_token") or None`
		`+ if not self.dist_git_token:`
		`+ raise Exception(`
		+ "Invalid toddler configuration, no `dist_git_token` defined"
		`+ )`
		`+`
		`+ _log.info("Querying PDC")`
		`+ branch_info = self.requests_session.get(pdc_active_branches).json()`
		`+`
		`+ _log.info("Retrieving list of package retired from PDC")`
		`+ fully_retired = collections.defaultdict(list)`
		`+ for namespace in sorted(branch_info):`
		`+ for package in sorted(branch_info[namespace]):`
		`+ if True not in [el[1] for el in branch_info[namespace][package]]:`
		`+ fully_retired[namespace].append(package)`
		`+`
		`+ for namespace in fully_retired:`
		`+ _log.info(`
		`+ "%s fully retired packages found in %s",`
		`+ len(fully_retired[namespace]),`
		`+ namespace,`
		`+ )`
		`+`
		`+ logs = []`
		`+`
		`+ for namespace in fully_retired:`
		`+ # for namespace in ["rpm"]:`
		`+ if namespace not in pdc_namespace_to_dist_git:`
		`+ _log.info(`
		`+ "Ignoring namespace: %s - not mapped from PDC to dist-git",`
		`+ namespace,`
		`+ )`
		`+ continue`
		`+`
		`+ _log.info("Processing: %s", namespace)`
		`+ for package in fully_retired[namespace]:`
		`+ # _log.info(" %s/%s", namespace, package)`
		`+`
		`+ ns = pdc_namespace_to_dist_git[namespace]`
		`+`
		`+ url = f"{self.distgit_api_base_url}{ns}/{package}"`
		`+ data = self.requests_session.get(url).json()`
		`+`
		`+ if data["user"]["name"] != "orphan":`
		`+ log = f"Orphaning {ns}/{package} from {data['user']['name']}"`
		`+ print(log)`
		`+ logs.append(log)`
		`+ self.orphan_package(ns, package, current_poc=data["user"]["name"])`
		`+`
		`+ packagers = set()`
		`+ for lvl in data["access_users"]:`
		`+ packagers.update([u for u in data["access_users"][lvl]])`
		`+`
		`+ for packager in sorted(packagers):`
		`+ if packager == "orphan":`
		`+ continue`
		`+ log = f"Removing access on {ns}/{package} to {packager}"`
		`+ print(log)`
		`+ logs.append(log)`
		`+ self.remove_access(ns, package, packager, "user")`
		`+`
		`+ groups = set()`
		`+ for lvl in data["access_groups"]:`
		`+ groups.update([u for u in data["access_groups"][lvl]])`
		`+`
		`+ for group in sorted(groups):`
		`+ log = f"Removing access on {ns}/{package} to @{group}"`
		`+ print(log)`
		`+ logs.append(log)`
		`+ self.remove_access(ns, package, group, "group")`
		`+`
		`+ if logs:`
		`+ logs_text = "\n- ".join(logs)`
		`+ message = f"""Dear Admin,`
		`+`
		`+ The clean_retired_packages toddler just ran and adjusted some retired packages`
		`+ for recording purposes, here is what it did:`
		`+ - {logs_text}`
		`+`
		`+ In case someone asks, you can come back to this email but there isn't really`
		`+ anything else to do with it.`
		`+`
		`+ Have a wonderful day and see you (maybe?) at the next run!`
		`+`
		`+ """`
		`+ send_email(`
		`+ to_addresses=[config["admin_email"]],`
		`+ from_address=config["admin_email"],`
		`+ subject="Toddlers cleaned up some retired packages",`
		`+ content=message,`
		`+ mail_server=config["mail_server"],`
		`+ )`
		`+`
		`+ def orphan_package(self, namespace, name, current_poc):`
		+ """ Give the specified project on dist_git to the ``orphan`` user.
		`+ """`
		`+`
		`+ # Orphan the package`
		`+ url = f"{self.dist_git_url}/_dg/orphan/{namespace}/{name}"`
		`+ headers = {"Authorization": f"token {self.dist_git_token}"}`
		`+ data = {`
		`+ "orphan_reason": "other",`
		`+ "orphan_reason_info": "Package was retired before it was orphaned",`
		`+ }`
		`+`
		`+ req = self.requests_session.post(url, data=data, headers=headers)`
		`+ if not req.ok:`
		`+ _log.debug("**** REQUEST FAILED")`
		`+ _log.debug(" - Orphan package")`
		`+ _log.debug(req.url)`
		`+ _log.debug(data)`
		`+ _log.debug(headers)`
		`+ _log.debug(req.text)`
		`+`
		`+ # Reset the bugzilla overrides`
		`+ url = f"{self.dist_git_url}/_dg/bzoverrides/{namespace}/{name}"`
		`+ data = {"fedora_assignee": None, "epel_assignee": None}`
		`+`
		`+ req = self.requests_session.post(url, data=data, headers=headers)`
		`+ if not req.ok:`
		`+ _log.debug("**** REQUEST FAILED")`
		`+ _log.debug(" - Reset bugzilla overrides")`
		`+ _log.debug(req.url)`
		`+ _log.debug(data)`
		`+ _log.debug(headers)`
		`+ _log.debug(req.text)`
		`+`
		`+ def remove_access(self, namespace, name, username, usertype):`
		`+ """ Remove the ACL of the specified user/group on the specified project. """`
		`+`
		`+ # Remove user/group from the package`
		`+ url = f"{self.distgit_api_base_url}/{namespace}/{name}/git/modifyacls"`
		`+ headers = {"Authorization": f"token {self.dist_git_token}"}`
		`+ data = {`
		`+ "user_type": usertype,`
		`+ "name": username,`
		`+ }`
		`+`
		`+ req = self.requests_session.patch(url, data=data, headers=headers)`
		`+ if not req.ok:`
		`+ _log.debug("**** REQUEST FAILED")`
		`+ _log.debug(" - Orphan package")`
		`+ _log.debug(req.url)`
		`+ _log.debug(data)`
		`+ _log.debug(headers)`
		`+ _log.debug(req.text)`
		`+`
		`+ if usertype == "user":`
		`+ # Reset the watching status`
		`+ url = f"{self.distgit_api_base_url}/{namespace}/{name}/watchers/update"`
		`+ data = {"status": -1, "watcher": username}`
		`+`
		`+ req = self.requests_session.post(url, data=data, headers=headers)`
		`+ if not req.ok:`
		`+ _log.debug("**** REQUEST FAILED")`
		`+ _log.debug(" - Unwatch package")`
		`+ _log.debug(req.url)`
		`+ _log.debug(data)`
		`+ _log.debug(headers)`
		`+ _log.debug(req.text)`

pingou commented 4 years ago

This toddler ensures retired packages are properly orphaned and
have no (co)maintainers set to properly reflect in the dist-git's UI
that these packages are not maintained by anyone.

Fixes https://pagure.io/fedora-infrastructure/issue/8600
(which contains all the references to how we came to do this,
including link to the FESCo ticket approving this).

Signed-off-by: Pierre-Yves Chibon pingou@pingoured.fr

zuul commented 4 years ago

Build succeeded.

tox : SUCCESS in 7m 27s

rebased onto e10907b

4 years ago

zuul commented 4 years ago

Build succeeded.

tox : SUCCESS in 9m 11s

zlopez commented on line 25 of toddlers/plugins/clean_retired_packages.py 4 years ago

Do we want to only support rpms for now?

zlopez commented on line 117 of toddlers/plugins/clean_retired_packages.py 4 years ago

Do you want to left this print? Same for the others bellow.

zlopez commented on line 170 of toddlers/plugins/clean_retired_packages.py 4 years ago

Wouldn't it be better to use the orphan API call?

pingou commented on line 117 of toddlers/plugins/clean_retired_packages.py 4 years ago

yes, I want to see this in the toddlers' log in openshift as well as having it sent by email to the admins

zlopez commented 4 years ago

LGTM, only few questions

pingou commented on line 25 of toddlers/plugins/clean_retired_packages.py 4 years ago

yes, I need to ask FESCo what we want to do for the other namespaces

pingou commented on line 170 of toddlers/plugins/clean_retired_packages.py 4 years ago

now that it is there, we could yeah

pingou commented on line 170 of toddlers/plugins/clean_retired_packages.py 4 years ago

IIRC, it doesn't reset the bugzilla overrides, does it?

zlopez commented on line 170 of toddlers/plugins/clean_retired_packages.py 4 years ago

No it doesn't and you need to call it as the current admin of the project

pingou commented on line 170 of toddlers/plugins/clean_retired_packages.py 4 years ago

hm, ok so this won't work here as the API token we use will clearly not be the one of the current main admin :(

zlopez commented on line 170 of toddlers/plugins/clean_retired_packages.py 4 years ago

To be sure, I looked at the orphan_endpoint (https://pagure.io/fork/zlopez/pagure-dist-git/blob/orphan/f/pagure_distgit/plugin.py#_228) and I'm wrong, you just need to be logged in and have correct ACLs to orphan package.

EDIT: This is just restriction for the button on Pagure to show.

Edited 4 years ago by zlopez

1 new commit added