PR#17: Fix the _process_dist_git method so it actually updates PDC - fedora-infra/toddlers

fedora-infra / toddlers

#17 Fix the _process_dist_git method so it actually updates PDC

Merged 3 years ago by nphilipp. Opened 3 years ago by pingou.

fix_retire_entire_dist_git into master

Fix the _process_dist_git method so it actually updates PDC

Pierre-Yves Chibon • 3 years ago

49a38f6

tests/plugins/test_pdc_retired_packages.py

file modified

+112 -1

		`@@ -1,6 +1,6 @@`
		`import datetime`
		`import logging`
		`- from unittest.mock import call, MagicMock, Mock, patch`
		`+ from unittest.mock import ANY, call, MagicMock, Mock, patch`

		`import fedora_messaging.api`
		`import pytest`
		`@@ -199,6 +199,117 @@`
		`in caplog.text`
		`)`

		`+ @patch("toddlers.plugins.pdc_retired_packages._retire_branch")`
		`+ @patch("toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git")`
		`+ def test_process_dist_git_full_distgit(self, retired_in_dg, retire_branch):`
		`+ page_component_branches = [`
		`+ {`
		`+ "id": 44,`
		`+ "global_component": "0ad",`
		`+ "name": "epel7",`
		`+ "slas": [`
		`+ {"id": 88, "sla": "bug_fixes", "eol": "2024-06-30"},`
		`+ {"id": 89, "sla": "security_fixes", "eol": "2024-06-30"},`
		`+ {"id": 90, "sla": "stable_api", "eol": "2024-06-30"},`
		`+ ],`
		`+ "type": "rpm",`
		`+ "active": True,`
		`+ "critical_path": False,`
		`+ },`
		`+ {`
		`+ "id": 39,`
		`+ "global_component": "0ad",`
		`+ "name": "f16",`
		`+ "slas": [`
		`+ {"id": 78, "sla": "bug_fixes", "eol": "2013-02-12"},`
		`+ {"id": 79, "sla": "security_fixes", "eol": "2013-02-12"},`
		`+ ],`
		`+ "type": "rpm",`
		`+ "active": False,`
		`+ "critical_path": False,`
		`+ },`
		`+ {`
		`+ "id": 396194,`
		`+ "global_component": "0ad",`
		`+ "name": "master",`
		`+ "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}],`
		`+ "type": "rpm",`
		`+ "active": True,`
		`+ "critical_path": False,`
		`+ },`
		`+ ]`
		`+ client = MagicMock()`
		`+ client["component-branches"]._ = page_component_branches`
		`+ client.get_paged.return_value = client["component-branches"]._`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages"`
		`+ msg.body = {}`
		`+`
		`+ toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git(`
		`+ {}, client`
		`+ )`
		`+`
		`+ client.get_paged.assert_has_calls(calls=[call(page_component_branches)])`
		`+ retired_in_dg.assert_has_calls(`
		`+ calls=[`
		`+ call(namespace="rpms", repo="0ad", branch="epel7"),`
		`+ call().__bool__(),`
		`+ call(namespace="rpms", repo="0ad", branch="f16"),`
		`+ call().__bool__(),`
		`+ call(namespace="rpms", repo="0ad", branch="master"),`
		`+ call().__bool__(),`
		`+ ]`
		`+ )`
		`+ retire_branch.assert_has_calls(`
		`+ calls=[`
		`+ call(`
		`+ ANY,`
		`+ {`
		`+ "id": 44,`
		`+ "global_component": "0ad",`
		`+ "name": "epel7",`
		`+ "slas": [`
		`+ {"id": 88, "sla": "bug_fixes", "eol": "2024-06-30"},`
		`+ {"id": 89, "sla": "security_fixes", "eol": "2024-06-30"},`
		`+ {"id": 90, "sla": "stable_api", "eol": "2024-06-30"},`
		`+ ],`
		`+ "type": "rpm",`
		`+ "active": True,`
		`+ "critical_path": False,`
		`+ },`
		`+ ),`
		`+ call(`
		`+ ANY,`
		`+ {`
		`+ "id": 39,`
		`+ "global_component": "0ad",`
		`+ "name": "f16",`
		`+ "slas": [`
		`+ {"id": 78, "sla": "bug_fixes", "eol": "2013-02-12"},`
		`+ {"id": 79, "sla": "security_fixes", "eol": "2013-02-12"},`
		`+ ],`
		`+ "type": "rpm",`
		`+ "active": False,`
		`+ "critical_path": False,`
		`+ },`
		`+ ),`
		`+ call(`
		`+ ANY,`
		`+ {`
		`+ "id": 396194,`
		`+ "global_component": "0ad",`
		`+ "name": "master",`
		`+ "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}],`
		`+ "type": "rpm",`
		`+ "active": True,`
		`+ "critical_path": False,`
		`+ },`
		`+ ),`
		`+ ]`
		`+ )`
		`+`
		`def test__process_single_package_regular_commit(self, caplog):`
		`caplog.set_level(logging.INFO)`
		`client = MagicMock()`

toddlers/plugins/pdc_retired_packages.py

file modified

+5 -14

		`@@ -60,18 +60,16 @@`
		`PDCRetiredPackages._process_dist_git(config, pdc)`

		`def _process_dist_git(config, pdc):`
		`- """ Returns the difference in retirement status in PDC and dist-git.`
		`+ """ Updates PDC retirement status from analyzing dist-git.`

		`- This function compares the status in PDC and the status in the`
		`- "real world" (i.e., in dist-git) and return the difference.`
		`+ This steps over all the branches in dist-git and retires any branches`
		`+ in PDC that have a dead.package file in dist-git.`
		`"""`
		`- branches_retired_in_distgit = set()`
		`- branches_retired_in_pdc = set()`
		`-`
		`_log.info("Looking up all branches from PDC.")`
		`for branch in pdc.get_paged(pdc["component-branches"]._):`
		`branch_str = "{type}/{global_component}#{name}".format(**branch)`
		`_log.debug("Considering {0}".format(branch_str))`
		`+ retired_in_dist_git = False`
		`try:`
		`retired_in_dist_git = _is_retired_in_dist_git(`
		`namespace=_pdc_to_namespace(branch["type"]),`
		`@@ -83,14 +81,7 @@`
		`continue`

		`if retired_in_dist_git:`
		`- branches_retired_in_distgit.add(branch_str)`
		`- if not branch["active"]:`
		`- branches_retired_in_pdc.add(branch_str)`
		`-`
		`- present = branches_retired_in_pdc - branches_retired_in_distgit`
		`- absent = branches_retired_in_distgit - branches_retired_in_pdc`
		`-`
		`- return present, absent`
		`+ _retire_branch(pdc, branch)`

		`def _process_single_package(config, pdc, message):`
		`"""Handle an incoming bus message.`

pingou commented 3 years ago

The pdc-updater code that we rely upon here had basically three methods.
One was to react on a specific message, one was to audit the PDC vs the
source of info and the third one was to go through the entire source of
info and update PDC accordingly.

In the port of this code to toddler the methods 2 and 3 were inverted.
In toddlers we do not really care about the method #2 but we do about
method #3 which allows to periodically check that everything is in sync.

So this commit changes the _process_dist_git method to ensure that it
actually updates PDC based on the information it finds on dist-git.

Signed-off-by: Pierre-Yves Chibon pingou@pingoured.fr