From d2a7459c13960d4c95208364611b1d2d7ef9db77 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 03 2020 10:04:07 +0000 Subject: Add a new toddler to update PDC when packages get retired This is a port of the RetireComponentHandler handler of pdc-updater to toddlers. This toddler accepts two message topics: - org.fedoraproject.*.git.receive which runs on this specific repository when a ``dead.package`` file is added to it. - org.fedoraproject.*.toddlers.trigger.pdc_retired_packages which runs over all the repositories and all branches to syncs entirely PDC and koji. Signed-off-by: Pierre-Yves Chibon --- diff --git a/requirements.txt b/requirements.txt index bf0f57a..4735038 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ koji requests python-fedora python-bugzilla>=2.4.0 +pdc-client diff --git a/tests/plugins/test_pdc_retired_packages.py b/tests/plugins/test_pdc_retired_packages.py new file mode 100644 index 0000000..095f9b1 --- /dev/null +++ b/tests/plugins/test_pdc_retired_packages.py @@ -0,0 +1,571 @@ +import datetime +import logging +from unittest.mock import ANY, call, MagicMock, Mock, patch + +import fedora_messaging.api +import pytest + +import toddlers.plugins.pdc_retired_packages + + +class TestPDCRetiredPackagesToddler: + def test_accepts_topic_invalid(self): + assert ( + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.accepts_topic( + "foo.bar" + ) + is False + ) + + @pytest.mark.parametrize( + "topic", + [ + "org.fedoraproject.*.toddlers.trigger.pdc_retired_packages", + "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages", + "org.fedoraproject.stg.toddlers.trigger.pdc_retired_packages", + "org.fedoraproject.*.git.receive", + "org.fedoraproject.prod.git.receive", + "org.fedoraproject.stg.git.receive", + ], + ) + def test_accepts_topic_valid(self, topic): + assert toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.accepts_topic( + topic + ) + + @patch("toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git") + @patch("pdc_client.PDCClient") + def test_process_full_dist_git(self, pdc, process_dg): + client = Mock() + pdc.return_value = client + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = {} + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.process( + {"config": "foobar"}, msg + ) + + process_dg.assert_called_once_with({"config": "foobar"}, client) + + @patch( + "toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package" + ) + @patch("pdc_client.PDCClient") + def test_process_single_package(self, pdc, process_dg): + client = Mock() + pdc.return_value = client + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.git.receive" + msg.body = {} + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.process( + {"config": "foobar"}, msg + ) + + process_dg.assert_called_once_with({"config": "foobar"}, client, msg) + + @patch("toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git") + def test_process_dist_git(self, retired_in_dg): + page_component_branches = [ + { + "id": 44, + "global_component": "0ad", + "name": "epel7", + "slas": [ + {"id": 88, "sla": "bug_fixes", "eol": "2024-06-30"}, + {"id": 89, "sla": "security_fixes", "eol": "2024-06-30"}, + {"id": 90, "sla": "stable_api", "eol": "2024-06-30"}, + ], + "type": "rpm", + "active": True, + "critical_path": False, + }, + { + "id": 39, + "global_component": "0ad", + "name": "f16", + "slas": [ + {"id": 78, "sla": "bug_fixes", "eol": "2013-02-12"}, + {"id": 79, "sla": "security_fixes", "eol": "2013-02-12"}, + ], + "type": "rpm", + "active": False, + "critical_path": False, + }, + { + "id": 396194, + "global_component": "0ad", + "name": "master", + "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}], + "type": "rpm", + "active": True, + "critical_path": False, + }, + ] + client = MagicMock() + client["component-branches"]._ = page_component_branches + client.get_paged.return_value = client["component-branches"]._ + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = {} + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git( + {}, client + ) + + client.get_paged.assert_has_calls(calls=[call(page_component_branches)]) + retired_in_dg.assert_has_calls( + calls=[ + call( + namespace="rpms", repo="0ad", branch="epel7", requests_session=ANY + ), + call().__bool__(), + call(namespace="rpms", repo="0ad", branch="f16", requests_session=ANY), + call().__bool__(), + call( + namespace="rpms", repo="0ad", branch="master", requests_session=ANY + ), + call().__bool__(), + ] + ) + + @patch("toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git") + def test_process_dist_git_invalid_pdc_namespace(self, retired_in_dg): + page_component_branches = [ + { + "id": 44, + "global_component": "0ad", + "name": "epel7", + "slas": [ + {"id": 88, "sla": "bug_fixes", "eol": "2024-06-30"}, + {"id": 89, "sla": "security_fixes", "eol": "2024-06-30"}, + {"id": 90, "sla": "stable_api", "eol": "2024-06-30"}, + ], + "type": "rpm", + "active": True, + "critical_path": False, + }, + { + "id": 39, + "global_component": "0ad", + "name": "f16", + "slas": [ + {"id": 78, "sla": "bug_fixes", "eol": "2013-02-12"}, + {"id": 79, "sla": "security_fixes", "eol": "2013-02-12"}, + ], + "type": "rpm", + "active": False, + "critical_path": False, + }, + { + "id": 396194, + "global_component": "0ad", + "name": "master", + "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}], + "type": "flatpack", + "active": True, + "critical_path": False, + }, + ] + client = MagicMock() + client["component-branches"]._ = page_component_branches + client.get_paged.return_value = client["component-branches"]._ + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = {} + + with pytest.raises( + ValueError, match=r'The PDC type "flatpack" is not supported' + ): + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git( + {}, client + ) + + client.get_paged.assert_has_calls(calls=[call(page_component_branches)]) + retired_in_dg.assert_has_calls( + calls=[ + call( + namespace="rpms", repo="0ad", branch="epel7", requests_session=ANY + ), + call().__bool__(), + call(namespace="rpms", repo="0ad", branch="f16", requests_session=ANY), + call().__bool__(), + ] + ) + + def test__process_single_package_regular_commit(self, caplog): + caplog.set_level(logging.INFO) + client = MagicMock() + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 5, "deletions": 2}, + "sources": {"additions": 1, "deletions": 1}, + } + } + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + {}, client, msg + ) + + assert caplog.records[-1].message == "No dead.package in the commit, bailing" + + def test__process_single_package_un_retirement(self, caplog): + caplog.set_level(logging.INFO) + client = MagicMock() + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 70, "deletions": 0}, + "sources": {"additions": 1, "deletions": 0}, + "dead.package": {"additions": 0, "deletions": 3}, + } + } + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + {}, client, msg + ) + + assert caplog.records[-1].message == "dead.package file was not added, bailing" + + def test__process_single_package_incomplete_config(self, caplog): + caplog.set_level(logging.INFO) + client = MagicMock() + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 0, "deletions": 120}, + "sources": {"additions": 0, "deletions": 1}, + "dead.package": {"additions": 3, "deletions": 0}, + } + }, + "branch": "f33", + "namespace": "rpms", + "repo": "toddlers", + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + {}, client, msg + ) + + assert caplog.records[-1].message == "No check URL configured, ignoring" + + def test__process_single_package_package_not_in_pdc(self, caplog): + caplog.set_level(logging.INFO) + + page_component_branches = { + "count": 4, + } + client = MagicMock() + client["component-branches"]._ = Mock(return_value=page_component_branches) + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 0, "deletions": 120}, + "sources": {"additions": 0, "deletions": 1}, + "dead.package": {"additions": 3, "deletions": 0}, + } + }, + "branch": "f33", + "namespace": "rpms", + "repo": "0ad", + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + { + "file_check_url": "https://src.fedoraproject.org/" + "%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s", + }, + client, + msg, + ) + + assert caplog.records[-1].message == '"rpms/0ad" was not found in PDC' + + def test__process_single_package_namespace_not_in_pdc(self): + + page_component_branches = { + "count": 4, + } + client = MagicMock() + client["component-branches"]._ = Mock(return_value=page_component_branches) + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 0, "deletions": 120}, + "sources": {"additions": 0, "deletions": 1}, + "dead.package": {"additions": 3, "deletions": 0}, + } + }, + "branch": "f33", + "namespace": "flatpack", + "repo": "0ad", + } + } + + with pytest.raises( + ValueError, match=r'The namespace "flatpack" is not supported' + ): + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + { + "file_check_url": "https://src.fedoraproject.org/" + "%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s", + }, + client, + msg, + ) + + def test__process_single_package_package_inactive(self, caplog): + caplog.set_level(logging.INFO) + + page_component_branches = { + "count": 1, + "results": [ + { + "id": 396194, + "global_component": "0ad", + "name": "master", + "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}], + "type": "rpm", + "active": False, + "critical_path": False, + }, + ], + } + client = MagicMock() + client["component-branches"]._ = Mock(return_value=page_component_branches) + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 0, "deletions": 120}, + "sources": {"additions": 0, "deletions": 1}, + "dead.package": {"additions": 3, "deletions": 0}, + } + }, + "branch": "f33", + "namespace": "rpms", + "repo": "0ad", + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + { + "file_check_url": "https://src.fedoraproject.org/" + "%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s", + }, + client, + msg, + ) + + assert caplog.records[-1].message == '"rpms/0ad" not active in PDC in master' + + @patch("requests.head") + def test__process_single_package_package_not_retired(self, req, caplog): + caplog.set_level(logging.INFO) + resp = Mock() + resp.status_code = 404 + req.return_value = resp + + page_component_branches = { + "count": 1, + "results": [ + { + "id": 396194, + "global_component": "0ad", + "name": "master", + "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}], + "type": "rpm", + "active": True, + "critical_path": False, + }, + ], + } + client = MagicMock() + client["component-branches"]._ = Mock(return_value=page_component_branches) + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 0, "deletions": 120}, + "sources": {"additions": 0, "deletions": 1}, + "dead.package": {"additions": 3, "deletions": 0}, + } + }, + "branch": "f33", + "namespace": "rpms", + "repo": "0ad", + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + { + "file_check_url": "https://src.fedoraproject.org/" + "%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s", + }, + client, + msg, + ) + + assert ( + caplog.records[-1].message + == "Seems not to actually be retired, possibly a merge commit?" + ) + + @patch("toddlers.plugins.pdc_retired_packages._retire_branch") + @patch("requests.head") + def test__process_single_package(self, req, retire, caplog): + caplog.set_level(logging.INFO) + resp = Mock() + resp.status_code = 200 + req.return_value = resp + + page_component_branches = { + "count": 1, + "results": [ + { + "id": 396194, + "global_component": "0ad", + "name": "master", + "slas": [{"id": 789078, "sla": "rawhide", "eol": "2222-01-01"}], + "type": "rpm", + "active": True, + "critical_path": False, + }, + ], + } + client = MagicMock() + client["component-branches"]._ = Mock(return_value=page_component_branches) + + msg = fedora_messaging.api.Message() + msg.id = 123 + msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages" + msg.body = { + "commit": { + "stats": { + "files": { + "foobar.spec": {"additions": 0, "deletions": 120}, + "sources": {"additions": 0, "deletions": 1}, + "dead.package": {"additions": 3, "deletions": 0}, + } + }, + "branch": "f33", + "namespace": "rpms", + "repo": "0ad", + } + } + + toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package( + { + "file_check_url": "https://src.fedoraproject.org/" + "%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s", + }, + client, + msg, + ) + + retire.assert_called_once_with(client, page_component_branches["results"][0]) + + @patch("toddlers.plugins.pdc_retired_packages.requests_session.head") + def test__is_retired_in_dist_git(self, req): + resp = Mock() + resp.status_code = 200 + req.return_value = resp + + assert toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git( + "rpms", "guake", "f33" + ) + req.assert_called_once_with( + "https://src.fedoraproject.org//rpms/guake/raw/f33/f/dead.package" + ) + + @patch("toddlers.plugins.pdc_retired_packages.requests_session.head") + def test__is_retired_in_dist_git_internal_error(self, req): + resp = Mock() + resp.status_code = 500 + req.return_value = resp + + with pytest.raises( + ValueError, + match=r"The connection to dist_git failed. Retirement status could " + "not be determined. The status code was: 500. The content was: .*", + ): + toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git( + "rpms", "guake", "f33" + ) + req.assert_called_once_with( + "https://src.fedoraproject.org//rpms/guake/raw/f33/f/dead.package" + ) + + def test_retire_branch(self, caplog): + caplog.set_level(logging.INFO) + today = datetime.datetime.utcnow().date() + eol = (datetime.datetime.utcnow() + datetime.timedelta(days=60)).date() + pdc = MagicMock() + branch = { + "id": 396194, + "global_component": "0ad", + "name": "master", + "slas": [{"id": 789078, "sla": "rawhide", "eol": str(eol)}], + "type": "rpm", + "active": True, + "critical_path": False, + } + + toddlers.plugins.pdc_retired_packages._retire_branch(pdc, branch) + + assert ( + caplog.records[-1].message + == "Setting eol of branch: {'id': 396194, 'global_component': " + "'0ad', 'name': 'master', 'slas': [{'id': 789078, 'sla': " + "'rawhide', 'eol': '%s'}], 'type': 'rpm', 'active': True, " + "'critical_path': False} to %s" % (eol, today) + ) diff --git a/tests/plugins/test_plugins.py b/tests/plugins/test_plugins.py index 04fb6cd..2502066 100644 --- a/tests/plugins/test_plugins.py +++ b/tests/plugins/test_plugins.py @@ -3,13 +3,16 @@ import toddlers.plugins def test_toddlers_plugins(): variables = [var for var in dir(toddlers.plugins) if not var.startswith("__")] - assert sorted(variables) == [ - "debug", - "flag_ci_pr", - "flag_commit_build", - "here", - "importlib", - "name", - "os", - "packager_bugzilla_sync", - ] + assert sorted(variables) == sorted( + [ + "debug", + "flag_ci_pr", + "flag_commit_build", + "here", + "importlib", + "name", + "os", + "packager_bugzilla_sync", + "pdc_retired_packages", + ] + ) diff --git a/tests/test_runner.py b/tests/test_runner.py index d303aad..30aa452 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,12 +11,15 @@ import toddlers.runner class TestRunningToddler: def test___init__(self): runner = toddlers.runner.RunningToddler() - assert sorted([t.name for t in runner.toddlers]) == [ - "debug", - "flag_ci_pr", - "flag_commit_build", - "packager_bugzilla_sync", - ] + assert sorted([t.name for t in runner.toddlers]) == sorted( + [ + "debug", + "flag_ci_pr", + "flag_commit_build", + "packager_bugzilla_sync", + "pdc_retired_packages", + ] + ) @patch("toddlers.runner.ToddlerBase") def test___init__no_toddlers(self, mock_base, caplog): @@ -27,12 +30,15 @@ class TestRunningToddler: match=r".* No toddlers found to run .*", ): runner = toddlers.runner.RunningToddler() - assert sorted([t.name for t in runner.toddlers]) == [ - "debug", - "flag_ci_pr", - "flag_commit_build", - "packager_bugzilla_sync", - ] + assert sorted([t.name for t in runner.toddlers]) == sorted( + [ + "debug", + "flag_ci_pr", + "flag_commit_build", + "packager_bugzilla_sync", + "pdc_retired_packages", + ] + ) assert caplog.records[-1].message == "Loaded: []" @patch.dict( @@ -41,10 +47,9 @@ class TestRunningToddler: ) def test___init__blockedlist(self, caplog): runner = toddlers.runner.RunningToddler() - assert sorted([t.name for t in runner.toddlers]) == [ - "flag_commit_build", - "packager_bugzilla_sync", - ] + assert sorted([t.name for t in runner.toddlers]) == sorted( + ["flag_commit_build", "packager_bugzilla_sync", "pdc_retired_packages"] + ) def test___call__(self, caplog): caplog.set_level(logging.INFO) diff --git a/toddlers.toml.example b/toddlers.toml.example index 1b5321a..0d206b0 100644 --- a/toddlers.toml.example +++ b/toddlers.toml.example @@ -57,6 +57,25 @@ pagure_token = "private random string to change" pagure_url = "https://src.fedoraproject.org" koji_url = "https://koji.fedoraproject.org" +[consumer_config.pdc_retired_packages] +file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s" + +# XXX - getting the token is a bit of a pain, but here's a walk through +# 1) go to https://pdc.fedoraproject.org/ in your browser and login. +# 2) go to https://pdc.fedoraproject.org/rest_api/v1/auth/token/obtain/ +# 3) open up the devtools console in your browser, and find the request for the current page. +# 4) right click to open a context menu and select 'copy as cURL' +# 5) paste that into a terminal. It should have your saml cookie. +# 6) before hitting enter, edit the command to add the following option +# -H 'Accept: application/json' # to tell the API you want data +# 7) the command should print out your token. + +# Credentials to talk to PDC +[consumer_config.pdc_retired_packages.pdc_config] +server = "https://pdc.fedoraproject.org/rest_api/v1/" +ssl_verify = false # Enable if using a self-signed cert +token = "PUT_HERE_THE_TOKEN_OBTAINED_MANUALLY" + [qos] prefetch_size = 0 prefetch_count = 25 diff --git a/toddlers/plugins/pdc_retired_packages.py b/toddlers/plugins/pdc_retired_packages.py new file mode 100644 index 0000000..ba4cf0e --- /dev/null +++ b/toddlers/plugins/pdc_retired_packages.py @@ -0,0 +1,216 @@ +""" +When a component's branch is retired, EOL all SLAs on that branch. + +This toddler is succeeding to the pdc-updater handlers: +https://github.com/fedora-infra/pdc-updater/blob/develop/pdcupdater/handlers/retirement.py + +Authors: Pierre-Yves Chibon + +""" + +from datetime import datetime +import logging + +import pdc_client +import requests +from requests.packages.urllib3.util import retry + +from toddlers.base import ToddlerBase + +_log = logging.getLogger(__name__) + +timeout = (30, 30) +retries = 3 +requests_session = requests.Session() +retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1) +retry_conf.BACKOFF_MAX = 5 +requests_session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retry_conf)) +requests_session.mount( + "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf) +) + + +class PDCRetiredPackages(ToddlerBase): + """ Listens to messages sent by playtime (which lives in toddlers) to sync + all the packagers found in FAS to bugzilla. + """ + + name = "pdc_retired_packages" + + amqp_topics = [ + "org.fedoraproject.*.toddlers.trigger.pdc_retired_packages", + "org.fedoraproject.*.git.receive", + ] + + def accepts_topic(topic): + """Returns a boolean whether this toddler is interested in messages + from this specific topic. + """ + return topic.startswith("org.fedoraproject.") and topic.endswith( + ("toddlers.trigger.pdc_retired_packages", "git.receive") + ) + + def process(config, message): + """Process a given message.""" + pdc = pdc_client.PDCClient(**config.get("pdc_config", {})) + + if message.topic.endswith("git.receive"): + PDCRetiredPackages._process_single_package(config, pdc, message) + else: + PDCRetiredPackages._process_dist_git(config, pdc) + + def _process_dist_git(config, pdc): + """ Returns the difference in retirement status in PDC and dist-git. + + This function compares the status in PDC and the status in the + "real world" (i.e., in dist-git) and return the difference. + """ + branches_retired_in_distgit = set() + branches_retired_in_pdc = set() + + _log.info("Looking up all branches from PDC.") + for branch in pdc.get_paged(pdc["component-branches"]._): + branch_str = "{type}/{global_component}#{name}".format(**branch) + _log.debug("Considering {0}".format(branch_str)) + retired_in_dist_git = _is_retired_in_dist_git( + namespace=_pdc_to_namespace(branch["type"]), + repo=branch["global_component"], + branch=branch["name"], + requests_session=requests_session, + ) + + if retired_in_dist_git: + branches_retired_in_distgit.add(branch_str) + if not branch["active"]: + branches_retired_in_pdc.add(branch_str) + + present = branches_retired_in_pdc - branches_retired_in_distgit + absent = branches_retired_in_distgit - branches_retired_in_pdc + + return present, absent + + def _process_single_package(config, pdc, message): + """Handle an incoming bus message. + + The message should be a dist-git retirement message (where someone adds + a dead.package file to the repo). In response, this method will retire + the package in PDC. + """ + + msg = message.body + + # If there is no dead.package in the commit, then it can be ignored + if "dead.package" not in msg["commit"]["stats"]["files"]: + _log.info("No dead.package in the commit, bailing") + return + + dead_package_commit = msg["commit"]["stats"]["files"]["dead.package"] + if ( + dead_package_commit["additions"] == 0 + and dead_package_commit["deletions"] != 0 + ): + _log.info("dead.package file was not added, bailing") + return + + branchname = msg["commit"]["branch"] + repo = msg["commit"]["repo"] + namespace = msg["commit"]["namespace"] + component_type = _namespace_to_pdc(namespace) + checkurl = config.get("file_check_url") + if not checkurl: + _log.error("No check URL configured, ignoring") + return + + # This query guarantees a unique component branch, so a count of 1 is + # expected + branch_query_rv = pdc["component-branches"]._( + name=branchname, type=component_type, global_component=repo + ) + + if branch_query_rv["count"] != 1: + _log.error('"{0}/{1}" was not found in PDC'.format(namespace, repo)) + return + + branch = branch_query_rv["results"][0] + # If the branch is already EOL in PDC, don't do anything + if branch["active"] is False: + _log.error( + '"{0}/{1}" not active in PDC in {2}'.format( + namespace, repo, branch["name"] + ) + ) + return + + # Make sure that the file is still retired right this moment. + # This fixes cases where people merge from master to an older branch, and one of the + # intermediate commits contained a dead.package. + fileurl = checkurl % { + "namespace": namespace, + "repo": repo, + "branch": branchname, + "file": "dead.package", + } + _log.info("Checking for file: %s", fileurl) + resp = requests.head(fileurl, timeout=15) + if resp.status_code != 200: + _log.info("Seems not to actually be retired, possibly a merge commit?") + return + + _retire_branch(pdc, branch) + + +def _namespace_to_pdc(namespace): + """ Internal method to translate a dist-git namespace to a PDC + component type. """ + namespace_to_pdc = { + "rpms": "rpm", + "modules": "module", + "container": "container", + } + if namespace not in namespace_to_pdc: + raise ValueError('The namespace "{0}" is not supported'.format(namespace)) + else: + return namespace_to_pdc[namespace] + + +def _pdc_to_namespace(pdc_type): + """ Internal method to translate a PDC component type to a dist-git + namespace. """ + pdc_to_namespace = { + "rpm": "rpms", + "module": "modules", + "container": "container", + } + if pdc_type not in pdc_to_namespace: + raise ValueError('The PDC type "{0}" is not supported'.format(pdc_type)) + else: + return pdc_to_namespace[pdc_type] + + +def _is_retired_in_dist_git(namespace, repo, branch): + base = "https://src.fedoraproject.org/" + # Check to see if they have a dead.package file in dist-git + url = "{base}/{namespace}/{repo}/raw/{branch}/f/dead.package" + response = requests_session.head( + url.format(base=base, namespace=namespace, repo=repo, branch=branch,) + ) + + # If there is a dead.package, then the branch is retired in dist_git + if response.status_code in [200, 404]: + return response.status_code == 200 + else: + raise ValueError( + "The connection to dist_git failed. Retirement status could not " + "be determined. The status code was: {0}. The content was: " + "{1}".format(response.status_code, response.content) + ) + + +def _retire_branch(pdc, branch): + """ Internal method for retiring a branch in PDC. """ + today = datetime.utcnow().date() + for sla in branch["slas"]: + sla_eol = datetime.strptime(sla["eol"], "%Y-%m-%d").date() + if sla_eol > today: + _log.info("Setting eol of branch: %s to %s", branch, today) + pdc["component-branch-slas"][sla["id"]]._ += {"eol": str(today)}