PR#18: Add the pdc_import_compose toddler - fedora-infra/toddlers

fedora-infra / toddlers

#18 Add the pdc_import_compose toddler

Merged 3 years ago by nphilipp. Opened 3 years ago by pingou.

pdc_import_compose into master

Pierre-Yves Chibon • 3 years ago

6d8934c

requirements.txt

file modified

		`@@ -1,3 +1,4 @@`
		`+ bs4`
		`fedora-messaging`
		`koji`
		`requests`

tests/plugins/test_pdc_import_compose.py

file added

+866

		`@@ -0,0 +1,866 @@`
		`+ import logging`
		`+ from unittest.mock import call, MagicMock, Mock, patch`
		`+`
		`+ import beanbag.bbexcept`
		`+ import fedora_messaging.api`
		`+ import pytest`
		`+`
		`+ import toddlers.plugins.pdc_import_compose`
		`+`
		`+`
		`+ APACHE_FILE_LIST = """`
		`+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">`
		`+ <html>`
		`+ <head>`
		`+ <title>Index of /compose/rawhide/Fedora-Rawhide-20200715.n.2/compose/metadata</title>`
		`+ </head>`
		`+ <body>`
		`+ <h1>Index of /compose/rawhide/Fedora-Rawhide-20200715.n.2/compose/metadata</h1>`
		`+ <pre><img src="/icons/blank.gif" alt="Icon "> <a href="?C=N;O=D">Name</a> <a href="?C=M;O=A">Last modified</a> <a href="?C=S;O=A">Size</a> <a href="?C=D;O=A">Description</a>`
		`+ <hr><img src="/icons/back.gif" alt="[PARENTDIR]"> <a href="/compose/rawhide/Fedora-Rawhide-20200715.n.2/compose/">Parent Directory</a> -`
		`+ <img src="/icons/unknown.gif" alt="[ ]"> <a href="composeinfo.json">composeinfo.json</a> 2020-07-16 03:40 15K`
		`+ <img src="/icons/unknown.gif" alt="[ ]"> <a href="images.json">images.json</a> 2020-07-16 03:40 39K`
		`+ <img src="/icons/unknown.gif" alt="[ ]"> <a href="modules.json">modules.json</a> 2020-07-16 00:42 1.4M`
		`+ <img src="/icons/unknown.gif" alt="[ ]"> <a href="rpms.json">rpms.json</a> 2020-07-16 00:17 168M`
		`+ <img src="/icons/unknown.gif" alt="[ ]"> <a>invalid</a> 2020-07-16 00:18 0M`
		`+ <hr></pre>`
		`+ </body></html>`
		`+ """ # noqa: E501`
		`+`
		`+`
		`+ class TestPDCImportComposeToddler:`
		`+`
		`+ toddler_cls = toddlers.plugins.pdc_import_compose.PDCImportCompose`
		`+`
		`+ def test_accepts_topic_invalid(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is False`
		`+`
		`+ @pytest.mark.parametrize(`
		`+ "topic",`
		`+ [`
		`+ "org.fedoraproject.*.toddlers.trigger.pdc_import_compose",`
		`+ "org.fedoraproject.prod.toddlers.trigger.pdc_import_compose",`
		`+ "org.fedoraproject.stg.toddlers.trigger.pdc_import_compose",`
		`+ "org.fedoraproject.*.pungi.compose.status.change",`
		`+ "org.fedoraproject.prod.pungi.compose.status.change",`
		`+ "org.fedoraproject.stg.pungi.compose.status.change",`
		`+ ],`
		`+ )`
		`+ def test_accepts_topic_valid(self, topic, toddler):`
		`+ assert toddler.accepts_topic(topic)`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.PDCImportCompose._process_all_composes")`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test_process_full_dist_git(self, pdc, process_dg, toddler):`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages"`
		`+ msg.body = {}`
		`+`
		`+ toddler.process({"config": "foobar"}, msg)`
		`+`
		`+ process_dg.assert_called_once_with({"config": "foobar"}, client)`
		`+`
		`+ @patch(`
		`+ "toddlers.plugins.pdc_import_compose.PDCImportCompose._process_single_compose"`
		`+ )`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test_process_single_package(self, pdc, process_dg, toddler):`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.pungi.compose.status.change"`
		`+ msg.body = {}`
		`+`
		`+ toddler.process({"config": "foobar"}, msg)`
		`+`
		`+ process_dg.assert_called_once_with({"config": "foobar"}, client, msg)`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.PDCImportCompose._import_compose")`
		`+ @patch("toddlers.plugins.pdc_import_compose._old_composes")`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test__process_all_composes(self, pdc, old_composes, import_compose, toddler):`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ old_composes.return_value = [`
		`+ ("branch", 42, "https://kojipkgs.../42"),`
		`+ ("branch", 123, "https://kojipkgs.../123"),`
		`+ ]`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.pungi.compose.status.change"`
		`+ msg.body = {}`
		`+`
		`+ toddler._process_all_composes(`
		`+ {"old_composes_url": "https://kojipkgs.fedoraproject.org/compose/"}, client`
		`+ )`
		`+ old_composes.assert_called_once_with(`
		`+ "https://kojipkgs.fedoraproject.org/compose/"`
		`+ )`
		`+ import_compose.assert_has_calls(`
		`+ calls=[`
		`+ call(client, 42, "https://kojipkgs.../42"),`
		`+ call(client, 123, "https://kojipkgs.../123"),`
		`+ ]`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.PDCImportCompose._import_compose")`
		`+ @patch("toddlers.plugins.pdc_import_compose._old_composes")`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test__process_all_composes_fail(`
		`+ self, pdc, old_composes, import_compose, caplog, toddler`
		`+ ):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+ import_compose.side_effect = Exception("A generic error")`
		`+`
		`+ old_composes.return_value = [`
		`+ ("branch", 42, "https://kojipkgs.../42"),`
		`+ ("branch", 123, "https://kojipkgs.../123"),`
		`+ ]`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.pungi.compose.status.change"`
		`+ msg.body = {}`
		`+`
		`+ toddler._process_all_composes(`
		`+ {"old_composes_url": "https://kojipkgs.fedoraproject.org/compose/"}, client`
		`+ )`
		`+ old_composes.assert_called_once_with(`
		`+ "https://kojipkgs.fedoraproject.org/compose/"`
		`+ )`
		`+ import_compose.assert_has_calls(`
		`+ calls=[`
		`+ call(client, 42, "https://kojipkgs.../42"),`
		`+ call(client, 123, "https://kojipkgs.../123"),`
		`+ ]`
		`+ )`
		`+`
		`+ assert len(caplog.records) == 3`
		`+ assert (`
		`+ caplog.records[-1].message == "Failed to import 'https://kojipkgs.../123'"`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.PDCImportCompose._import_compose")`
		`+ @patch("toddlers.plugins.pdc_import_compose._old_composes")`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test__process_all_composes_fail_request(`
		`+ self, pdc, old_composes, import_compose, caplog, toddler`
		`+ ):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ error = Exception("Failed to query resource")`
		`+ error.response = Mock()`
		`+ error.response.url = "https://kojipkgs/..../42"`
		`+ error.response.text = "AHAH"`
		`+ import_compose.side_effect = error`
		`+`
		`+ old_composes.return_value = [`
		`+ ("branch", 42, "https://kojipkgs.../42"),`
		`+ ("branch", 123, "https://kojipkgs.../123"),`
		`+ ]`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.pungi.compose.status.change"`
		`+ msg.body = {}`
		`+`
		`+ toddler._process_all_composes(`
		`+ {"old_composes_url": "https://kojipkgs.fedoraproject.org/compose/"}, client`
		`+ )`
		`+ old_composes.assert_called_once_with(`
		`+ "https://kojipkgs.fedoraproject.org/compose/"`
		`+ )`
		`+ import_compose.assert_has_calls(`
		`+ calls=[`
		`+ call(client, 42, "https://kojipkgs.../42"),`
		`+ call(client, 123, "https://kojipkgs.../123"),`
		`+ ]`
		`+ )`
		`+`
		`+ assert len(caplog.records) == 3`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Failed to import 'https://kojipkgs.../123' - 'https://kojipkgs/..../42' 'AHAH'"`
		`+ )`
		`+`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test__process_single_compose_invalid_status(self, pdc, caplog, toddler):`
		`+ caplog.set_level(logging.INFO)`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.pungi.compose.status.change"`
		`+ msg.body = {"status": "INVALID"}`
		`+`
		`+ toddler._process_single_compose(`
		`+ {"old_composes_url": "https://kojipkgs.fedoraproject.org/compose/"},`
		`+ pdc,`
		`+ msg,`
		`+ )`
		`+`
		`+ assert len(caplog.records) == 1`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "The compose that finished isn't in a state we care: INVALID, bailing"`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.PDCImportCompose._import_compose")`
		`+ @patch("pdc_client.PDCClient")`
		`+ def test__process_single_compose(self, pdc, import_compose, toddler):`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = 123`
		`+ msg.topic = "org.fedoraproject.prod.pungi.compose.status.change"`
		`+ msg.body = {`
		`+ "status": "FINISHED",`
		`+ "compose_id": 42,`
		`+ "location": "http://kojipkgs.fedoraproject.org/compose/rawhide/42/compose",`
		`+ }`
		`+`
		`+ toddler._process_single_compose(`
		`+ {"old_composes_url": "https://kojipkgs.fedoraproject.org/compose/"},`
		`+ pdc,`
		`+ msg,`
		`+ )`
		`+`
		`+ import_compose.assert_called_once_with(`
		`+ pdc, 42, "http://kojipkgs.fedoraproject.org/compose/rawhide/42"`
		`+ )`
		`+`
		`+ def test__import_compose_failed_to_retrieve_composeinfo(self, caplog, toddler):`
		`+ caplog.set_level(logging.INFO)`
		`+ req = Mock()`
		`+ resp = Mock()`
		`+ resp.ok = False`
		`+ resp.status_code = 404`
		`+ req.get.return_value = resp`
		`+ toddler.requests_session.get.return_value = resp`
		`+`
		`+ toddler._import_compose(`
		`+ Mock(),`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2",`
		`+ )`
		`+ assert len(caplog.records) == 1`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Failed to get https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/composeinfo.json: 404"`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.compose_exists")`
		`+ def test__import_compose_compose_exists(self, compose_exists, caplog, toddler):`
		`+ caplog.set_level(logging.INFO)`
		`+ req = Mock()`
		`+ resp = Mock()`
		`+ resp.ok = True`
		`+ resp.json.return_value = {`
		`+ "payload": {"compose": {"id": "Fedora-Rawhide-20200715.n.2"}}`
		`+ }`
		`+ req.get.return_value = resp`
		`+ toddler.requests_session = req`
		`+ compose_exists.return_value = True`
		`+`
		`+ toddler._import_compose(`
		`+ Mock(),`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2",`
		`+ )`
		`+ assert len(caplog.records) == 2`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "'Fedora-Rawhide-20200715.n.2' already exists in PDC."`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.compose_exists")`
		`+ def test__import_compose_failed_to_retrieve_images(`
		`+ self, compose_exists, caplog, toddler`
		`+ ):`
		`+ caplog.set_level(logging.INFO)`
		`+ req = Mock()`
		`+ resp_composeinfo = Mock()`
		`+ resp_composeinfo.ok = True`
		`+ resp_composeinfo.json.return_value = {`
		`+ "payload": {"compose": {"id": "Fedora-Rawhide-20200715.n.2"}}`
		`+ }`
		`+ resp_images = Mock()`
		`+ resp_images.ok = False`
		`+ resp_images.status_code = 404`
		`+ req.get.side_effect = (resp_composeinfo, resp_images)`
		`+ toddler.requests_session = req`
		`+ compose_exists.return_value = False`
		`+`
		`+ toddler._import_compose(`
		`+ Mock(),`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2",`
		`+ )`
		`+ assert len(caplog.records) == 2`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Failed to get https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/images.json: 404"`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.compose_exists")`
		`+ def test__import_compose_failed_to_retrieve_rpms(`
		`+ self, compose_exists, caplog, toddler`
		`+ ):`
		`+ caplog.set_level(logging.INFO)`
		`+ req = Mock()`
		`+ resp_composeinfo = Mock()`
		`+ resp_composeinfo.ok = True`
		`+ resp_composeinfo.json.return_value = {`
		`+ "payload": {"compose": {"id": "Fedora-Rawhide-20200715.n.2"}}`
		`+ }`
		`+ resp_images = Mock()`
		`+ resp_images.ok = True`
		`+ resp_images.json.return_value = {"foor": "bar"}`
		`+ resp_rpms = Mock()`
		`+ resp_rpms.ok = False`
		`+ resp_rpms.status_code = 500`
		`+ req.get.side_effect = (resp_composeinfo, resp_images, resp_rpms)`
		`+ toddler.requests_session = req`
		`+ compose_exists.return_value = False`
		`+`
		`+ toddler._import_compose(`
		`+ Mock(),`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2",`
		`+ )`
		`+ assert len(caplog.records) == 2`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Failed to get https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/rpms.json: 500"`
		`+ )`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.ensure_release_exists")`
		`+ @patch("toddlers.plugins.pdc_import_compose.compose_exists")`
		`+ def test__import_compose(`
		`+ self, compose_exists, ensure_release_exists, caplog, toddler`
		`+ ):`
		`+ caplog.set_level(logging.INFO)`
		`+ req = Mock()`
		`+ resp_composeinfo = Mock()`
		`+ resp_composeinfo.ok = True`
		`+ resp_composeinfo.json.return_value = {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "Fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ }`
		`+ resp_images = Mock()`
		`+ resp_images.ok = True`
		`+ resp_images.json.return_value = {"foor": "bar"}`
		`+ resp_rpms = Mock()`
		`+ resp_rpms.ok = True`
		`+ resp_rpms.json.return_value = {"foo": "bar"}`
		`+ req.get.side_effect = (resp_composeinfo, resp_images, resp_rpms)`
		`+ toddler.requests_session = req`
		`+ compose_exists.return_value = False`
		`+`
		`+ pdc = MagicMock()`
		`+`
		`+ toddler._import_compose(`
		`+ pdc,`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2",`
		`+ )`
		`+ ensure_release_exists.assert_called_once_with(`
		`+ pdc,`
		`+ "fedora-Rawhide",`
		`+ {`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "version": "Rawhide",`
		`+ "release_type": "ga",`
		`+ },`
		`+ )`
		`+`
		`+ # They are not actually called twice, this is a side-effect of using Mock/MagicMock`
		`+ pdc["compose-images"]._.assert_has_calls(`
		`+ calls=[`
		`+ call(`
		`+ {`
		`+ "release_id": "fedora-Rawhide",`
		`+ "composeinfo": {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ },`
		`+ "image_manifest": {"foor": "bar"},`
		`+ }`
		`+ ),`
		`+ call(`
		`+ {`
		`+ "release_id": "fedora-Rawhide",`
		`+ "composeinfo": {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ },`
		`+ "rpm_manifest": {"foo": "bar"},`
		`+ }`
		`+ ),`
		`+ ]`
		`+ )`
		`+ pdc["compose-rpms"]._.assert_has_calls(`
		`+ calls=[`
		`+ call(`
		`+ {`
		`+ "release_id": "fedora-Rawhide",`
		`+ "composeinfo": {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ },`
		`+ "image_manifest": {"foor": "bar"},`
		`+ }`
		`+ ),`
		`+ call(`
		`+ {`
		`+ "release_id": "fedora-Rawhide",`
		`+ "composeinfo": {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ },`
		`+ "rpm_manifest": {"foo": "bar"},`
		`+ }`
		`+ ),`
		`+ ]`
		`+ )`
		`+ assert len(caplog.records) == 2`
		`+ assert (`
		`+ caplog.records[-2].message`
		`+ == "Importing compose 'Fedora-Rawhide-20200715.n.2'"`
		`+ )`
		`+ assert caplog.records[-1].message == "Import done."`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose.ensure_release_exists")`
		`+ @patch("toddlers.plugins.pdc_import_compose.compose_exists")`
		`+ def test__import_compose_no_rpms(`
		`+ self, compose_exists, ensure_release_exists, caplog, toddler`
		`+ ):`
		`+ caplog.set_level(logging.INFO)`
		`+ req = Mock()`
		`+ resp_composeinfo = Mock()`
		`+ resp_composeinfo.ok = True`
		`+ resp_composeinfo.json.return_value = {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "Fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ }`
		`+ resp_images = Mock()`
		`+ resp_images.ok = True`
		`+ resp_images.json.return_value = {"foor": "bar"}`
		`+ resp_rpms = Mock()`
		`+ resp_rpms.ok = False`
		`+ resp_rpms.status_code = 404`
		`+ req.get.side_effect = (resp_composeinfo, resp_images, resp_rpms)`
		`+ toddler.requests_session = req`
		`+ compose_exists.return_value = False`
		`+`
		`+ pdc = MagicMock()`
		`+`
		`+ toddler._import_compose(`
		`+ pdc,`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2",`
		`+ )`
		`+ ensure_release_exists.assert_called_once_with(`
		`+ pdc,`
		`+ "fedora-Rawhide",`
		`+ {`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "version": "Rawhide",`
		`+ "release_type": "ga",`
		`+ },`
		`+ )`
		`+`
		`+ pdc["compose-images"]._.assert_called_once_with(`
		`+ {`
		`+ "release_id": "fedora-Rawhide",`
		`+ "composeinfo": {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ },`
		`+ "image_manifest": {"foor": "bar"},`
		`+ }`
		`+ )`
		`+ # It's not actually called, this is a side-effect of using Mock/MagicMock`
		`+ pdc["compose-rpms"]._.assert_called_once_with(`
		`+ {`
		`+ "release_id": "fedora-Rawhide",`
		`+ "composeinfo": {`
		`+ "payload": {`
		`+ "compose": {"id": "Fedora-Rawhide-20200715.n.2"},`
		`+ "release": {`
		`+ "internal": False,`
		`+ "name": "Fedora",`
		`+ "short": "fedora",`
		`+ "type": "ga",`
		`+ "version": "Rawhide",`
		`+ },`
		`+ }`
		`+ },`
		`+ "image_manifest": {"foor": "bar"},`
		`+ }`
		`+ )`
		`+ assert len(caplog.records) == 3`
		`+ assert (`
		`+ caplog.records[-2].message`
		`+ == "Found no rpms.json file at 'https://kojipkgs.fedoraproject.org/compose/"`
		`+ "rawhide/Fedora-Rawhide-20200715.n.2/compose/metadata/rpms.json'"`
		`+ )`
		`+ assert caplog.records[-1].message == "Import done."`
		`+`
		`+ def test__scrape_links_failed(self, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ session = Mock()`
		`+ resp = Mock()`
		`+ resp.ok = False`
		`+ resp.status_code = 500`
		`+ session.get.return_value = resp`
		`+`
		`+ output = list(`
		`+ toddlers.plugins.pdc_import_compose._scrape_links(`
		`+ session=session, url="https://kojipkgs.fedoraproject.org/..",`
		`+ )`
		`+ )`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Couldn't talk to 'https://kojipkgs.fedoraproject.org/..', 500"`
		`+ )`
		`+ assert output == []`
		`+`
		`+ def test__scrape_links(self, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ session = Mock()`
		`+ resp = Mock()`
		`+ resp.ok = True`
		`+ resp.text = APACHE_FILE_LIST`
		`+ session.get.return_value = resp`
		`+`
		`+ output = list(`
		`+ toddlers.plugins.pdc_import_compose._scrape_links(`
		`+ session=session,`
		`+ url="https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata",`
		`+ )`
		`+ )`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Scraping https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata"`
		`+ )`
		`+ assert output == [`
		`+ (`
		`+ "Parent Directory",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose",`
		`+ ),`
		`+ (`
		`+ "composeinfo.json",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/composeinfo.json",`
		`+ ),`
		`+ (`
		`+ "images.json",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/images.json",`
		`+ ),`
		`+ (`
		`+ "modules.json",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/modules.json",`
		`+ ),`
		`+ (`
		`+ "rpms.json",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/compose/metadata/rpms.json",`
		`+ ),`
		`+ ]`
		`+`
		`+ @patch("toddlers.plugins.pdc_import_compose._scrape_links")`
		`+ @patch("toddlers.plugins.pdc_import_compose.make_session")`
		`+ def test__old_composes(self, make_sess, scrape_links, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+`
		`+ scrape_links.side_effect = (`
		`+ (`
		`+ ["F31", "https://kojipkgs.fedoraproject.org/compose/F31"],`
		`+ ["F32", "https://kojipkgs.fedoraproject.org/compose/F32"],`
		`+ ["rawhide", "https://kojipkgs.fedoraproject.org/compose/rawhide"],`
		`+ ),`
		`+ (`
		`+ [`
		`+ "Parent Directory",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F31/",`
		`+ ],`
		`+ [`
		`+ "Fedora-31-20190715.n.1/",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F31/Fedora-31-20190715.n.1/",`
		`+ ],`
		`+ ),`
		`+ (`
		`+ [`
		`+ "Parent Directory",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F32/",`
		`+ ],`
		`+ [`
		`+ "Fedora-Rawhide-20200715.n.1/",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F32/Fedora-32-20200715.n.1/",`
		`+ ],`
		`+ [`
		`+ "Fedora-Rawhide-20200715.n.2/",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F32/Fedora-32-20200715.n.2/",`
		`+ ],`
		`+ ),`
		`+ (`
		`+ [`
		`+ "Parent Directory",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/",`
		`+ ],`
		`+ [`
		`+ "Fedora-Rawhide-20200715.n.1",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.1/",`
		`+ ],`
		`+ [`
		`+ "Fedora-Rawhide-20200715.n.2",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-"`
		`+ "Rawhide-20200715.n.2/",`
		`+ ],`
		`+ [`
		`+ "latest-Fedora-Rawhide",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/latest-Fedora-Rawhide/",`
		`+ ],`
		`+ ),`
		`+ )`
		`+`
		`+ req = Mock()`
		`+ resp = Mock()`
		`+ resp.ok = True`
		`+ resp.text = "FINISHED"`
		`+ resp_not_ok = Mock()`
		`+ resp_not_ok.ok = False`
		`+ resp_not_done = Mock()`
		`+ resp_not_done.ok = True`
		`+ resp_not_done.text = "ONGOING"`
		`+ req.get.side_effect = (`
		`+ resp, # F31`
		`+ resp_not_ok,`
		`+ resp,`
		`+ resp, # F32`
		`+ resp_not_ok,`
		`+ resp,`
		`+ resp_not_done, # rawhide`
		`+ )`
		`+ req.head.side_effect = (`
		`+ resp,`
		`+ resp_not_ok,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp, # F31`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp, # F32`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp,`
		`+ resp, # rawhide`
		`+ )`
		`+ make_sess.return_value = req`
		`+`
		`+ output = list(`
		`+ toddlers.plugins.pdc_import_compose._old_composes(`
		`+ "https://kojipkgs.fedoraproject.org/compose/"`
		`+ )`
		`+ )`
		`+ assert output == [`
		`+ (`
		`+ "F31",`
		`+ "Fedora-31-20190715.n.1/",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F31/Fedora-31-20190715.n.1/",`
		`+ ),`
		`+ (`
		`+ "F32",`
		`+ "Fedora-Rawhide-20200715.n.1/",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F32/Fedora-32-20200715.n.1/",`
		`+ ),`
		`+ (`
		`+ "F32",`
		`+ "Fedora-Rawhide-20200715.n.2/",`
		`+ "https://kojipkgs.fedoraproject.org/compose/F32/Fedora-32-20200715.n.2/",`
		`+ ),`
		`+ (`
		`+ "rawhide",`
		`+ "Fedora-Rawhide-20200715.n.1",`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.1/",`
		`+ ),`
		`+ ]`
		`+`
		`+ assert (`
		`+ "Looking for old composes at: https://kojipkgs.fedoraproject.org/compose/"`
		`+ in caplog.text`
		`+ )`
		`+ assert (`
		`+ "Skipping https://kojipkgs.fedoraproject.org/compose/rawhide/"`
		`+ "latest-Fedora-Rawhide/. Just a symlink." in caplog.text`
		`+ )`
		`+ assert (`
		`+ "Bailing, could not retrieve the STATUS file at: "`
		`+ "https://kojipkgs.fedoraproject.org/compose/F32/" in caplog.text`
		`+ )`
		`+ assert (`
		`+ "Bailing, STATUS is not final at: "`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/Fedora-Rawhide-20200715.n.2/"`
		`+ in caplog.text`
		`+ )`
		`+ assert (`
		`+ "Bailing, could not retrieve the STATUS file at: "`
		`+ "https://kojipkgs.fedoraproject.org/compose/rawhide/" in caplog.text`
		`+ )`
		`+`
		`+ def test_compose_exists_failed(self, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ pdc = MagicMock()`
		`+ resp = Mock()`
		`+ resp.status_code = 500`
		`+ pdc["composes"]["compose-id"]._.side_effect = beanbag.bbexcept.BeanBagException(`
		`+ msg="Failed to do something", response=resp`
		`+ )`
		`+`
		`+ assert (`
		`+ toddlers.plugins.pdc_import_compose.compose_exists(pdc, "compose-id")`
		`+ is False`
		`+ )`
		`+ assert caplog.records[-1].message.startswith(`
		`+ "Failed to check if the compose compose-id exists: Failed to do something - response:"`
		`+ )`
		`+`
		`+ def test_compose_exists(self, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ pdc = MagicMock()`
		`+ resp = Mock()`
		`+ resp.status_code = 200`
		`+`
		`+ assert (`
		`+ toddlers.plugins.pdc_import_compose.compose_exists(pdc, "compose-id")`
		`+ is True`
		`+ )`
		`+`
		`+ def test_ensure_release_exists_failed(self, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ pdc = MagicMock()`
		`+ resp = Mock()`
		`+ resp.status_code = 500`
		`+ pdc["releases"]["release-id"]._.side_effect = beanbag.bbexcept.BeanBagException(`
		`+ msg="Failed to do something", response=resp`
		`+ )`
		`+`
		`+ toddlers.plugins.pdc_import_compose.ensure_release_exists(`
		`+ pdc, "release-id", {"foo": "bar"}`
		`+ )`
		`+ assert caplog.records[-1].message.startswith(`
		`+ "Failed to check if the release release-id exists: Failed to do something - response:"`
		`+ )`
		`+`
		`+ def test_ensure_release_exists_create(self, caplog):`
		`+ caplog.set_level(logging.DEBUG)`
		`+ pdc = MagicMock()`
		`+ resp = Mock()`
		`+ resp.status_code = 404`
		`+ pdc["releases"]["release-id"]._.side_effect = beanbag.bbexcept.BeanBagException(`
		`+ msg="Failed to do something", response=resp`
		`+ )`
		`+`
		`+ toddlers.plugins.pdc_import_compose.ensure_release_exists(`
		`+ pdc, "release-id", {"foo": "bar"}`
		`+ )`
		`+ pdc["releases"]._.assert_called_once_with({"foo": "bar", "active": True})`
		`+ assert (`
		`+ caplog.records[-1].message`
		`+ == "Creating release {'foo': 'bar', 'active': True}"`
		`+ )`

toddlers.toml.example

file modified

		`@@ -119,6 +119,10 @@`
		`[consumer_config.pdc_retired_packages]`
		`file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s"`

		`+ [consumer_config.pdc_import_compose]`
		`+ old_composes_url = "https://kojipkgs.fedoraproject.org/compose/"`
		`+`
		`+`
		`[qos]`
		`prefetch_size = 0`
		`prefetch_count = 25`

toddlers/plugins/pdc_import_compose.py

file added

+284

		`@@ -0,0 +1,284 @@`
		`+ """`
		`+ When a new compose is announce by Pungi, we want to import it in PDC.`
		`+`
		`+ This toddler replaces the respective handler of pdc-updater:`
		`+ https://github.com/fedora-infra/pdc-updater/blob/develop/pdcupdater/handlers/compose.py`
		`+`
		`+ Authors: Pierre-Yves Chibon <pingou@pingoured.fr>`
		`+`
		`+ """`
		`+`
		`+ import logging`
		`+`
		`+ import beanbag.bbexcept`
		`+ import bs4`
		`+ import pdc_client`
		`+`
		`+ from ..base import ToddlerBase`
		`+ from ..utils.requests import make_session`
		`+`
		`+ _log = logging.getLogger(__name__)`
		`+`
		`+`
		`+ # These are the states of a pungi4 compose that we care about.`
		`+ # There are other states that we don't care about.. like DOOMED, etc..`
		`+ _FINAL = [`
		`+ "FINISHED",`
		`+ "FINISHED_INCOMPLETE",`
		`+ ]`
		`+`
		`+`
		`+ class PDCImportCompose(ToddlerBase):`
		`+ """ Listens to messages sent by playtime (which lives in toddlers) or pungi`
		`+ and import compose(s) into PDC.`
		`+ """`
		`+`
		`+ name = "pdc_import_compose"`
		`+`
		`+ amqp_topics = [`
		`+ "org.fedoraproject.*.toddlers.trigger.pdc_import_compose",`
		`+ "org.fedoraproject.*.pungi.compose.status.change",`
		`+ ]`
		`+`
		`+ def __init__(self):`
		`+ self.requests_session = make_session(timeout=600)`
		`+`
		`+ def accepts_topic(self, topic):`
		`+ """Returns a boolean whether this toddler is interested in messages`
		`+ from this specific topic.`
		`+ """`
		`+ return topic.startswith("org.fedoraproject.") and topic.endswith(`
		`+ ("toddlers.trigger.pdc_import_compose", "pungi.compose.status.change")`
		`+ )`
		`+`
		`+ def process(self, config, message):`
		`+ """Process a given message."""`
		`+ pdc = pdc_client.PDCClient(**config.get("pdc_config", {}))`
		`+`
		`+ if message.topic.endswith("pungi.compose.status.change"):`
		`+ self._process_single_compose(config, pdc, message)`
		`+ else:`
		`+ self._process_all_composes(config, pdc)`
		`+`
		`+ def _process_all_composes(self, config, pdc):`
		`+ """ Updates PDC composes from the data gathered in koji.`
		`+`
		`+ This steps over all the composes found in koji and insert them into PDC.`
		`+ """`
		`+ _log.info("Looking up all composes from koji and importing them to PDC.")`
		`+ old_composes = _old_composes(config["old_composes_url"])`
		`+ for _, compose_id, url in old_composes:`
		`+ try:`
		`+ self._import_compose(pdc, compose_id, url)`
		`+ except Exception as e:`
		`+ if getattr(e, "response", None):`
		`+ _log.exception(`
		`+ "Failed to import %r - %r %r",`
		`+ url,`
		`+ e.response.url,`
		`+ e.response.text,`
		`+ )`
		`+ else:`
		`+ _log.exception("Failed to import %r", url)`
		`+`
		`+ def _process_single_compose(self, config, pdc, message):`
		`+ """Handle an incoming bus message.`
		`+`
		`+ The message should be a pungi notification about a new compose having`
		`+ finished. That compose will be added to PDC.`
		`+ """`
		`+`
		`+ msg = message.body`
		`+`
		`+ if not msg["status"] in _FINAL:`
		`+ _log.info(`
		`+ "The compose that finished isn't in a state we care: %s, bailing",`
		`+ msg["status"],`
		`+ )`
		`+ return`
		`+`
		`+ # This is something like Fedora-24-20151130.n.2 or Fedora-Rawhide-201..`
		`+ compose_id = msg["compose_id"]`
		`+`
		`+ # The URL given looks like`
		`+ # http://kojipkgs.fedoraproject.org/compose/rawhide/COMPOSEID/compose`
		`+ # but we want`
		`+ # http://kojipkgs.fedoraproject.org/compose/rawhide/COMPOSEID`
		`+ # So handle it carefully, like this`
		`+ compose_url = msg["location"].strip("/").strip("compose").strip("/")`
		`+`
		`+ self._import_compose(pdc, compose_id, compose_url)`
		`+`
		`+ def _import_compose(self, pdc, compose_id, compose_url):`
		`+ base = compose_url + "/compose/metadata"`
		`+`
		`+ url = base + "/composeinfo.json"`
		`+ response = self.requests_session.get(url)`
		`+ if not response.ok:`
		`+ _log.info("Failed to get %s: %s", url, response.status_code)`
		`+ return`
		`+ composeinfo = response.json()`
		`+`
		`+ # Before we waste any more time pulling down 100MB files from koji and`
		`+ # POSTing them back to PDC, let's check to see if we already know about`
		`+ # this compose.`
		`+ compose_id = composeinfo["payload"]["compose"]["id"]`
		`+ _log.info("Importing compose %r", compose_id)`
		`+`
		`+ if compose_exists(pdc, compose_id):`
		`+ _log.warning("%r already exists in PDC.", compose_id)`
		`+ return`
		`+`
		`+ # OK, go ahead and pull down these gigantic files.`
		`+ url = base + "/images.json"`
		`+ response = self.requests_session.get(url)`
		`+ if not response.ok:`
		`+ _log.info("Failed to get %s: %s", url, response.status_code)`
		`+ return`
		`+ images = response.json()`
		`+`
		`+ url = base + "/rpms.json"`
		`+ response = self.requests_session.get(url)`
		`+ # Check first for a 404...`
		`+ if response.status_code == 404:`
		`+ # Not all composes have rpms. In particular, atomic ones.`
		`+ # https://github.com/fedora-infra/pdc-updater/issues/11`
		`+ _log.warning("Found no rpms.json file at %r", url)`
		`+ rpms = None`
		`+ elif not response.ok:`
		`+ # Something other than a 404 means real failure, so complain.`
		`+ _log.info("Failed to get %s: %s", url, response.status_code)`
		`+ return`
		`+ else:`
		`+ rpms = response.json()`
		`+`
		`+ # PDC demands lowercase`
		`+ composeinfo["payload"]["release"]["short"] = composeinfo["payload"]["release"][`
		`+ "short"`
		`+ ].lower()`
		`+ release = composeinfo["payload"]["release"].copy()`
		`+ release["release_type"] = release.pop("type", "ga")`
		`+`
		`+ # PDC doesn't know about this field which showed up recently in pungi`
		`+ # productmd metadata here.`
		`+ release.pop("internal", None)`
		`+`
		`+ release_id = "{short}-{version}".format(**release)`
		`+`
		`+ ensure_release_exists(pdc, release_id, release)`
		`+`
		`+ # https://github.com/product-definition-center/product-definition-center/issues/228`
		`+ # https://pdc.fedoraproject.org/rest_api/v1/compose-images/`
		`+ pdc["compose-images"]._(`
		`+ {`
		`+ "release_id": release_id,`
		`+ "composeinfo": composeinfo,`
		`+ "image_manifest": images,`
		`+ }`
		`+ )`
		`+`
		`+ # https://pdc.fedoraproject.org/rest_api/v1/compose-rpms/`
		`+ if rpms:`
		`+ pdc["compose-rpms"]._(`
		`+ {`
		`+ "release_id": release_id,`
		`+ "composeinfo": composeinfo,`
		`+ "rpm_manifest": rpms,`
		`+ }`
		`+ )`
		`+`
		`+ _log.info("Import done.")`
		`+`
		`+`
		`+ def _scrape_links(session, url):`
		`+ """ Utility to scrape links from a <pre> tag. """`
		`+ _log.debug("Scraping %s", url)`
		`+ response = session.get(url)`
		`+ if not response.ok:`
		`+ _log.error("Couldn't talk to %r, %r", url, response.status_code)`
		`+ return []`
		`+ soup = bs4.BeautifulSoup(response.text, "html.parser")`
		`+ pre = soup.find("pre")`
		`+ for link in pre.findAll("a"):`
		`+ href = link.attrs.get("href", "").strip("/")`
		`+ if not href:`
		`+ continue`
		`+ if href.startswith("?"):`
		`+ continue`
		`+ href = url.strip("/") + "/" + href.strip("/")`
		`+ yield link.text.strip("/"), href`
		`+`
		`+`
		`+ def _old_composes(base_url):`
		`+ """ Screen-scrape the list of old composes from kojipkgs. """`
		`+ _log.info("Looking for old composes at: %s", base_url)`
		`+`
		`+ requests_session = make_session(timeout=30)`
		`+`
		`+ # Get the initial list of branches`
		`+ branch_links = _scrape_links(requests_session, base_url)`
		`+ for branch, branch_link in branch_links:`
		`+ _log.debug("Checking branch: %s at %s", branch, branch_link)`
		`+ # Find all composes for that branch`
		`+ compose_links = _scrape_links(requests_session, branch_link)`
		`+ for compose, compose_link in compose_links:`
		`+ _log.debug("Checking compose file: %s at %s", compose, compose_link)`
		`+ # Some of these are symlinks to others`
		`+ if compose.startswith("latest"):`
		`+ _log.debug("Skipping %s. Just a symlink.", compose_link)`
		`+ continue`
		`+`
		`+ # Some of these failed mid-way and didn't complete.`
		`+ metadata_link = compose_link + "/compose/metadata/"`
		`+ filenames = ["composeinfo", "images", "rpms"]`
		`+ urls = [metadata_link + fname + ".json" for fname in filenames]`
		`+ _log.info("Querying: %s", urls)`
		`+ good = all([requests_session.head(url).ok for url in urls])`
		`+ if not good:`
		`+ _log.debug("Bailing, one of the urls failed in %s", urls)`
		`+ continue`
		`+`
		`+ # But really, the real check is to see if the status is good.`
		`+ response = requests_session.get(compose_link + "/STATUS")`
		`+ if not response.ok:`
		`+ _log.debug(`
		`+ "Bailing, could not retrieve the STATUS file at: %s", compose_link`
		`+ )`
		`+ continue`
		`+ if response.text.strip() not in _FINAL:`
		`+ _log.debug("Bailing, STATUS is not final at: %s", compose_link)`
		`+ continue`
		`+`
		`+ # If we got this far, then return it`
		`+ _log.info(" found %s/%s", branch, compose)`
		`+ yield branch, compose, compose_link`
		`+`
		`+ # Finally, close the requests session.`
		`+ requests_session.close()`
		`+`
		`+`
		`+ def compose_exists(pdc, compose_id):`
		`+ """ Return True if a compose exists in PDC. False if not. """`
		`+ try:`
		`+ pdc["composes"][compose_id]._()`
		`+ return True`
		`+ except beanbag.bbexcept.BeanBagException as e:`
		`+ if e.response.status_code != 404:`
		`+ _log.error("Failed to check if the compose %s exists: %s", compose_id, e)`
		`+ return False`
		`+`
		`+`
		`+ def ensure_release_exists(pdc, release_id, release):`
		`+ """ Create a release in PDC if it doesn't already exist. """`
		`+ try:`
		`+ pdc["releases"][release_id]._()`
		`+ except beanbag.bbexcept.BeanBagException as e:`
		`+ if e.response.status_code != 404:`
		`+ _log.error("Failed to check if the release %s exists: %s", release_id, e)`
		`+ return`
		`+ _log.warning("No release %r exists. Creating.", release_id)`
		`+`
		`+ release_payload = release.copy()`
		`+ release_payload.update({"active": True})`
		`+ _log.info("Creating release %r", release_payload)`
		`+ pdc["releases"]._(release_payload)`

pingou commented 3 years ago

This toddler reacts to messages coming from pungi or playtime and inserts
into PDC either the compose that was just done (when the notification
comes from pungi) or finds in koji all the old composes and inserts them
all (when the notification is coming from playtime).

This toddler is the equivalent of the following handler in pdc-updater:
https://github.com/fedora-infra/pdc-updater/blob/develop/pdcupdater/handlers/compose.py

Signed-off-by: Pierre-Yves Chibon pingou@pingoured.fr

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 17s

Metadata Update from @nphilipp:
- Request assigned

3 years ago

nphilipp commented on line 21 of toddlers/plugins/pdc_import_compose.py 3 years ago

Let's do a relative import here:

from ..base import ToddlerBase

nphilipp commented on line 47 of toddlers/plugins/pdc_import_compose.py 3 years ago

PR #13 has a generic version of this as ..utils.requests.make_session()

nphilipp commented on line 62 of toddlers/plugins/pdc_import_compose.py 3 years ago

This one and the other methods are missing self. By chance, this worked previously but PR #13 fixes this as well (needs being adapted in the tests).

nphilipp commented on line 72 of toddlers/plugins/pdc_import_compose.py 3 years ago

PR#13 also makes this a little easier:

from ..utils.pdc import pdc_client_for_config
...
        pdc = pdc_client_for_config(config)

nphilipp commented on line 131 of toddlers/plugins/pdc_import_compose.py 3 years ago

Using real instance methods (with self), the session object can be constructed once and then used for the lifetime of the toddler process:

from ..utils.requests import make_session
...
class PDCImportCompose(ToddlerBase):
...
    def __init__(self):
        self.requests_session = make_session(timeout=600, total=3)

pingou commented on line 21 of toddlers/plugins/pdc_import_compose.py 3 years ago

will do

nphilipp commented on line 131 of toddlers/plugins/pdc_import_compose.py 3 years ago

Using real instance methods (with self), the session object can be constructed once and then used for the lifetime of the toddler process:

from ..utils.requests import make_session
...
class PDCImportCompose(ToddlerBase):
...
    def __init__(self):
        self.requests_session = make_session(timeout=600, total=3)
...
   response = self.requests_session.get(url)

pingou commented on line 47 of toddlers/plugins/pdc_import_compose.py 3 years ago

I'll adjust once #13 is in

pingou commented on line 62 of toddlers/plugins/pdc_import_compose.py 3 years ago

yup saw this :)

nphilipp commented on line 12 of toddlers/plugins/pdc_import_compose.py 3 years ago

Only FYI: We seem to be (shallow-)copying dicts only below, they have their own .copy() method which can be used instead.

pingou commented on line 72 of toddlers/plugins/pdc_import_compose.py 3 years ago

Ah cool, hm, pdc_client_for_config or pdc_client_from_config?

pingou commented on line 12 of toddlers/plugins/pdc_import_compose.py 3 years ago

I copied this over from pdc-updater, happy to adjust

nphilipp commented on line 15 of tests/plugins/test_pdc_import_compose.py 3 years ago

With #13 in, this would get a fixture for the toddler object and use that instead:

@pytest.fixture
def toddler(monkeypatch):
    monkeypatch.setattr(toddlers.plugins.pdc_import_compose, "make_session", MagicMock())
    return toddlers.plugins.pdc_import_compose.PDCImportCompose()
...
    def test_accepts_topic_invalid(self, toddler):
        assert toddler.accepts_topic("foo.bar") is False
...

...and likewise in other test methods.

nphilipp commented on line 72 of toddlers/plugins/pdc_import_compose.py 3 years ago

I think that both versions would be correct. The function name is "for_config". shrugs

zlopez commented on line 6 of tests/test_runner.py 3 years ago

There is ordered_set library, if you need to have the set sorted.

nphilipp commented on line 6 of tests/test_runner.py 3 years ago

We don't care for sorting order here, but (just as an observation) it could use a set comprehension rather than a list comprehension as the argument for set() because the list object is just used once and then thrown away:

assert {t.name for t in runner.toddlers} >= {"debug", "flag_ci_pr", ...}

Edited 3 years ago by nphilipp

rebased onto 15e0a2e78b2b9de3a0250c1fd72eacb0bedafd39

3 years ago

rebased onto 8daad22e3e60ea967ac303a62725e993bd4c3bbf

3 years ago

rebased onto 95b8a68805d2bbed978780e05f4d0ab423b91814

3 years ago

rebased onto 98f9c63e8e84b52547e33e62650b00f5361a86b7

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 31s

nphilipp commented on line 5 of toddlers/plugins/pdc_import_compose.py 3 years ago

Better:

"""
When a new compose is announced by Pungi, we want to import it into PDC.

This toddler replaces the respective handler of pdc-updater:
...

Edited 3 years ago by nphilipp

nphilipp commented on line 285 of toddlers/plugins/pdc_import_compose.py 3 years ago

I wonder if we shouldn't move compose_exists() and ensure_release_exists() into toddlers.utils.pdc, e.g. by subclassing PDCClient and adding them as methods there.

Also, is there a reason to have these, _scrape_links() and _old_composes() as module-global functions instead of methods in the toddler?

nphilipp commented on line 27 of tests/plugins/test_pdc_import_compose.py 3 years ago

What's the reason for testing topics containing asterisks?

nphilipp commented on line 607 of tests/plugins/test_pdc_import_compose.py 3 years ago

Hmm, interspersing code with large text blocks breaks the reading flow, we could use a global or class variable instead, e.g.:

import toddlers.plugins.pdc_import_compose

SCRAPED_LINKS_SOURCE = """
<!DOCTYPE ...
...
"""  # noqa: E501

class TestPDCImportComposeToddler:
    ...
    def test__scrape_links(self, caplog):
        ...
        resp.text = SCRAPED_LINKS_SOURCE
        ...

We also should list the flagged error/warning with # noqa: ....

Edited 3 years ago by nphilipp

rebased onto 794aba4045e6e8e02a2ce4c1469fedae06f999e3

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 21s

pingou commented on line 27 of tests/plugins/test_pdc_import_compose.py 3 years ago

None really, it's just the amqp_topic the toddler has

pingou commented on line 607 of tests/plugins/test_pdc_import_compose.py 3 years ago

will move

pingou commented on line 5 of toddlers/plugins/pdc_import_compose.py 3 years ago

on it

pingou commented on line 285 of toddlers/plugins/pdc_import_compose.py 3 years ago

For the first two, if we need them somewhere else we could sure.

For the later, they don't need anything from the toddler so I've kept them out.

One reason for keeping the code used by the toddler in the toddler is that removing the toddler will remove all the code it uses. As soon as other toddler start to rely on that code, then refactoring it to the toddlers.utils module make sense I think, but if it's only one toddler, then I like the idea to just do a rm of the toddler and have the entire code gone :)

rebased onto 6d8934c

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 09s

nphilipp commented 3 years ago

None really, it's just the amqp_topic the toddler has

Nah, it's a wildcard pattern, we shouldn't ever see a topic with an asterisk in them. The parametrized topic is one that the toddler is fed with and the test checks that it accepts it. Likewise, we could add e.g. org.fedoraproject.thisisallkindsofinvalid.pungi.compose.status.change as a topic to test with, which wouldn't make the test fail (I think) but wouldn't make any sense either. :wink:

Maybe we should test that a toddler refuses unknown topics, but that's probably better in tests for the base class otherwise we end up repeating that test for all the toddlers with no tangible benefit.

Edited 3 years ago by nphilipp

nphilipp commented 3 years ago

One reason for keeping the code used by the toddler in the toddler is that removing the toddler will remove all the code it uses. As soon as other toddler start to rely on that code, then refactoring it to the toddlers.utils module make sense I think, but if it's only one toddler, then I like the idea to just do a rm of the toddler and have the entire code gone :)

Alright, as long as we remember that when adding a new toddler which would use it. :stuck_out_tongue_winking_eye:

pingou commented 3 years ago

Maybe we should test that a toddler refuses unknown topics, but that's probably better in tests for the base class otherwise we end up repeating that test for all the toddlers with no tangible benefit.

Isn't that the test_accepts_topic_invalid test?

nphilipp commented 3 years ago