PR#55: Add a way to manually import a compose using the JSON sent on the bus - fedora-infra/toddlers

fedora-infra / toddlers

#55 Add a way to manually import a compose using the JSON sent on the bus

Merged 3 years ago by pingou. Opened 3 years ago by pingou.

manual_import into master

Code-style fixes

Pierre-Yves Chibon • 3 years ago

89a7759

Add a way to manually import composes into PDC if needed

Pierre-Yves Chibon • 3 years ago

b7224b4

tests/conftest.py

file modified

+30

		`@@ -1,3 +1,5 @@`
		`+ import json`
		`+ import os`
		`import sys`
		`from unittest.mock import MagicMock`

		`@@ -29,3 +31,31 @@`

		`toddler_obj = toddler_cls()`
		`return toddler_obj`
		`+`
		`+`
		`+ @pytest.fixture`
		`+ def msg_file(request, monkeypatch):`
		`+ """Fixture creating a msg.json file containing the data sent by pungi upon`
		`+ successful compose.`
		`+`
		`+ """`
		`+ data = {`
		`+ "status": "FINISHED",`
		`+ "release_type": "ga",`
		`+ "compose_label": "RC-1.2",`
		`+ "compose_respin": 0,`
		`+ "compose_date": "20201019",`
		`+ "release_short": "Fedora",`
		`+ "release_version": "33",`
		`+ "location": "https://kojipkgs.fedoraproject.org/compose/33/Fedora-33-20201019.0/compose",`
		`+ "compose_type": "production",`
		`+ "release_is_layered": False,`
		`+ "release_name": "Fedora",`
		`+ "compose_path": "/mnt/koji/compose/33/Fedora-33-20201019.0",`
		`+ "compose_id": "Fedora-33-20201019.0",`
		`+ }`
		`+ here = os.path.join(os.path.dirname(os.path.abspath(__file__)))`
		`+ msg_file = os.path.join(here, "msg.json")`
		`+ with open(msg_file, "w") as stream:`
		`+ json.dump(data, stream)`
		`+ return msg_file`

tests/plugins/test_pdc_import_compose.py

file modified

+89 -1

		`@@ -1,5 +1,5 @@`
		`import logging`
		`- from unittest.mock import call, MagicMock, Mock, patch`
		`+ from unittest.mock import ANY, call, MagicMock, Mock, patch`

		`import beanbag.bbexcept`
		`import fedora_messaging.api`
		`@@ -929,3 +929,91 @@`
		`pdc["releases"]._.assert_called_once_with({"foo": "bar", "active": True})`
		`assert caplog.records[-2].message.startswith("Failed to create the release")`
		`assert caplog.records[-1].message.startswith("html content")`
		`+`
		`+`
		`+ class TestPDCImportComposeToddlerCLI:`
		`+`
		`+ toddler_cls = toddlers.plugins.pdc_import_compose.PDCImportCompose`
		`+`
		`+ def test_main_no_args(self, capsys):`
		`+ with pytest.raises(SystemExit):`
		`+ toddlers.plugins.pdc_import_compose.main([])`
		`+`
		`+ out, err = capsys.readouterr()`
		`+ assert out == ""`
		`+ # Expecting something along these lines, but don't make the test too tight:`
		`+ #`
		`+ # usage: pytest [-h] [--dry-run] [-q \| --debug] conf [username]`
		`+ # pytest: error: the following arguments are required: conf`
		`+ assert err.startswith("usage:")`
		`+ assert "error: the following arguments are required:" in err`
		`+`
		`+ @patch(`
		`+ "toddlers.plugins.pdc_import_compose.PDCImportCompose._process_single_compose"`
		`+ )`
		`+ @patch("toddlers.plugins.pdc_import_compose.pdc_client_for_config")`
		`+ @patch("toml.load", new=Mock(return_value={}))`
		`+ def test_main_debug(self, pdc, process_dg, msg_file, capsys):`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = ANY`
		`+ msg.topic = "toddlers.cli.pungi.compose.status.change"`
		`+ msg.body = {`
		`+ "status": "FINISHED",`
		`+ "release_type": "ga",`
		`+ "compose_label": "RC-1.2",`
		`+ "compose_respin": 0,`
		`+ "compose_date": "20201019",`
		`+ "release_short": "Fedora",`
		`+ "release_version": "33",`
		`+ "location": "https://kojipkgs.fedoraproject.org/compose/33/"`
		`+ "Fedora-33-20201019.0/compose",`
		`+ "compose_type": "production",`
		`+ "release_is_layered": False,`
		`+ "release_name": "Fedora",`
		`+ "compose_path": "/mnt/koji/compose/33/Fedora-33-20201019.0",`
		`+ "compose_id": "Fedora-33-20201019.0",`
		`+ }`
		`+`
		`+ toddlers.plugins.pdc_import_compose.main(["test.cfg", "--debug", msg_file])`
		`+ process_dg.assert_called_once_with({}, client, msg)`
		`+ out, err = capsys.readouterr()`
		`+ assert out == ""`
		`+ assert err == ""`
		`+`
		`+ @patch(`
		`+ "toddlers.plugins.pdc_import_compose.PDCImportCompose._process_single_compose"`
		`+ )`
		`+ @patch("toddlers.plugins.pdc_import_compose.pdc_client_for_config")`
		`+ @patch("toml.load", new=Mock(return_value={}))`
		`+ def test_main(self, pdc, process_dg, msg_file, capsys):`
		`+ client = Mock()`
		`+ pdc.return_value = client`
		`+`
		`+ msg = fedora_messaging.api.Message()`
		`+ msg.id = ANY`
		`+ msg.topic = "toddlers.cli.pungi.compose.status.change"`
		`+ msg.body = {`
		`+ "status": "FINISHED",`
		`+ "release_type": "ga",`
		`+ "compose_label": "RC-1.2",`
		`+ "compose_respin": 0,`
		`+ "compose_date": "20201019",`
		`+ "release_short": "Fedora",`
		`+ "release_version": "33",`
		`+ "location": "https://kojipkgs.fedoraproject.org/compose/33/"`
		`+ "Fedora-33-20201019.0/compose",`
		`+ "compose_type": "production",`
		`+ "release_is_layered": False,`
		`+ "release_name": "Fedora",`
		`+ "compose_path": "/mnt/koji/compose/33/Fedora-33-20201019.0",`
		`+ "compose_id": "Fedora-33-20201019.0",`
		`+ }`
		`+`
		`+ toddlers.plugins.pdc_import_compose.main(["test.cfg", msg_file])`
		`+ process_dg.assert_called_once_with({}, client, msg)`
		`+ out, err = capsys.readouterr()`
		`+ assert out == ""`
		`+ assert err == ""`

toddlers/plugins/pdc_import_compose.py

file modified

+93 -2

		`@@ -8,10 +8,15 @@`

		`"""`

		`+ import argparse`
		`+ import json`
		`import logging`
		`+ import sys`

		`import beanbag.bbexcept`
		`import bs4`
		`+ import fedora_messaging.api`
		`+ import toml`

		`from ..base import ToddlerBase`
		`from ..utils.pdc import pdc_client_for_config`
		`@@ -184,7 +189,9 @@`
		`}`
		`)`
		`except beanbag.bbexcept.BeanBagException as e:`
		`- _log.exception("Failed to import compose-images (BeenBagException): %s", url)`
		`+ _log.exception(`
		`+ "Failed to import compose-images (BeenBagException): %s", url`
		`+ )`
		`_log.error(e.msg)`
		`_log.error(e.response.url)`
		`_log.error(e.response.text)`
		`@@ -201,7 +208,9 @@`
		`}`
		`)`
		`except beanbag.bbexcept.BeanBagException as e:`
		`- _log.exception("Failed to import compose-images (BeenBagException): %s", url)`
		`+ _log.exception(`
		`+ "Failed to import compose-images (BeenBagException): %s", url`
		`+ )`
		`_log.error(e.msg)`
		`_log.error(e.response.url)`
		`_log.error(e.response.text)`
		`@@ -310,3 +319,85 @@`
		`_log.error(e.msg)`
		`_log.error(e.response.text)`
		`raise`
		`+`
		`+`
		`+ # We have had the situation in the past where we've had to check a specific`
		`+ # account, so the following code allows to run this script stand-alone if`
		`+ # needed.`
		`+`
		`+`
		`+ def setup_logging(log_level: int):`
		`+ handlers = []`
		`+`
		`+ _log.setLevel(log_level)`
		`+ # We want all messages logged at level INFO or lower to be printed to stdout`
		`+ info_handler = logging.StreamHandler(stream=sys.stdout)`
		`+ handlers.append(info_handler)`
		`+`
		`+ if log_level == logging.INFO:`
		`+ # In normal operation, don't decorate messages`
		`+ for handler in handlers:`
		`+ handler.setFormatter(logging.Formatter("%(message)s"))`
		`+`
		`+ logging.basicConfig(level=log_level, handlers=handlers)`
		`+`
		`+`
		`+ def get_arguments(args):`
		`+ """ Load and parse the CLI arguments."""`
		`+ parser = argparse.ArgumentParser(`
		`+ description="Manually import a compose into PDC via a file storing the "`
		`+ "JSON body of the message about the successful compose"`
		`+ )`
		`+ parser.add_argument(`
		`+ "conf",`
		`+ help="Configuration file",`
		`+ )`
		`+ parser.add_argument(`
		`+ "json_message",`
		`+ help="A text file with the body of the message to process in JSON",`
		`+ )`
		`+`
		`+ log_level_group = parser.add_mutually_exclusive_group()`
		`+ log_level_group.add_argument(`
		`+ "-q",`
		`+ "--quiet",`
		`+ action="store_const",`
		`+ dest="log_level",`
		`+ const=logging.WARNING,`
		`+ default=logging.INFO,`
		`+ help="Be less talkative",`
		`+ )`
		`+ log_level_group.add_argument(`
		`+ "--debug",`
		`+ action="store_const",`
		`+ dest="log_level",`
		`+ const=logging.DEBUG,`
		`+ help="Enable debugging output",`
		`+ )`
		`+`
		`+ return parser.parse_args(args)`
		`+`
		`+`
		`+ def main(args):`
		`+ """ Schedule the first test and run the scheduler. """`
		`+ args = get_arguments(args)`
		`+ setup_logging(log_level=args.log_level)`
		`+`
		`+ config = toml.load(args.conf)`
		`+`
		`+ with open(args.json_message) as stream:`
		`+ message = fedora_messaging.api.Message(`
		`+ body=json.load(stream), topic="toddlers.cli.pungi.compose.status.change"`
		`+ )`
		`+`
		`+ PDCImportCompose().process(`
		`+ config=config.get("consumer_config", {}).get("pdc_import_compose", {}),`
		`+ message=message,`
		`+ )`
		`+`
		`+`
		`+ if __name__ == "__main__": # pragma: no cover`
		`+ try:`
		`+ main(sys.argv[1:])`
		`+ except KeyboardInterrupt:`
		`+ pass`