PR#13: Misc: tests and default configuration - fedora-infra/toddlers

		`@@ -0,0 +1,31 @@`
		`+ import sys`
		`+ from unittest.mock import MagicMock`
		`+`
		`+ import pytest`
		`+`
		`+ from toddlers.utils.requests import make_session`
		`+`
		`+`
		`+ @pytest.fixture`
		`+ def toddler(request, monkeypatch):`
		`+ """Fixture creating a toddler for a class testing it`
		`+`
		+ The test class must set the toddler class in its `toddler_cls` attribute.
		+ The fixture will mock out the `make_session` function before creating the
		`+ object so that any request objects the toddler creates in its constructor`
		`+ won't be real.`
		`+ """`
		`+ test_cls = request.cls`
		`+ if not hasattr(test_cls, "toddler_cls"):`
		+ raise RuntimeError(f"{test_cls} needs to set `toddler_cls`")
		`+`
		`+ toddler_cls = test_cls.toddler_cls`
		`+ toddler_module = sys.modules[toddler_cls.__module__]`
		`+`
		`+ for name in dir(toddler_module):`
		`+ obj = getattr(toddler_module, name)`
		`+ if obj is make_session:`
		`+ monkeypatch.setattr(toddler_module, name, MagicMock())`
		`+`
		`+ toddler_obj = toddler_cls()`
		`+ return toddler_obj`

tests/plugins/test_debug.py

file modified

+8 -7

		`@@ -1,16 +1,17 @@`
		`import fedora_messaging.api`

		`- import toddlers.plugins.debug`
		`+ from toddlers.plugins.debug import DebugToddler`


		`class TestDebugToddler:`
		`- def test_accepts_topic(self):`
		`- assert toddlers.plugins.debug.DebugToddler.accepts_topic("foo.bar")`

		`- def test_process(self):`
		`+ toddler_cls = DebugToddler`
		`+`
		`+ def test_accepts_topic(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar")`
		`+`
		`+ def test_process(self, toddler):`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "toddlers.test.topic"`
		`- assert (`
		`- toddlers.plugins.debug.DebugToddler.process(config={}, message=msg) is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`

tests/plugins/test_flag_ci_pr.py

file modified

+33 -46

		`@@ -1,15 +1,18 @@`
		`import logging`
		`- from unittest.mock import MagicMock, patch`
		`+ from unittest.mock import MagicMock`

		`import fedora_messaging.api`
		`import pytest`

		`- import toddlers.plugins.flag_ci_pr`
		`+ from toddlers.plugins.flag_ci_pr import FlagCIPR`


		`class TestFlagCIPRToddler:`
		`- def test_accepts_topic_invalid(self):`
		`- assert toddlers.plugins.flag_ci_pr.FlagCIPR.accepts_topic("foo.bar") is False`
		`+`
		`+ toddler_cls = FlagCIPR`
		`+`
		`+ def test_accepts_topic_invalid(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is False`

		`@pytest.mark.parametrize(`
		`"topic",`
		`@@ -25,55 +28,47 @@`
		`"org.centos.stg.ci.dist-git-pr.test.running",`
		`],`
		`)`
		`- def test_accepts_topic_valid(self, topic):`
		`- assert toddlers.plugins.flag_ci_pr.FlagCIPR.accepts_topic(topic)`
		`+ def test_accepts_topic_valid(self, toddler, topic):`
		`+ assert toddler.accepts_topic(topic)`

		`- def test_process_invalid(self):`
		`+ def test_process_invalid(self, toddler):`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "toddlers.test.topic"`
		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config={}, message=msg) is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`

		`- def test_process_invalid_status(self, caplog):`
		`+ def test_process_invalid_status(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "org.centos.stg.ci.dist-git-pr.test.invalid"`
		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config={}, message=msg) is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Pipeline state is not 'complete' or 'running' or 'error'."`
		`)`

		`- def test_process_no_version(self, caplog):`
		`+ def test_process_no_version(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "org.centos.stg.ci.dist-git-pr.test.complete"`
		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config={}, message=msg) is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert caplog.records[-1].message == "Unsupported msg version, ignoring"`

		`- def test_process_invalid_complete_status(self, caplog):`
		`+ def test_process_invalid_complete_status(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "org.centos.stg.ci.dist-git-pr.test.complete"`
		`msg.body = {"version": "0.2.1", "test": {"result": "invalid"}}`
		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config={}, message=msg) is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Build is not in one of the expected status, ignoring"`
		`)`

		`- def test_process_no_artifact(self, caplog):`
		`+ def test_process_no_artifact(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -83,20 +78,18 @@`
		`"test": {"result": "passed"},`
		`"artifact": {"commit_hash": "abcdefghijklmnopqrst"},`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config={}, message=msg) is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Invalid message: {'commit_hash': 'abcdefghijklmnopqrst'}, could not extract "`
		`"the PR id from it"`
		`)`

		`- @patch("toddlers.plugins.flag_ci_pr.requests_session")`
		`- def test_process_request_failed(self, mock_requests, caplog):`
		`- mock_requests.request.return_value = MagicMock(`
		`+ def test_process_request_failed(self, toddler, caplog):`
		`+ toddler.requests_session.request.return_value = MagicMock(`
		`ok=False, status_code=401, text="invalid"`
		`)`
		`+`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -112,24 +105,21 @@`
		`"run": {"url": "https://example.com/testing"},`
		`}`
		`config = {`
		`- "pagure_token_seed": "example_seed",`
		`- "pagure_url": "https://pagure.io",`
		`- "pagure_token": "ahah",`
		`+ "dist_git_token_seed": "example_seed",`
		`+ "dist_git_url": "https://pagure.io",`
		`+ "dist_git_token": "ahah",`
		`}`

		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config=config, message=msg)`
		`- == 1`
		`- )`
		`+ assert toddler.process(config=config, message=msg) == 1`
		`assert (`
		`caplog.records[-1].message == "Request to https://pagure.io returned: 401"`
		`)`

		`- @patch("toddlers.plugins.flag_ci_pr.requests_session")`
		`- def test_process_valid(self, mock_requests, caplog):`
		`- mock_requests.request.return_value = MagicMock(`
		`+ def test_process_valid(self, toddler, caplog):`
		`+ toddler.requests_session.request.return_value = MagicMock(`
		`ok=True, status_code=200, text="invalid"`
		`)`
		`+`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -145,15 +135,12 @@`
		`"run": {"url": "https://example.com/testing"},`
		`}`
		`config = {`
		`- "pagure_token_seed": "example_seed",`
		`- "pagure_url": "https://pagure.io",`
		`- "pagure_token": "ahah",`
		`+ "dist_git_token_seed": "example_seed",`
		`+ "dist_git_url": "https://pagure.io",`
		`+ "dist_git_token": "ahah",`
		`}`

		`- assert (`
		`- toddlers.plugins.flag_ci_pr.FlagCIPR.process(config=config, message=msg)`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert (`
		`caplog.records[-3].message == "Request to https://pagure.io returned: 200"`
		`)`

tests/plugins/test_flag_commit_build.py

file modified

+48 -94

		`@@ -4,15 +4,15 @@`
		`import fedora_messaging.api`
		`import pytest`

		`- import toddlers.plugins.flag_commit_build`
		`+ from toddlers.plugins.flag_commit_build import FlagCommitBuild`


		`class TestFlagCommitBuildToddler:`
		`- def test_accepts_topic_invalid(self):`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.accepts_topic("foo.bar")`
		`- is False`
		`- )`
		`+`
		`+ toddler_cls = FlagCommitBuild`
		`+`
		`+ def test_accepts_topic_invalid(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is False`

		`@pytest.mark.parametrize(`
		`"topic",`
		`@@ -22,38 +22,28 @@`
		`"org.fedoraproject.prod.buildsys.build.state.change",`
		`],`
		`)`
		`- def test_accepts_topic_valid(self, topic):`
		`- assert toddlers.plugins.flag_commit_build.FlagCommitBuild.accepts_topic(topic)`
		`+ def test_accepts_topic_valid(self, toddler, topic):`
		`+ assert toddler.accepts_topic(topic)`

		`- def test_process_containerbuild(self, caplog):`
		`+ def test_process_containerbuild(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "org.fedoraproject.prod.buildsys.build.state.change"`
		`msg.body = {"owner": "containerbuild"}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config={}, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert caplog.records[-1].message == "Skipping container build"`

		`- def test_process_mbs_build(self, caplog):`
		`+ def test_process_mbs_build(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`msg.topic = "org.fedoraproject.prod.buildsys.build.state.change"`
		`msg.body = {"owner": "mbs/mbs.fedoraproject.org"}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config={}, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert caplog.records[-1].message == "Skipping MBS builds"`

		`- def test_process_secondary_instance(self, caplog):`
		`+ def test_process_secondary_instance(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -62,15 +52,10 @@`
		`"owner": "username",`
		`"instance": "secondary",`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config={}, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert caplog.records[-1].message == "Ignoring secondary arch task..."`

		`- def test_process_uninteresting_status(self, caplog):`
		`+ def test_process_uninteresting_status(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -80,19 +65,14 @@`
		`"instance": "primary",`
		`"new": 2,`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config={}, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config={}, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Build is not in a state we care about, skipping"`
		`)`

		`@patch("toddlers.plugins.flag_commit_build.koji")`
		`- def test_process_no_git_url(self, mock_koji, caplog):`
		`+ def test_process_no_git_url(self, mock_koji, toddler, caplog):`
		`client = Mock()`
		`client.getBuild = Mock(return_value={})`
		`mock_koji.ClientSession = Mock(return_value=client)`
		`@@ -107,19 +87,14 @@`
		`"build_id": 42,`
		`}`
		`config = {"koji_url": "https://koji.fedoraproject.org"}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config=config, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "No git url found in the extra information: None"`
		`)`

		`@patch("toddlers.plugins.flag_commit_build.koji")`
		`- def test_process_invalid_git_url(self, mock_koji, caplog):`
		`+ def test_process_invalid_git_url(self, mock_koji, toddler, caplog):`
		`client = Mock()`
		`client.getBuild = Mock(`
		`return_value={"extra": {"source": {"original_url": "foobar"}}}`
		`@@ -136,20 +111,15 @@`
		`"build_id": 42,`
		`}`
		`config = {"koji_url": "https://koji.fedoraproject.org"}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config=config, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert caplog.records[-1].message == "No # in the git_url: foobar"`

		`- @patch("toddlers.plugins.flag_commit_build.requests_session")`
		`@patch("toddlers.plugins.flag_commit_build.koji")`
		`- def test_process_flag_no_task_id(self, mock_koji, mock_requests, caplog):`
		`- mock_requests.request.return_value = MagicMock(`
		`+ def test_process_flag_no_task_id(self, mock_koji, toddler, caplog):`
		`+ toddler.requests_session.request.return_value = MagicMock(`
		`ok=False, status_code=401, text="invalid"`
		`)`
		`+`
		`client = Mock()`
		`client.getBuild = Mock(`
		`return_value={`
		`@@ -162,6 +132,7 @@`
		`}`
		`)`
		`mock_koji.ClientSession = Mock(return_value=client)`
		`+`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -179,26 +150,21 @@`
		`}`
		`config = {`
		`"koji_url": "https://koji.fedoraproject.org",`
		`- "pagure_url": "https://src.fedoraproject.org",`
		`- "pagure_token": "ahah",`
		`+ "dist_git_url": "https://src.fedoraproject.org",`
		`+ "dist_git_token": "ahah",`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config=config, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Request to https://src.fedoraproject.org returned: 401"`
		`)`

		`- @patch("toddlers.plugins.flag_commit_build.requests_session")`
		`@patch("toddlers.plugins.flag_commit_build.koji")`
		`- def test_process_flag_failed(self, mock_koji, mock_requests, caplog):`
		`- mock_requests.request.return_value = MagicMock(`
		`+ def test_process_flag_failed(self, mock_koji, toddler, caplog):`
		`+ toddler.requests_session.request.return_value = MagicMock(`
		`ok=False, status_code=401, text="invalid"`
		`)`
		`+`
		`client = Mock()`
		`client.getBuild = Mock(`
		`return_value={`
		`@@ -211,6 +177,7 @@`
		`}`
		`)`
		`mock_koji.ClientSession = Mock(return_value=client)`
		`+`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -228,26 +195,21 @@`
		`}`
		`config = {`
		`"koji_url": "https://koji.fedoraproject.org",`
		`- "pagure_url": "https://src.fedoraproject.org",`
		`- "pagure_token": "ahah",`
		`+ "dist_git_url": "https://src.fedoraproject.org",`
		`+ "dist_git_token": "ahah",`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config=config, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Request to https://src.fedoraproject.org returned: 401"`
		`)`

		`- @patch("toddlers.plugins.flag_commit_build.requests_session")`
		`@patch("toddlers.plugins.flag_commit_build.koji")`
		`- def test_process_flag_failed_cancel(self, mock_koji, mock_requests, caplog):`
		`- mock_requests.request.return_value = MagicMock(`
		`+ def test_process_flag_failed_cancel(self, mock_koji, toddler, caplog):`
		`+ toddler.requests_session.request.return_value = MagicMock(`
		`ok=False, status_code=401, text="invalid"`
		`)`
		`+`
		`client = Mock()`
		`client.getBuild = Mock(`
		`return_value={`
		`@@ -260,6 +222,7 @@`
		`}`
		`)`
		`mock_koji.ClientSession = Mock(return_value=client)`
		`+`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -277,26 +240,21 @@`
		`}`
		`config = {`
		`"koji_url": "https://koji.fedoraproject.org",`
		`- "pagure_url": "https://src.fedoraproject.org",`
		`- "pagure_token": "ahah",`
		`+ "dist_git_url": "https://src.fedoraproject.org",`
		`+ "dist_git_token": "ahah",`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config=config, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert (`
		`caplog.records[-1].message`
		`== "Request to https://src.fedoraproject.org returned: 401"`
		`)`

		`- @patch("toddlers.plugins.flag_commit_build.requests_session")`
		`@patch("toddlers.plugins.flag_commit_build.koji")`
		`- def test_process_flag_pass(self, mock_koji, mock_requests, caplog):`
		`- mock_requests.request.return_value = MagicMock(`
		`+ def test_process_flag_pass(self, mock_koji, toddler, caplog):`
		`+ toddler.requests_session.request.return_value = MagicMock(`
		`ok=True, status_code=200, text="woohoo!"`
		`)`
		`+`
		`client = Mock()`
		`client.getBuild = Mock(`
		`return_value={`
		`@@ -309,6 +267,7 @@`
		`}`
		`)`
		`mock_koji.ClientSession = Mock(return_value=client)`
		`+`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -325,15 +284,10 @@`
		`}`
		`config = {`
		`"koji_url": "https://koji.fedoraproject.org",`
		`- "pagure_url": "https://src.fedoraproject.org",`
		`- "pagure_token": "ahah",`
		`+ "dist_git_url": "https://src.fedoraproject.org",`
		`+ "dist_git_token": "ahah",`
		`}`
		`- assert (`
		`- toddlers.plugins.flag_commit_build.FlagCommitBuild.process(`
		`- config=config, message=msg`
		`- )`
		`- is None`
		`- )`
		`+ assert toddler.process(config=config, message=msg) is None`
		`assert (`
		`caplog.records[-3].message`
		`== "Request to https://src.fedoraproject.org returned: 200"`

tests/plugins/test_packager_bugzilla_sync.py

file modified

+25 -25

		`@@ -2,17 +2,15 @@`

		`import pytest`

		`- import toddlers.plugins.packager_bugzilla_sync`
		`+ from toddlers.plugins.packager_bugzilla_sync import main, PackagerBugzillaSync`


		`class TestPackagerBugzillaSyncToddler:`
		`- def test_accepts_topic_invalid(self):`
		`- assert (`
		`- toddlers.plugins.packager_bugzilla_sync.PackagerBugzillaSync.accepts_topic(`
		`- "foo.bar"`
		`- )`
		`- is False`
		`- )`
		`+`
		`+ toddler_cls = PackagerBugzillaSync`
		`+`
		`+ def test_accepts_topic_invalid(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is False`

		`@pytest.mark.parametrize(`
		`"topic",`
		`@@ -22,26 +20,22 @@`
		`"org.fedoraproject.stg.toddlers.trigger.packager_bugzilla_sync",`
		`],`
		`)`
		`- def test_accepts_topic_valid(self, topic):`
		`- assert toddlers.plugins.packager_bugzilla_sync.PackagerBugzillaSync.accepts_topic(`
		`- topic`
		`- )`
		`+ def test_accepts_topic_valid(self, toddler, topic):`
		`+ assert toddler.accepts_topic(topic)`

		`- def test_process_no_email_override(self, capsys):`
		`+ def test_process_no_email_override(self, toddler, capsys):`
		`with pytest.raises(KeyError, match=r"'email_overrides_file'"):`
		`- toddlers.plugins.packager_bugzilla_sync.PackagerBugzillaSync.process(`
		`- config={}, message=None, username=False, dry_run=True`
		`- )`
		`+ toddler.process(config={}, message=None, username=False, dry_run=True)`

		`out, err = capsys.readouterr()`
		`assert out == "Failed to load the file containing the email-overrides\n"`
		`assert err == ""`

		`- def test_process_no_email_override_file(self, capsys):`
		`+ def test_process_no_email_override_file(self, toddler, capsys):`
		`with pytest.raises(`
		`FileNotFoundError, match=r"No such file or directory: 'test'"`
		`):`
		`- toddlers.plugins.packager_bugzilla_sync.PackagerBugzillaSync.process(`
		`+ toddler.process(`
		`config={"email_overrides_file": "test"},`
		`message=None,`
		`username=False,`
		`@@ -60,14 +54,20 @@`
		`@patch("toddlers.utils.bugzilla_system.add_user_to_group")`
		`@patch("toml.load")`
		`def test_process(`
		`- self, toml_load, bz_user_grp, get_bz_grp_mbr, get_bz_email, get_fas_grp_mbr`
		`+ self,`
		`+ toml_load,`
		`+ bz_user_grp,`
		`+ get_bz_grp_mbr,`
		`+ get_bz_email,`
		`+ get_fas_grp_mbr,`
		`+ toddler,`
		`):`
		`toml_load.return_value = {}`
		`get_fas_grp_mbr.return_value = ["pingou", "nils"]`
		`get_bz_email.side_effect = ["pingou@fp.o", "nils@fp.o"]`
		`get_bz_grp_mbr.return_value = ["pingou@fp.o", "nphilipp@fp.o"]`

		`- toddlers.plugins.packager_bugzilla_sync.PackagerBugzillaSync.process(`
		`+ toddler.process(`
		`config={"email_overrides_file": "test", "bugzilla_group": "fedora_contrib"},`
		`message=None,`
		`username=False,`
		`@@ -92,13 +92,13 @@`
		`@patch("toddlers.utils.bugzilla_system.add_user_to_group")`
		`@patch("toml.load")`
		`def test_process_username(`
		`- self, toml_load, bz_user_grp, get_bz_grp_mbr, get_bz_email`
		`+ self, toml_load, bz_user_grp, get_bz_grp_mbr, get_bz_email, toddler`
		`):`
		`toml_load.return_value = {}`
		`get_bz_email.side_effect = ["nils@fp.o"]`
		`get_bz_grp_mbr.return_value = ["pingou@fp.o"]`

		`- toddlers.plugins.packager_bugzilla_sync.PackagerBugzillaSync.process(`
		`+ toddler.process(`
		`config={"email_overrides_file": "test", "bugzilla_group": "fedora_contrib"},`
		`message=None,`
		`username="nils",`
		`@@ -117,7 +117,7 @@`

		`def test_main_no_args(self, capsys):`
		`with pytest.raises(SystemExit):`
		`- toddlers.plugins.packager_bugzilla_sync.main([])`
		`+ main([])`

		`out, err = capsys.readouterr()`
		`assert out == ""`
		`@@ -131,7 +131,7 @@`
		`@patch("toml.load", new=Mock(return_value={}))`
		`def test_main_debug(self, capsys):`
		`with pytest.raises(KeyError, match=r"'email_overrides_file'"):`
		`- toddlers.plugins.packager_bugzilla_sync.main(["test.cfg", "--debug"])`
		`+ main(["test.cfg", "--debug"])`
		`out, err = capsys.readouterr()`
		`assert out == "Failed to load the file containing the email-overrides\n"`
		`assert err == ""`
		`@@ -139,7 +139,7 @@`
		`@patch("toml.load", new=Mock(return_value={}))`
		`def test_main(self, capsys):`
		`with pytest.raises(KeyError, match=r"'email_overrides_file'"):`
		`- toddlers.plugins.packager_bugzilla_sync.main(["test.cfg"])`
		`+ main(["test.cfg"])`
		`out, err = capsys.readouterr()`
		`assert out == "Failed to load the file containing the email-overrides\n"`
		`assert err == ""`

tests/plugins/test_pdc_retired_packages.py

file modified

+55 -81

		`@@ -9,13 +9,11 @@`


		`class TestPDCRetiredPackagesToddler:`
		`- def test_accepts_topic_invalid(self):`
		`- assert (`
		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.accepts_topic(`
		`- "foo.bar"`
		`- )`
		`- is False`
		`- )`
		`+`
		`+ toddler_cls = toddlers.plugins.pdc_retired_packages.PDCRetiredPackages`
		`+`
		`+ def test_accepts_topic_invalid(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is False`

		`@pytest.mark.parametrize(`
		`"topic",`
		`@@ -28,14 +26,12 @@`
		`"org.fedoraproject.stg.git.receive",`
		`],`
		`)`
		`- def test_accepts_topic_valid(self, topic):`
		`- assert toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.accepts_topic(`
		`- topic`
		`- )`
		`+ def test_accepts_topic_valid(self, toddler, topic):`
		`+ assert toddler.accepts_topic(topic)`

		`@patch("toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git")`
		`- @patch("pdc_client.PDCClient")`
		`- def test_process_full_dist_git(self, pdc, process_dg):`
		`+ @patch("toddlers.plugins.pdc_retired_packages.pdc_client_for_config")`
		`+ def test_process_full_dist_git(self, pdc, process_dg, toddler):`
		`client = Mock()`
		`pdc.return_value = client`

		`@@ -44,17 +40,15 @@`
		`msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages"`
		`msg.body = {}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.process(`
		`- {"config": "foobar"}, msg`
		`- )`
		`+ toddler.process({"config": "foobar"}, msg)`

		`process_dg.assert_called_once_with({"config": "foobar"}, client)`

		`@patch(`
		`"toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package"`
		`)`
		`- @patch("pdc_client.PDCClient")`
		`- def test_process_single_package(self, pdc, process_dg):`
		`+ @patch("toddlers.plugins.pdc_retired_packages.pdc_client_for_config")`
		`+ def test_process_single_package(self, pdc, process_dg, toddler):`
		`client = Mock()`
		`pdc.return_value = client`

		`@@ -63,14 +57,12 @@`
		`msg.topic = "org.fedoraproject.prod.git.receive"`
		`msg.body = {}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages.process(`
		`- {"config": "foobar"}, msg`
		`- )`
		`+ toddler.process({"config": "foobar"}, msg)`

		`process_dg.assert_called_once_with({"config": "foobar"}, client, msg)`

		`- @patch("toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git")`
		`- def test_process_dist_git(self, retired_in_dg):`
		`+ def test_process_dist_git(self, toddler):`
		`+ retired_in_dg = toddler._is_retired_in_dist_git = MagicMock()`
		`page_component_branches = [`
		`{`
		`"id": 44,`
		`@@ -116,9 +108,7 @@`
		`msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages"`
		`msg.body = {}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git(`
		`- {}, client`
		`- )`
		`+ toddler._process_dist_git({}, client)`

		`client.get_paged.assert_has_calls(calls=[call(page_component_branches)])`
		`retired_in_dg.assert_has_calls(`
		`@@ -132,8 +122,8 @@`
		`]`
		`)`

		`- @patch("toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git")`
		`- def test_process_dist_git_invalid_pdc_namespace(self, retired_in_dg, caplog):`
		`+ def test_process_dist_git_invalid_pdc_namespace(self, caplog, toddler):`
		`+ retired_in_dg = toddler._is_retired_in_dist_git = MagicMock()`
		`caplog.set_level(logging.DEBUG)`
		`page_component_branches = [`
		`{`
		`@@ -180,9 +170,7 @@`
		`msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages"`
		`msg.body = {}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git(`
		`- {}, client`
		`- )`
		`+ toddler._process_dist_git({}, client)`

		`client.get_paged.assert_has_calls(calls=[call(page_component_branches)])`
		`retired_in_dg.assert_has_calls(`
		`@@ -199,9 +187,9 @@`
		`in caplog.text`
		`)`

		`- @patch("toddlers.plugins.pdc_retired_packages._retire_branch")`
		`- @patch("toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git")`
		`- def test_process_dist_git_full_distgit(self, retired_in_dg, retire_branch):`
		`+ def test_process_dist_git_full_distgit(self, toddler):`
		`+ retired_in_dg = toddler._is_retired_in_dist_git = MagicMock()`
		`+ retire_branch = toddler._retire_branch = MagicMock()`
		`page_component_branches = [`
		`{`
		`"id": 44,`
		`@@ -247,9 +235,7 @@`
		`msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_retired_packages"`
		`msg.body = {}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_dist_git(`
		`- {}, client`
		`- )`
		`+ toddler._process_dist_git({}, client)`

		`client.get_paged.assert_has_calls(calls=[call(page_component_branches)])`
		`retired_in_dg.assert_has_calls(`
		`@@ -310,7 +296,7 @@`
		`]`
		`)`

		`- def test__process_single_package_regular_commit(self, caplog):`
		`+ def test__process_single_package_regular_commit(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`client = MagicMock()`

		`@@ -328,13 +314,11 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`- {}, client, msg`
		`- )`
		`+ toddler._process_single_package({}, client, msg)`

		`assert caplog.records[-1].message == "No dead.package in the commit, bailing"`

		`- def test__process_single_package_un_retirement(self, caplog):`
		`+ def test__process_single_package_un_retirement(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`client = MagicMock()`

		`@@ -353,13 +337,11 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`- {}, client, msg`
		`- )`
		`+ toddler._process_single_package({}, client, msg)`

		`assert caplog.records[-1].message == "dead.package file was not added, bailing"`

		`- def test__process_single_package_incomplete_config(self, caplog):`
		`+ def test__process_single_package_incomplete_config(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`client = MagicMock()`

		`@@ -381,13 +363,11 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`- {}, client, msg`
		`- )`
		`+ toddler._process_single_package({}, client, msg)`

		`assert caplog.records[-1].message == "No check URL configured, ignoring"`

		`- def test__process_single_package_package_not_in_pdc(self, caplog):`
		`+ def test__process_single_package_package_not_in_pdc(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`

		`page_component_branches = {`
		`@@ -414,7 +394,7 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`+ toddler._process_single_package(`
		`{`
		`"file_check_url": "https://src.fedoraproject.org/"`
		`"%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s",`
		`@@ -425,7 +405,7 @@`

		`assert caplog.records[-1].message == '"rpms/0ad" was not found in PDC'`

		`- def test__process_single_package_namespace_not_in_pdc(self, caplog):`
		`+ def test__process_single_package_namespace_not_in_pdc(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`

		`page_component_branches = {`
		`@@ -452,7 +432,7 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`+ toddler._process_single_package(`
		`{`
		`"file_check_url": "https://src.fedoraproject.org/"`
		`"%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s",`
		`@@ -466,7 +446,7 @@`
		`== "Ignoring namespace 'flatpack', error: The namespace \"flatpack\" is not supported"`
		`)`

		`- def test__process_single_package_package_inactive(self, caplog):`
		`+ def test__process_single_package_package_inactive(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`

		`page_component_branches = {`
		`@@ -504,7 +484,7 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`+ toddler._process_single_package(`
		`{`
		`"file_check_url": "https://src.fedoraproject.org/"`
		`"%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s",`
		`@@ -515,12 +495,11 @@`

		`assert caplog.records[-1].message == '"rpms/0ad" not active in PDC in master'`

		`- @patch("requests.head")`
		`- def test__process_single_package_package_not_retired(self, req, caplog):`
		`+ def test__process_single_package_package_not_retired(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`resp = Mock()`
		`resp.status_code = 404`
		`- req.return_value = resp`
		`+ toddler.requests_session.head.return_value = resp`

		`page_component_branches = {`
		`"count": 1,`
		`@@ -557,7 +536,7 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`+ toddler._process_single_package(`
		`{`
		`"file_check_url": "https://src.fedoraproject.org/"`
		`"%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s",`
		`@@ -571,13 +550,12 @@`
		`== "Seems not to actually be retired, possibly a merge commit?"`
		`)`

		`- @patch("toddlers.plugins.pdc_retired_packages._retire_branch")`
		`- @patch("requests.head")`
		`- def test__process_single_package(self, req, retire, caplog):`
		`+ def test__process_single_package(self, toddler, caplog):`
		`+ toddler._retire_branch = MagicMock()`
		`caplog.set_level(logging.INFO)`
		`resp = Mock()`
		`resp.status_code = 200`
		`- req.return_value = resp`
		`+ toddler.requests_session.head.return_value = resp`

		`page_component_branches = {`
		`"count": 1,`
		`@@ -614,7 +592,7 @@`
		`}`
		`}`

		`- toddlers.plugins.pdc_retired_packages.PDCRetiredPackages._process_single_package(`
		`+ toddler._process_single_package(`
		`{`
		`"file_check_url": "https://src.fedoraproject.org/"`
		`"%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s",`
		`@@ -623,40 +601,36 @@`
		`msg,`
		`)`

		`- retire.assert_called_once_with(client, page_component_branches["results"][0])`
		`+ toddler._retire_branch.assert_called_once_with(`
		`+ client, page_component_branches["results"][0]`
		`+ )`

		`- @patch("toddlers.plugins.pdc_retired_packages.requests_session.head")`
		`- def test__is_retired_in_dist_git(self, req):`
		`+ def test__is_retired_in_dist_git(self, toddler):`
		`resp = Mock()`
		`resp.status_code = 200`
		`- req.return_value = resp`
		`+ toddler.requests_session.head.return_value = resp`

		`- assert toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git(`
		`- "rpms", "guake", "f33"`
		`- )`
		`- req.assert_called_once_with(`
		`+ assert toddler._is_retired_in_dist_git("rpms", "guake", "f33")`
		`+ toddler.requests_session.head.assert_called_once_with(`
		`"https://src.fedoraproject.org//rpms/guake/raw/f33/f/dead.package"`
		`)`

		`- @patch("toddlers.plugins.pdc_retired_packages.requests_session.head")`
		`- def test__is_retired_in_dist_git_internal_error(self, req):`
		`+ def test__is_retired_in_dist_git_internal_error(self, toddler):`
		`resp = Mock()`
		`resp.status_code = 500`
		`- req.return_value = resp`
		`+ toddler.requests_session.head.return_value = resp`

		`with pytest.raises(`
		`ValueError,`
		`match=r"The connection to dist_git failed. Retirement status could "`
		`"not be determined. The status code was: 500. The content was: .*",`
		`):`
		`- toddlers.plugins.pdc_retired_packages._is_retired_in_dist_git(`
		`- "rpms", "guake", "f33"`
		`- )`
		`- req.assert_called_once_with(`
		`+ toddler._is_retired_in_dist_git("rpms", "guake", "f33")`
		`+ toddler.requests_session.head.assert_called_once_with(`
		`"https://src.fedoraproject.org//rpms/guake/raw/f33/f/dead.package"`
		`)`

		`- def test_retire_branch(self, caplog):`
		`+ def test_retire_branch(self, toddler, caplog):`
		`caplog.set_level(logging.INFO)`
		`today = datetime.datetime.utcnow().date()`
		`eol = (datetime.datetime.utcnow() + datetime.timedelta(days=60)).date()`
		`@@ -671,7 +645,7 @@`
		`"critical_path": False,`
		`}`

		`- toddlers.plugins.pdc_retired_packages._retire_branch(pdc, branch)`
		`+ toddler._retire_branch(pdc, branch)`

		`assert (`
		`caplog.records[-1].message`

tests/plugins/test_plugins.py

file modified

+8 -14

		`@@ -2,17 +2,11 @@`


		`def test_toddlers_plugins():`
		`- variables = [var for var in dir(toddlers.plugins) if not var.startswith("__")]`
		`- assert sorted(variables) == sorted(`
		`- [`
		`- "debug",`
		`- "flag_ci_pr",`
		`- "flag_commit_build",`
		`- "here",`
		`- "importlib",`
		`- "name",`
		`- "os",`
		`- "packager_bugzilla_sync",`
		`- "pdc_retired_packages",`
		`- ]`
		`- )`
		`+ plugin_module_names = toddlers.plugins.__all__`
		`+ assert not any(n.startswith("__") for n in plugin_module_names)`
		`+ assert {`
		`+ "flag_ci_pr",`
		`+ "flag_commit_build",`
		`+ "packager_bugzilla_sync",`
		`+ "pdc_retired_packages",`
		`+ } <= set(plugin_module_names)`

tests/test_base.py

file modified

+12 -9

		`@@ -1,15 +1,18 @@`
		`- import toddlers.base`
		`+ from toddlers.base import ToddlerBase`


		`class TestToddlerBase:`
		`- def test_name(self):`
		`- assert toddlers.base.ToddlerBase.name.fget() == "base"`

		`- def test_amqp_topics(self):`
		`- assert toddlers.base.ToddlerBase.amqp_topics.fget() == []`
		`+ toddler_cls = ToddlerBase`

		`- def test_accepts_topic(self):`
		`- assert toddlers.base.ToddlerBase.accepts_topic("foo.bar") is None`
		`+ def test_name(self, toddler):`
		`+ assert toddler.name == "base"`

		`- def test_process(self):`
		`- assert toddlers.base.ToddlerBase.process(config={}, message={}) is None`
		`+ def test_amqp_topics(self, toddler):`
		`+ assert toddler.amqp_topics == []`
		`+`
		`+ def test_accepts_topic(self, toddler):`
		`+ assert toddler.accepts_topic("foo.bar") is None`
		`+`
		`+ def test_process(self, toddler):`
		`+ assert toddler.process(config={}, message={}) is None`

tests/test_runner.py

file modified

+38 -26

		`@@ -6,21 +6,33 @@`
		`import pytest`

		`import toddlers.runner`
		`+ from toddlers.utils.misc import merge_dicts`
		`+`
		`+`
		`+ def patch_messaging_config(*add_dicts):`
		`+ mock_config = {`
		`+ "consumer_config": {"blocked_toddlers": ["debug"], "toddlers": {}},`
		`+ }`
		`+ for add_dict in add_dicts:`
		`+ mock_config = merge_dicts(mock_config, add_dict)`
		`+`
		`+ return patch.dict("toddlers.runner.fedora_messaging.config.conf", mock_config)`


		`class TestRunningToddler:`
		`+ @patch_messaging_config()`
		`def test___init__(self):`
		`runner = toddlers.runner.RunningToddler()`
		`- assert sorted([t.name for t in runner.toddlers]) == sorted(`
		`- [`
		`- "debug",`
		`- "flag_ci_pr",`
		`- "flag_commit_build",`
		`- "packager_bugzilla_sync",`
		`- "pdc_retired_packages",`
		`- ]`
		`- )`
		`+ # Check that at least these toddlers exist (but don't fail`
		`+ # automatically just because new ones are added).`
		`+ assert {`
		`+ "flag_ci_pr",`
		`+ "flag_commit_build",`
		`+ "packager_bugzilla_sync",`
		`+ "pdc_retired_packages",`
		`+ } <= {t.name for t in runner.toddlers}`

		`+ @patch_messaging_config()`
		`@patch("toddlers.runner.ToddlerBase")`
		`def test___init__no_toddlers(self, mock_base, caplog):`
		`mock_base.__subclasses__ = Mock(return_value=[])`
		`@@ -29,29 +41,28 @@`
		`fedora_messaging.exceptions.ConfigurationException,`
		`match=r".* No toddlers found to run .*",`
		`):`
		`- runner = toddlers.runner.RunningToddler()`
		`- assert sorted([t.name for t in runner.toddlers]) == sorted(`
		`- [`
		`- "debug",`
		`- "flag_ci_pr",`
		`- "flag_commit_build",`
		`- "packager_bugzilla_sync",`
		`- "pdc_retired_packages",`
		`- ]`
		`- )`
		`+ toddlers.runner.RunningToddler()`
		`assert caplog.records[-1].message == "Loaded: []"`

		`- @patch.dict(`
		`- "toddlers.runner.fedora_messaging.config.conf",`
		`- {"consumer_config": {"blocked_toddlers": ["debug", "flag_ci_pr"]}},`
		`+ @patch_messaging_config(`
		`+ {"consumer_config": {"blocked_toddlers": ["debug", "flag_ci_pr"]}}`
		`)`
		`def test___init__blockedlist(self, caplog):`
		`+ blocked_toddlers = {"debug", "flag_ci_pr"}`
		`+ expected_toddlers = {`
		`+ "flag_commit_build",`
		`+ "packager_bugzilla_sync",`
		`+ "pdc_retired_packages",`
		`+ }`
		`+`
		`runner = toddlers.runner.RunningToddler()`
		`- assert sorted([t.name for t in runner.toddlers]) == sorted(`
		`- ["flag_commit_build", "packager_bugzilla_sync", "pdc_retired_packages"]`
		`- )`
		`+ running_toddlers = {t.name for t in runner.toddlers}`
		`+`
		`+ assert not running_toddlers & blocked_toddlers`
		`+ assert running_toddlers & expected_toddlers == expected_toddlers`

		`- def test___call__(self, caplog):`
		`+ @patch_messaging_config({"consumer_config": {"blocked_toddlers": []}})`
		`+ def test___call__debug(self, caplog):`
		`caplog.set_level(logging.INFO)`
		`msg = fedora_messaging.api.Message()`
		`msg.id = 123`
		`@@ -69,6 +80,7 @@`
		`== "Toddler 'debug' accepted to process message id: 123"`
		`)`

		`+ @patch_messaging_config({"consumer_config": {"blocked_toddlers": []}})`
		`@patch(`
		`"toddlers.plugins.debug.DebugToddler.process",`
		`new=Mock(side_effect=Exception("haha")),`

toddlers.toml.example

file modified

+45 -33

		`@@ -45,32 +45,25 @@`
		`# more of them.`
		`blocked_toddlers = ["debug"]`

		`-`
		`- [consumer_config.flag_ci_pr]`
		`- # flag_ci_pr`
		`- pagure_token_seed = "private random string to change"`
		`- pagure_token = "private random string to change"`
		`- pagure_url = "https://src.fedoraproject.org"`
		`-`
		`-`
		`- [consumer_config.flag_commit_build]`
		`- # flag_commit_build`
		`- pagure_token = "private random string to change"`
		`- pagure_url = "https://src.fedoraproject.org"`
		`- koji_url = "https://koji.fedoraproject.org"`
		`-`
		`-`
		`- [consumer_config.packager_bugzilla_sync]`
		`- # Configuration file storing all the email overrides in the form of:`
		`- # "foo@bar.com" = "bar@foo.org"`
		`- # This is the same format as used by the distgit_bugzilla_sync cron/app`
		`- email_overrides_file = "/path/to/email_overrides.toml"`
		`-`
		`- # Base url of dist-git`
		`- dist_git_url = "https://src.fedoraproject.org"`
		`-`
		`- # List of accounts we do not want to report about`
		`- ignorable_accounts = ["packagerbot", "zuul"]`
		`+ [consumer_config.default]`
		`+ # Configuration common to all toddlers.`
		`+ #`
		`+ # You can override any of these in the section of a particular toddler, e.g.:`
		`+ #`
		`+ # [consumer_config.default]`
		`+ # somekey = "somevalue"`
		`+ #`
		`+ # [consumer_config.default.subsection]`
		`+ # key1 = "value1"`
		`+ # key2 = "value2"`
		`+ #`
		`+ # ...`
		`+ #`
		`+ # [consumer_config.sometoddler]`
		`+ # somekey = "someothervalue"`
		`+ #`
		`+ # [consumer_config.sometoddler.subsection]`
		`+ # key2 = "othervalue2"`

		`# Configuration used when sending notifications:`
		`mail_server = "bastion.fedoraproject.org"`
		`@@ -87,10 +80,18 @@`
		`bugzilla_password = "Bugzilla password for that account"`
		`bugzilla_group = "fedora_contrib"`

		`+ # Base URL for the Koji build system`
		`+ koji_url = "https://koji.fedoraproject.org"`

		`- [consumer_config.pdc_retired_packages]`
		`- file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s"`
		`+ # Account to use to connect to Pagure-as-dist-git`
		`+ dist_git_url = "https://src.fedoraproject.org"`
		`+ dist_git_token_seed = "private random string to change"`
		`+ dist_git_token = "private random string to change"`

		`+ [consumer_config.default.pdc_config]`
		`+ # Configuration to talk to PDC, as understood by pdc-client.`
		`+ server = "https://pdc.fedoraproject.org/rest_api/v1/"`
		`+ ssl_verify = false # Enable if using a self-signed cert`
		`# XXX - getting the token is a bit of a pain, but here's a walk through`
		`# 1) go to https://pdc.fedoraproject.org/ in your browser and login.`
		`# 2) go to https://pdc.fedoraproject.org/rest_api/v1/auth/token/obtain/`
		`@@ -100,13 +101,24 @@`
		`# 6) before hitting enter, edit the command to add the following option`
		`# -H 'Accept: application/json' # to tell the API you want data`
		`# 7) the command should print out your token.`
		`-`
		`- # Credentials to talk to PDC`
		`- [consumer_config.pdc_retired_packages.pdc_config]`
		`- server = "https://pdc.fedoraproject.org/rest_api/v1/"`
		`- ssl_verify = false # Enable if using a self-signed cert`
		`token = "PUT_HERE_THE_TOKEN_OBTAINED_MANUALLY"`

		`+ [consumer_config.flag_ci_pr]`
		`+`
		`+ [consumer_config.flag_commit_build]`
		`+`
		`+ [consumer_config.packager_bugzilla_sync]`
		`+ # Configuration file storing all the email overrides in the form of:`
		`+ # "foo@bar.com" = "bar@foo.org"`
		`+ # This is the same format as used by the distgit_bugzilla_sync cron/app`
		`+ email_overrides_file = "/path/to/email_overrides.toml"`
		`+`
		`+ # List of accounts we do not want to report about`
		`+ ignorable_accounts = ["packagerbot", "zuul"]`
		`+`
		`+ [consumer_config.pdc_retired_packages]`
		`+ file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s"`
		`+`
		`[qos]`
		`prefetch_size = 0`
		`prefetch_count = 25`

toddlers/base.py

file modified

+4 -4

		`@@ -6,13 +6,13 @@`

		`@property`
		`@abc.abstractmethod`
		`- def name():`
		`+ def name(self):`
		`"""Returns name of the plugin."""`
		`return "base"`

		`@property`
		`@abc.abstractmethod`
		`- def amqp_topics():`
		`+ def amqp_topics(self):`
		`"""Returns the list of topics of interest for this toddler in a format`
		`that can be used directly when connecting to amqp.`
		`For example, it supports items like:`
		`@@ -24,13 +24,13 @@`
		`return []`

		`@abc.abstractmethod`
		`- def accepts_topic(topic):`
		`+ def accepts_topic(self, topic):`
		`"""Returns a boolean whether this toddler is interested in messages`
		`from this specific topic.`
		`"""`
		`return`

		`@abc.abstractmethod`
		`- def process(config, message):`
		`+ def process(self, config, message):`
		`"""Process a given message."""`
		`return`

toddlers/plugins/__init__.py

file modified

		`@@ -2,8 +2,11 @@`
		`import os`


		`+ __all__ = [] # fill below`
		`+`
		`here = os.path.abspath(os.path.dirname(__file__))`
		`for name in os.listdir(here):`
		`if name.endswith(".py") and not name.startswith("__"):`
		`name = name[:-3]`
		`importlib.import_module(f".{name}", package=__name__)`
		`+ __all__.append(name)`

toddlers/plugins/debug.py

file modified

+2 -2

		`@@ -24,13 +24,13 @@`

		`amqp_topics = ["#"]`

		`- def accepts_topic(topic):`
		`+ def accepts_topic(self, topic):`
		`"""Returns a boolean whether this toddler is interested in messages`
		`from this specific topic.`
		`"""`
		`return True`

		`- def process(config, message):`
		`+ def process(self, config, message):`
		`"""Process a given message."""`
		`print(f"Debug Toddler received message: id:{message.id}, topic:{message.topic}")`
		`print("Debug Toddler - Processing done")`

toddlers/plugins/flag_ci_pr.py

file modified

+14 -22

		`@@ -10,24 +10,12 @@`
		`import hashlib`
		`import logging`

		`- import requests`
		`- from requests.packages.urllib3.util import retry`
		`-`
		`from ..base import ToddlerBase`
		`+ from ..utils.requests import make_session`


		`_log = logging.getLogger(__name__)`

		`- timeout = (30, 30)`
		`- retries = 3`
		`- requests_session = requests.Session()`
		`- retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1)`
		`- retry_conf.BACKOFF_MAX = 5`
		`- requests_session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retry_conf))`
		`- requests_session.mount(`
		`- "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)`
		`- )`
		`-`
		`done_states = {`
		`"passed": {"api": "success", "human": "passed"},`
		`"error": {"api": "error", "human": "errored"},`
		`@@ -49,7 +37,10 @@`
		`"org.centos.*.ci.dist-git-pr.test.running",`
		`]`

		`- def accepts_topic(topic):`
		`+ def __init__(self):`
		`+ self.requests_session = make_session()`
		`+`
		`+ def accepts_topic(self, topic):`
		`"""Returns a boolean whether this toddler is interested in messages`
		`from this specific topic.`
		`"""`
		`@@ -59,7 +50,7 @@`
		`and topic.endswith(("complete", "error", "running"))`
		`)`

		`- def process(config, message):`
		`+ def process(self, config, message):`
		`"""Process a given message."""`

		`pipeline_state = None`
		`@@ -106,7 +97,7 @@`
		`pr_id = str(msg["artifact"]["id"])`

		`commit_hash_text = " for %s" % msg["artifact"]["commit_hash"][:8]`
		`- seed = config.get("pagure_token_seed") or ""`
		`+ seed = config.get("dist_git_token_seed", config.get("pagure_token_seed")) or ""`

		`data = {`
		`"username": "Fedora CI",`
		`@@ -116,7 +107,7 @@`
		`"uid": hashlib.md5((pr_id + seed).encode("utf-8")).hexdigest(),`
		`}`

		`- pagure_url = config["pagure_url"]`
		`+ dist_git_url = config.get("dist_git_url") or config["pagure_url"]`

		`namespace = msg["artifact"]["repository"].split("/")[-2]`
		`repo = msg["artifact"]["repository"].split("/")[-1]`
		`@@ -124,20 +115,21 @@`
		`target_url = "/".join(`
		`["api", "0", namespace, repo, "pull-request", pr_id, "flag"]`
		`)`
		`- flag_url = pagure_url + "/" + target_url`
		`+ flag_url = dist_git_url + "/" + target_url`
		`_log.info("Flagging commit at: %s" % flag_url)`

		`headers = {`
		`- "Authorization": "token " + config["pagure_token"],`
		`+ "Authorization": "token " + config.get("dist_git_token")`
		`+ or config["pagure_token"],`
		`"User-Agent": "loopabull@fedora-infra",`
		`}`

		`_log.info("payload: %s" % data)`

		`- req = requests_session.request(`
		`- method="POST", url=flag_url, headers=headers, data=data,`
		`+ req = self.requests_session.request(`
		`+ method="POST", url=flag_url, headers=headers, data=data`
		`)`
		`- _log.info("Request to %s returned: %s" % (pagure_url, req.status_code))`
		`+ _log.info("Request to %s returned: %s" % (dist_git_url, req.status_code))`
		`if not req.ok:`
		`_log.debug(req.text)`
		`return 1`

toddlers/plugins/flag_commit_build.py

file modified

+14 -21

		`@@ -10,24 +10,13 @@`
		`import logging`

		`import koji`
		`- import requests`
		`- from requests.packages.urllib3.util import retry`

		`from ..base import ToddlerBase`
		`+ from ..utils.requests import make_session`


		`_log = logging.getLogger(__name__)`

		`- timeout = (30, 30)`
		`- retries = 3`
		`- requests_session = requests.Session()`
		`- retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1)`
		`- retry_conf.BACKOFF_MAX = 5`
		`- requests_session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retry_conf))`
		`- requests_session.mount(`
		`- "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)`
		`- )`
		`-`
		`# see koji.TASK_STATES for all values`
		`done_states = {`
		`1: "completed",`
		`@@ -56,7 +45,10 @@`
		`"org.fedoraproject.*.buildsys.build.state.change",`
		`]`

		`- def accepts_topic(topic):`
		`+ def __init__(self):`
		`+ self.requests_session = make_session()`
		`+`
		`+ def accepts_topic(self, topic):`
		`"""Returns a boolean whether this toddler is interested in messages`
		`from this specific topic.`
		`"""`
		`@@ -64,7 +56,7 @@`
		`"buildsys.build.state.change"`
		`)`

		`- def process(config, message):`
		`+ def process(self, config, message):`
		`"""Process a given message."""`

		`msg = message.body`
		`@@ -127,23 +119,24 @@`
		`"url": _koji_link(config, msg),`
		`}`

		`- pagure_url = config["pagure_url"]`
		`+ dist_git_url = config.get("dist_git_url") or config["pagure_url"]`

		`target_url = "/".join(["api", "0", "rpms", msg["name"], "c", commit, "flag"])`
		`- flag_url = pagure_url + "/" + target_url`
		`+ flag_url = dist_git_url + "/" + target_url`
		`_log.info("Flagging commit at: %s" % flag_url)`

		`headers = {`
		`- "Authorization": "token " + config["pagure_token"],`
		`+ "Authorization": "token " + config.get("dist_git_token")`
		`+ or config["pagure_token"],`
		`"User-Agent": "toddlers@fedora-infra",`
		`}`

		`_log.debug("payload: %s" % data)`

		`- req = requests_session.request(`
		`- method="POST", url=flag_url, headers=headers, data=data,`
		`+ req = self.requests_session.request(`
		`+ method="POST", url=flag_url, headers=headers, data=data`
		`)`
		`- _log.info("Request to %s returned: %s" % (pagure_url, req.status_code))`
		`+ _log.info("Request to %s returned: %s" % (dist_git_url, req.status_code))`
		`if not req.ok:`
		`_log.debug(" -- FAILED -- ")`
		`_log.debug(req.text)`
		`@@ -151,7 +144,7 @@`
		`else:`
		`_log.info("All clear")`
		`_log.info(`
		`- "User-URL: %s" % pagure_url`
		`+ "User-URL: %s" % dist_git_url`
		`+ "/"`
		`+ "/".join(["rpms", msg["name"], "c", commit])`
		`)`

toddlers/plugins/packager_bugzilla_sync.py

file modified

+3 -15

		`@@ -12,8 +12,6 @@`
		`import logging`
		`import sys`

		`- import requests`
		`- from requests.packages.urllib3.util import retry`
		`import toml`

		`try:`
		`@@ -28,16 +26,6 @@`

		`_log = logging.getLogger(__name__)`

		`- timeout = (30, 30)`
		`- retries = 3`
		`- requests_session = requests.Session()`
		`- retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1)`
		`- retry_conf.BACKOFF_MAX = 5`
		`- requests_session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retry_conf))`
		`- requests_session.mount(`
		`- "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)`
		`- )`
		`-`

		`class PackagerBugzillaSync(ToddlerBase):`
		`""" Listens to messages sent by playtime (which lives in toddlers) to sync`
		`@@ -50,7 +38,7 @@`
		`"org.fedoraproject.*.toddlers.trigger.packager_bugzilla_sync",`
		`]`

		`- def accepts_topic(topic):`
		`+ def accepts_topic(self, topic):`
		`"""Returns a boolean whether this toddler is interested in messages`
		`from this specific topic.`
		`"""`
		`@@ -58,7 +46,7 @@`
		`"toddlers.trigger.packager_bugzilla_sync"`
		`)`

		`- def process(config, message, username=False, dry_run=False):`
		`+ def process(self, config, message, username=False, dry_run=False):`
		`"""Process a given message."""`

		`try:`
		`@@ -206,7 +194,7 @@`
		`setup_logging(log_level=args.log_level)`

		`config = toml.load(args.conf)`
		`- PackagerBugzillaSync.process(`
		`+ PackagerBugzillaSync().process(`
		`config=config.get("consumer_config", {}).get("packager_bugzilla_sync", {}),`
		`message={},`
		`username=args.username,`

toddlers/plugins/pdc_retired_packages.py

file modified

+67 -80

		`@@ -11,24 +11,12 @@`
		`from datetime import datetime`
		`import logging`

		`- import pdc_client`
		`- import requests`
		`- from requests.packages.urllib3.util import retry`
		`-`
		`- from toddlers.base import ToddlerBase`
		`+ from ..base import ToddlerBase`
		`+ from ..utils.pdc import pdc_client_for_config`
		`+ from ..utils.requests import make_session`

		`_log = logging.getLogger(__name__)`

		`- timeout = (30, 30)`
		`- retries = 3`
		`- requests_session = requests.Session()`
		`- retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1)`
		`- retry_conf.BACKOFF_MAX = 5`
		`- requests_session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retry_conf))`
		`- requests_session.mount(`
		`- "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)`
		`- )`
		`-`

		`class PDCRetiredPackages(ToddlerBase):`
		`""" Listens to messages sent by playtime (which lives in toddlers) to sync`
		`@@ -42,7 +30,10 @@`
		`"org.fedoraproject.*.git.receive",`
		`]`

		`- def accepts_topic(topic):`
		`+ def __init__(self):`
		`+ self.requests_session = make_session()`
		`+`
		`+ def accepts_topic(self, topic):`
		`"""Returns a boolean whether this toddler is interested in messages`
		`from this specific topic.`
		`"""`
		`@@ -50,16 +41,16 @@`
		`("toddlers.trigger.pdc_retired_packages", "git.receive")`
		`)`

		`- def process(config, message):`
		`+ def process(self, config, message):`
		`"""Process a given message."""`
		`- pdc = pdc_client.PDCClient(**config.get("pdc_config", {}))`
		`+ pdc = pdc_client_for_config(config)`

		`if message.topic.endswith("git.receive"):`
		`PDCRetiredPackages._process_single_package(config, pdc, message)`
		`else:`
		`PDCRetiredPackages._process_dist_git(config, pdc)`

		`- def _process_dist_git(config, pdc):`
		`+ def _process_dist_git(self, config, pdc):`
		`""" Updates PDC retirement status from analyzing dist-git.`

		`This steps over all the branches in dist-git and retires any branches`
		`@@ -71,8 +62,8 @@`
		`_log.debug("Considering {0}".format(branch_str))`
		`retired_in_dist_git = False`
		`try:`
		`- retired_in_dist_git = _is_retired_in_dist_git(`
		`- namespace=_pdc_to_namespace(branch["type"]),`
		`+ retired_in_dist_git = self._is_retired_in_dist_git(`
		`+ namespace=self._pdc_to_namespace(branch["type"]),`
		`repo=branch["global_component"],`
		`branch=branch["name"],`
		`)`
		`@@ -81,9 +72,9 @@`
		`continue`

		`if retired_in_dist_git:`
		`- _retire_branch(pdc, branch)`
		`+ self._retire_branch(pdc, branch)`

		`- def _process_single_package(config, pdc, message):`
		`+ def _process_single_package(self, config, pdc, message):`
		`"""Handle an incoming bus message.`

		`The message should be a dist-git retirement message (where someone adds`
		`@@ -110,7 +101,7 @@`
		`repo = msg["commit"]["repo"]`
		`namespace = msg["commit"]["namespace"]`
		`try:`
		`- component_type = _namespace_to_pdc(namespace)`
		`+ component_type = self._namespace_to_pdc(namespace)`
		`except ValueError as err:`
		`_log.info("Ignoring namespace '%s', error: %s", namespace, err)`
		`return`
		`@@ -150,66 +141,62 @@`
		`"file": "dead.package",`
		`}`
		`_log.info("Checking for file: %s", fileurl)`
		`- resp = requests.head(fileurl, timeout=15)`
		`+ resp = self.requests_session.head(fileurl, timeout=15)`
		`if resp.status_code != 200:`
		`_log.info("Seems not to actually be retired, possibly a merge commit?")`
		`return`

		`- _retire_branch(pdc, branch)`
		`-`
		`-`
		`- def _namespace_to_pdc(namespace):`
		`- """ Internal method to translate a dist-git namespace to a PDC`
		`- component type. """`
		`- namespace_to_pdc = {`
		`- "rpms": "rpm",`
		`- "modules": "module",`
		`- "container": "container",`
		`- }`
		`- if namespace not in namespace_to_pdc:`
		`- raise ValueError('The namespace "{0}" is not supported'.format(namespace))`
		`- else:`
		`- return namespace_to_pdc[namespace]`
		`-`
		`-`
		`- def _pdc_to_namespace(pdc_type):`
		`- """ Internal method to translate a PDC component type to a dist-git`
		`- namespace. """`
		`- pdc_to_namespace = {`
		`- "rpm": "rpms",`
		`- "module": "modules",`
		`- "container": "container",`
		`- }`
		`- if pdc_type not in pdc_to_namespace:`
		`- raise ValueError('The PDC type "{0}" is not supported'.format(pdc_type))`
		`- else:`
		`- return pdc_to_namespace[pdc_type]`
		`-`
		`-`
		`- def _is_retired_in_dist_git(namespace, repo, branch):`
		`- base = "https://src.fedoraproject.org/"`
		`- # Check to see if they have a dead.package file in dist-git`
		`- url = "{base}/{namespace}/{repo}/raw/{branch}/f/dead.package"`
		`- response = requests_session.head(`
		`- url.format(base=base, namespace=namespace, repo=repo, branch=branch,)`
		`- )`
		`-`
		`- # If there is a dead.package, then the branch is retired in dist_git`
		`- if response.status_code in [200, 404]:`
		`- return response.status_code == 200`
		`- else:`
		`- raise ValueError(`
		`- "The connection to dist_git failed. Retirement status could not "`
		`- "be determined. The status code was: {0}. The content was: "`
		`- "{1}".format(response.status_code, response.content)`
		`+ self._retire_branch(pdc, branch)`
		`+`
		`+ def _namespace_to_pdc(self, namespace):`
		`+ """ Internal method to translate a dist-git namespace to a PDC`
		`+ component type. """`
		`+ namespace_to_pdc = {`
		`+ "rpms": "rpm",`
		`+ "modules": "module",`
		`+ "container": "container",`
		`+ }`
		`+ if namespace not in namespace_to_pdc:`
		`+ raise ValueError('The namespace "{0}" is not supported'.format(namespace))`
		`+ else:`
		`+ return namespace_to_pdc[namespace]`
		`+`
		`+ def _pdc_to_namespace(self, pdc_type):`
		`+ """ Internal method to translate a PDC component type to a dist-git`
		`+ namespace. """`
		`+ pdc_to_namespace = {`
		`+ "rpm": "rpms",`
		`+ "module": "modules",`
		`+ "container": "container",`
		`+ }`
		`+ if pdc_type not in pdc_to_namespace:`
		`+ raise ValueError('The PDC type "{0}" is not supported'.format(pdc_type))`
		`+ else:`
		`+ return pdc_to_namespace[pdc_type]`
		`+`
		`+ def _is_retired_in_dist_git(self, namespace, repo, branch):`
		`+ base = "https://src.fedoraproject.org/"`
		`+ # Check to see if they have a dead.package file in dist-git`
		`+ url = "{base}/{namespace}/{repo}/raw/{branch}/f/dead.package"`
		`+ response = self.requests_session.head(`
		`+ url.format(base=base, namespace=namespace, repo=repo, branch=branch)`
		`)`

		`+ # If there is a dead.package, then the branch is retired in dist_git`
		`+ if response.status_code in [200, 404]:`
		`+ return response.status_code == 200`
		`+ else:`
		`+ raise ValueError(`
		`+ "The connection to dist_git failed. Retirement status could not "`
		`+ "be determined. The status code was: {0}. The content was: "`
		`+ "{1}".format(response.status_code, response.content)`
		`+ )`

		`- def _retire_branch(pdc, branch):`
		`- """ Internal method for retiring a branch in PDC. """`
		`- today = datetime.utcnow().date()`
		`- for sla in branch["slas"]:`
		`- sla_eol = datetime.strptime(sla["eol"], "%Y-%m-%d").date()`
		`- if sla_eol > today:`
		`- _log.info("Setting eol of branch: %s to %s", branch, today)`
		`- pdc["component-branch-slas"][sla["id"]]._ += {"eol": str(today)}`
		`+ def _retire_branch(self, pdc, branch):`
		`+ """ Internal method for retiring a branch in PDC. """`
		`+ today = datetime.utcnow().date()`
		`+ for sla in branch["slas"]:`
		`+ sla_eol = datetime.strptime(sla["eol"], "%Y-%m-%d").date()`
		`+ if sla_eol > today:`
		`+ _log.info("Setting eol of branch: %s to %s", branch, today)`
		`+ pdc["component-branch-slas"][sla["id"]]._ += {"eol": str(today)}`

toddlers/runner.py

file modified

+21 -7

		`@@ -5,6 +5,7 @@`

		`from . import plugins # noqa: F401`
		`from .base import ToddlerBase`
		`+ from .utils.misc import merge_dicts`


		`_log = logging.getLogger(__name__)`
		`@@ -32,13 +33,22 @@`
		`if toddler.name in blocked_toddlers:`
		`_log.info("Toddler '%s' is blocked, skipping it", toddler.name)`
		`continue`
		`- self.toddlers.append(toddler)`
		`+ self.toddlers.append(toddler())`
		`_log.info("Loaded: %s", self.toddlers)`
		`if not self.toddlers:`
		`raise fedora_messaging.exceptions.ConfigurationException(`
		`"No toddlers found to run :("`
		`)`

		`+ self.toddler_config = {}`
		`+ for toddler in self.toddlers:`
		`+ name = toddler.name`
		`+ # Bail out if two toddlers have the same name.`
		`+ if name in self.toddler_config:`
		`+ raise ValueError(f"Toddler named {name} already exists.")`
		`+`
		`+ self.toddler_config[name] = self._get_toddler_config(name)`
		`+`
		`topics = set()`
		`for toddler in self.toddlers:`
		`_log.debug(`
		`@@ -57,6 +67,15 @@`
		`)`
		`binding["routing_keys"] = topics`

		`+ def _get_toddler_config(self, toddler_name: str) -> dict:`
		`+ """Merge default and private configuration for a toddler."""`
		`+ base_config = fedora_messaging.config.conf["consumer_config"]`
		`+`
		`+ default_config = base_config.get("default", {})`
		`+ private_config = base_config.get(toddler_name, {})`
		`+`
		`+ return merge_dicts(default_config, private_config)`
		`+`
		`def __call__(self, message):`
		`"""`
		`Invoked when a message is received by the consumer.`
		`@@ -74,12 +93,7 @@`
		`toddler.name,`
		`message.id,`
		`)`
		`- toddler.process(`
		`- fedora_messaging.config.conf["consumer_config"].get(`
		`- toddler.name`
		`- ),`
		`- message,`
		`- )`
		`+ toddler.process(self.toddler_config[toddler.name], message)`
		`except Exception:`
		`_log.exception(`
		`"Toddler '%s' failed to process message id: %s -- putting it back in the queue",`

toddlers/utils/bugzilla_system.py

file modified

+1 -1

		`@@ -80,7 +80,7 @@`
		`raise`


		`- def user_exists(user_email: str,):`
		`+ def user_exists(user_email: str):`
		`""" Returns a boolean specifying if the given user exists in bugzilla or not. """`

		`server = get_bz()`

toddlers/utils/misc.py

file added

+26

		`@@ -0,0 +1,26 @@`
		`+ def merge_dicts(d1: dict, d2: dict) -> dict:`
		`+ """Merge nested dictionaries in depth.`
		`+`
		`+ :param d1: First dictionary to merge`
		`+ :param d2: Second dictionary to merge, takes precedence over the first`
		`+ :return: A dictionary merged from d1 and d2`
		`+ """`
		`+ d3 = {}`
		`+ d1keys = set(d1)`
		`+ d2keys = set(d2)`
		`+`
		`+ for k in d1keys - d2keys:`
		`+ d3[k] = d1[k]`
		`+`
		`+ for k in d2keys - d1keys:`
		`+ d3[k] = d2[k]`
		`+`
		`+ for k in d1keys & d2keys:`
		`+ v1 = d1[k]`
		`+ v2 = d2[k]`
		`+ if isinstance(v1, dict) and isinstance(v2, dict):`
		`+ d3[k] = merge_dicts(v1, v2)`
		`+ else:`
		`+ d3[k] = v2`
		`+`
		`+ return d3`

toddlers/utils/pdc.py

file added

		`@@ -0,0 +1,8 @@`
		`+ from pdc_client import PDCClient`
		`+`
		`+`
		`+ __all__ = ["pdc_client_for_config"]`
		`+`
		`+`
		`+ def pdc_client_for_config(config):`
		`+ return PDCClient(**config.get("pdc_config", {}))`

toddlers/utils/requests.py

file added

+83

		`@@ -0,0 +1,83 @@`
		`+ """Create requests session objects with defaults"""`
		`+`
		`+ from typing import Optional, Tuple, Union`
		`+`
		`+ import requests`
		`+ from requests.packages.urllib3.util import retry`
		`+`
		`+`
		`+ __all__ = ["make_session"]`
		`+`
		`+ RETRY_DEFAULTS = {`
		`+ "total": 3,`
		`+ "backoff_factor": 1,`
		`+ "BACKOFF_MAX": 5,`
		`+ }`
		`+`
		`+ TIMEOUT_DEFAULT = 5`
		`+`
		`+`
		`+ class _UnsetSentinel:`
		`+ pass`
		`+`
		`+`
		`+ _UNSET = _UnsetSentinel()`
		`+`
		`+`
		`+ class TimeoutHTTPAdapter(requests.adapters.HTTPAdapter):`
		`+ """A requests adapter with default timeout values"""`
		`+`
		`+ def __init__(`
		`+ self,`
		`+ *args: list,`
		`+ timeout: Optional[Union[Tuple[float], float, _UnsetSentinel]] = _UNSET,`
		`+ **kwargs: dict,`
		`+ ):`
		`+ if timeout is not _UNSET:`
		`+ self.timeout = timeout`
		`+ else:`
		`+ self.timeout = TIMEOUT_DEFAULT`
		`+ super().__init__(args, *kwargs)`
		`+`
		`+ def send(self, request: requests.Request, **kwargs: dict) -> requests.Response:`
		`+ if "timeout" not in kwargs:`
		`+ kwargs["timeout"] = self.timeout`
		`+ return super().send(request, **kwargs)`
		`+`
		`+`
		`+ def make_session(`
		`+ timeout: Optional[Union[Tuple[float], float, _UnsetSentinel]] = _UNSET,`
		`+ retry_conf: dict = None,`
		`+ ) -> requests.Session:`
		`+ """Make a requests session object`
		`+`
		`+ Args:`
		`+ timeout: a default timeout value`
		`+ retry_conf: retry configuration settings, its keys/values will be`
		`+ passed as arguments to the constructor of`
		+ :py:class:`requests.packages.urllib3.util.retry.Retry`, or (in the
		+ case of ``BACKOFF_MAX``), set as an attribute on the created retry
		`+ configuration object.`
		`+ Return:`
		`+ a session object`
		`+ """`
		`+ if not retry_conf:`
		`+ retry_conf = {}`
		`+ else:`
		`+ retry_conf = retry_conf.copy()`
		`+`
		`+ for k, v in RETRY_DEFAULTS.items():`
		`+ retry_conf.setdefault(k, v)`
		`+`
		`+ BACKOFF_MAX = retry_conf.pop("BACKOFF_MAX")`
		`+`
		`+ retry_conf_obj = retry.Retry(**retry_conf)`
		`+ retry_conf_obj.BACKOFF_MAX = BACKOFF_MAX`
		`+`
		`+ http_adapter = TimeoutHTTPAdapter(timeout=timeout, max_retries=retry_conf_obj)`
		`+`
		`+ session = requests.Session()`
		`+ for prefix in ("http://", "https://"):`
		`+ session.mount(prefix, http_adapter)`
		`+`
		`+ return session`

nphilipp commented 3 years ago

This contains some fixes for ~~the runner~~ some tests and introduces default configuration which can be shared between and overridden in toddlers.

zuul commented 3 years ago

Build failed.

tox : FAILURE in 3m 40s

pingou commented on line 53 of tests/test_runner.py 3 years ago

why the _ =?

pingou commented on line 10 of toddlers.toml.example 3 years ago

I'm not sure we need to put the config inside a toddlers section, I think putting it directly in consumer_config.* should be fine.

pingou commented 3 years ago

Looks like black is not happy in zuul

rebased onto 8fb78e622ea261db8f24d7f54864353a6772c060

3 years ago

zuul commented 3 years ago

Build failed.

tox : FAILURE in 3m 33s

rebased onto 5c9baae5c697929afa95e4c1ff93bf2c33f7fd31

3 years ago

nphilipp commented on line 53 of tests/test_runner.py 3 years ago

I could also just call toddlers.runner.RunningToddler(), assigning it to _ makes it clear that it's purposefully thrown away. What do you prefer?

zuul commented 3 years ago

Build failed.

tox : FAILURE in 3m 24s

nphilipp commented on line 10 of toddlers.toml.example 3 years ago

Hah, that came from my initial idea of putting default configuration directly into consumer_config, not in a default block, and then running haywire with doing things cleanly. I'll move it back.

rebased onto cad7d7c958dc467e3484859c336df32c4dd7bcb1

3 years ago

nphilipp commented 3 years ago

The latest rebase fixes a test failing because a bug was fixed in the meantime and should take care of black, too. BTW, what do you think of allowing lines of 100 characters for black? Would cause some churn when introducing it.

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 26s

4 new commits added

Tests: Don't trip over added toddlers
Fix test to follow suit with bug fix
Introduce config defaults, valid for all toddlers
Tests: Make tweaking messaging config easier

3 years ago

nphilipp commented 3 years ago

And this one gets rid of the toddlers section in configuration and relaxes some tests so we don't have to update them when adding new toddlers.

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 23s

rebased onto 50e7040e34c6aa2a31b060fb059ba2a66d71af90

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 16s

nphilipp commented 3 years ago

Latest one is rebased on latest changes (#14) and resolved conflicts and inconsistencies.

pingou commented 3 years ago

I could also just call toddlers.runner.RunningToddler(), assigning it to _ makes it clear that it's purposefully thrown away. What do you prefer?

I think dropping the _ is just as simple, whatever is returned we do not handle.

pingou commented on line 6 of toddlers/plugins/flag_ci_pr.py 3 years ago

Hm, this is not backward compatible so we'll need to be careful to update the configuration file in ansible before we roll out these changes in production

nphilipp commented on line 6 of toddlers/plugins/flag_ci_pr.py 3 years ago

Let me make it compatible then. We can leave pagure_... in for a while and remove when Ansible has followed suit. If, at some point, we want configuration for pagure.io, we could use pagure_io_... as a prefix.

rebased onto 759a5c559888a28e8feed1242ecf7d1af81a1204

3 years ago

nphilipp commented 3 years ago

The latest rebase removes the _ = construct and falls back to the previous pagure_* configuration keys if dist_git_* keys are missing from the configuration (in the toddlers that used them previously).

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 09s

2 new commits added

Use one global requests session object
Use relative imports within toddlers consistently

3 years ago

zuul commented 3 years ago

Build failed.

tox : FAILURE in 3m 16s

6 new commits added

Remove trailing comma in argument list
Utils: Use one global requests session object
Use relative imports within toddlers consistently
Tests: Don't trip over added toddlers
Introduce config defaults, valid for all toddlers
Tests: Make tweaking messaging config easier

3 years ago

pingou commented on line 17 of toddlers/utils/requests.py 3 years ago

it may be less change, more explicit and backward compatible to name it request_session (then we're sure it won't conflict with a koji session or a fas session that could also be named session).

zuul commented 3 years ago

Build failed.

tox : FAILURE in 2m 55s

2 new commits added

Utils: Add and use helper to create PDC objects
Add pre-configured dogpile cache regions

3 years ago

zuul commented 3 years ago

Build failed.

tox : FAILURE in 3m 27s

8 new commits added

Utils: Add and use helper to create PDC objects
Add pre-configured dogpile cache regions
Remove trailing comma in argument list
Utils: Use one global requests session object
Use relative imports within toddlers consistently
Tests: Don't trip over added toddlers
Introduce config defaults, valid for all toddlers
Tests: Make tweaking messaging config easier

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 22s

8 new commits added

Utils: Add and use helper to create PDC objects
Add pre-configured dogpile cache regions
Remove trailing comma in argument list
Utils: Use one global requests session object
Use relative imports within toddlers consistently
Tests: Don't trip over added toddlers
Introduce config defaults, valid for all toddlers
Tests: Make tweaking messaging config easier

3 years ago

nphilipp commented 3 years ago

The latest push uses requests_session again and fixes the wrong import order tripping up the tests.

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 4m 46s

pingou commented on line 12 of toddlers/utils/cache.py 3 years ago

This file doesn't seem to be used at the moment, pushed by accident?

rebased onto 45e43a98948fe5a985f2e2266940e08d03ffec59

3 years ago

nphilipp commented 3 years ago

This file doesn't seem to be used at the moment, pushed by accident?

Yeah, I planned to use it for something but meanwhile noticed we can create all long-lived resources in the constructors of the toddler objects instead. If we actually instantiate objects, which we seem to have gotten away without so far... :stuck_out_tongue_winking_eye:

The latest push remedies this and uses a requests session factory we discussed before, where we can just override retry, timeout defaults for the toddlers that need different values.

Edited 3 years ago by nphilipp

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 4m 51s

rebased onto 3f081a6

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 27s

pingou commented 3 years ago

So:
- https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/3f081a66ac5a9482991d1bd02c1ff036c45cbd24 no problem
- for https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/1960eafcf4197d23ca51bd9bbef8fbeed7df66aa why not doing a self.toddler = toddlers.plugins...() in the setUp()? It would avoid relying on fixture for this and avoid having to specify it to every test function definition
- https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/27c6484e7abcaddfafb2726fce3a9575826d3e4c ok, I'd specify in the docstring that the values of the second dict take precedence over the values in the first dict
- https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/5501d533d8253ba74860265b934d19af81baa8d5 I'd raise an exception instead of relying on the assert so we can provide a more descriptive error message (in toddlers/runner.py)
- https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/46785fc93a58aa013dabef80be2d10a4b736e6f9 all good for me
- https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/81a7faaefa894dc00c4fa3df039b3c9aa634466d all good for me

Edited 3 years ago by pingou

8 new commits added

Utils: Add and use helper to create PDC objects
Use factory for requests session objects
Use relative imports within toddlers consistently
Tests: Don't trip over added toddlers
Introduce config defaults, valid for all toddlers
Tests: Make tweaking messaging config easier
Using real objects is considered polite
Remove trailing comma in argument list

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 19s

8 new commits added

Utils: Add and use helper to create PDC objects
Use factory for requests session objects
Use relative imports within toddlers consistently
Tests: Don't trip over added toddlers
Introduce config defaults, valid for all toddlers
Tests: Make tweaking messaging config easier
Using real objects is considered polite
Remove trailing comma in argument list

3 years ago

zuul commented 3 years ago

Build succeeded.

tox : SUCCESS in 3m 10s

nphilipp commented 3 years ago

for https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/1960eafcf4197d23ca51bd9bbef8fbeed7df66aa why not doing a self.toddler = toddlers.plugins...() in the setUp()? It would avoid relying on fixture for this and avoid having to specify it to every test function definition

The reason I used a fixture and not a create the object in setUp() or setup_method() is that by listing the fixture in the list of arguments it is clear a method uses it. If the fixture isn't use, the toddler object doesn't have to be created, cf. e.g. the test_main*() methods in tests/plugins/test_packager_bugzilla_sync.py.

Having the fixture implemented for each test module again bugged me a little, so I've written a generic version (in tests/conftest.py) which only requires that toddler_cls is set in the test class using it to work.

https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/27c6484e7abcaddfafb2726fce3a9575826d3e4c ok, I'd specify in the docstring that the values of the second dict take precedence over the values in the first dict

Good catch, done.

https://pagure.io/fork/nphilipp/fedora-infra/toddlers/c/5501d533d8253ba74860265b934d19af81baa8d5 I'd raise an exception instead of relying on the assert so we can provide a more descriptive error message (in toddlers/runner.py)

Agreed, assert shouldn't be used for this, because it could be optimized out.

Edited 3 years ago by nphilipp

pingou commented on line 17 of toddlers/utils/requests.py 3 years ago

Hm, looks like we've lost setting the timeout values

nphilipp commented 3 years ago

We didn't lose it, it was never there to begin with. All the files set a variable timeout which is never used... I'll add it to make_session().

1 new commit added

Support default timeouts in requests sessions

3 years ago

nphilipp commented 3 years ago

@pingou I've added the changes we discussed as its own commit. As you prefer it, either squash it into its corresponding commit d53bd2e, or merge as is.

zuul commented 3 years ago

Build succeeded.