#178 Add new toddler for syncing Pagure groups with FAS
Merged 10 months ago by zlopez. Opened 2 years ago by zlopez.
fedora-infra/ zlopez/toddlers sync_fas_group_membership  into  main

@@ -0,0 +1,18 @@ 

+ # Sync Groups From FAS to Pagure

+ 

+ Toddler for synchronizing FAS groups with Pagure groups was created from https://pagure.io/pagure-utility/blob/master/f/sync_fas_group_membership.py script. It is triggered either by group membership messages or by `toddlers.trigger` message.

+ 

+ In case of group membership message it works as following:

+ 

+ 1. Check if we care about the group (it's in configuration for toddler)

+ 1. Add member from group (currently there is no message about user being removed from group)

+ 

+ In case of `toddlers.trigger` message it compares all the groups in configuration

+ and remove/add users to pagure group based on the changes in FAS.

+ 

+ ## Accepted topics

+ 

+ The sync toddler accepts following topics.

+ 

+ * org.fedoraproject.*.fas.group.member.sponsor - New member added to group

+ * org.fedoraproject.*.toddlers.trigger.pagure_fas_groups_sync - Message triggered by toddlers cron

file modified
+1
@@ -8,6 +8,7 @@ 

  GitPython

  koji

  requests

+ noggin-messages

  pagure-messages

  pyGObject

  python-fedora

@@ -0,0 +1,409 @@ 

+ """

+ Unit tests for `toddlers.plugins.scm_request_processor`

+ """

+ 

+ import logging

+ from unittest.mock import Mock, patch

+ 

+ from fedora_messaging.api import Message

+ from noggin_messages import MemberSponsorV1

+ import pytest

+ 

+ from toddlers.exceptions import PagureError

+ from toddlers.plugins import pagure_fas_groups_sync

+ 

+ 

+ class TestAcceptsTopic:

+     """

+     Test class for `toddlers.plugins.pagure_fas_groups_sync.PagureFASGroupsSync.accepts_topic`

+     method.

+     """

+ 

+     toddler_cls = pagure_fas_groups_sync.PagureFASGroupsSync

+ 

+     def test_accepts_topic_invalid(self, toddler):

+         """

+         Assert that invalid topic is not accepted.

+         """

+         assert toddler.accepts_topic("foo.bar") is False

+ 

+     @pytest.mark.parametrize(

+         "topic",

+         [

+             "org.fedoraproject.*.fas.group.member.sponsor",

+             "org.fedoraproject.*.toddlers.trigger.pagure_fas_groups_sync",

+             "org.fedoraproject.stg.fas.group.member.sponsor",

+             "org.fedoraproject.stg.toddlers.trigger.pagure_fas_groups_sync",

+             "org.fedoraproject.prod.fas.group.member.sponsor",

+             "org.fedoraproject.prod.toddlers.trigger.pagure_fas_groups_sync",

+         ],

+     )

+     def test_accepts_topic_valid(self, topic, toddler):

+         """

+         Assert that valid topics are accepted.

+         """

+         assert toddler.accepts_topic(topic)

+ 

+ 

+ class TestProcess:

+     """

+     Test class for `toddlers.plugins.pagure_fas_groups_sync.PagureFASGroupsSync.process` method.

+     """

+ 

+     toddler_cls = pagure_fas_groups_sync.PagureFASGroupsSync

+ 

+     @patch("toddlers.utils.pagure.set_pagure")

+     @patch("toddlers.utils.fedora_account.set_fasjson")

+     def test_process_trigger_message(self, mock_fas, mock_pagure, toddler):

+         """

+         Assert that trigger message is processed correctly.

+         """

+         # Preparation

+         config = {"group_map": {"fas_group": "pagure_group"}}

+         msg = Message()

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pagure_fas_groups_sync"

+ 

+         # Test

+         with patch(

+             "toddlers.plugins.pagure_fas_groups_sync.PagureFASGroupsSync.sync_group"

+         ) as mock_sync_group:

+             toddler.process(config, msg)

+ 

+             # Assertions

+             mock_sync_group.assert_called_with("fas_group", "pagure_group")

+ 

+         mock_fas.assert_called_with(config)

+         mock_pagure.assert_called_with(config)

+ 

+     @patch("toddlers.utils.pagure.set_pagure")

+     @patch("toddlers.utils.fedora_account.set_fasjson")

+     def test_process_sponsor_message(self, mock_fas, mock_pagure, toddler):

+         """

+         Assert that sponsor message is processed correctly.

+         """

+         # Preparation

+         mock_pagure_obj = Mock()

+         mock_pagure.return_value = mock_pagure_obj

+         config = {"group_map": {"fas_group": "pagure_group"}}

+         msg = MemberSponsorV1(

+             {"msg": {"agent": "agent", "user": "user", "group": "fas_group"}}

+         )

+ 

+         # Test

+         toddler.process(config, msg)

+ 

+         # Assertions

+         mock_pagure_obj.add_member_to_group.assert_called_with("user", "pagure_group")

+ 

+         mock_fas.assert_called_with(config)

+         mock_pagure.assert_called_with(config)

+ 

+     @patch("toddlers.utils.pagure.set_pagure")

+     @patch("toddlers.utils.fedora_account.set_fasjson")

+     def test_process_sponsor_message_user_not_in_pagure(

+         self, mock_fas, mock_pagure, toddler

+     ):

+         """

+         Assert that sponsor message is processed correctly when user

+         doesn't exist in pagure.

+         """

+         # Preparation

+         mock_pagure_obj = Mock()

+         mock_pagure_obj.user_exists.return_value = False

+         mock_pagure.return_value = mock_pagure_obj

+         config = {"group_map": {"fas_group": "pagure_group"}}

+         msg = MemberSponsorV1(

+             {"msg": {"agent": "agent", "user": "user", "group": "fas_group"}}

+         )

+ 

+         # Test

+         toddler.process(config, msg)

+ 

+         # Assertions

+         mock_pagure_obj.add_member_to_group.assert_not_called()

+ 

+         mock_fas.assert_called_with(config)

+         mock_pagure.assert_called_with(config)

+ 

+     @patch("toddlers.utils.pagure.set_pagure")

+     @patch("toddlers.utils.fedora_account.set_fasjson")

+     def test_process_sponsor_message_failure(

+         self, mock_fas, mock_pagure, caplog, toddler

+     ):

+         """

+         Assert that failure during processing sponsor message is handled correctly.

+         """

+         # Preparation

+         caplog.set_level(logging.ERROR)

+         mock_pagure_obj = Mock()

+         mock_pagure_obj.add_member_to_group.side_effect = PagureError("PagureError")

+         mock_pagure.return_value = mock_pagure_obj

+         config = {"group_map": {"fas_group": "pagure_group"}}

+         msg = MemberSponsorV1(

+             {"msg": {"agent": "agent", "user": "user", "group": "fas_group"}}

+         )

+ 

+         # Test

+         toddler.process(config, msg)

+ 

+         # Assertions

+         mock_pagure_obj.add_member_to_group.assert_called_with("user", "pagure_group")

+ 

+         mock_fas.assert_called_with(config)

+         mock_pagure.assert_called_with(config)

+ 

+         assert caplog.records[-1].message == "PagureError"

+ 

+     @patch("toddlers.utils.pagure.set_pagure")

+     @patch("toddlers.utils.fedora_account.set_fasjson")

+     def test_process_sponsor_message_skipping(

+         self, mock_fas, mock_pagure, caplog, toddler

+     ):

+         """

+         Assert that group is skipped when we don't care about it.

+         """

+         # Preparation

+         caplog.set_level(logging.INFO)

+         mock_pagure_obj = Mock()

+         mock_pagure.return_value = mock_pagure_obj

+         config = {"group_map": {"fas_group": "pagure_group"}}

+         msg = MemberSponsorV1(

+             {"msg": {"agent": "agent", "user": "user", "group": "some_group"}}

+         )

+ 

+         # Test

+         toddler.process(config, msg)

+ 

+         # Assertions

+         mock_pagure_obj.add_member_to_group.assert_not_called()

+ 

+         mock_fas.assert_called_with(config)

+         mock_pagure.assert_called_with(config)

+ 

+         assert caplog.records[-1].message.startswith(

+             "User 'user' was added to group 'some_group'"

+         )

+ 

+ 

+ class TestSyncGroup:

+     """

+     Test class for `toddlers.plugins.pagure_fas_groups_sync.sync_group` function.

+     """

+ 

+     def setup_method(self):

+         """Prepare the toddler object."""

+         self.sync_object = pagure_fas_groups_sync.PagureFASGroupsSync()

+         self.sync_object.pagure = Mock()

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     def test_sync_group_add_member(self, mock_fas):

+         """Assert that adding member to group works as intended."""

+         # Preparation

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = ["user1"]

+         mock_fas.get_group_member.return_value = ["user1", "user2"]

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.add_member_to_group.assert_called_with(

+             "user2", pagure_group

+         )

+         self.sync_object.pagure.remove_member_from_group.assert_not_called()

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     def test_sync_group_add_member_user_not_in_pagure(self, mock_fas):

+         """

+         Assert that adding member to group works as intended

+         when the user doesn't exist in pagure.

+         """

+         # Preparation

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = ["user1"]

+         self.sync_object.pagure.user_exists.return_value = False

+         mock_fas.get_group_member.return_value = ["user1", "user2"]

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.add_member_to_group.assert_not_called()

+         self.sync_object.pagure.remove_member_from_group.assert_not_called()

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     def test_sync_group_add_member_failure(self, mock_fas, caplog):

+         """Assert that failing to adding member to group works as intended."""

+         # Preparation

+         caplog.set_level(logging.ERROR)

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = ["user1"]

+         mock_fas.get_group_member.return_value = ["user1", "user2"]

+         self.sync_object.pagure.add_member_to_group.side_effect = PagureError(

+             "PagureError"

+         )

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.add_member_to_group.assert_called_with(

+             "user2", pagure_group

+         )

+         self.sync_object.pagure.remove_member_from_group.assert_not_called()

+         assert caplog.records[-1].message == "PagureError"

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     def test_sync_group_remove_member(self, mock_fas):

+         """Assert that removing member from group works as intended."""

+         # Preparation

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = ["user1", "user2"]

+         mock_fas.get_group_member.return_value = ["user1"]

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.remove_member_from_group.assert_called_with(

+             "user2", pagure_group

+         )

+         self.sync_object.pagure.add_member_to_group.assert_not_called()

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     def test_sync_group_remove_member_user_not_in_pagure(self, mock_fas):

+         """

+         Assert that removing member from group works

+         as intended when user doesn't exist in pagure.

+         """

+         # Preparation

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = ["user1", "user2"]

+         self.sync_object.pagure.user_exists.return_value = False

+         mock_fas.get_group_member.return_value = ["user1"]

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.remove_member_from_group.assert_not_called()

+         self.sync_object.pagure.add_member_to_group.assert_not_called()

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     def test_sync_group_remove_member_failure(self, mock_fas, caplog):

+         """Assert that removing member from group works as intended."""

+         # Preparation

+         caplog.set_level(logging.ERROR)

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = ["user1", "user2"]

+         mock_fas.get_group_member.return_value = ["user1"]

+         self.sync_object.pagure.remove_member_from_group.side_effect = PagureError(

+             "PagureError"

+         )

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.remove_member_from_group.assert_called_with(

+             "user2", pagure_group

+         )

+         self.sync_object.pagure.add_member_to_group.assert_not_called()

+         assert caplog.records[-1].message == "PagureError"

+ 

+     @patch("toddlers.plugins.pagure_fas_groups_sync.fedora_account")

+     @pytest.mark.parametrize(

+         "mock_fas_group, mock_pagure_group", [([], ["user"]), (["user"], [])]

+     )

+     def test_sync_group_group_empty(self, mock_fas, mock_fas_group, mock_pagure_group):

+         """

+         Assert that nothing is done if any of the retrieved groups is empty.

+         That should not happen, there always needs to be at least one member

+         in the group.

+         """

+         # Preparation

+         fas_group = "fas_group"

+         pagure_group = "pagure_group"

+         self.sync_object.pagure.get_group_members.return_value = mock_pagure_group

+         mock_fas.get_group_member.return_value = mock_fas_group

+ 

+         # Test

+         self.sync_object.sync_group(fas_group, pagure_group)

+ 

+         # Assertions

+         mock_fas.get_group_member.assert_called_with(fas_group)

+         self.sync_object.pagure.get_group_members.assert_called_with(pagure_group)

+ 

+         self.sync_object.pagure.add_member_to_group.assert_not_called()

+         self.sync_object.pagure.remove_member_from_group.assert_not_called()

+ 

+ 

+ class TestMain:

+     """

+     Test class for `toddlers.plugins.pagure_fas_groups_sync.main` function.

+     """

+ 

+     def test_main_no_args(self, capsys):

+         """Assert that help is printed if no arg is provided."""

+         # Test

+         with pytest.raises(SystemExit):

+             pagure_fas_groups_sync.main([])

+ 

+         # Assertions

+         out, err = capsys.readouterr()

+         assert out == ""

+         # Expecting something along these lines, but don't make the test too tight:

+         #

+         # usage: pytest [-h] [--dry-run] [-q | --debug] conf [username]

+         # pytest: error: the following arguments are required: conf

+         assert err.startswith("usage:")

+         assert "error: the following arguments are required:" in err

+ 

+     @patch("toddlers.utils.pagure.set_pagure")

+     @patch("toddlers.utils.fedora_account.set_fasjson")

+     @patch("toddlers.plugins.pagure_fas_groups_sync.PagureFASGroupsSync.sync_group")

+     def test_main(self, mock_sync_group, mock_fas, mock_pagure):

+         """Assert that main is processed correctly."""

+         # Test

+         pagure_fas_groups_sync.main(

+             [

+                 "--api-key",

+                 "api_key",

+                 "--group",

+                 "fas_group",

+                 "--target-group",

+                 "pagure_group",

+                 "--debug",

+             ]

+         )

+ 

+         # Assertions

+         mock_pagure.assert_called_with(

+             {"pagure_url": "https://pagure.io", "pagure_api_key": "api_key"}

+         )

+         mock_fas.assert_called_with({"fas_url": "https://fasjson.fedoraproject.org"})

+         mock_sync_group.assert_called_with("fas_group", "pagure_group")

file modified
+214 -1
@@ -3,7 +3,7 @@ 

  """

  

  import json

- from unittest.mock import call, Mock, patch

+ from unittest.mock import call, MagicMock, Mock, patch

  

  import pytest

  
@@ -1455,3 +1455,216 @@ 

              data=json.dumps(payload),

              headers=self.pagure.get_auth_header(),

          )

+ 

+ 

+ class TestPagureAddMemberToGroup:

+     """

+     Test class for `toddlers.pagure.Pagure.add_member_to_group` method.

+     """

+ 

+     def setup_method(self):

+         """

+         Setup method for the test class.

+         """

+         config = {

+             "pagure_url": "https://pagure.io",

+             "pagure_api_key": "Very secret key",

+         }

+         self.pagure = pagure.set_pagure(config)

+         self.pagure._requests_session = Mock()

+ 

+     def test_add_member_to_group(self):

+         """

+         Assert that adding member to group is processed correctly.

+         """

+         response_mock = Mock()

+ 

+         self.pagure._requests_session.post.return_value = response_mock

+ 

+         user = "conrad_kurze"

+         group = "adeptus_astartes"

+         payload = {

+             "user": user,

+         }

+ 

+         self.pagure.add_member_to_group(user, group)

+ 

+         self.pagure._requests_session.post.assert_called_with(

+             "https://pagure.io/api/0/group/" + group + "/add",

+             data=json.dumps(payload),

+             headers=self.pagure.get_auth_header(),

+         )

+ 

+     def test_add_member_to_group_failure(self):

+         """

+         Assert that failing to add member to group is handled correctly.

+         """

+         response_mock = MagicMock()

+         response_mock.ok = False

+ 

+         self.pagure._requests_session.post.return_value = response_mock

+ 

+         user = "conrad_kurze"

+         group = "adeptus_astartes"

+         payload = {

+             "user": user,

+         }

+ 

+         expected_error = "Couldn't add user '{0}' to group '{1}'".format(user, group)

+ 

+         with pytest.raises(PagureError, match=expected_error):

+             self.pagure.add_member_to_group(user, group)

+ 

+         self.pagure._requests_session.post.assert_called_with(

+             "https://pagure.io/api/0/group/" + group + "/add",

+             data=json.dumps(payload),

+             headers=self.pagure.get_auth_header(),

+         )

+ 

+ 

+ class TestPagureRemoveMemberFromGroup:

+     """

+     Test class for `toddlers.pagure.Pagure.remove_member_from_group` method.

+     """

+ 

+     def setup_method(self):

+         """

+         Setup method for the test class.

+         """

+         config = {

+             "pagure_url": "https://pagure.io",

+             "pagure_api_key": "Very secret key",

+         }

+         self.pagure = pagure.set_pagure(config)

+         self.pagure._requests_session = Mock()

+ 

+     def test_remove_member_from_group(self):

+         """

+         Assert that removing member from group is processed correctly.

+         """

+         response_mock = Mock()

+ 

+         self.pagure._requests_session.post.return_value = response_mock

+ 

+         user = "conrad_kurze"

+         group = "adeptus_astartes"

+         payload = {

+             "user": user,

+         }

+ 

+         self.pagure.remove_member_from_group(user, group)

+ 

+         self.pagure._requests_session.post.assert_called_with(

+             "https://pagure.io/api/0/group/" + group + "/remove",

+             data=json.dumps(payload),

+             headers=self.pagure.get_auth_header(),

+         )

+ 

+     def test_remove_member_from_group_failure(self):

+         """

+         Assert that failing to remove member from group is handled correctly.

+         """

+         response_mock = MagicMock()

+         response_mock.ok = False

+ 

+         self.pagure._requests_session.post.return_value = response_mock

+ 

+         user = "conrad_kurze"

+         group = "adeptus_astartes"

+         payload = {

+             "user": user,

+         }

+ 

+         expected_error = "Couldn't remove user '{0}' from group '{1}'".format(

+             user, group

+         )

+ 

+         with pytest.raises(PagureError, match=expected_error):

+             self.pagure.remove_member_from_group(user, group)

+ 

+         self.pagure._requests_session.post.assert_called_with(

+             "https://pagure.io/api/0/group/" + group + "/remove",

+             data=json.dumps(payload),

+             headers=self.pagure.get_auth_header(),

+         )

+ 

+ 

+ class TestPagureGetGroupMembers:

+     """

+     Test class for `toddlers.pagure.Pagure.get_group_members` method.

+     """

+ 

+     def setup_method(self):

+         """

+         Setup method for the test class.

+         """

+         config = {

+             "pagure_url": "https://pagure.io",

+             "pagure_api_key": "Very secret key",

+         }

+         self.pagure = pagure.set_pagure(config)

+         self.pagure._requests_session = Mock()

+ 

+     def test_get_group_members(self):

+         """

+         Assert that retrieving members in group is processed correctly.

+         """

+         response_mock = Mock()

+         user = "conrad_kurze"

+         data = {"members": [user]}

+         response_mock.json.return_value = data

+ 

+         self.pagure._requests_session.get.return_value = response_mock

+ 

+         group = "adeptus_astartes"

+ 

+         result = self.pagure.get_group_members(group)

+ 

+         assert result == [user]

+ 

+         self.pagure._requests_session.get.assert_called_with(

+             "https://pagure.io/api/0/group/" + group,

+             headers=self.pagure.get_auth_header(),

+         )

+ 

+     def test_get_group_members_no_group(self):

+         """

+         Assert that when group doesn't exist is handled correctly.

+         """

+         response_mock = MagicMock()

+         response_mock.ok = False

+         response_mock.status_code = 404

+ 

+         self.pagure._requests_session.get.return_value = response_mock

+ 

+         group = "adeptus_astartes"

+ 

+         result = self.pagure.get_group_members(group)

+ 

+         assert not result

+ 

+         self.pagure._requests_session.get.assert_called_with(

+             "https://pagure.io/api/0/group/" + group,

+             headers=self.pagure.get_auth_header(),

+         )

+ 

+     def test_get_group_members_failure(self):

+         """

+         Assert that failing to obtain members of group is handled correctly.

+         """

+         response_mock = MagicMock()

+         response_mock.ok = False

+ 

+         self.pagure._requests_session.get.return_value = response_mock

+ 

+         group = "adeptus_astartes"

+ 

+         expected_error = "Couldn't get members of group '{0}'".format(group)

+ 

+         with pytest.raises(PagureError, match=expected_error):

+             self.pagure.get_group_members(group)

+ 

+         self.pagure._requests_session.get.assert_called_with(

+             "https://pagure.io/api/0/group/" + group,

+             headers=self.pagure.get_auth_header(),

+         )

file modified
+12
@@ -295,6 +295,18 @@ 

  security_fixes = '2022-03-08'

  bug_fixes = '2022-03-08'

  

+ [consumer_config.pagure_fas_groups_sync]

+ pagure_url = "https://pagure.io"

+ # Token needs to have following permissions:

+ # - adding_member_to_group

+ # - removing_member_from_group

+ pagure_api_key = "API token for pagure"

+ 

+ [consumer_config.pagure_fas_groups_sync.group_map]

+ #Mapping of FAS groups to Pagure groups

+ infra-sig = 'fedora-infra'

+ 

+ 

  [qos]

  prefetch_size = 0

  prefetch_count = 25

@@ -0,0 +1,205 @@ 

+ """

+ This toddler synchronizes groups from FAS to pagure.io.

+ 

+ Authors:    Michal Konecny <mkonecny@redhat.com>

+ 

+ """

+ 

+ import argparse

+ import json

+ import logging

+ import sys

+ from typing import Dict

+ 

+ from fedora_messaging.api import Message

+ from noggin_messages import MemberSponsorV1

+ 

+ from toddlers.base import ToddlerBase

+ from toddlers.exceptions import PagureError

+ from toddlers.utils import fedora_account, pagure

+ 

+ _log = logging.getLogger(__name__)

+ 

+ 

+ class PagureFASGroupsSync(ToddlerBase):

+     """

+     Synchronize FAS groups with Pagure.

+ 

+     Consumes group membership change messages and process them

+     and consumes trigger to sync all the configured groups.

+     """

+ 

+     name: str = "pagure_fas_groups_sync"

+ 

+     amqp_topics: list = [

+         "org.fedoraproject.*.fas.group.member.sponsor",

+         "org.fedoraproject.*.toddlers.trigger.pagure_fas_groups_sync",

+     ]

+ 

+     # Pagure object

+     pagure: pagure.Pagure

+ 

+     # Group mapping from FAS to pagure

+     group_map: Dict[str, str]

+ 

+     def accepts_topic(self, topic: str) -> bool:

+         """

+         Return a boolean if this toddler consumes messages from topic.

+ 

+         :arg topic: Topic to check.

+ 

+         :returns: True if topic is accepted, False otherwise.

+         """

+         if topic.startswith("org.fedoraproject."):

+             if topic.endswith("fas.group.member.sponsor"):

+                 return True

+             if topic.endswith("toddlers.trigger.pagure_fas_groups_sync"):

+                 return True

+ 

+         return False

+ 

+     def process(

+         self,

+         config: dict,

+         message: Message,

+     ) -> None:

+         """Process a given message.

+ 

+         :arg config: Toddlers configuration

+         :arg message: Message to process

+         """

+         if _log.isEnabledFor(logging.DEBUG):

+             _log.debug("Processing message:\n%s", json.dumps(message.body, indent=2))

+         topic = message.topic

+ 

+         self.pagure = pagure.set_pagure(config)

+         fedora_account.set_fasjson(config)

+ 

+         group_map = config.get("group_map", {})

+ 

+         if topic.endswith("toddlers.trigger.pagure_fas_groups_sync"):

+             for group in group_map:

+                 self.sync_group(group, group_map[group])

+ 

+         if topic.endswith("fas.group.member.sponsor"):

+             member_sponsor_message = MemberSponsorV1(message.body)

+             user = member_sponsor_message.user_name

+             for group in member_sponsor_message.groups:

+                 if group in group_map:

+                     try:

+                         if self.pagure.user_exists(user):

+                             self.pagure.add_member_to_group(user, group_map[group])

+                     except PagureError as e:

+                         _log.exception(str(e))

+                 else:

+                     _log.info(

+                         "User '%s' was added to group '%s', "

+                         "but we don't care about this group. "

+                         "Skipping.",

+                         user,

+                         group,

+                     )

+ 

+     def sync_group(self, fas_group: str, pagure_group: str):

+         """

+         Synchronize FAS group with Pagure group.

+ 

+         Compares FAS group and Pagure group and modifies the Pagure

+         group to correspond with the specified FAS group.

+ 

+         :arg fas_group: FAS group to check

+         :arg pagure_group: Pagure group to sync

+         """

+         _log.info(

+             "Syncing FAS group '%s' with pagure group '%s'", fas_group, pagure_group

+         )

+         group_members_fas = fedora_account.get_group_member(fas_group)

+         _log.debug("FAS group members: [%s]", group_members_fas)

+         group_members_pagure = self.pagure.get_group_members(pagure_group)

+         _log.debug("Pagure group members: [%s]", group_members_pagure)

+ 

+         if not group_members_fas or not group_members_pagure:

+             _log.warning(

+                 "FAS group or Pagure group is empty. This shouldn't happen. Skipping sync."

+             )

+             return

+         add_members = [

+             user for user in group_members_fas if user not in group_members_pagure

+         ]

+         remove_members = [

+             user for user in group_members_pagure if user not in group_members_fas

+         ]

+ 

+         for user in add_members:

+             try:

+                 if self.pagure.user_exists(user):

+                     self.pagure.add_member_to_group(user, pagure_group)

+             except PagureError as e:

+                 _log.exception(str(e))

+ 

+         for user in remove_members:

+             try:

+                 if self.pagure.user_exists(user):

+                     self.pagure.remove_member_from_group(user, pagure_group)

+             except PagureError as e:

+                 _log.exception(str(e))

+ 

+         _log.info("Sync complete")

+ 

+ 

+ def main(args):

+     """Run toddler class without trigger message."""

+     parser = argparse.ArgumentParser(

+         description="Sync the group membership from FAS to pagure."

+     )

+     parser.add_argument(

+         "--fas-url",

+         default="https://fasjson.fedoraproject.org",

+         help="URL to the FAS instance to use.",

+     )

+     parser.add_argument(

+         "--pagure-url",

+         default="https://pagure.io",

+         help="URL to the Pagure instance to use.",

+     )

+     parser.add_argument(

+         "--api-key",

+         help="Pagure API key to use.",

+         required=True,

+     )

+     parser.add_argument(

+         "--group",

+         help="Sync a specific group from FAS to pagure.",

+         required=True,

+     )

+     parser.add_argument(

+         "--target-group",

+         help="Name of the group on the pagure side that the group specified "

+         "in --group should be synced to.",

+         required=True,

+     )

+     parser.add_argument(

+         "--debug",

+         dest="debug",

+         action="store_true",

+         default=False,

+         help="Print the debugging output",

+     )

+     args = parser.parse_args(args)

+     _log.addHandler(logging.StreamHandler(sys.stdout))

+     if args.debug:

+         _log.setLevel(logging.DEBUG)

+ 

+     pagure_fas_sync_toddler = PagureFASGroupsSync()

+     pagure_fas_sync_toddler.pagure = pagure.set_pagure(

+         {"pagure_url": args.pagure_url, "pagure_api_key": args.api_key}

+     )

+     fedora_account.set_fasjson({"fas_url": args.fas_url})

+     pagure_fas_sync_toddler.sync_group(args.group, args.target_group)

+ 

+ 

+ if __name__ == "__main__":  # pragma: no cover

+     try:

+         main(sys.argv[1:])

+     except KeyboardInterrupt:

+         pass

file modified
+167 -1
@@ -18,7 +18,7 @@ 

  

  import json

  import logging

- from typing import Any, Optional

+ from typing import Any, List, Optional

  

  from toddlers.exceptions import PagureError

  from toddlers.utils import requests
@@ -936,3 +936,169 @@ 

                      namespace, repo

                  )

              )

+ 

+     def add_member_to_group(self, user: str, group: str) -> None:

+         """

+         Add member to pagure group.

+ 

+         Params:

+             user: User to add

+             group: Group to add user to

+ 

+         Raises:

+             `toddlers.utils.exceptions.PagureError``: When user can't be added to group.

+         """

+         api_url = "{0}/api/0/group/{1}/add".format(self._pagure_url, group)

+ 

+         payload = {"user": user}

+         headers = self.get_auth_header()

+ 

+         log.debug("Adding user '%s' to group '%s'", user, group)

+         response = self._requests_session.post(

+             api_url, data=json.dumps(payload), headers=headers

+         )

+ 

+         if not response.ok:

+             log.error(

+                 "Error when adding user '%s' to group '%s'. " "Got status_code '%s'.",

+                 user,

+                 group,

+                 response.status_code,

+             )

+ 

+             response_json = None

+             if response.headers.get("content-type") == "application/json":

+                 response_json = response.json()

+                 log.error("Received response: %s", response.json())

+ 

+             raise PagureError(

+                 (

+                     "Couldn't add user '{0}' to group '{1}'\n\n"

+                     "Request to '{2}':\n\n"

+                     "Response:\n"

+                     "{3}\n\n"

+                     "Status code: {4}"

+                 ).format(

+                     user,

+                     group,

+                     api_url,

+                     response_json,

+                     response.status_code,

+                 )

+             )

+ 

+         log.debug("User '%s' added to group '%s'", user, group)

+ 

+     def remove_member_from_group(self, user: str, group: str) -> None:

+         """

+         Remove member from pagure group.

+ 

+         Params:

+             user: User to remove

+             group: Group to remove user from

+ 

+         Raises:

+             `toddlers.utils.exceptions.PagureError``: When user can't be removed to group.

+         """

+         api_url = "{0}/api/0/group/{1}/remove".format(self._pagure_url, group)

+         payload = {"user": user}

+         headers = self.get_auth_header()

+ 

+         log.debug("Removing user '%s' from group '%s'", user, group)

+         response = self._requests_session.post(

+             api_url, data=json.dumps(payload), headers=headers

+         )

+ 

+         if not response.ok:

+             log.error(

+                 "Error when removing user '%s' from group '%s'. "

+                 "Got status_code '%s'.",

+                 user,

+                 group,

+                 response.status_code,

+             )

+ 

+             response_json = None

+             if response.headers.get("content-type") == "application/json":

+                 response_json = response.json()

+                 log.error("Received response: %s", response.json())

+ 

+             raise PagureError(

+                 (

+                     "Couldn't remove user '{0}' from group '{1}'\n\n"

+                     "Request to '{2}':\n\n"

+                     "Response:\n"

+                     "{3}\n\n"

+                     "Status code: {4}"

+                 ).format(

+                     user,

+                     group,

+                     api_url,

+                     response_json,

+                     response.status_code,

+                 )

+             )

+ 

+         log.debug("User '%s' removed from group '%s'", user, group)

+ 

+     def get_group_members(self, group: str) -> List[str]:

+         """

+         Get members of group.

+ 

+         Params:

+           group: Group name

+ 

+         Returns:

+           List of users or empty list if group doesn't exists..

+ 

+         Raises:

+           `toddlers.utils.exceptions.PagureError``: When getting members fail.

+         """

+         result = []

+         api_url = "{0}/api/0/group/{1}".format(self._pagure_url, group)

+         headers = self.get_auth_header()

+ 

+         log.debug("Getting members of group '%s'", group)

+         response = self._requests_session.get(api_url, headers=headers)

+ 

+         if response.ok:

+             if "members" in response.json():

+                 result = response.json()["members"]

+             else:

+                 raise PagureError(

+                     "The 'members' parameter is missing in response. "

+                     "JSON response: {0}".format(response.json())

+                 )

+         elif response.status_code == 404:

+             return result

+         else:

+             log.error(

+                 "Error when retrieving members from group '%s'. "

+                 "Got status_code '%s'.",

+                 group,

+                 response.status_code,

+             )

+ 

+             response_json = None

+             if response.headers.get("content-type") == "application/json":

+                 response_json = response.json()

+                 log.error("Received response: %s", response.json())

+ 

+             raise PagureError(

+                 (

+                     "Couldn't get members of group '{0}'\n\n"

+                     "Request to '{1}:\n\n"

+                     "Response:\n"

+                     "{2}\n\n"

+                     "Status code: {3}"

+                 ).format(

+                     group,

+                     api_url,

+                     response_json,

+                     response.status_code,

+                 )

+             )

+ 

+         log.debug("Members retrieved for group '%s'", group)

+ 

+         return result

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/30455d1fdeb249edb8a17dd6fe27121a

6 new commits added

  • Fix the last failing test
  • Fix flake8 tests
  • Fix mypy issues
  • Add documentation for pagure_fas_groups_sync toddler
  • Add configuration example for pagure_fas_groups_sync toddler
  • Add tests for pagure_fas_groups_sync toddler
2 years ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/f3ac24aac42e4a6ebcc48030e768fb16

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/793672d479de4025acb189047b8ead41

rebased onto f254995

10 months ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/465db134bbf04c46b8a7d1c4a565868d

1 new commit added

  • Fix the error when adding user which is not in Pagure
10 months ago

As the work on https://pagure.io/pagure/issue/5333 is finished and this toddler tested on staging with remaining issues fixed, this PR is now ready to be merged.

1 new commit added

  • Fix format issues
10 months ago

rebased onto 6b4f8e0

10 months ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/b4cbda55d6484bcc98f9d7781803a714

rebased onto a6dfc2e

10 months ago

Metadata Update from @nphilipp:
- Request assigned

10 months ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/a4c09cb5463b41bbbbc17ede874477a8

For formatting logged strings, it’s recommended to use %-style formats and pass the variables as additional positional arguments (so they only have to be interpolated if logging is configured for the level):

_log.debug(
    "Processing message:\n%s", json.dumps(message.body, indent=2)
)

In this case, json.dumps() might be even more expensive than string interpolation, so you could even do it like this:

if _log.isEnabledFor(logging.DEBUG):
    _log.debug(
        "Processing message:\n%s", json.dumps(message.body, indent=2)
    )
)

Is there a reason to store the group map as a member variable? It’s not used outside of process() and always taken from config as passed into the method.

As above, I suggest using %-style string formats for delayed interpolation in this module.

If the above is addressed, I would like the commits of this PR to be squashed into one before merging.

rebased onto a6dfc2e

10 months ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/33b2d3dfe0cf44c08384779c53611ccf

1 new commit added

  • Address the review comments
10 months ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/75fac5343f75494cb8e7af8c1aaf13ab

11 new commits added

  • Address the review comments
  • Fix format issues
  • Fix the error when adding user which is not in Pagure
  • Fix the last failing test
  • Fix flake8 tests
  • Fix mypy issues
  • Add documentation for pagure_fas_groups_sync toddler
  • Add configuration example for pagure_fas_groups_sync toddler
  • Add tests for pagure_fas_groups_sync toddler
  • Add new methods to pagure module
  • Add pagure_fas_groups_sync toddler
10 months ago

@nphilipp I addressed all the comments from you and added that as new commit. Let me know if this is OK now and I will squash the commits and merge the PR.

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/c0cc0669ef224844bd6fc17649051c17

@nphilipp I addressed all the comments from you and added that as new commit. Let me know if this is OK now and I will squash the commits and merge the PR.

Looks good to me, thanks! Please squash and merge.

rebased onto cc277ea

10 months ago

The commits were squashed and I will merge it and work on deploying this on staging.

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/779e954a20d74dfc90a40ad5bd51fbd3

Pull-Request has been merged by zlopez

10 months ago