fedora-infra / toddlers

Clone
Source Code
GIT

#238 Replace pdc calls
Closed 2 months ago by zlopez. Opened 2 months ago by humaton.
fedora-infra/ humaton/toddlers distgit_bz_sync  into  main

file modified
+102 -4 
@@ -24,6 +24,8 @@ 
 

          "pdc_config": {
 

              "server": "https://pdc.fedoraproject.org/rest_api/v1",
 

          },
 

+         "pagure_api_key": "some api key",
 

+         "pagure_url": "https://src.fedoraproject.org",
 

          # distgit_bugzilla_sync config values
 

          "ignorable_accounts": [],
 

          "fasjson": False,
 
@@ -66,6 +68,16 @@ 
 

  class TestDistgitBugzillaSyncToddler:
 

      toddler_cls = DistgitBugzillaSync
 

  

+     def setup_method(self):
 

+         """
 

+         Initialize toddler.
 

+         """
 

+         #self.toddler = DistgitBugzillaSync
 

+         self.pagure_io = Mock()
 

+         self.dist_git = Mock()
 

+         self.bodhi = Mock()
 

+         self.requests_session = Mock()
 

+  

      def test_accepts_topic_invalid(self, toddler):
 

          assert toddler.accepts_topic("foo.bar") is False
 

  
@@ -80,8 +92,26 @@ 
 

      def test_accepts_topic_valid(self, toddler, topic):
 

          assert toddler.accepts_topic(topic)
 

  

-     def test_process_no_email_override_file(self, toddler):
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 

+     def test_process_no_email_override_file(self, toddler, mock_set_pagure, mock_set_bodhi):
 

          """Assert that the exception is raised when e-mail overrides file is not provided."""
 

+         mock_dist_git = MagicMock()
 

+         mock_dist_git.get_retired_packages.side_effect = (
 

+             ["package01", "package02"],
 

+             ["package01", "package03"],
 

+         )
 

+         mock_dist_git.get_branches.return_value = ["f39", "f40"]
 

+         mock_dist_git.get_project.return_value = {
 

+             "user": {"name": "Gavriel Loken"},
 

+             "access_users": {"admin": ["Fulgrim", "orphan"]},
 

+             "access_groups": {"admin": ["Adeptus Astartes"]},
 

+         }
 

+         mock_set_pagure.return_value = mock_dist_git
 

+         mock_bodhi = MagicMock()
 

+         mock_bodhi.get_active_branches.return_value = ["f39", "f40"]
 

+         mock_set_bodhi.return_value = mock_bodhi
 

+ 

          with pytest.raises(KeyError) as exc:
 

              toddler.process(
 

                  config={},
 
@@ -95,6 +125,8 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_dry_run_edit_project(
 

          self,
 
@@ -234,6 +266,8 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_dry_run_add_project(
 

          self,
 
@@ -370,6 +404,8 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_dry_run_specific_project(
 

          self,
 
@@ -493,6 +529,8 @@ 
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_report_protocol_error(
 

          self,
 
@@ -606,6 +644,8 @@ 
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_report_client_error(
 

          self,
 
@@ -739,6 +779,8 @@ 
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_report_missing_mails(
 

          self,
 
@@ -835,6 +877,8 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_dry_run_verbose(
 

          self,
 
@@ -932,6 +976,8 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_missing_namespace(
 

          self,
 
@@ -984,6 +1030,8 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load")
 

      def test_process_missing_pdc_branches(
 

          self,
 
@@ -1041,6 +1089,8 @@ 
 

      @patch(
 

          "toddlers.plugins.distgit_bugzilla_sync.PackageSummaries.get_package_summaries"
 

      )
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toml.load")
 
@@ -1157,6 +1207,8 @@ 
 

      @patch(
 

          "toddlers.plugins.distgit_bugzilla_sync.PackageSummaries.get_package_summaries"
 

      )
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toml.load")
 
@@ -1165,6 +1217,10 @@ 
 

          mock_toml,
 

          mock_bugzilla,
 

          mock_fas,
 

+         mock_dist_git,
 

+         mock_set_pagure,
 

+         mock_set_bodhi,
 

+         mock_bodhi,
 

          mock_summaries,
 

          config,
 

          toddler,
 
@@ -1179,6 +1235,23 @@ 
 

          # Mock package summaries response
 

          mock_summaries.return_value = {"foo": "Summary"}
 

  

+         mock_dist_git = MagicMock()
 

+         mock_dist_git.get_retired_packages.side_effect = (
 

+             ["package01", "package02"],
 

+             ["package01", "package03"],
 

+         )
 

+         mock_dist_git.get_branches.return_value = ["f39", "f40"]
 

+         mock_dist_git.get_project.return_value = {
 

+             "user": {"name": "Gavriel Loken"},
 

+             "access_users": {"admin": ["Fulgrim", "orphan"]},
 

+             "access_groups": {"admin": ["Adeptus Astartes"]},
 

+         }
 

+         mock_set_pagure.return_value = mock_dist_git
 

+         mock_bodhi = MagicMock()
 

+         mock_bodhi.get_active_branches.return_value = ["f39", "f40"]
 

+         mock_set_bodhi.return_value = mock_bodhi
 

+ 

+ 

          # Mock pagure responses
 

          toddler.requests_session = Mock()
 

          response_pagure_poc = MagicMock()
 
@@ -1252,6 +1325,7 @@ 
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")
 

      @patch("toml.load")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

      def test_process_notify_user_cache_exists(
 

          self,
 

          mock_toml,
 
@@ -1331,9 +1405,29 @@ 
 

  

  class TestMain:
 

      """Test class for `toddler.plugins.distgit_bugzilla_sync.main`."""
 

- 

-     def test_main_no_args(self, capsys):
 

+     

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

+     def test_main_no_args(self, capsys, mock_set_pagure,
 

+         mock_set_bodhi):
 

          """Assert that help is printed if no arg is provided."""
 

+         mock_dist_git = MagicMock()
 

+         mock_dist_git.get_retired_packages.side_effect = (
 

+             ["package01", "package02"],
 

+             ["package01", "package03"],
 

+         )
 

+         mock_dist_git.get_branches.return_value = ["f39", "f40"]
 

+         mock_dist_git.get_project.return_value = {
 

+             "user": {"name": "Gavriel Loken"},
 

+             "access_users": {"admin": ["Fulgrim", "orphan"]},
 

+             "access_groups": {"admin": ["Adeptus Astartes"]},
 

+         }
 

+         mock_set_pagure.return_value = mock_dist_git
 

+         mock_bodhi = MagicMock()
 

+         mock_bodhi.get_active_branches.return_value = ["f39", "f40"]
 

+         mock_set_bodhi.return_value = mock_bodhi
 

+ 

+ 

          with pytest.raises(SystemExit):
 

              main([])
 

  
@@ -1346,13 +1440,17 @@ 
 

          assert err.startswith("usage:")
 

          assert "error: the following arguments are required:" in err
 

  

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load", new=Mock(return_value={}))
 

      def test_main_debug(self, caplog):
 

          """Assert that debug is set correctly."""
 

          with pytest.raises(KeyError, match=r"'email_overrides_file'"):
 

              main(["test.cfg", "--debug"])
 

          assert "Failed to load the file containing the email-overrides" in caplog.text
 

- 

+     

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.utils.bodhi.set_bodhi")
 

      @patch("toml.load", new=Mock(return_value={}))
 

      def test_main(self, caplog):
 

          """Assert that INFO log level is handled correctly."""

file modified
+22 -16 
@@ -25,7 +25,7 @@ 
 

      tqdm = None
 

  

  from ..base import ToddlerBase
 

- from ..utils import bugzilla_system, fedora_account, notify
 

+ from ..utils import bugzilla_system, fedora_account, notify, pagure, bodhi
 

  from ..utils.package_summaries import PackageSummaries
 

  from ..utils.requests import make_session
 

  
@@ -38,7 +38,6 @@ 
 

      """
 

  

      name = "distgit_bugzilla_sync"
 

- 

      amqp_topics = ["org.fedoraproject.*.toddlers.trigger.distgit_bugzilla_sync"]
 

  

      def __init__(self):
 
@@ -54,6 +53,9 @@ 
 

          # Mapping of product to branch regex, will be filled from configuration
 

          self.product_to_branch_regex = {}
 

  

+         self.dist_git = None
 

+         self.bodhi = None
 

+ 

      def accepts_topic(self, topic: str) -> bool:
 

          """Returns a boolean whether this toddler is interested in messages
 

          from this specific topic.
 
@@ -86,6 +88,15 @@ 
 

              where no change was done
 

          :arg dry_run: Don't do any change in bugzilla
 

          """
 

+         self.dist_git = pagure.set_pagure(
 

+             {
 

+                 "pagure_url": config.get("dist_git_url"),
 

+                 "pagure_api_key": config.get("dist_git_token"),
 

+             }
 

+         )
 

+ 

+         self.bodhi = bodhi.set_bodhi(config)
 

+ 

          try:
 

              email_overrides = toml.load(config["email_overrides_file"])
 

          except Exception:
 
@@ -369,9 +380,6 @@ 
 

          """
 

          _log.debug("Querying PDC for EOL information.")
 

  

-         pdc_branches = self.requests_session.get(
 

-             pdc_url + "extras/active_branches.json"
 

-         ).json()
 

          for idx, project in enumerate(self.pagure_projects):
 

              # Summary
 

              summary = None
 
@@ -408,9 +416,9 @@ 
 

                  continue
 

  

              pdc_type = pdc_types[project["namespace"]]
 

-             project["branches"] = pdc_branches.get(pdc_type, {}).get(
 

-                 project["name"], []
 

-             )
 

+ 

+             project["branches"] = self.dist_git.get_branches(project["namespace"],project["name"])
 

+             

              if not project["branches"]:
 

                  self.errors["PDC"].append(
 

                      f"No PDC branch found for {project['namespace']}/{project['name']}"
 
@@ -481,15 +489,13 @@ 
 

          """
 

          branches = project["branches"]
 

          branch_regex = self.product_to_branch_regex.get(product)
 

-         for branch, active in branches:
 

-             if branch_regex:
 

-                 if branch_regex.match(branch) and active:
 

-                     return False
 

+         active_branches = bodhi.get_active_branches()
 

+         for branch in branches:
 

+             if branch_regex.match(branch) and branch in active_branches:
 

+                 if pagure.is_retired_on_branch(project["name"], branch):
 

+                     return True
 

              else:
 

-                 if active:
 

-                     return False
 

-         # No active branches means it is retired.
 

-         return True
 

+                 return False
 

  

  

  # In case this code needs to be run manually for some projects

file modified
+19 
@@ -1175,6 +1175,25 @@ 
 

  

          return result
 

  

+     def is_retired_on_branch(self, package, branch) -> bool:
 

+         """
 

+         Check if the given package is retired on branch
 

+ 

+         Params:
 

+           package: Package name
 

+           branch: Branch name
 

+ 

+         Raises:
 

+           `toddlers.utils.exceptions.PagureError``: When orphaning the package fails.
 

+         """
 

+ 

+         retired_on_branch = self.get_retired_packages(branch)
 

+ 

+         if package in retired_on_branch:
 

+             return True
 

+         else:
 

+             return False
 

+ 

      def orphan_package(
 

          self, namespace: str, package: str, reason: str, info: str
 

      ) -> None:

Signed-off-by: Tomas Hrcka thrcka@redhat.com

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/e889d46a1cc0414f8335d8ede048837f

I guess this is still failing some tests?

The lack of sync was noted on the devel list and in https://pagure.io/fedora-infrastructure/issue/12127

So we should try and get this fixed up soon. ;)

rebased onto de8c703
2 months ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/ebf2ca7efbfc422b90a8e1732809d8e3

@amedvede mind taking a look at this?

@smilner I will look into this as it's urgent. Hopefully I will get to it today.

I was able to fix one of the tests, but I'm unable to push to @humaton fork.

I will continue working on this on Monday, as @humaton seems to be on vacation from next week on I will probably create a new PR for this one.

Pull-Request has been closed by zlopez
2 months ago
Metadata
None
No Tags
Changes Summary 3
+102 -4
file changed
tests/plugins/test_distgit_bugzilla_sync.py
+22 -16
file changed
toddlers/plugins/distgit_bugzilla_sync.py
+19 -0
file changed
toddlers/utils/pagure.py
Powered by Pagure 5.14.1
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.