#243 Replace PDC calls in distgit_bugzilla_sync
Merged 2 months ago by zlopez. Opened 2 months ago by zlopez.
fedora-infra/ zlopez/toddlers distgit_bz_sync  into  main

@@ -21,19 +21,19 @@ 

          "dist_git_url": "https://src.fedoraproject.org",

          "mail_server": "bastion",

          "admin_email": "root@localhost",

-         "pdc_config": {

-             "server": "https://pdc.fedoraproject.org/rest_api/v1",

-         },

+         "pagure_api_key": "some api key",

+         "pagure_url": "https://src.fedoraproject.org",

          # distgit_bugzilla_sync config values

          "ignorable_accounts": [],

          "fasjson": False,

          "default_qa_contact": "nurgle@fedoraproject.org",

          "notify_admins": ["root@localhost.localdomain"],

-         "pdc_types": {

-             "rpms": "rpm",

-         },

          "products": {

-             "Fedora": {"namespace": "rpms", "versions": ["rawhide", 33, 32, 31]},

+             "Fedora": {

+                 "branch_regex": r"^((f\d+)|rawhide)$",

+                 "namespace": "rpms",

+                 "versions": ["rawhide", 33, 32, 31],

+             },

              "Fedora EPEL": {

                  "branch_regex": r"^epel\d+$",

                  "versions": ["epel9", "epel8"],
@@ -66,6 +66,12 @@ 

  class TestDistgitBugzillaSyncToddler:

      toddler_cls = DistgitBugzillaSync

  

+     def setup_method(self):

+         """

+         Initialize toddler.

+         """

+         self.requests_session = Mock()

+ 

      def test_accepts_topic_invalid(self, toddler):

          assert toddler.accepts_topic("foo.bar") is False

  
@@ -80,8 +86,28 @@ 

      def test_accepts_topic_valid(self, toddler, topic):

          assert toddler.accepts_topic(topic)

  

-     def test_process_no_email_override_file(self, toddler):

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

+     def test_process_no_email_override_file(

+         self, mock_set_bodhi, mock_set_pagure, toddler

+     ):

          """Assert that the exception is raised when e-mail overrides file is not provided."""

+         mock_dist_git = MagicMock()

+         mock_dist_git.get_retired_packages.side_effect = (

+             ["package01", "package02"],

+             ["package01", "package03"],

+         )

+         mock_dist_git.get_branches.return_value = ["f39", "f40"]

+         mock_dist_git.get_project.return_value = {

+             "user": {"name": "Gavriel Loken"},

+             "access_users": {"admin": ["Fulgrim", "orphan"]},

+             "access_groups": {"admin": ["Adeptus Astartes"]},

+         }

+         mock_set_pagure.return_value = mock_dist_git

+         mock_bodhi = MagicMock()

+         mock_bodhi.get_active_branches.return_value = ["f39", "f40"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

          with pytest.raises(KeyError) as exc:

              toddler.process(

                  config={},
@@ -95,10 +121,14 @@ 

      )

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_dry_run_edit_project(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_bugzilla,

          mock_fas,

          mock_summaries,
@@ -121,34 +151,48 @@ 

          }

          response_pagure_cc = MagicMock()

          response_pagure_cc.json.return_value = {"rpms": {"foo": ["Slaanesh"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {"foo": [["f32", True], ["epel8", True]]}

-         }

+ 

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS

          mock_fas.get_bz_email_user.side_effect = mock_bz_mail

+ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {"foo": "dummy"}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=True)

  

+         # Asserts

          mock_toml.assert_called_with("dummy_file")

  

          assert toddler.namespace_to_product == {"rpms": "Fedora"}

  

          assert toddler.product_to_branch_regex == {

-             "Fedora EPEL": re.compile(r"^epel\d+$")

+             "Fedora": re.compile(r"^((f\d+)|rawhide)$"),

+             "Fedora EPEL": re.compile(r"^epel\d+$"),

          }

  

          assert toddler.branch_regex_to_product == {

-             re.compile(r"^epel\d+$"): "Fedora EPEL"

+             re.compile(r"^epel\d+$"): "Fedora EPEL",

+             re.compile(r"^((f\d+)|rawhide)$"): "Fedora",

          }

  

          assert len(toddler.errors) == 0
@@ -158,9 +202,13 @@ 

          assert toddler.requests_session.get.mock_calls == [

              call(config["dist_git_url"] + "/extras/pagure_poc.json", timeout=120),

              call(config["dist_git_url"] + "/extras/pagure_bz.json", timeout=120),

-             call("https://pdc.fedoraproject.org/extras/active_branches.json"),

          ]

  

+         mock_pagure.get_branches.assert_called_once_with("rpms", "foo")

+ 

+         # One call for each product

+         mock_bodhi.get_active_branches.call_count == 2

+ 

          assert toddler.pagure_projects == [

              {

                  "namespace": "rpms",
@@ -169,7 +217,7 @@ 

                  "epelpoc": "Tzeentch",

                  "watchers": ["Slaanesh"],

                  "summary": "Summary",

-                 "branches": [["f32", True], ["epel8", True]],

+                 "branches": ["f32", "epel8"],

                  "products": [

                      "Fedora",

                      "Fedora EPEL",
@@ -234,10 +282,14 @@ 

      )

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_dry_run_add_project(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_bugzilla,

          mock_fas,

          mock_summaries,
@@ -260,15 +312,10 @@ 

          }

          response_pagure_cc = MagicMock()

          response_pagure_cc.json.return_value = {"rpms": {"foo": ["Slaanesh"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {"foo": [["f32", True], ["epel8", True]]}

-         }

+ 

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -277,18 +324,36 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=True)

  

+         # Asserts

          mock_toml.assert_called_with("dummy_file")

  

          assert toddler.namespace_to_product == {"rpms": "Fedora"}

  

          assert toddler.product_to_branch_regex == {

-             "Fedora EPEL": re.compile(r"^epel\d+$")

+             "Fedora": re.compile(r"^((f\d+)|rawhide)$"),

+             "Fedora EPEL": re.compile(r"^epel\d+$"),

          }

  

          assert toddler.branch_regex_to_product == {

-             re.compile(r"^epel\d+$"): "Fedora EPEL"

+             re.compile(r"^epel\d+$"): "Fedora EPEL",

+             re.compile(r"^((f\d+)|rawhide)$"): "Fedora",

          }

  

          assert len(toddler.errors) == 0
@@ -298,7 +363,6 @@ 

          assert toddler.requests_session.get.mock_calls == [

              call(config["dist_git_url"] + "/extras/pagure_poc.json", timeout=120),

              call(config["dist_git_url"] + "/extras/pagure_bz.json", timeout=120),

-             call("https://pdc.fedoraproject.org/extras/active_branches.json"),

          ]

  

          assert toddler.pagure_projects == [
@@ -309,7 +373,7 @@ 

                  "epelpoc": "Tzeentch",

                  "watchers": ["Slaanesh"],

                  "summary": "Summary",

-                 "branches": [["f32", True], ["epel8", True]],

+                 "branches": ["f32", "epel8"],

                  "products": [

                      "Fedora",

                      "Fedora EPEL",
@@ -370,10 +434,14 @@ 

      )

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_dry_run_specific_project(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_bugzilla,

          mock_fas,

          mock_summaries,
@@ -407,18 +475,9 @@ 

                  "bar": ["Slaanesh"],

              }

          }

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {

-                 "foo": [["f32", True], ["epel8", True]],

-                 "bar": [["f32", True], ["epel8", True]],

-             }

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -427,7 +486,24 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, projects=["rpms/foo"], dry_run=True)

+ 

+         # Asserts

          assert toddler.pagure_projects == [

              {

                  "namespace": "rpms",
@@ -436,7 +512,7 @@ 

                  "epelpoc": "Tzeentch",

                  "watchers": ["Slaanesh"],

                  "summary": "Summary",

-                 "branches": [["f32", True], ["epel8", True]],

+                 "branches": ["f32", "epel8"],

                  "products": [

                      "Fedora",

                      "Fedora EPEL",
@@ -493,10 +569,14 @@ 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_report_protocol_error(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_notify,

          mock_bugzilla,

          mock_fas,
@@ -508,6 +588,7 @@ 

          """Assert that `xmlrpc.client.ProtocolError` is reported correctly."""

          # Adjust config

          config["temp_folder"] = tmpdir

+ 

          # Mock toml load

          email_overrides = Mock()

          mock_toml.return_value = email_overrides
@@ -529,15 +610,9 @@ 

                  "foo": ["Slaanesh"],

              }

          }

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {"foo": [["f32", True], ["epel8", True]]}

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -551,9 +626,24 @@ 

              "The name khorne@fedoraproject.org is not a valid username",

              {},

          )

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

  

+         # Run test

          toddler.process(config=config, message={}, dry_run=False)

  

+         # Asserts

          assert mock_bugzilla.add_component.mock_calls == [

              call(

                  product="Fedora",
@@ -581,7 +671,6 @@ 

                  "The name khorne@fedoraproject.org is not a valid username>\n"

                  "    ()"

              ],

-             "PDC": [],

              "configuration": [],

              "mails": [],

          }
@@ -606,10 +695,14 @@ 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_report_client_error(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_notify,

          mock_bugzilla,

          mock_fas,
@@ -621,6 +714,7 @@ 

          """Assert that `xmlrpc.client.Error` is reported correctly."""

          # Adjust config

          config["temp_folder"] = tmpdir

+ 

          # Mock toml load

          email_overrides = Mock()

          mock_toml.return_value = email_overrides
@@ -642,15 +736,9 @@ 

                  "foo": ["Slaanesh"],

              }

          }

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {"foo": [["f32", True], ["epel8", True]]}

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -662,8 +750,24 @@ 

              "The name khorne@fedoraproject.org is not a valid username"

          )

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=False)

  

+         # Assert

          assert mock_bugzilla.add_component.mock_calls == [

              call(

                  product="Fedora",
@@ -712,7 +816,6 @@ 

                  "    Error('The name khorne@fedoraproject.org is not a valid username')\n"

                  "    ('The name khorne@fedoraproject.org is not a valid username',)",

              ],

-             "PDC": [],

              "configuration": [],

              "mails": [],

          }
@@ -739,10 +842,14 @@ 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_report_missing_mails(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_notify,

          mock_bugzilla,

          mock_fas,
@@ -789,8 +896,24 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=False)

  

+         # Asserts

          assert mock_bugzilla.add_component.mock_calls == [

              call(

                  product="Fedora EPEL",
@@ -808,7 +931,6 @@ 

  

          assert toddler.errors == {

              "bugzilla": [],

-             "PDC": [],

              "configuration": [],

              "mails": [

                  "`Slaanesh` has no bugzilla_email or mailing_list set on `Fedora/foo`",
@@ -835,10 +957,14 @@ 

      )

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_dry_run_verbose(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_bugzilla,

          mock_fas,

          mock_summaries,
@@ -862,15 +988,9 @@ 

          }

          response_pagure_cc = MagicMock()

          response_pagure_cc.json.return_value = {"rpms": {"foo": ["Slaanesh"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {"foo": [["f32", True], ["epel8", True]]}

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -879,9 +999,25 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {"foo": "dummy"}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          with caplog.at_level(logging.DEBUG):

              toddler.process(config=config, message={}, dry_run=True)

  

+         # Asserts

          assert "Building a cache of the rpm package summaries" in caplog.text

  

          assert (
@@ -898,8 +1034,6 @@ 

              in caplog.text

          )

  

-         assert "Querying PDC for EOL information." in caplog.text

- 

          assert "Building bugzilla's products in-memory cache" in caplog.text

  

          assert "Querying bugzilla but not doing anything" in caplog.text
@@ -932,10 +1066,14 @@ 

      )

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_missing_namespace(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_bugzilla,

          mock_fas,

          mock_summaries,
@@ -958,15 +1096,9 @@ 

          }

          response_pagure_cc = MagicMock()

          response_pagure_cc.json.return_value = {"modules": {"foo": ["Slaanesh"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {"foo": [["f32", True], ["epel8", True]]}

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -975,67 +1107,20 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {"foo": "dummy"}

  

-         toddler.process(config=config, message={}, dry_run=True)

- 

-         mock_bugzilla.edit_component.assert_not_called()

- 

-     @patch(

-         "toddlers.plugins.distgit_bugzilla_sync.PackageSummaries.get_package_summaries"

-     )

-     @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

-     @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

-     @patch("toml.load")

-     def test_process_missing_pdc_branches(

-         self,

-         mock_toml,

-         mock_bugzilla,

-         mock_fas,

-         mock_summaries,

-         config,

-         toddler,

-     ):

-         """Assert that dry run with edit is processed correctly."""

-         # Mock toml load

-         email_overrides = Mock()

-         mock_toml.return_value = email_overrides

- 

-         # Mock package summaries response

-         mock_summaries.return_value = {"foo": "Summary"}

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = []

+         mock_set_pagure.return_value = mock_pagure

  

-         # Mock pagure responses

-         toddler.requests_session = Mock()

-         response_pagure_poc = MagicMock()

-         response_pagure_poc.json.return_value = {

-             "rpms": {"foo": {"fedora": "Khorne", "epel": "Tzeentch"}}

-         }

-         response_pagure_cc = MagicMock()

-         response_pagure_cc.json.return_value = {"rpms": {"foo": ["Slaanesh"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {}

-         toddler.requests_session.get.side_effect = (

-             response_pagure_poc,

-             response_pagure_cc,

-             response_pdc,

-         )

- 

-         # Mock FAS

-         mock_fas.get_bz_email_user.side_effect = mock_bz_mail

- 

-         # Mock bugzilla

-         mock_bugzilla.get_product_info_packages.return_value = {"foo": "dummy"}

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

  

+         # Run test

          toddler.process(config=config, message={}, dry_run=True)

  

-         assert toddler.errors == {

-             "bugzilla": [],

-             "PDC": [

-                 "No PDC branch found for rpms/foo",

-             ],

-             "configuration": [],

-             "mails": [],

-         }

- 

+         # Asserts

          mock_bugzilla.edit_component.assert_not_called()

  

      @patch(
@@ -1043,10 +1128,14 @@ 

      )

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_orphaned_project(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_bugzilla,

          mock_fas,

          mock_summaries,
@@ -1069,17 +1158,9 @@ 

          }

          response_pagure_cc = MagicMock()

          response_pagure_cc.json.return_value = {"rpms": {"foo": ["@Chaos"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {

-                 "foo": [["f32", False], ["epel8", False]],

-             }

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -1089,8 +1170,24 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {"foo": "dummy"}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = True

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=True)

  

+         # Asserts

          assert toddler.pagure_projects == [

              {

                  "namespace": "rpms",
@@ -1099,7 +1196,7 @@ 

                  "epelpoc": "Tzeentch",

                  "watchers": ["@Chaos"],

                  "summary": "Summary",

-                 "branches": [["f32", False], ["epel8", False]],

+                 "branches": ["f32", "epel8"],

                  "products": [

                      "Fedora",

                      "Fedora EPEL",
@@ -1115,6 +1212,11 @@ 

              }

          ]

  

+         mock_pagure.is_retired_on_branch.calls == [

+             call("foo", "f32"),

+             call("foo", "epel8"),

+         ]

+ 

          assert mock_bugzilla.edit_component.mock_calls == [

              call(

                  owner="orphan@fedoraproject.org",
@@ -1157,6 +1259,8 @@ 

      @patch(

          "toddlers.plugins.distgit_bugzilla_sync.PackageSummaries.get_package_summaries"

      )

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

      @patch("toml.load")
@@ -1165,6 +1269,8 @@ 

          mock_toml,

          mock_bugzilla,

          mock_fas,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_summaries,

          config,

          toddler,
@@ -1179,6 +1285,22 @@ 

          # Mock package summaries response

          mock_summaries.return_value = {"foo": "Summary"}

  

+         mock_dist_git = MagicMock()

+         mock_dist_git.get_retired_packages.side_effect = (

+             ["package01", "package02"],

+             ["package01", "package03"],

+         )

+         mock_dist_git.get_branches.return_value = ["f39", "f40"]

+         mock_dist_git.get_project.return_value = {

+             "user": {"name": "Gavriel Loken"},

+             "access_users": {"admin": ["Fulgrim", "orphan"]},

+             "access_groups": {"admin": ["Adeptus Astartes"]},

+         }

+         mock_set_pagure.return_value = mock_dist_git

+         mock_bodhi = MagicMock()

+         mock_bodhi.get_active_branches.return_value = ["f39", "f40"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

          # Mock pagure responses

          toddler.requests_session = Mock()

          response_pagure_poc = MagicMock()
@@ -1187,17 +1309,9 @@ 

          }

          response_pagure_cc = MagicMock()

          response_pagure_cc.json.return_value = {"rpms": {"foo": ["Slaanesh"]}}

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {

-                 "foo": [["f32", True], ["epel8", True]],

-             }

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -1206,8 +1320,24 @@ 

          # Mock bugzilla

          mock_bugzilla.get_product_info_packages.return_value = {"foo": "dummy"}

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=True)

  

+         # Asserts

          assert mock_bugzilla.edit_component.mock_calls == [

              call(

                  owner="khorne@fedoraproject.org",
@@ -1251,10 +1381,14 @@ 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load")

      def test_process_notify_user_cache_exists(

          self,

          mock_toml,

+         mock_set_bodhi,

+         mock_set_pagure,

          mock_notify,

          mock_bugzilla,

          mock_fas,
@@ -1299,17 +1433,9 @@ 

                  "foo": ["Slaanesh"],

              }

          }

-         # Mock PDC response

-         response_pdc = MagicMock()

-         response_pdc.json.return_value = {

-             "rpm": {

-                 "foo": [["f32", True], ["epel8", True]],

-             }

-         }

          toddler.requests_session.get.side_effect = (

              response_pagure_poc,

              response_pagure_cc,

-             response_pdc,

          )

  

          # Mock FAS
@@ -1321,8 +1447,24 @@ 

              "The name khorne@fedoraproject.org is not a valid username"

          )

  

+         # Mock pagure

+         mock_pagure = Mock()

+         mock_pagure.get_branches.return_value = [

+             "f32",

+             "epel8",

+         ]

+         mock_pagure.is_retired_on_branch.return_value = False

+         mock_set_pagure.return_value = mock_pagure

+ 

+         # Mock bodhi

+         mock_bodhi = Mock()

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel8"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

+         # Run test

          toddler.process(config=config, message={}, dry_run=False)

  

+         # Asserts

          mock_notify.notify_packager_distgit_sync_error.assert_not_called()

  

          with open(os.path.join(tmpdir, "user_cache.json")) as stream:
@@ -1332,8 +1474,26 @@ 

  class TestMain:

      """Test class for `toddler.plugins.distgit_bugzilla_sync.main`."""

  

-     def test_main_no_args(self, capsys):

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

+     def test_main_no_args(self, mock_set_bodhi, mock_set_pagure, capsys):

          """Assert that help is printed if no arg is provided."""

+         mock_dist_git = MagicMock()

+         mock_dist_git.get_retired_packages.side_effect = (

+             ["package01", "package02"],

+             ["package01", "package03"],

+         )

+         mock_dist_git.get_branches.return_value = ["f39", "f40"]

+         mock_dist_git.get_project.return_value = {

+             "user": {"name": "Gavriel Loken"},

+             "access_users": {"admin": ["Fulgrim", "orphan"]},

+             "access_groups": {"admin": ["Adeptus Astartes"]},

+         }

+         mock_set_pagure.return_value = mock_dist_git

+         mock_bodhi = MagicMock()

+         mock_bodhi.get_active_branches.return_value = ["f39", "f40"]

+         mock_set_bodhi.return_value = mock_bodhi

+ 

          with pytest.raises(SystemExit):

              main([])

  
@@ -1346,15 +1506,19 @@ 

          assert err.startswith("usage:")

          assert "error: the following arguments are required:" in err

  

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load", new=Mock(return_value={}))

-     def test_main_debug(self, caplog):

+     def test_main_debug(self, mock_set_bodhi, mock_set_pagure, caplog):

          """Assert that debug is set correctly."""

          with pytest.raises(KeyError, match=r"'email_overrides_file'"):

              main(["test.cfg", "--debug"])

          assert "Failed to load the file containing the email-overrides" in caplog.text

  

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")

      @patch("toml.load", new=Mock(return_value={}))

-     def test_main(self, caplog):

+     def test_main(self, mock_set_bodhi, mock_set_pagure, caplog):

          """Assert that INFO log level is handled correctly."""

          with pytest.raises(KeyError, match=r"'email_overrides_file'"):

              main(["test.cfg"])

@@ -108,10 +108,12 @@ 

          self.toddler_cls.process_block_retired(config, message)

          assert caplog.records[-1].message == "dead.package file was not added, bailing"

  

-     def test_dead_package_added_to_main_branch(self, caplog):

+     @patch("toddlers.plugins.koji_block_retired.KojiBlockRetired.get_rawhide_tag")

+     def test_dead_package_added_to_main_branch(self, mock_get_rawhide_tag, caplog):

          """

          Assert that that main branch will be changed to rawhide tag

          """

+         mock_get_rawhide_tag.return_value = "f41"

          caplog.set_level(logging.INFO)

  

          message = MagicMock()

@@ -1777,6 +1777,61 @@ 

          )

  

  

+ class TestPagureIsRetiredOnBranch:

+     """

+     Test class for `toddlers.utils.pagure.Pagure.is_retired_on_branch` method.

+     """

+ 

+     def setup_method(self):

+         """

+         Setup method for the test class.

+         """

+         config = {

+             "pagure_url": "https://pagure.io",

+             "pagure_api_key": "Very secret key",

+         }

+         self.pagure = pagure.set_pagure(config)

+         self.pagure.get_retired_packages = Mock()

+ 

+     def test_is_retired_on_branch_true(self):

+         """

+         Assert that retired package is correctly processed.

+         """

+         # Prepare

+         branch = "f40"

+         package_list = ["package01", "package02"]

+         package = "package01"

+ 

+         self.pagure.get_retired_packages.return_value = package_list

+ 

+         # Run test

+         result = self.pagure.is_retired_on_branch(package, branch)

+ 

+         # Asserts

+         assert result is True

+ 

+         self.pagure.get_retired_packages.assert_called_with(branch)

+ 

+     def test_is_retired_on_branch_false(self):

+         """

+         Assert that non-retired package is correctly processed.

+         """

+         # Prepare

+         branch = "f40"

+         package_list = ["package02"]

+         package = "package01"

+ 

+         self.pagure.get_retired_packages.return_value = package_list

+ 

+         # Run test

+         result = self.pagure.is_retired_on_branch(package, branch)

+ 

+         # Asserts

+         assert result is False

+ 

+         self.pagure.get_retired_packages.assert_called_with(branch)

+ 

+ 

  class TestPagureOrphanPackage:

      """

      Test class for `toddlers.utils.pagure.Pagure.orphan_package` method.

file modified
+1 -8
@@ -151,17 +151,10 @@ 

  [consumer_config.distgit_bugzilla_sync.products]

  

  [consumer_config.distgit_bugzilla_sync.products.Fedora]

+ branch_regex = '^((f\d+)|rawhide)$'

  namespace = "rpms"

  versions = ["rawhide", "33", "32", "31"]

  

- [consumer_config.distgit_bugzilla_sync.products."Fedora Container Images"]

- namespace = "container"

- versions = ["rawhide", "33", "32", "31"]

- 

- [consumer_config.distgit_bugzilla_sync.products."Fedora Modules"]

- namespace = "modules"

- versions = []

- 

  [consumer_config.distgit_bugzilla_sync.products."Fedora EPEL"]

  branch_regex = '^epel\d+$'

  versions = ["epel9", "epel8"]

@@ -25,7 +25,7 @@ 

      tqdm = None

  

  from ..base import ToddlerBase

- from ..utils import bugzilla_system, fedora_account, notify

+ from ..utils import bodhi, bugzilla_system, fedora_account, notify, pagure

  from ..utils.package_summaries import PackageSummaries

  from ..utils.requests import make_session

  
@@ -38,7 +38,6 @@ 

      """

  

      name = "distgit_bugzilla_sync"

- 

      amqp_topics = ["org.fedoraproject.*.toddlers.trigger.distgit_bugzilla_sync"]

  

      def __init__(self):
@@ -54,6 +53,9 @@ 

          # Mapping of product to branch regex, will be filled from configuration

          self.product_to_branch_regex = {}

  

+         self.dist_git = None

+         self.bodhi = None

+ 

      def accepts_topic(self, topic: str) -> bool:

          """Returns a boolean whether this toddler is interested in messages

          from this specific topic.
@@ -86,6 +88,15 @@ 

              where no change was done

          :arg dry_run: Don't do any change in bugzilla

          """

+         self.dist_git = pagure.set_pagure(

+             {

+                 "pagure_url": config.get("dist_git_url"),

+                 "pagure_api_key": config.get("dist_git_token"),

+             }

+         )

+ 

+         self.bodhi = bodhi.set_bodhi(config)

+ 

          try:

              email_overrides = toml.load(config["email_overrides_file"])

          except Exception:
@@ -115,8 +126,6 @@ 

          self._get_pagure_projects(config["dist_git_url"], project_list=projects)

  

          self._add_branches_products_and_summaries(

-             config["pdc_config"]["server"].split("rest_api")[0],

-             config["pdc_types"],

              rpm_summaries,

              config["ignorable_accounts"],

          )
@@ -265,7 +274,7 @@ 

              # Build the report for the admins

              report = ["ERROR REPORT"]

              final_keys = []

-             for key in ["configuration", "PDC", "mails", "bugzilla"]:

+             for key in ["configuration", "mails", "bugzilla"]:

                  if self.errors[key]:

                      final_keys.append(key)

                      report.append(key)
@@ -350,28 +359,18 @@ 

  

      def _add_branches_products_and_summaries(

          self,

-         pdc_url: str,

-         pdc_types: dict,

          rpm_summaries: Mapping[str, str],

          ignorable_accounts: list,

      ) -> None:

          """For each project retrieved, this method adds branches, products

          and summary information.

-         The branches are retrieved from PDC.

          The products are determined based on the branches.

          The summaries are coming from the primary.xml file of the Rawhide repodata

          in Koji.

  

-         :arg pdc_url: PDC url to get data from

-         :arg pdc_types: Mapping of types of PDC to distgit namespaces

          :arg rpm_summaries: Package summaries from koji

          :arg ignorable_accounts: Accounts to ignore when filling up watchers list.

          """

-         _log.debug("Querying PDC for EOL information.")

- 

-         pdc_branches = self.requests_session.get(

-             pdc_url + "extras/active_branches.json"

-         ).json()

          for idx, project in enumerate(self.pagure_projects):

              # Summary

              summary = None
@@ -379,46 +378,18 @@ 

                  summary = rpm_summaries.get(project["name"])

              project["summary"] = summary

  

-             # Branches

-             if project["namespace"] not in pdc_types:

-                 project["branches"] = []

-                 project["products"] = []

-                 error = (

-                     f'Namespace `{project["namespace"]}` not found in the pdc_type '

-                     f"configuration key -- ignoring it"

-                 )

-                 if error not in self.errors["configuration"]:

-                     self.errors["configuration"].append(error)

-                     _log.debug(

-                         "Namespace `%s` not found in the pdc_type "

-                         "configuration key, project %s/%s "

-                         "ignored",

-                         project["namespace"],

-                         project["namespace"],

-                         project["name"],

-                     )

-                 _log.debug(

-                     "Namespace `%s` not found in the pdc_type "

-                     "configuration key, project %s/%s "

-                     "ignored",

-                     project["namespace"],

-                     project["namespace"],

-                     project["name"],

-                 )

-                 continue

- 

-             pdc_type = pdc_types[project["namespace"]]

-             project["branches"] = pdc_branches.get(pdc_type, {}).get(

-                 project["name"], []

+             project["branches"] = self.dist_git.get_branches(

+                 project["namespace"], project["name"]

              )

+ 

              if not project["branches"]:

-                 self.errors["PDC"].append(

-                     f"No PDC branch found for {project['namespace']}/{project['name']}"

+                 self.errors["dist-git"].append(

+                     f"No branch found for {project['namespace']}/{project['name']}"

                  )

  

              # Products

              products = set()

-             for branch, active in project.get("branches"):

+             for branch in project.get("branches"):

                  for regex, product in self.branch_regex_to_product.items():

                      if regex.match(branch):

                          products.add(product)
@@ -481,14 +452,15 @@ 

          """

          branches = project["branches"]

          branch_regex = self.product_to_branch_regex.get(product)

-         for branch, active in branches:

-             if branch_regex:

-                 if branch_regex.match(branch) and active:

-                     return False

-             else:

-                 if active:

+         active_branches = self.bodhi.get_active_branches()

+         for branch in branches:

+             # Only check the branches that match the regex for Product in bugzilla

+             if branch_regex.match(branch) and branch in active_branches:

+                 # One branch matching the product regex is still active

+                 # The project is not retired yet

+                 if not self.dist_git.is_retired_on_branch(project["name"], branch):

                      return False

-         # No active branches means it is retired.

+         # No branch matching the regex is active, the project is retired in the product

          return True

  

  

file modified
+17
@@ -1175,6 +1175,23 @@ 

  

          return result

  

+     def is_retired_on_branch(self, package: str, branch: str) -> bool:

+         """

+         Check if the given package is retired on branch.

+ 

+         Works only with dist-git

+ 

+         Params:

+           package: Package name

+           branch: Branch name

+         """

+         retired_on_branch = self.get_retired_packages(branch)

+ 

+         if package in retired_on_branch:

+             return True

+         else:

+             return False

+ 

      def orphan_package(

          self, namespace: str, package: str, reason: str, info: str

      ) -> None:

This is the effort to get rid of PDC from distgit_bugzilla_sync toddler.

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/715400fcd34f47429a842ae3cf08ee55

Would it be possible to mock the get_rawhide_tag function for all of the tests, e.g. moving the mock to the setup_method?

@lenkaseg Is that needed? I only saw one test failing.

I was thinking it might be better to avoid unnecessary api calls from inside the tests, but I guess it's fine.
The only other test, that gets to call bodhi through get_rawhide_tag would be test_dead_package_added_to_different_branch.

Another thing:
The function get_rawhide_tag limits the search only to active branches. Is it desired behavior or we want to be able to block packages in koji for any release tag, regardless of the status of the branch?

That will be probably question for releng folks.

rebased onto 74cacd4

2 months ago

Pull-Request has been merged by zlopez

2 months ago

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/3094ea26405d49f4b069729668b1c9d9