fedora-infra / toddlers

Clone
Source Code
GIT

#253 Fix for container namespace
Closed a year ago by zlopez. Opened a year ago by zlopez.
fedora-infra/ zlopez/toddlers distgit_bz_sync  into  main

file modified
+313 -20 
@@ -35,8 +35,14 @@ 
 

                  "versions": ["rawhide", 33, 32, 31],
 

              },
 

              "Fedora EPEL": {
 

-                 "branch_regex": r"^epel\d+$",
 

-                 "versions": ["epel9", "epel8"],
 

+                 "branch_regex": r"^epel\d+(?:\.\d+)?$",
 

+                 "namespace": "rpms",
 

+                 "versions": ["epel10.0", "epel9", "epel8"],
 

+             },
 

+             "Fedora Container Images": {
 

+                 "branch_regex": r"^((f\d+)|rawhide)$",
 

+                 "namespace": "container",
 

+                 "versions": ["rawhide", 33, 32, 31],
 

              },
 

          },
 

      }
 
@@ -183,16 +189,15 @@ 
 

          # Asserts
 

          mock_toml.assert_called_with("dummy_file")
 

  

-         assert toddler.namespace_to_product == {"rpms": "Fedora"}
 

+         assert toddler.namespace_to_product == {
 

+             "rpms": ["Fedora", "Fedora EPEL"],
 

+             "container": ["Fedora Container Images"],
 

+         }
 

  

          assert toddler.product_to_branch_regex == {
 

              "Fedora": re.compile(r"^((f\d+)|rawhide)$"),
 

-             "Fedora EPEL": re.compile(r"^epel\d+$"),
 

-         }
 

- 

-         assert toddler.branch_regex_to_product == {
 

-             re.compile(r"^epel\d+$"): "Fedora EPEL",
 

-             re.compile(r"^((f\d+)|rawhide)$"): "Fedora",
 

+             "Fedora Container Images": re.compile(r"^((f\d+)|rawhide)$"),
 

+             "Fedora EPEL": re.compile(r"^epel\d+(?:\.\d+)?$"),
 

          }
 

  

          assert len(toddler.errors) == 0
 
@@ -236,6 +241,7 @@ 
 

          assert mock_bugzilla.get_product_info_packages.mock_calls == [
 

              call("Fedora"),
 

              call("Fedora EPEL"),
 

+             call("Fedora Container Images"),
 

          ]
 

  

          assert mock_bugzilla.edit_component.mock_calls == [
 
@@ -263,7 +269,7 @@ 
 

                  package="foo",
 

                  component="dummy",
 

                  cc_list=["slaanesh@fedoraproject.org", "tzeentch@fedoraproject.org"],
 

-                 versions=["epel9", "epel8"],
 

+                 versions=["epel10.0", "epel9", "epel8"],
 

                  description="Summary",
 

                  fas_users_info={
 

                      "khorne@fedoraproject.org": "Khorne",
 
@@ -324,7 +330,10 @@ 
 

          # Asserts
 

          mock_toml.assert_called_with("dummy_file")
 

  

-         assert toddler.namespace_to_product == {"rpms": "Fedora"}
 

+         assert toddler.namespace_to_product == {
 

+             "rpms": ["Fedora", "Fedora EPEL"],
 

+             "container": ["Fedora Container Images"],
 

+         }
 

  

          assert len(toddler.errors) == 0
 

  
@@ -406,16 +415,15 @@ 
 

          # Asserts
 

          mock_toml.assert_called_with("dummy_file")
 

  

-         assert toddler.namespace_to_product == {"rpms": "Fedora"}
 

+         assert toddler.namespace_to_product == {
 

+             "rpms": ["Fedora", "Fedora EPEL"],
 

+             "container": ["Fedora Container Images"],
 

+         }
 

  

          assert toddler.product_to_branch_regex == {
 

              "Fedora": re.compile(r"^((f\d+)|rawhide)$"),
 

-             "Fedora EPEL": re.compile(r"^epel\d+$"),
 

-         }
 

- 

-         assert toddler.branch_regex_to_product == {
 

-             re.compile(r"^epel\d+$"): "Fedora EPEL",
 

-             re.compile(r"^((f\d+)|rawhide)$"): "Fedora",
 

+             "Fedora Container Images": re.compile(r"^((f\d+)|rawhide)$"),
 

+             "Fedora EPEL": re.compile(r"^epel\d+(?:\.\d+)?$"),
 

          }
 

  

          assert len(toddler.errors) == 0
 
@@ -454,6 +462,7 @@ 
 

          assert mock_bugzilla.get_product_info_packages.mock_calls == [
 

              call("Fedora"),
 

              call("Fedora EPEL"),
 

+             call("Fedora Container Images"),
 

          ]
 

  

          assert mock_bugzilla.add_component.mock_calls == [
 
@@ -630,6 +639,290 @@ 
 

      )
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 

+     @patch("toml.load")
 

+     def test_process_epel10(
 

+         self,
 

+         mock_toml,
 

+         mock_set_bodhi,
 

+         mock_set_pagure,
 

+         mock_bugzilla,
 

+         mock_fas,
 

+         mock_summaries,
 

+         config,
 

+         toddler,
 

+     ):
 

+         """Assert that epel 10 project is processed correctly."""
 

+         # Mock toml load
 

+         email_overrides = Mock()
 

+         mock_toml.return_value = email_overrides
 

+ 

+         # Mock package summaries response
 

+         mock_summaries.return_value = {"foo": "Summary"}
 

+ 

+         # Mock pagure responses
 

+         toddler.requests_session = Mock()
 

+         response_pagure_poc = MagicMock()
 

+         response_pagure_poc.json.return_value = {
 

+             "rpms": {"foo": {"fedora": "Khorne", "epel": "Tzeentch"}}
 

+         }
 

+         response_pagure_cc = MagicMock()
 

+         response_pagure_cc.json.return_value = {"rpms": {"foo": ["Slaanesh"]}}
 

+ 

+         toddler.requests_session.get.side_effect = (
 

+             response_pagure_poc,
 

+             response_pagure_cc,
 

+         )
 

+ 

+         # Mock FAS
 

+         mock_fas.get_bz_email_user.side_effect = mock_bz_mail
 

+ 

+         # Mock bugzilla
 

+         mock_bugzilla.get_product_info_packages.return_value = {}
 

+ 

+         # Mock pagure
 

+         mock_pagure = Mock()
 

+         mock_pagure.get_branches.return_value = [
 

+             "f32",
 

+             "epel10.0",
 

+         ]
 

+         mock_pagure.is_retired_on_branch.return_value = False
 

+         mock_set_pagure.return_value = mock_pagure
 

+ 

+         # Mock bodhi
 

+         mock_bodhi = Mock()
 

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel10.0"]
 

+         mock_set_bodhi.return_value = mock_bodhi
 

+ 

+         # Run test
 

+         toddler.process(config=config, message={}, dry_run=True)
 

+ 

+         # Asserts
 

+         mock_toml.assert_called_with("dummy_file")
 

+ 

+         assert toddler.namespace_to_product == {
 

+             "rpms": ["Fedora", "Fedora EPEL"],
 

+             "container": ["Fedora Container Images"],
 

+         }
 

+ 

+         assert toddler.product_to_branch_regex == {
 

+             "Fedora": re.compile(r"^((f\d+)|rawhide)$"),
 

+             "Fedora Container Images": re.compile(r"^((f\d+)|rawhide)$"),
 

+             "Fedora EPEL": re.compile(r"^epel\d+(?:\.\d+)?$"),
 

+         }
 

+ 

+         assert len(toddler.errors) == 0
 

+ 

+         mock_summaries.assert_called_with(config)
 

+ 

+         assert toddler.requests_session.get.mock_calls == [
 

+             call(config["dist_git_url"] + "/extras/pagure_poc.json", timeout=120),
 

+             call(config["dist_git_url"] + "/extras/pagure_bz.json", timeout=120),
 

+         ]
 

+ 

+         assert toddler.pagure_projects == [
 

+             {
 

+                 "namespace": "rpms",
 

+                 "name": "foo",
 

+                 "poc": "Khorne",
 

+                 "epelpoc": "Tzeentch",
 

+                 "watchers": ["Slaanesh"],
 

+                 "summary": "Summary",
 

+                 "branches": ["f32", "epel10.0"],
 

+                 "products": [
 

+                     "Fedora",
 

+                     "Fedora EPEL",
 

+                 ],
 

+                 "products_poc": {
 

+                     "Fedora": "Khorne",
 

+                     "Fedora EPEL": "Tzeentch",
 

+                 },
 

+                 "products_retired": {
 

+                     "Fedora": False,
 

+                     "Fedora EPEL": False,
 

+                 },
 

+             }
 

+         ]
 

+ 

+         assert mock_bugzilla.get_product_info_packages.mock_calls == [
 

+             call("Fedora"),
 

+             call("Fedora EPEL"),
 

+             call("Fedora Container Images"),
 

+         ]
 

+ 

+         assert mock_bugzilla.add_component.mock_calls == [
 

+             call(
 

+                 product="Fedora",
 

+                 owner="khorne@fedoraproject.org",
 

+                 package="foo",
 

+                 qa_contact="nurgle@fedoraproject.org",
 

+                 cc_list=["slaanesh@fedoraproject.org", "khorne@fedoraproject.org"],
 

+                 fas_users_info={
 

+                     "khorne@fedoraproject.org": "Khorne",
 

+                     "slaanesh@fedoraproject.org": "Slaanesh",
 

+                     "tzeentch@fedoraproject.org": "Tzeentch",
 

+                 },
 

+                 description="Summary",
 

+                 retired=False,
 

+                 print_fas_names=False,
 

+                 dry_run=True,
 

+             ),
 

+             call(
 

+                 product="Fedora EPEL",
 

+                 owner="tzeentch@fedoraproject.org",
 

+                 package="foo",
 

+                 qa_contact="nurgle@fedoraproject.org",
 

+                 cc_list=["slaanesh@fedoraproject.org", "tzeentch@fedoraproject.org"],
 

+                 fas_users_info={
 

+                     "khorne@fedoraproject.org": "Khorne",
 

+                     "slaanesh@fedoraproject.org": "Slaanesh",
 

+                     "tzeentch@fedoraproject.org": "Tzeentch",
 

+                 },
 

+                 description="Summary",
 

+                 retired=False,
 

+                 print_fas_names=False,
 

+                 dry_run=True,
 

+             ),
 

+         ]
 

+ 

+     @patch(
 

+         "toddlers.plugins.distgit_bugzilla_sync.PackageSummaries.get_package_summaries"
 

+     )
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 

+     @patch("toml.load")
 

+     def test_process_container(
 

+         self,
 

+         mock_toml,
 

+         mock_set_bodhi,
 

+         mock_set_pagure,
 

+         mock_bugzilla,
 

+         mock_fas,
 

+         mock_summaries,
 

+         config,
 

+         toddler,
 

+     ):
 

+         """Assert that container project is processed correctly."""
 

+         # Mock toml load
 

+         email_overrides = Mock()
 

+         mock_toml.return_value = email_overrides
 

+ 

+         # Mock package summaries response
 

+         mock_summaries.return_value = {"foo": "Summary"}
 

+ 

+         # Mock pagure responses
 

+         toddler.requests_session = Mock()
 

+         response_pagure_poc = MagicMock()
 

+         response_pagure_poc.json.return_value = {
 

+             "container": {"foo": {"fedora": "Khorne", "epel": "Tzeentch"}}
 

+         }
 

+         response_pagure_cc = MagicMock()
 

+         response_pagure_cc.json.return_value = {"container": {"foo": ["Slaanesh"]}}
 

+ 

+         toddler.requests_session.get.side_effect = (
 

+             response_pagure_poc,
 

+             response_pagure_cc,
 

+         )
 

+ 

+         # Mock FAS
 

+         mock_fas.get_bz_email_user.side_effect = mock_bz_mail
 

+ 

+         # Mock bugzilla
 

+         mock_bugzilla.get_product_info_packages.return_value = {}
 

+ 

+         # Mock pagure
 

+         mock_pagure = Mock()
 

+         mock_pagure.get_branches.return_value = [
 

+             "f32",
 

+             "epel10.0",
 

+         ]
 

+         mock_pagure.is_retired_on_branch.return_value = False
 

+         mock_set_pagure.return_value = mock_pagure
 

+ 

+         # Mock bodhi
 

+         mock_bodhi = Mock()
 

+         mock_bodhi.get_active_branches.return_value = ["f32", "epel10.0"]
 

+         mock_set_bodhi.return_value = mock_bodhi
 

+ 

+         # Run test
 

+         toddler.process(config=config, message={}, dry_run=True)
 

+ 

+         # Asserts
 

+         mock_toml.assert_called_with("dummy_file")
 

+ 

+         assert toddler.namespace_to_product == {
 

+             "rpms": ["Fedora", "Fedora EPEL"],
 

+             "container": ["Fedora Container Images"],
 

+         }
 

+ 

+         assert toddler.product_to_branch_regex == {
 

+             "Fedora": re.compile(r"^((f\d+)|rawhide)$"),
 

+             "Fedora Container Images": re.compile(r"^((f\d+)|rawhide)$"),
 

+             "Fedora EPEL": re.compile(r"^epel\d+(?:\.\d+)?$"),
 

+         }
 

+ 

+         assert len(toddler.errors) == 0
 

+ 

+         mock_summaries.assert_called_with(config)
 

+ 

+         assert toddler.requests_session.get.mock_calls == [
 

+             call(config["dist_git_url"] + "/extras/pagure_poc.json", timeout=120),
 

+             call(config["dist_git_url"] + "/extras/pagure_bz.json", timeout=120),
 

+         ]
 

+ 

+         assert toddler.pagure_projects == [
 

+             {
 

+                 "namespace": "container",
 

+                 "name": "foo",
 

+                 "poc": "Khorne",
 

+                 "epelpoc": "Tzeentch",
 

+                 "watchers": ["Slaanesh"],
 

+                 "summary": None,
 

+                 "branches": ["f32", "epel10.0"],
 

+                 "products": [
 

+                     "Fedora Container Images",
 

+                 ],
 

+                 "products_poc": {
 

+                     "Fedora Container Images": "Khorne",
 

+                 },
 

+                 "products_retired": {
 

+                     "Fedora Container Images": False,
 

+                 },
 

+             }
 

+         ]
 

+ 

+         assert mock_bugzilla.get_product_info_packages.mock_calls == [
 

+             call("Fedora"),
 

+             call("Fedora EPEL"),
 

+             call("Fedora Container Images"),
 

+         ]
 

+ 

+         assert mock_bugzilla.add_component.mock_calls == [
 

+             call(
 

+                 product="Fedora Container Images",
 

+                 owner="khorne@fedoraproject.org",
 

+                 package="foo",
 

+                 qa_contact="nurgle@fedoraproject.org",
 

+                 cc_list=["slaanesh@fedoraproject.org", "khorne@fedoraproject.org"],
 

+                 fas_users_info={
 

+                     "khorne@fedoraproject.org": "Khorne",
 

+                     "slaanesh@fedoraproject.org": "Slaanesh",
 

+                 },
 

+                 description=None,
 

+                 retired=False,
 

+                 print_fas_names=False,
 

+                 dry_run=True,
 

+             )
 

+         ]
 

+ 

+     @patch(
 

+         "toddlers.plugins.distgit_bugzilla_sync.PackageSummaries.get_package_summaries"
 

+     )
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.fedora_account")
 

+     @patch("toddlers.plugins.distgit_bugzilla_sync.bugzilla_system")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.notify")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.pagure.set_pagure")
 

      @patch("toddlers.plugins.distgit_bugzilla_sync.bodhi.set_bodhi")
 
@@ -1304,7 +1597,7 @@ 
 

                  package="foo",
 

                  component="dummy",
 

                  cc_list=["chaos@fedoraproject.org", "tzeentch@fedoraproject.org"],
 

-                 versions=["epel9", "epel8"],
 

+                 versions=["epel10.0", "epel9", "epel8"],
 

                  description="Summary",
 

                  fas_users_info={
 

                      "orphan@fedoraproject.org": "orphan",
 
@@ -1424,7 +1717,7 @@ 
 

                  package="foo",
 

                  component="dummy",
 

                  cc_list=["tzeentch@fedoraproject.org"],
 

-                 versions=["epel9", "epel8"],
 

+                 versions=["epel10.0", "epel9", "epel8"],
 

                  description="Summary",
 

                  fas_users_info={
 

                      "khorne@fedoraproject.org": "Khorne",

file modified
+25 -18 
@@ -46,8 +46,6 @@ 
 

          self.pagure_projects = []
 

          # Dictionary of errors to send notifications about
 

          self.errors = collections.defaultdict(list)
 

-         # Mapping of branch regex to product, will be filled from configuration
 

-         self.branch_regex_to_product = {}
 

          # Mapping of namespace to product, will be filled from configuration
 

          self.namespace_to_product = {}
 

          # Mapping of product to branch regex, will be filled from configuration
 
@@ -104,17 +102,18 @@ 
 

              raise
 

  

          # Fill the class attributes we will need later
 

-         self.namespace_to_product = {
 

-             p["namespace"]: n for n, p in config["products"].items() if "namespace" in p
 

-         }
 

+         for product, values in config["products"].items():
 

+             if "namespace" in values:
 

+                 if values["namespace"] in self.namespace_to_product:
 

+                     self.namespace_to_product[values["namespace"]].append(product)
 

+                 else:
 

+                     self.namespace_to_product[values["namespace"]] = [product]
 

+ 

          self.product_to_branch_regex = {
 

              n: re.compile(p["branch_regex"])
 

              for n, p in config["products"].items()
 

              if "branch_regex" in p
 

          }
 

-         self.branch_regex_to_product = {
 

-             n: r for r, n in self.product_to_branch_regex.items()
 

-         }
 

          times = {"start": time.time()}
 

  

          # Clean the errors
 
@@ -390,21 +389,29 @@ 
 

  

              # Products
 

              products = set()
 

-             for branch in project.get("branches"):
 

-                 for regex, product in self.branch_regex_to_product.items():
 

-                     if regex.match(branch):
 

-                         products.add(product)
 

-                         break
 

-                 else:
 

-                     products.add(self.namespace_to_product[project["namespace"]])
 

+             # Determine product for project
 

+             # First based on namespace
 

+             namespace = project.get("namespace")
 

+             if len(self.namespace_to_product[namespace]) > 1:
 

+                 # When we have more than one entry for the namespace, use branch regex
 

+                 for product in self.namespace_to_product[namespace]:
 

+                     for branch in project.get("branches"):
 

+                         regex = self.product_to_branch_regex[product]
 

+                         if regex.match(branch):
 

+                             products.add(product)
 

+                             break
 

+             else:
 

+                 products.add(self.namespace_to_product[project["namespace"]][0])
 

              project["products"] = list(sorted(products))
 

  

              products_poc = {}
 

              products_retired = {}
 

              for product in products:
 

                  owner = project["poc"]
 

+                 # We need the active branches only once
 

+                 active_branches = self.bodhi.get_active_branches()
 

                  # Check if the project is retired in PDC, and if so set assignee to orphan.
 

-                 if self._is_retired(product, project):
 

+                 if self._is_retired(product, project, active_branches):
 

                      owner = "orphan"
 

                      products_retired[product] = True
 

                  else:
 
@@ -442,18 +449,18 @@ 
 

              return fedora_account.get_bz_email_group(username[1:], email_overrides)
 

          return fedora_account.get_bz_email_user(username, email_overrides)
 

  

-     def _is_retired(self, product: str, project: dict) -> bool:
 

+     def _is_retired(self, product: str, project: dict, active_branches: list) -> bool:
 

          """Check if the project is retired.
 

          The project is considered retired if there is no active branch.
 

  

          :arg product: Product to determine branches for which to look (example: "Fedora")
 

          :arg project: Project to check
 

+         :arg active_branches: Active branches in Bodhi
 

  

          :returns: Retired status.
 

          """
 

          branches = project["branches"]
 

          branch_regex = self.product_to_branch_regex.get(product)
 

-         active_branches = self.bodhi.get_active_branches()
 

          for branch in branches:
 

              # Only check the branches that match the regex for Product in bugzilla
 

              if branch_regex.match(branch) and branch in active_branches:

First run of distgit_bugzilla_sync in production created plenty of container
bugzilla entries from projects that were meant to end up in Fedora product on
Bugzilla. This should fix the issue.

Signed-off-by: Michal Konecny mkonecny@redhat.com

Build succeeded.
https://fedora.softwarefactory-project.io/zuul/buildset/b91ca263ffe145f9aef819e8e3f9da69

This is already merged, not sure why it's still in pull requests.

Pull-Request has been closed by zlopez
a year ago
Metadata
None
No Tags
Changes Summary 2
+313 -20
file changed
tests/plugins/test_distgit_bugzilla_sync.py
+25 -18
file changed
toddlers/plugins/distgit_bugzilla_sync.py
Powered by Pagure 5.14.1
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.