#73 Add a toddler to update the list of critpath packages in PDC
Merged 2 years ago by asaleh. Opened 3 years ago by pingou.

@@ -0,0 +1,626 @@ 

+ import logging

+ from unittest.mock import ANY, call, MagicMock, Mock, patch

+ 

+ import fedora_messaging.api

+ import pytest

+ import requests

+ 

+ import toddlers.plugins.pdc_update_critpath

+ 

+ if toddlers.plugins.pdc_update_critpath.dnf is None:

+     pytest.skip(

+         "We can't test this toddler without importing DNF", allow_module_level=True

+     )

+ 

+ 

+ class TestPDCUpdateCritpath:

+ 

+     toddler_cls = toddlers.plugins.pdc_update_critpath.PDCUpdateCritpath

+ 

+     def test_accepts_topic_invalid(self, toddler):

+         assert toddler.accepts_topic("foo.bar") is False

+ 

+     @pytest.mark.parametrize(

+         "topic",

+         [

+             "org.fedoraproject.*.toddlers.trigger.pdc_update_critpath",

+             "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath",

+             "org.fedoraproject.stg.toddlers.trigger.pdc_update_critpath",

+         ],

+     )

+     def test_accepts_topic_valid(self, topic, toddler):

+         assert toddler.accepts_topic(topic)

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.pdc_client_for_config")

+     def test_process_no_primary_arch(self, pdc, toddler):

+         client = Mock()

+         pdc.return_value = client

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = 123

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         with pytest.raises(KeyError, match=r"primary_arches"):

+             toddler.process({"config": "foobar"}, msg)

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.pdc_client_for_config")

+     def test_process_no_alternate_arches(self, pdc, toddler):

+         client = Mock()

+         pdc.return_value = client

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = 123

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         with pytest.raises(KeyError, match=r"alternate_arches"):

+             toddler.process({"primary_arches": ["armhfp", "aarch64", "x86_64"]}, msg)

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.pdc_client_for_config")

+     def test_process_no_releases(self, pdc, toddler):

+         client = Mock()

+         pdc.return_value = client

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = 123

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+         }

+ 

+         with pytest.raises(KeyError, match=r"releases"):

+             toddler.process(config, msg)

+ 

+     @patch("dnf.const.VERSION")

+     @patch("toddlers.plugins.pdc_update_critpath.pdc_client_for_config")

+     def test_process_old_dnf(self, pdc, dnf_vers, toddler):

+         client = Mock()

+         pdc.return_value = client

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = 123

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide", "33", "32"],

+         }

+ 

+         with pytest.raises(

+             Exception, match=r"This script requires the DNF version 2.0 API."

+         ):

+             toddler.process(config, msg)

+ 

+     @patch(

+         "toddlers.plugins.pdc_update_critpath.PDCUpdateCritpath.get_critpath_packages"

+     )

+     @patch("toddlers.plugins.pdc_update_critpath.pdc_client_for_config")

+     def test_process_no_critpath_pkgs_nopdc(self, pdc, get_c_pkgs, toddler, caplog):

+         caplog.set_level(logging.INFO)

+ 

+         client = Mock()

+         pdc.return_value = client

+         get_c_pkgs.return_value = []

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = 123

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide", "33", "32"],

+         }

+         toddler.process(config, msg, nopdc=True)

+         get_c_pkgs.assert_has_calls(

+             calls=[

+                 call(

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide", "33", "32"],

+                     },

+                     "rawhide",

+                     check_arches=["armhfp", "aarch64", "x86_64"],

+                     alternate_check_arches=["ppc64le", "s390x"],

+                 ),

+                 call(

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide", "33", "32"],

+                     },

+                     "33",

+                     check_arches=["armhfp", "aarch64", "x86_64"],

+                     alternate_check_arches=["ppc64le", "s390x"],

+                 ),

+                 call(

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide", "33", "32"],

+                     },

+                     "32",

+                     check_arches=["armhfp", "aarch64", "x86_64"],

+                     alternate_check_arches=["ppc64le", "s390x"],

+                 ),

+             ]

+         )

+ 

+         assert caplog.records[-3].message == "0 packages found in rawhide"

+         assert caplog.records[-2].message == "0 packages found in 33"

+         assert caplog.records[-1].message == "0 packages found in 32"

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.update_critpath")

+     @patch("toddlers.plugins.pdc_update_critpath.list_critpath")

+     @patch("toddlers.plugins.pdc_update_critpath.prepend_pdc_branch_ids")

+     @patch(

+         "toddlers.plugins.pdc_update_critpath.PDCUpdateCritpath.get_critpath_packages"

+     )

+     @patch("toddlers.plugins.pdc_update_critpath.pdc_client_for_config")

+     def test_process(

+         self,

+         pdc,

+         get_c_pkgs,

+         prepend_pdc_branch_ids,

+         list_critpath,

+         update_critpath,

+         toddler,

+         caplog,

+     ):

+         caplog.set_level(logging.INFO)

+ 

+         client = MagicMock()

+         pdc.return_value = client

+         prepend_pdc_branch_ids.side_effect = [

+             requests.RequestException("ahah"),

+             set(["543:dnf", "321:kernel"]),

+         ]

+         list_critpath.side_effect = [

+             requests.RequestException("ahah"),

+             set(["543:dnf", "321:kernel", "876:python3"]),

+         ]

+         update_critpath.side_effect = [Exception("nope nope nope")]

+         get_c_pkgs.return_value = []

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = 123

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+         }

+         with pytest.raises(Exception, match=r"nope nope nope"):

+             toddler.process(config, msg)

+ 

+         get_c_pkgs.assert_has_calls(

+             calls=[

+                 call(

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide"],

+                     },

+                     "rawhide",

+                     check_arches=["armhfp", "aarch64", "x86_64"],

+                     alternate_check_arches=["ppc64le", "s390x"],

+                 ),

+             ]

+         )

+         update_critpath.assert_has_calls(

+             calls=[

+                 call(

+                     client,

+                     {"543:dnf", "321:kernel", "876:python3"},

+                     {"543:dnf", "321:kernel"},

+                     "rawhide",

+                 )

+             ]

+         )

+ 

+         assert "0 packages found in rawhide" in caplog.text

+ 

+     def test_get_critpath_packages_invalid_config_dl_baseurl(self, toddler, caplog):

+         caplog.set_level(logging.INFO)

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+         }

+         with pytest.raises(Exception, match=r"fedora_dl_baseurl"):

+             toddler.get_critpath_packages(

+                 config, "invalid", config["primary_arches"], config["alternate_arches"]

+             )

+ 

+     def test_get_critpath_packages_invalid_config_fakearch(self, toddler, caplog):

+         caplog.set_level(logging.INFO)

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+             "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+         }

+         with pytest.raises(Exception, match=r"fakearch"):

+             toddler.get_critpath_packages(

+                 config, "invalid", config["primary_arches"], config["alternate_arches"]

+             )

+ 

+     def test_get_critpath_packages_invalid_config_releasepath(self, toddler, caplog):

+         caplog.set_level(logging.INFO)

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+             "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+             "fakearch": {

+                 "i386": "i686",

+                 "x86_64": "x86_64",

+                 "ppc64": "ppc64",

+                 "ppc": "ppc64",

+                 "armhfp": "armv7hl",

+                 "aarch64": "aarch64",

+                 "ppc64le": "ppc64le",

+                 "s390x": "s390x",

+             },

+         }

+         with pytest.raises(Exception, match=r"releasepath"):

+             toddler.get_critpath_packages(

+                 config, "invalid", config["primary_arches"], config["alternate_arches"]

+             )

+ 

+     def test_get_critpath_packages_invalid_arch(self, toddler, caplog):

+         caplog.set_level(logging.INFO)

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+             "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+             "fakearch": {

+                 "i386": "i686",

+                 "x86_64": "x86_64",

+                 "ppc64": "ppc64",

+                 "ppc": "ppc64",

+                 "armhfp": "armv7hl",

+                 "aarch64": "aarch64",

+                 "ppc64le": "ppc64le",

+                 "s390x": "s390x",

+             },

+             "releasepath": {

+                 "rawhide": "development/rawhide/Everything/$basearch/os/",

+                 "33": "releases/33/Everything/$basearch/os/",

+             },

+         }

+         with pytest.raises(Exception, match=r"invalid"):

+             toddler.get_critpath_packages(

+                 config, "invalid", config["primary_arches"], config["alternate_arches"]

+             )

+ 

+     def test_get_critpath_packages_invalid_config_critpath_groups(

+         self, toddler, caplog

+     ):

+         caplog.set_level(logging.INFO)

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+             "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+             "fakearch": {

+                 "i386": "i686",

+                 "x86_64": "x86_64",

+                 "ppc64": "ppc64",

+                 "ppc": "ppc64",

+                 "armhfp": "armv7hl",

+                 "aarch64": "aarch64",

+                 "ppc64le": "ppc64le",

+                 "s390x": "s390x",

+             },

+             "releasepath": {

+                 "rawhide": "development/rawhide/Everything/$basearch/os/",

+                 "33": "releases/33/Everything/$basearch/os/",

+             },

+         }

+         with pytest.raises(Exception, match=r"critpath_groups"):

+             toddler.get_critpath_packages(

+                 config, "rawhide", config["primary_arches"], config["alternate_arches"]

+             )

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.expand_dnf_critpath")

+     def test_get_critpath_packages(self, expand_dnf_critpath, toddler, caplog):

+         caplog.set_level(logging.INFO)

+         p1 = MagicMock()

+         p1.name = "dnf"

+         p2 = MagicMock()

+         p2.name = "kernel"

+         expand_dnf_critpath.return_value = [p1, p2]

+ 

+         config = {

+             "primary_arches": ["armhfp", "aarch64", "x86_64"],

+             "alternate_arches": ["ppc64le", "s390x"],

+             "releases": ["rawhide"],

+             "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+             "fedora_dl_alternateurl": "http://dl.fedoraproject.org/pub/fedora-secondary/",

+             "fakearch": {

+                 "i386": "i686",

+                 "x86_64": "x86_64",

+                 "ppc64": "ppc64",

+                 "ppc": "ppc64",

+                 "armhfp": "armv7hl",

+                 "aarch64": "aarch64",

+                 "ppc64le": "ppc64le",

+                 "s390x": "s390x",

+             },

+             "releasepath": {

+                 "rawhide": "development/rawhide/Everything/$basearch/os/",

+                 "33": "releases/33/Everything/$basearch/os/",

+             },

+             "critpath_groups": [

+                 "@core",

+                 "@critical-path-apps",

+                 "@critical-path-base",

+                 "@critical-path-gnome",

+                 "@critical-path-kde",

+                 "@critical-path-lxde",

+                 "@critical-path-xfce",

+             ],

+         }

+         toddler.get_critpath_packages(

+             config, "rawhide", config["primary_arches"], config["alternate_arches"]

+         )

+ 

+         expand_dnf_critpath.assert_has_calls(

+             calls=[

+                 call(

+                     "http://dl.fedoraproject.org/pub/fedora/linux/",

+                     "rawhide",

+                     "armhfp",

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide"],

+                         "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+                         "fedora_dl_alternateurl": "http://dl.fedoraproject.org/"

+                         "pub/fedora-secondary/",

+                         "fakearch": {

+                             "i386": "i686",

+                             "x86_64": "x86_64",

+                             "ppc64": "ppc64",

+                             "ppc": "ppc64",

+                             "armhfp": "armv7hl",

+                             "aarch64": "aarch64",

+                             "ppc64le": "ppc64le",

+                             "s390x": "s390x",

+                         },

+                         "releasepath": {

+                             "rawhide": "development/rawhide/Everything/$basearch/os/",

+                             "33": "releases/33/Everything/$basearch/os/",

+                         },

+                         "critpath_groups": [

+                             "@core",

+                             "@critical-path-apps",

+                             "@critical-path-base",

+                             "@critical-path-gnome",

+                             "@critical-path-kde",

+                             "@critical-path-lxde",

+                             "@critical-path-xfce",

+                         ],

+                     },

+                 ),

+                 call(

+                     "http://dl.fedoraproject.org/pub/fedora/linux/",

+                     "rawhide",

+                     "aarch64",

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide"],

+                         "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+                         "fedora_dl_alternateurl": "http://dl.fedoraproject.org/"

+                         "pub/fedora-secondary/",

+                         "fakearch": {

+                             "i386": "i686",

+                             "x86_64": "x86_64",

+                             "ppc64": "ppc64",

+                             "ppc": "ppc64",

+                             "armhfp": "armv7hl",

+                             "aarch64": "aarch64",

+                             "ppc64le": "ppc64le",

+                             "s390x": "s390x",

+                         },

+                         "releasepath": {

+                             "rawhide": "development/rawhide/Everything/$basearch/os/",

+                             "33": "releases/33/Everything/$basearch/os/",

+                         },

+                         "critpath_groups": [

+                             "@core",

+                             "@critical-path-apps",

+                             "@critical-path-base",

+                             "@critical-path-gnome",

+                             "@critical-path-kde",

+                             "@critical-path-lxde",

+                             "@critical-path-xfce",

+                         ],

+                     },

+                 ),

+                 call(

+                     "http://dl.fedoraproject.org/pub/fedora/linux/",

+                     "rawhide",

+                     "x86_64",

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide"],

+                         "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+                         "fedora_dl_alternateurl": "http://dl.fedoraproject.org/"

+                         "pub/fedora-secondary/",

+                         "fakearch": {

+                             "i386": "i686",

+                             "x86_64": "x86_64",

+                             "ppc64": "ppc64",

+                             "ppc": "ppc64",

+                             "armhfp": "armv7hl",

+                             "aarch64": "aarch64",

+                             "ppc64le": "ppc64le",

+                             "s390x": "s390x",

+                         },

+                         "releasepath": {

+                             "rawhide": "development/rawhide/Everything/$basearch/os/",

+                             "33": "releases/33/Everything/$basearch/os/",

+                         },

+                         "critpath_groups": [

+                             "@core",

+                             "@critical-path-apps",

+                             "@critical-path-base",

+                             "@critical-path-gnome",

+                             "@critical-path-kde",

+                             "@critical-path-lxde",

+                             "@critical-path-xfce",

+                         ],

+                     },

+                 ),

+                 call(

+                     "http://dl.fedoraproject.org/pub/fedora-secondary/",

+                     "rawhide",

+                     "ppc64le",

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide"],

+                         "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+                         "fedora_dl_alternateurl": "http://dl.fedoraproject.org/"

+                         "pub/fedora-secondary/",

+                         "fakearch": {

+                             "i386": "i686",

+                             "x86_64": "x86_64",

+                             "ppc64": "ppc64",

+                             "ppc": "ppc64",

+                             "armhfp": "armv7hl",

+                             "aarch64": "aarch64",

+                             "ppc64le": "ppc64le",

+                             "s390x": "s390x",

+                         },

+                         "releasepath": {

+                             "rawhide": "development/rawhide/Everything/$basearch/os/",

+                             "33": "releases/33/Everything/$basearch/os/",

+                         },

+                         "critpath_groups": [

+                             "@core",

+                             "@critical-path-apps",

+                             "@critical-path-base",

+                             "@critical-path-gnome",

+                             "@critical-path-kde",

+                             "@critical-path-lxde",

+                             "@critical-path-xfce",

+                         ],

+                     },

+                 ),

+                 call(

+                     "http://dl.fedoraproject.org/pub/fedora-secondary/",

+                     "rawhide",

+                     "s390x",

+                     {

+                         "primary_arches": ["armhfp", "aarch64", "x86_64"],

+                         "alternate_arches": ["ppc64le", "s390x"],

+                         "releases": ["rawhide"],

+                         "fedora_dl_baseurl": "http://dl.fedoraproject.org/pub/fedora/linux/",

+                         "fedora_dl_alternateurl": "http://dl.fedoraproject.org/"

+                         "pub/fedora-secondary/",

+                         "fakearch": {

+                             "i386": "i686",

+                             "x86_64": "x86_64",

+                             "ppc64": "ppc64",

+                             "ppc": "ppc64",

+                             "armhfp": "armv7hl",

+                             "aarch64": "aarch64",

+                             "ppc64le": "ppc64le",

+                             "s390x": "s390x",

+                         },

+                         "releasepath": {

+                             "rawhide": "development/rawhide/Everything/$basearch/os/",

+                             "33": "releases/33/Everything/$basearch/os/",

+                         },

+                         "critpath_groups": [

+                             "@core",

+                             "@critical-path-apps",

+                             "@critical-path-base",

+                             "@critical-path-gnome",

+                             "@critical-path-kde",

+                             "@critical-path-lxde",

+                             "@critical-path-xfce",

+                         ],

+                     },

+                 ),

+             ]

+         )

+ 

+ 

+ class TestPDCUpdateCritpathCLI:

+ 

+     toddler_cls = toddlers.plugins.pdc_update_critpath.PDCUpdateCritpath

+ 

+     def test_main_no_args(self, capsys):

+         with pytest.raises(SystemExit):

+             toddlers.plugins.pdc_update_critpath.main([])

+ 

+         out, err = capsys.readouterr()

+         assert out == ""

+         # Expecting something along these lines, but don't make the test too tight:

+         #

+         # usage: pytest [-h] [--dry-run] [-q | --debug] conf [username]

+         # pytest: error: the following arguments are required: conf

+         assert err.startswith("usage:")

+         assert "error: the following arguments are required:" in err

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.PDCUpdateCritpath.process")

+     @patch("toddlers.plugins.pdc_import_compose.pdc_client_for_config")

+     @patch("toml.load", new=Mock(return_value={}))

+     def test_main_debug(self, pdc, process, capsys):

+         client = Mock()

+         pdc.return_value = client

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = ANY

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         toddlers.plugins.pdc_update_critpath.main(["test.cfg", "--debug"])

+         process.assert_called_once_with(

+             config={}, message=None, releases=None, nopdc=False

+         )

+         out, err = capsys.readouterr()

+         assert out == ""

+         assert err == ""

+ 

+     @patch("toddlers.plugins.pdc_update_critpath.PDCUpdateCritpath.process")

+     @patch("toddlers.plugins.pdc_import_compose.pdc_client_for_config")

+     @patch("toml.load", new=Mock(return_value={}))

+     def test_main(self, pdc, process, capsys):

+         client = Mock()

+         pdc.return_value = client

+ 

+         msg = fedora_messaging.api.Message()

+         msg.id = ANY

+         msg.topic = "org.fedoraproject.prod.toddlers.trigger.pdc_update_critpath"

+         msg.body = {}

+ 

+         toddlers.plugins.pdc_update_critpath.main(["test.cfg"])

+         process.assert_called_once_with(

+             config={}, message=None, releases=None, nopdc=False

+         )

+         out, err = capsys.readouterr()

+         assert out == ""

+         assert err == ""

@@ -235,7 +235,7 @@ 

  

      @patch("toddlers.utils.bugzilla_system.get_bz")

      def test_get_product_info_packages(self, mock_bz):

-         """ Assert that product info is received. """

+         """Assert that product info is received."""

          server = MagicMock()

          server.product_get.return_value = [

              {
@@ -282,7 +282,7 @@ 

  

      @patch("toddlers.utils.bugzilla_system.get_bz")

      def test_get_product_info_packages_exception(self, mock_bz):

-         """ Assert that exception is raised when data are not in correct format. """

+         """Assert that exception is raised when data are not in correct format."""

          server = MagicMock()

          server.product_get.return_value = []

          mock_bz.return_value = server

file modified
+39
@@ -180,6 +180,45 @@ 

  [consumer_config.packagers_without_bugzilla]

  ignorable_namespaces = ["tests"]

  

+ [consumer_config.pdc_update_critpath]

+ critpath_groups = [

+     '@core', '@critical-path-apps', '@critical-path-base',

+     '@critical-path-gnome', '@critical-path-kde', '@critical-path-lxde',

+     '@critical-path-xfce'

+ ]

+ primary_arches = ['armhfp', 'aarch64', 'x86_64']

+ alternate_arches = ['ppc64le','s390x']

+ releases = ['rawhide', 33, 32]

+ 

+ fedora_dl_baseurl = 'http://dl.fedoraproject.org/pub/fedora/linux/'

+ fedora_dl_alternateurl = 'http://dl.fedoraproject.org/pub/fedora-secondary/'

+ 

+ # Package excluded

+ excludes = [ 'tzdata' ]

+ 

+ [consumer_config.pdc_update_critpath.releasepath]

+ devel = 'development/rawhide/Everything/$basearch/os/'

+ rawhide = 'development/rawhide/Everything/$basearch/os/'

+ 33 = 'releases/33/Everything/$basearch/os/'

+ 32 = 'releases/32/Everything/$basearch/os/'

+ 31 = 'releases/31/Everything/$basearch/os/'

+ 

+ [consumer_config.pdc_update_critpath.updatepath]

+ devel = ''

+ rawhide = ''

+ 33 = 'updates/33/$basearch/'

+ 32 = 'updates/32/$basearch/'

+ 31 = 'updates/31/$basearch/'

+ 

+ [consumer_config.pdc_update_critpath.fakearch]

+ i386 = 'i686'

+ x86_64 = 'x86_64'

+ ppc64 = 'ppc64'

+ ppc = 'ppc64'

+ armhfp = 'armv7hl'

+ aarch64 = 'aarch64'

+ ppc64le = 'ppc64le'

+ s390x = 's390x'

  

  

  [qos]

file modified
+1 -1
@@ -41,7 +41,7 @@ 

  

  

  def get_arguments(args):

-     """ Load and parse the CLI arguments."""

+     """Load and parse the CLI arguments."""

      parser = argparse.ArgumentParser(

          description="Sends a message on the fedora-messaging message bus to trigger a toddler "

          "(if that toddlers supports being triggered this way)."

@@ -108,7 +108,7 @@ 

  

  

  def get_arguments(args):

-     """ Load and parse the CLI arguments."""

+     """Load and parse the CLI arguments."""

      parser = argparse.ArgumentParser(

          description="Check the state of the email_overrides file"

      )
@@ -162,7 +162,7 @@ 

  

  

  def main(args):

-     """ Schedule the first test and run the scheduler. """

+     """Schedule the first test and run the scheduler."""

      args = get_arguments(args)

      setup_logging(log_level=args.log_level)

  

@@ -182,7 +182,7 @@ 

              _log.debug(req.text)

  

      def remove_access(self, namespace, name, username, usertype):

-         """ Remove the ACL of the specified user/group on the specified project. """

+         """Remove the ACL of the specified user/group on the specified project."""

  

          # Remove user/group from the package

          url = f"{self.distgit_api_base_url}/{namespace}/{name}/git/modifyacls"

@@ -577,7 +577,7 @@ 

  

  

  def main(args):

-     """ Main function. """

+     """Main function."""

      args = _get_arguments(args)

      _setup_logging(log_level=args.log_level)

  

@@ -129,7 +129,7 @@ 

  

  

  def get_arguments(args):

-     """ Load and parse the CLI arguments."""

+     """Load and parse the CLI arguments."""

      parser = argparse.ArgumentParser(

          description="Sync packagers access from FAS to bugzilla"

      )
@@ -190,7 +190,7 @@ 

  

  

  def main(args):

-     """ Schedule the first test and run the scheduler. """

+     """Schedule the first test and run the scheduler."""

      args = get_arguments(args)

      setup_logging(log_level=args.log_level)

  

@@ -90,7 +90,7 @@ 

          return (users, groups)

  

      def process(self, config, message, send_email=True, username=None):

-         """ Looks for packagers/groups without bugzilla email. """

+         """Looks for packagers/groups without bugzilla email."""

          self.logs = []

  

          try:
@@ -244,7 +244,7 @@ 

  

  

  def get_arguments(args):

-     """ Load and parse the CLI arguments."""

+     """Load and parse the CLI arguments."""

      parser = argparse.ArgumentParser(

          description="Checks that packagers have a valid bugzilla account"

      )
@@ -289,7 +289,7 @@ 

  

  

  def main(args):

-     """ Schedule the first test and run the scheduler. """

+     """Schedule the first test and run the scheduler."""

      args = get_arguments(args)

      setup_logging(log_level=args.log_level)

  

@@ -220,7 +220,7 @@ 

  

  

  def _scrape_links(session, url):

-     """ Utility to scrape links from a <pre> tag. """

+     """Utility to scrape links from a <pre> tag."""

      _log.debug("Scraping %s", url)

      response = session.get(url)

      if not response.ok:
@@ -239,7 +239,7 @@ 

  

  

  def _old_composes(base_url):

-     """ Screen-scrape the list of old composes from kojipkgs. """

+     """Screen-scrape the list of old composes from kojipkgs."""

      _log.info("Looking for old composes at: %s", base_url)

  

      requests_session = make_session(timeout=30)
@@ -287,7 +287,7 @@ 

  

  

  def compose_exists(pdc, compose_id):

-     """ Return True if a compose exists in PDC.  False if not. """

+     """Return True if a compose exists in PDC.  False if not."""

      try:

          pdc["composes"][compose_id]._()

          return True
@@ -298,7 +298,7 @@ 

  

  

  def ensure_release_exists(pdc, release_id, release):

-     """ Create a release in PDC if it doesn't already exist. """

+     """Create a release in PDC if it doesn't already exist."""

      try:

          pdc["releases"][release_id]._()

      except beanbag.bbexcept.BeanBagException as e:
@@ -343,7 +343,7 @@ 

  

  

  def get_arguments(args):

-     """ Load and parse the CLI arguments."""

+     """Load and parse the CLI arguments."""

      parser = argparse.ArgumentParser(

          description="Manually import a compose into PDC via a file storing the "

          "JSON body of the message about the successful compose"
@@ -379,7 +379,7 @@ 

  

  

  def main(args):

-     """ Schedule the first test and run the scheduler. """

+     """Schedule the first test and run the scheduler."""

      args = get_arguments(args)

      setup_logging(log_level=args.log_level)

  

@@ -67,7 +67,7 @@ 

                  raise

  

      def process(self, config, message):

-         """ Process a given message. """

+         """Process a given message."""

          pdc = pdc_client_for_config(config)

  

          _log.debug("Determining which PDC API to use.")
@@ -105,7 +105,7 @@ 

              pdc[pdc_api][uid]._ += {"active": True, "rpms": rpms}

  

      def _get_modulemd_by_mbs_id(self, config, idx):

-         """ Query MBS to get the modulemd """

+         """Query MBS to get the modulemd"""

          mbs_url = config["mbs_url"]

          module_url = "{0}/{1}?verbose=True".format(mbs_url, idx)

          response = self.requests_session.get(module_url, timeout=30)
@@ -192,7 +192,7 @@ 

          return uid

  

      def _koji_rpms_in_tag(self, url, tag):

-         """ Return the list of koji rpms in a tag. """

+         """Return the list of koji rpms in a tag."""

          if not url.endswith("kojihub"):

              url = f"{url.rstrip('/')}/kojihub"

          _log.info("Listing rpms in koji(%s) tag %s", url, tag)

@@ -193,7 +193,7 @@ 

              )

  

      def _retire_branch(self, pdc, branch):

-         """ Internal method for retiring a branch in PDC. """

+         """Internal method for retiring a branch in PDC."""

          today = datetime.utcnow().date()

          for sla in branch["slas"]:

              sla_eol = datetime.strptime(sla["eol"], "%Y-%m-%d").date()

@@ -0,0 +1,377 @@ 

+ """

+ Periodically updates in PDC the list of packages in the critical path.

+ 

+ This toddler is succeeding to two releng scripts:

+ https://pagure.io/releng/blob/d0817f0000fd05dc840902c115b8221594849e48/f/scripts/critpath.py

+ https://pagure.io/releng/blob/d0817f0000fd05dc840902c115b8221594849e48/f/scripts/update-critpath.py

+ 

+ Authors:    Pierre-Yves Chibon <pingou@pingoured.fr>

+ 

+ """

+ 

+ import argparse

+ import logging

+ import shutil

+ import sys

+ from tempfile import mkdtemp

+ import time

+ 

+ try:

+     import dnf

+ except ImportError:

+     dnf = None

+ import requests

+ import toml

+ 

+ from toddlers.base import ToddlerBase

+ from toddlers.utils.pdc import pdc_client_for_config

+ from toddlers.utils.requests import make_session

+ 

+ _log = logging.getLogger(__name__)

+ 

+ 

+ # A lot of the code here is not covered in the tests, it either interacts with

+ # the DNF API or PDC. Getting this code covered in the tests would mean a lot

+ # of mocking and we would end up testing that our mocking works

+ 

+ 

+ def expand_dnf_critpath(url, release, arch, config):  # pragma no cover

+     _log.info("Resolving %s dependencies with DNF" % arch)

+     base = dnf.Base()

+ 

+     temp_cache_dir = mkdtemp(suffix="-critpath")

+     temp_install_root = mkdtemp(suffix="-critpath-installroot")

+ 

+     conf = base.conf

+     # cache download data somewhere else

+     conf.cachedir = temp_cache_dir

+     # do not use the data from the previous runs of system dnf or groups will

+     # be marked incorrectly

+     conf.persistdir = temp_cache_dir

+     conf.installroot = temp_install_root

+     # dnf needs arches, not basearches to work

+     conf.arch = config["fakearch"][arch]

+ 

+     try:

+         packages = set()

+ 

+         # add a new repo requires an id, a conf object, and a baseurl

+         repo_url = url + config["releasepath"][release]

+ 

+         # make sure we don't load the system repo and get local data

+         _log.info("Basearch: %s" % conf.basearch)

+         _log.info("Arch:     %s" % conf.arch)

+         _log.info("%s repo %s" % (arch, repo_url))

+ 

+         # mark all critpath groups in base object

+         for group in config["critpath_groups"]:

+             base.reset(repos=True, goal=True, sack=True)

+             base.repos.add_new_repo(arch, conf, baseurl=[repo_url])

+             base.fill_sack(load_system_repo=False)

+             if base.repos[arch].enabled is False:

+                 raise Exception

+ 

+             # load up the comps data from configured repositories

+             base.read_comps()

+             group = group.replace("@", "")

+             base.group_install(

+                 group, ["mandatory", "default", "optional"], strict=False

+             )

+             # resolve the groups marked in base object

+             base.resolve()

+             packages = packages.union(base.transaction.install_set)

+ 

+         return packages

+ 

+     except Exception as ex:

+         template = "An exception of type {0} occurred. Arguments:\n{1!r}"

+         message = template.format(type(ex).__name__, ex.args)

+         _log.info(message)

+         _log.info("DNF failed to synchronize the repository and cannot proceed.")

+         raise

+     finally:

+         base.close()

+         del base

+         del conf

+         shutil.rmtree(temp_cache_dir)

+         shutil.rmtree(temp_install_root)

+ 

+ 

+ def list_critpath(pdc, branch):  # pragma no cover

+     """Return the set of packages flagged as critpatch in the specified

+     branch.

+ 

+     :arg branch: branch used to restrict the list of packages returned.

+     :type branch: str

+     :return: a set of package names with PDC branch IDs

+     :rtype: set

+     """

+     if not isinstance(branch, str):

+         raise TypeError("branch *must* be a string")

+ 

+     _log.debug("Retrieving existing critpath list from PDC.")

+     args = {"name": branch, "type": "rpm", "critical_path": True}

+     results = pdc.get_paged(pdc["component-branches"], **args)

+ 

+     pkgs = set()

+     for result in results:

+         pkgs.add("{id}:{global_component}".format(**result))

+     _log.debug("Done retrieving existing critpath list from PDC.")

+     return pkgs

+ 

+ 

+ def prepend_pdc_branch_ids(pdc, global_components, branch):  # pragma no cover

+     _log.debug("Finding PDC branch IDs for supplied critpath list.")

+     pkgs = set()

+     endpoint = pdc["component-branches"]

+ 

+     def _get_pdc_branch_id(cnt, **args):

+         try:

+             results = list(pdc.get_paged(endpoint, **args))

+         except requests.RequestException:

+             if cnt == 5:

+                 raise

+             cnt += 1

+             time.sleep(1)

+             _log.info("   Attempt %s" % cnt)

+             return _get_pdc_branch_id(cnt, **args)

+         # critpath.py generates subpackages as well.

+         # As there wont be any pdc entries for subpackages

+         # we need to skip them

+         if len(results) == 1:

+             return results[0]

+         else:

+             return None

+ 

+     for i, component in enumerate(global_components):

+         args = {"name": branch, "type": "rpm", "global_component": component}

+         _log.info("%s/%s  %s", i, len(global_components), args)

+         cnt = 0

+         result = _get_pdc_branch_id(cnt, **args)

+ 

+         if result:

+             pkgs.add("{id}:{global_component}".format(**result))

+     _log.debug("Done finding PDC branch IDs for supplied critpath list.")

+     return pkgs

+ 

+ 

+ def update_critpath(pdc, current_critpath, new_critpath, branch):  # pragma no cover

+     """Change critpath status of packages in PDC

+ 

+     :arg current_critpath: a set listing all the packages that currently have

+         the critpath package

+     """

+ 

+     # Remove the critpath flag from packages which do not have it, but should.

+     new_no = current_critpath - new_critpath

+     _log.info("%i packages need critpath removed." % len(new_no))

+     for item in new_no:

+         idx, name = item.split(":", 1)

+         _log.debug("  Sending PATCH for %s, (idx: %s)" % (name, idx))

+         pdc["component-branches"][idx + "/"] += dict(critical_path=False)

+ 

+     # Add the critpath flag to packages which should have it, but do not.

+     new_yes = new_critpath - current_critpath

+     _log.info("%i packages need critpath added." % len(new_yes))

+     for item in new_yes:

+         idx, name = item.split(":", 1)

+         _log.debug("  Sending PATCH for %s, (idx: %s)" % (name, idx))

+         pdc["component-branches"][idx + "/"] += dict(critical_path=True)

+ 

+ 

+ class PDCUpdateCritpath(ToddlerBase):

+     """Listens to messages sent by playtime (which lives in toddlers) to sync

+     all the packagers found in FAS to bugzilla.

+     """

+ 

+     name = "pdc_update_critpath"

+ 

+     amqp_topics = [

+         "org.fedoraproject.*.toddlers.trigger.pdc_update_critpath",

+     ]

+ 

+     def __init__(self):

+         self.requests_session = make_session()

+ 

+     def accepts_topic(self, topic):

+         """Returns a boolean whether this toddler is interested in messages

+         from this specific topic.

+         """

+         return topic.startswith("org.fedoraproject.") and topic.endswith(

+             "toddlers.trigger.pdc_update_critpath"

+         )

+ 

+     def process(self, config, message, releases=None, nopdc=False):

+         """Process a given message."""

+         pdc = pdc_client_for_config(config)

+ 

+         check_arches = config["primary_arches"]

+         alternate_check_arches = config["alternate_arches"]

+         if releases is None:

+             releases = config["releases"]

+         for release in releases:

+ 

+             dnf_version = tuple(map(int, dnf.const.VERSION.split(".")))

+             if dnf_version < (2, 0, 0):

+                 raise Exception("This script requires the DNF version 2.0 API.")

+ 

+             pkgs = self.get_critpath_packages(

+                 config,

+                 release,

+                 check_arches=check_arches,

+                 alternate_check_arches=alternate_check_arches,

+             )

+             _log.info("%s packages found in %s" % (len(pkgs), release))

+ 

+             if not nopdc:

+ 

+                 _log.info("Retrieving the PDC branch ids for the list of packages")

+                 new_critpath = None

+                 cnt = 0

+                 while new_critpath is None:

+                     try:

+                         new_critpath = prepend_pdc_branch_ids(pdc, pkgs, release)

+                     except requests.RequestException:

+                         if cnt == 5:  # pragma no cover

+                             raise

+                         cnt += 1

+                         time.sleep(1)

+                         _log.info("   Attempt %s" % cnt)

+ 

+                 _log.info("Retrieving the current list of packages stored in PDC")

+                 current_critpath = None

+                 cnt = 0

+                 while current_critpath is None:

+                     try:

+                         current_critpath = list_critpath(pdc, release)

+                     except requests.RequestException:

+                         if cnt == 5:  # pragma no cover

+                             raise

+                         cnt += 1

+                         time.sleep(1)

+                         _log.info("   Attempt %s" % cnt)

+ 

+                 _log.info("Updating PDC for changes in the list of critpath packages")

+                 try:

+                     update_critpath(

+                         pdc, set(current_critpath), set(new_critpath), release

+                     )

+                 except Exception:

+                     _log.exception("Caught generic error while updating PDC")

+                     raise

+ 

+     def get_critpath_packages(

+         self, config, release, check_arches, alternate_check_arches

+     ):

+         critpath = set()

+         for arch in check_arches + alternate_check_arches:

+             if arch in check_arches:

+                 url = config["fedora_dl_baseurl"]

+             elif arch in alternate_check_arches:

+                 url = config["fedora_dl_alternateurl"]

+             else:  # pragma no cover

+                 raise Exception("Invalid architecture")

+ 

+             _log.info("Expanding critical path for %s" % arch)

+ 

+             pkgs = None

+ 

+             pkgs = expand_dnf_critpath(url, release, arch, config)

+ 

+             if pkgs is None:

+                 package_count = 0

+             else:

+                 package_count = len(pkgs)

+ 

+             _log.info("%u packages for %s" % (package_count, arch))

+ 

+             critpath.update([p.name for p in pkgs])

+ 

+             del pkgs

+ 

+         return critpath

+ 

+ 

+ def setup_logging(log_level: int):

+     handlers = []

+ 

+     _log.setLevel(log_level)

+     # We want all messages logged at level INFO or lower to be printed to stdout

+     info_handler = logging.StreamHandler(stream=sys.stdout)

+     handlers.append(info_handler)

+ 

+     if log_level == logging.INFO:

+         # In normal operation, don't decorate messages

+         for handler in handlers:

+             handler.setFormatter(logging.Formatter("%(message)s"))

+ 

+     logging.basicConfig(level=log_level, handlers=handlers)

+ 

+ 

+ def get_arguments(args):

+     """Load and parse the CLI arguments."""

+     parser = argparse.ArgumentParser(description="Update critpath packages in PDC")

+     parser.add_argument(

+         "conf",

+         help="Configuration file",

+     )

+     parser.add_argument(

+         "release", default=None, nargs="?", help="Specific release to work on"

+     )

+     parser.add_argument(

+         "--nopdc",

+         dest="nopdc",

+         action="store_true",

+         default=False,

+         help="Do not update PDC, just resolve the package list",

+     )

+ 

+     log_level_group = parser.add_mutually_exclusive_group()

+     log_level_group.add_argument(

+         "-q",

+         "--quiet",

+         action="store_const",

+         dest="log_level",

+         const=logging.WARNING,

+         default=logging.INFO,

+         help="Be less talkative",

+     )

+     log_level_group.add_argument(

+         "--debug",

+         action="store_const",

+         dest="log_level",

+         const=logging.DEBUG,

+         help="Enable debugging output",

+     )

+ 

+     return parser.parse_args(args)

+ 

+ 

+ def main(args):

+     """Schedule the first test and run the scheduler."""

+     args = get_arguments(args)

+     setup_logging(log_level=args.log_level)

+ 

+     config = toml.load(args.conf)

+     parsed_config = config.get("consumer_config", {}).get("default", {})

+     parsed_config.update(

+         config.get("consumer_config", {}).get("pdc_update_critpath", ())

+     )

+ 

+     releases = None

+     if args.release:

+         releases = [args.release]

+ 

+     PDCUpdateCritpath().process(

+         config=parsed_config,

+         message=None,

+         releases=releases,

+         nopdc=args.nopdc,

+     )

+ 

+ 

+ if __name__ == "__main__":  # pragma: no cover

+     try:

+         main(sys.argv[1:])

+     except KeyboardInterrupt:

+         pass

@@ -64,7 +64,7 @@ 

  

  

  def remove_user_from_group(user_email: str, bz_group: str):

-     """ Remove the specified user from the specified group. """

+     """Remove the specified user from the specified group."""

  

      server = get_bz()

  
@@ -81,7 +81,7 @@ 

  

  

  def get_user(user_email: str):

-     """ Returns a user object if the given user exists in bugzilla or None. """

+     """Returns a user object if the given user exists in bugzilla or None."""

  

      server = get_bz()

  
@@ -101,7 +101,7 @@ 

  def add_user_to_group(

      user_email: str, bz_group: str, no_bz_account: list, dry_run: bool = False

  ):

-     """ Add the specified user to the specified group. """

+     """Add the specified user to the specified group."""

  

      server = get_bz()

  

@@ -45,7 +45,7 @@ 

  

  

  def get_group_member(group_name: str) -> set:

-     """ Return a list containing the name the members of the given group. """

+     """Return a list containing the name the members of the given group."""

      output = set()

      for user in __get_fas_grp_member(group_name):

          output.add(user.get("username"))

file modified
+3
@@ -8,6 +8,7 @@ 

  deps =

      -r requirements.txt

      -r test-requirements.txt

+ sitepackages = True

  setenv =

      PYTHONPATH={toxinidir}

  commands =
@@ -16,6 +17,7 @@ 

  [testenv:black]

  deps =

      black

+ sitepackages = False

  commands =

      black --check --diff .

  
@@ -23,5 +25,6 @@ 

  deps =

      flake8

      flake8-import-order

+ sitepackages = False

  commands =

      flake8 toddlers/ tests/ {posargs}

no initial comment

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

  • tox : FAILURE in 9m 40s

rebased onto c78344c2cce6e4c5f0f299d3a19efa19630ed06f

3 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

  • tox : FAILURE in 6m 10s

Nice the tests are all passing locally...

rebased onto 97a96a72909b19f2976f8314e76304d533873f8c

2 years ago

rebased onto d74524b65fca2b6a72285ff491fdf7b5a4ee88f1

2 years ago

1 new commit added

  • Attempt to fix the tests
2 years ago

1 new commit added

  • Adjust code-base for newer black version
2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci

  • tox : FAILURE in 6m 28s

4 new commits added

  • Adjust code-base for newer black version
  • Add unit-tests for the pdc_update_critpath toddler
  • pdc_update_critpath: Drop all the yum-related code
  • Add a toddler to update the list of critpath packages in PDC
2 years ago

1 new commit added

  • Attempt to fix the tests
2 years ago

5 new commits added

  • Attempt to fix the tests
  • Adjust code-base for newer black version
  • Add unit-tests for the pdc_update_critpath toddler
  • pdc_update_critpath: Drop all the yum-related code
  • Add a toddler to update the list of critpath packages in PDC
2 years ago

Build succeeded.

  • tox : SUCCESS in 7m 23s

4 new commits added

  • Adjust code-base for newer black version
  • Add unit-tests for the pdc_update_critpath toddler
  • pdc_update_critpath: Drop all the yum-related code
  • Add a toddler to update the list of critpath packages in PDC
2 years ago

rebased onto 5968cc9

2 years ago

Build succeeded.

  • tox : SUCCESS in 6m 26s

Pull-Request has been merged by asaleh

2 years ago