From 4cd78cb92767bc00eca158b6740f1371881c2caf Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Aug 22 2019 07:33:44 +0000 Subject: New sub-command: update-tag Add a new sub-command to provide the feature of updating a tag's inheritance data with latest build tags of modules in config. This command can check latest build tags for all modules in a tag's config, if there is any build tag missing from the tag's inheritance, the tag will be added, and old tags will be removed at the same time. --- diff --git a/README.rst b/README.rst index ddeeb4d..f1a8712 100644 --- a/README.rst +++ b/README.rst @@ -304,6 +304,28 @@ In this case, there are 3 build tags found for ``my-example-tag``, they are: Ursa-Major doesn't count it in, because it stops at tag 'module-123456-build' which name starts with 'module-'. +update-tag +---------- + +Update a tag's inheritance data with all latest module build tags of the +modules in tag's config. + +Arguments: + +* ``--tag`` (required): the tag to update +* ``--wait-regen-repo`` (optional): wait for regen-repo task to finish + +Example: + +.. code-block:: bash + + $ ursa-major update-tag --tag fedora-30-test-build --wait-regen-repo + +This will check the latest builds in MBS for all modules in config of tag +'fedora-30-test-build', if there is any build's tag is missing from tag's +inheritance data, the tag will be added into inheritance, and old tags +will be removed at the same time for the module. + add-tag ------- diff --git a/tests/test_update_tag.py b/tests/test_update_tag.py new file mode 100644 index 0000000..14a5be6 --- /dev/null +++ b/tests/test_update_tag.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi +# Qixiang Wan + +import json +import mock +import os +import six +import shutil +import tempfile +try: + import unittest2 as unittest +except ImportError: + import unittest + +from argparse import Namespace +from ursa_major import MBS_BUILD_STATES +from ursa_major.handlers.update_tag import UpdateTagHandler + + +class TestUpdateTagHandler(unittest.TestCase): + def setUp(self): + config = mock.MagicMock() + self.handler = UpdateTagHandler(config) + + self.handler.connect_koji = mock.MagicMock() + self.handler._koji = mock.MagicMock() + + self.handler.connect_mbs = mock.MagicMock() + self.handler._mbs = mock.MagicMock() + + self.tmpdir = tempfile.mkdtemp(suffix='_ursa_major_test') + self.tag_config_file = os.path.join(self.tmpdir, 'default.json') + tag_config = { + 'example-tag': { + 'owners': ['foo@example.com'], + 'modules': [ + {'name': 'testmodule', 'stream': 'f30', 'priority': 10, + 'requires': {'platform': 'f30'}}, + ] + }, + 'empty-tag': { + 'owners': ['foo@example.com'], + 'modules': [] + } + } + + with open(self.tag_config_file, 'w') as f: + json.dump(tag_config, f) + + self.set_args() + + def tearDown(self): + try: + shutil.rmtree(self.tmpdir) + except: # noqa + pass + + def set_args(self, **kwargs): + kwargs.setdefault('tag_config_file', self.tag_config_file) + kwargs.setdefault('wait_regen_repo', False) + kwargs.setdefault('dry_run', False) + kwargs.setdefault('debug', False) + kwargs.setdefault('tag', 'example-tag') + args = Namespace() + for k, v in kwargs.items(): + setattr(args, k, v) + self.handler.set_args(args) + + def mock_get_tag(self, tag): + taginfo = { + "example-tag": { + "name": "example-tag", + "id": 123 + }, + "module-testmodule-f30-20180101-abc123": { + "name": "module-testmodule-f30-20180101-abc123", + "id": 10034 + }, + "module-testmodule-f30-20161212-000000": { + "name": "module-testmodule-f30-20161212-000000", + "id": 8080 + }, + } + return taginfo.get(tag, {}) + + def test_terminate_if_tag_not_in_config(self): + self.set_args(tag='non-exist-tag') + with six.assertRaisesRegex(self, RuntimeError, 'is not found in tag config file'): + self.handler.run() + + def test_terminate_if_tag_not_in_koji(self): + self.handler.koji.get_tag.return_value = None + with six.assertRaisesRegex(self, RuntimeError, 'is not found in koji'): + self.handler.run() + + @mock.patch('ursa_major.handlers.update_tag.log') + def test_terminate_if_tag_has_no_module_in_config(self, log): + self.set_args(tag='empty-tag') + self.handler.run() + log.warning.assert_any_call( + "No module specified for tag '%s' in tag config file", "empty-tag") + + @mock.patch('ursa_major.handlers.update_tag.log') + def test_skip_if_no_ready_build_found(self, log): + self.handler.mbs.get_modules.return_value = [] + + self.handler.run() + log.warning.assert_has_calls([ + mock.call("There is no ready build found for %s, skipping", mock.ANY) + ]) + + @mock.patch('ursa_major.handlers.update_tag.log') + def test_skip_if_tag_in_inheritance_with_same_priority(self, log): + self.handler.mbs.get_modules.return_value = [ + {'koji_tag': 'module-testmodule-f30-20180101-abc123'} + ] + mock_inheritance = [ + {'name': 'module-testmodule-f30-20180101-abc123', + 'priority': 10} + ] + self.handler.koji.get_inheritance_data.return_value = mock_inheritance + self.handler.run() + log.info.assert_has_calls([ + mock.call("Tag '%s' is in inheritance data of %s with same priority, skipping", + "module-testmodule-f30-20180101-abc123", "example-tag") + ]) + + @mock.patch('ursa_major.handlers.update_tag.log') + def test_tag_in_inheritance_with_different_priority(self, log): + self.handler.mbs.get_modules.return_value = [ + {'koji_tag': 'module-testmodule-f30-20180101-abc123'} + ] + mock_inheritance = [ + {'name': 'module-testmodule-f30-20180101-abc123', + 'priority': 50} + ] + self.handler.koji.get_inheritance_data.return_value = mock_inheritance + self.handler.run() + self.handler.koji.set_inheritance_data.assert_has_calls([ + mock.call('example-tag', + [{'priority': 10, 'name': 'module-testmodule-f30-20180101-abc123'}]) + ]) + self.handler.koji.regen_repo.assert_has_calls([ + mock.call('example-tag', wait=False) + ]) + + @mock.patch('ursa_major.handlers.update_tag.log') + def test_tag_not_in_inheritance_and_no_old_tag_found(self, log): + self.handler.mbs.get_modules.return_value = [ + {'koji_tag': 'module-testmodule-f30-20180101-abc123'} + ] + + self.handler.koji.get_tag.side_effect = self.mock_get_tag + + self.handler.koji.get_inheritance_data.return_value = [] + self.handler.run() + inheritance_data = [ + {'intransitive': False, + 'name': 'module-testmodule-f30-20180101-abc123', + 'parent_id': 10034, + 'priority': 10, + 'maxdepth': None, + 'noconfig': False, + 'pkg_filter': '', + 'child_id': 123} + ] + self.handler.koji.set_inheritance_data.assert_has_calls([ + mock.call('example-tag', inheritance_data) + ]) + self.handler.koji.regen_repo.assert_has_calls([ + mock.call('example-tag', wait=False) + ]) + + def test_tag_not_in_inheritance_and_old_tag_exist(self): + mock_inheritance = [ + {'name': 'module-testmodule-f30-20161212-000000', + 'priority': 10} + ] + self.handler.koji.get_inheritance_data.return_value = mock_inheritance + self.handler.koji.get_tag.side_effect = self.mock_get_tag + + def mock_get_modules(**kwargs): + state = kwargs.get('state', None) + if isinstance(state, list) and MBS_BUILD_STATES['garbage'] in state: + # return all tags + return [ + {'koji_tag': 'module-testmodule-f30-20161212-000000'}, + {'koji_tag': 'module-testmodule-f30-20180101-abc123'}, + ] + else: + # only return the latest one + return [{'koji_tag': 'module-testmodule-f30-20180101-abc123'}] + + self.handler.mbs.get_modules.side_effect = mock_get_modules + + self.handler.run() + + call_args = self.handler.koji.set_inheritance_data.call_args.args + self.assertEqual('example-tag', call_args[0]) + tag_to_remove = [t['name'] for t in call_args[1] if t.get('delete link', False) is True] + tag_to_add = [t['name'] for t in call_args[1] if t.get('delete link', False) is False] + self.assertTrue("module-testmodule-f30-20161212-000000" in tag_to_remove) + self.assertTrue("module-testmodule-f30-20180101-abc123" in tag_to_add) + self.handler.koji.regen_repo.assert_has_calls([ + mock.call('example-tag', wait=False) + ]) diff --git a/ursa_major/cli.py b/ursa_major/cli.py index 04a097f..6eb2588 100644 --- a/ursa_major/cli.py +++ b/ursa_major/cli.py @@ -36,6 +36,7 @@ from ursa_major.handlers.add_tag import AddTagHandler from ursa_major.handlers.check_config import CheckConfigHandler from ursa_major.handlers.remove_module import RemoveModuleHandler from ursa_major.handlers.show_config import ShowConfigHandler +from ursa_major.handlers.update_tag import UpdateTagHandler if six.PY3: # SafeConfigParser == ConfigParser, former deprecated in >= 3.2 from six.moves.configparser import ConfigParser @@ -252,6 +253,22 @@ def main(args=None): action='store_true', default=False, help='wait for regen-repo task to finish') + update_tag_parser = subparser.add_parser( + 'update-tag', + parents=[global_args_parser], + help=" " + "module in MBS and add its tag to koji tag inheritance.") + update_tag_parser.set_defaults(_class=UpdateTagHandler, func='run') + update_tag_parser.add_argument( + '--tag', + metavar='TAG', + required=True, + help="koji tag to update inheritance data (required)") + update_tag_parser.add_argument( + '--wait-regen-repo', + action='store_true', default=False, + help='wait for regen-repo task(s) to finish') + args = parser.parse_args(args) debug = getattr(args, 'debug', False) if debug: diff --git a/ursa_major/handlers/update_tag.py b/ursa_major/handlers/update_tag.py new file mode 100644 index 0000000..3356f04 --- /dev/null +++ b/ursa_major/handlers/update_tag.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi +# Qixiang Wan + +from ursa_major import MBS_BUILD_STATES +from ursa_major.handlers.base import BaseHandler +from ursa_major.logger import log + + +class UpdateTagHandler(BaseHandler): + def __init__(self, *args, **kwargs): + super(UpdateTagHandler, self).__init__(*args, **kwargs) + self.dry_run = False + self.wait_regen_repo = False + self.inheritance_changed = False + + def run(self): + self.debug = self.args.debug + self.dry_run = self.args.dry_run + self.wait_regen_repo = self.args.wait_regen_repo + + self.tag = self.args.tag + self.tag_config_file = self.args.tag_config_file + self.load_tag_config() + + if self.tag not in self.tag_config.keys(): + raise RuntimeError("Tag '%s' is not found in tag config file %s" % (self.tag, + self.tag_config_file)) + + self.connect_koji(dry_run=self.dry_run) + self.connect_mbs() + + self.tag_info = self.koji.get_tag(self.tag) + if self.tag_info is None: + raise RuntimeError("Tag '%s' is not found in koji (profile: %s)" % (self.tag, + self.koji.profile)) + + modules = self.tag_config.get(self.tag).get('modules', {}) + if not modules: + log.warning("No module specified for tag '%s' in tag config file", self.tag) + return + + inheritance = self.koji.get_inheritance_data(self.tag) + + # convert inheritance list to dict for convenience + # dict key is tag name and value is the inheritance data of that tag + inheritance_dict = {} + for tag in inheritance: + inheritance_dict[tag['name']] = tag + + # for each module in config: + # 1. check whether there is any ready build exists in MBS + # 2. check whether the build tag found in step #1 is in inheritance data + # already and with same priority, if it is in inheritance but with + # different priority, update priority + # 3. if build tag is not inheritance, add this tag into inheritance + # 4. check whether there is any old build tags for this module (name+stream) + # exist in inheritance, if true, remove these tags from inheritance + for module in modules: + log.debug("Processing module in config: \n%s", str(module)) + module_name = module.get('name') + module_stream = module.get('stream') + module_priority = module.get('priority') + module_requires = module.get('requires', None) + module_buildrequires = module.get('buildrequires', None) + + latest_build_tag = None + # find out the latest ready module for this config from MBS + ready_modules = self.mbs.get_modules( + buildrequires=module_buildrequires, + requires=module_requires, + name=module_name, + stream=module_stream, + state=MBS_BUILD_STATES['ready'], + page=1) # we only need the default first page items + if ready_modules: + latest_build_tag = ready_modules[0]['koji_tag'] + + if latest_build_tag is None: + log.warning("There is no ready build found for %s, skipping", str(module)) + continue + + has_latest_tag = latest_build_tag in inheritance_dict + same_priority = False + if has_latest_tag: + same_priority = inheritance_dict[latest_build_tag]['priority'] == module_priority + + if has_latest_tag and same_priority: + log.info("Tag '%s' is in inheritance data of %s with same priority, skipping", + latest_build_tag, self.tag) + continue + + if has_latest_tag and not same_priority: + log.info("Tag '%s' is in inhertiance data of %s but has different priority, " + "will update inheritance data with new priority", + latest_build_tag, self.tag) + # update with new priority value + inheritance_dict[latest_build_tag]['priority'] = module_priority + self.inheritance_changed = True + continue + + # after check, the latest build tag is not in inheritance, we'll add this + # tag to inheritance + latest_tag_info = self.koji.get_tag(latest_build_tag) + if latest_tag_info is None: + raise RuntimeError("Tag '%s' is not found in koji (profile: %s)" % + (latest_build_tag, self.koji.profile)) + + # this is inheritance data for the latest module build we're going to add + module_tag_data = {} + module_tag_data['child_id'] = self.tag_info['id'] + module_tag_data['parent_id'] = latest_tag_info.get('id') + module_tag_data['name'] = latest_tag_info.get('name') + module_tag_data['priority'] = module_priority + module_tag_data['maxdepth'] = module.get('maxdepth', None) + module_tag_data['intransitive'] = module.get('intransitive', False) + module_tag_data['noconfig'] = module.get('noconfig', False) + module_tag_data['pkg_filter'] = module.get('pkg_filter', '') + + # check any old module tag exist in inheritance, for each + # name + stream combination, there is up to 1 tag can exists + # in inheritance data. Remove all old tags. + modules_with_name_stream = self.mbs.get_modules( + name=module_name, + stream=module_stream, + state=[MBS_BUILD_STATES['ready'], MBS_BUILD_STATES['garbage']]) + + old_build_tags = [m['koji_tag'] for m in modules_with_name_stream] + + inheritance_old_tags = set(old_build_tags) & set(inheritance_dict.keys()) + + # add new inheritance data for this module build, update the inheritance + # after checking the old tags in inheritance + log.info("Adding tag '%s' to inheritance data", latest_build_tag) + inheritance_dict[latest_build_tag] = module_tag_data + self.inheritance_changed = True + + if len(inheritance_old_tags) == 0: + # there is no old tag found for this module, do nothing + continue + + if len(inheritance_old_tags) > 0: + # found old tag(s) of this NAME:STREAM in inheritance, remove them + for old_tag in inheritance_old_tags: + log.info("Removing tag '%s' from inheritance data", old_tag) + inheritance_dict[old_tag]['delete link'] = True + + # modules loop finished, now we have the finalized inheritance data + if self.inheritance_changed: + # here we convert the inheritance dict back to list + inheritance_data = [v for k, v in inheritance_dict.items()] + self.koji.login() + log.info("Updating inheritance of tag %s with data: \n%s", + self.tag, str(inheritance_data)) + self.koji.set_inheritance_data(self.tag, inheritance_data) + self.koji.regen_repo(self.tag, wait=self.wait_regen_repo) + else: + log.info("No change to inheritance data of tag '%s', skipping", + self.tag)