From a33e08264640e3ca01c8558b86d1ac7a68036107 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Oct 17 2018 13:58:55 +0000 Subject: Merge #1038 `Add the ability to query by the base modules a module build buildrequires` --- diff --git a/README.rst b/README.rst index b885721..757dcf2 100644 --- a/README.rst +++ b/README.rst @@ -369,6 +369,15 @@ parameters:: { "items": [ { + "base_module_buildrequires": [ + { + "context": "00000000", + "name": "platform", + "stream": "f29", + "stream_version": 290000, + "version": "5" + } + ], "component_builds": [ 57047, 57048 @@ -479,6 +488,16 @@ Filtering module builds The module builds can be filtered by a variety of GET parameters. Some of these parameters include: +- ``base_module_br`` - the name:stream:version:context of a base module the module buildrequires +- ``base_module_br_context`` - the context of a base module the module buildrequires +- ``base_module_br_name`` - the name of a base module the module buildrequires +- ``base_module_br_stream`` - the stream of a base module the module buildrequires +- ``base_module_br_stream_version`` - the stream version of a base module the module buildrequires +- ``base_module_br_stream_version_lte`` - less than or equal to the stream version of a base module + the module buildrequires +- ``base_module_br_stream_version_gte`` - greater than or equal to the stream version of a base + module the module buildrequires +- ``base_module_br_version`` - the version of a base module the module buildrequires - ``batch`` - ``cg_build_koji_tag`` - ``completed_after`` - Zulu ISO 8601 format e.g. ``completed_after=2016-08-23T09:40:07Z`` diff --git a/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py b/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py new file mode 100644 index 0000000..b663936 --- /dev/null +++ b/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py @@ -0,0 +1,107 @@ +"""Add an association table for module buildrequires + +Revision ID: 526fb7d445f7 +Revises: 9d5e6938588f +Create Date: 2018-10-11 12:46:36.060460 + +""" + +# revision identifiers, used by Alembic. +revision = '526fb7d445f7' +down_revision = '9d5e6938588f' + +from alembic import op +import sqlalchemy as sa + +# Data migration imports +from module_build_service import Modulemd, conf +from module_build_service.models import ModuleBuild + +# Data migration tables +mb = sa.Table( + 'module_builds', + sa.MetaData(), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.String()), + sa.Column('stream', sa.String()), + sa.Column('version', sa.String()), + sa.Column('context', sa.String()), + sa.Column('stream_version', sa.Integer()), + sa.Column('modulemd', sa.String()) +) + +mb_to_mbr = sa.Table( + 'module_builds_to_module_buildrequires', + sa.MetaData(), + sa.Column('module_id', sa.Integer(), nullable=False), + sa.Column('module_buildrequire_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']), + sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_builds.id']), + sa.UniqueConstraint('module_id', 'module_buildrequire_id') +) + + +def upgrade(): + with op.batch_alter_table('module_builds', schema=None) as batch_op: + batch_op.add_column(sa.Column('stream_version', sa.Integer())) + + op.create_table( + 'module_builds_to_module_buildrequires', + sa.Column('module_id', sa.Integer(), nullable=False), + sa.Column('module_buildrequire_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_buildrequires.id']), + sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']), + sa.UniqueConstraint('module_id', 'module_buildrequire_id') + ) + + connection = op.get_bind() + # Create all the base module buildrequire entries + for build in connection.execute(mb.select()): + if not build.modulemd: + # If the modulemd is empty, skip this build + continue + + brs = None + try: + mmd = Modulemd.Module().new_from_string(build.modulemd) + mmd.upgrade() + brs = mmd.get_xmd()['mbs']['buildrequires'] + except Exception: + # If the modulemd isn't parseable then skip this build + continue + + for base_module in conf.base_module_names: + base_module_dict = brs.get(base_module) + if not base_module_dict: + # If this base module isn't a buildrequire, continue to see if the next one is + continue + + select = mb.select()\ + .where(mb.c.name == base_module)\ + .where(mb.c.stream == base_module_dict['stream'])\ + .where(mb.c.version == base_module_dict['version'])\ + .where(mb.c.context == base_module_dict['context']) + br = connection.execute(select).fetchone() + if not br: + # If the buildrequire isn't in the datbase, then skip it + continue + + connection.execute(mb_to_mbr.insert().values( + module_id=build.id, + module_buildrequire_id=br.id + )) + + for base_module in conf.base_module_names: + for build in connection.execute(mb.select().where(mb.c.name == base_module)): + stream_version = ModuleBuild.get_stream_version(build.stream) + if not stream_version: + # If a stream version isn't parseable, then skip it + continue + connection.execute(mb.update().where(mb.c.id == build.id).values( + stream_version=stream_version)) + + +def downgrade(): + with op.batch_alter_table('module_builds', schema=None) as batch_op: + batch_op.drop_column(sa.Column('stream_version')) + op.drop_table('module_builds_to_module_buildrequires') diff --git a/module_build_service/models.py b/module_build_service/models.py index 281738c..9b8ad9b 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -166,6 +166,15 @@ class MBSBase(db.Model): __abstract__ = True +module_builds_to_module_buildrequires = db.Table( + 'module_builds_to_module_buildrequires', + db.Column('module_id', db.Integer, db.ForeignKey('module_builds.id'), nullable=False), + db.Column('module_buildrequire_id', db.Integer, db.ForeignKey('module_builds.id'), + nullable=False), + db.UniqueConstraint('module_id', 'module_buildrequire_id', name='unique_buildrequires') +) + + class ModuleBuild(MBSBase): __tablename__ = "module_builds" id = db.Column(db.Integer, primary_key=True) @@ -195,6 +204,16 @@ class ModuleBuild(MBSBase): # components. Think like 'mockchain --recurse' batch = db.Column(db.Integer, default=0) + # This is only used for base modules for ordering purposes (f27.0.1 => 270001) + stream_version = db.Column(db.Integer) + buildrequires = db.relationship( + 'ModuleBuild', + secondary=module_builds_to_module_buildrequires, + primaryjoin=module_builds_to_module_buildrequires.c.module_id == id, + secondaryjoin=module_builds_to_module_buildrequires.c.module_buildrequire_id == id, + backref='buildrequire_for' + ) + rebuild_strategies = { 'all': 'All components will be rebuilt', 'changed-and-after': ('All components that have changed and those in subsequent batches ' @@ -429,6 +448,11 @@ class ModuleBuild(MBSBase): # Add a state transition to "init" mbt = ModuleBuildTrace(state_time=now, state=module.state) module.module_builds_trace.append(mbt) + + # Record the base modules this module buildrequires + for base_module in module.get_buildrequired_base_modules(): + module.buildrequires.append(base_module) + session.add(module) session.commit() if publish_msg: @@ -535,8 +559,8 @@ class ModuleBuild(MBSBase): return query.first() - def short_json(self): - return { + def short_json(self, show_stream_version=False): + rv = { 'id': self.id, 'state': self.state, 'state_name': INVERSE_BUILD_STATES[self.state], @@ -545,6 +569,9 @@ class ModuleBuild(MBSBase): 'name': self.name, 'context': self.context, } + if show_stream_version: + rv['stream_version'] = self.stream_version + return rv def json(self, show_tasks=True): json = self.short_json() @@ -576,7 +603,9 @@ class ModuleBuild(MBSBase): state_url = None if show_state_url: state_url = get_url_for('module_build', api_version=api_version, id=self.id) + json.update({ + 'base_module_buildrequires': [br.short_json(True) for br in self.buildrequires], 'build_context': self.build_context, 'modulemd': self.modulemd, 'ref_build_context': self.ref_build_context, @@ -620,6 +649,78 @@ class ModuleBuild(MBSBase): return ModuleBuildTrace.query.filter_by( module_id=module_id).order_by(ModuleBuildTrace.state_time).all() + @staticmethod + def get_stream_version(stream, right_pad=True): + """ + Parse the supplied stream to find its version. + + This will parse a stream such as "f27" and return 270000. Another example would be a stream + of "f27.0.1" and return 270001. + :param str stream: the module stream + :kwarg bool right_pad: determines if the right side of the stream version should be padded + with zeroes (e.g. `f27` => `27` vs `270000`) + :return: a stream version represented as an integer + :rtype: int or None if the stream doesn't have a valid version + """ + # The platform version (e.g. prefix1.2.0 => 010200) + version = '' + for char in stream: + # See if the current character is an integer, signifying the version has started + if char.isdigit(): + version += char + # If version isn't set, then a digit hasn't been encountered + elif version: + # If the character is a period and the version is set, then + # the loop is still processing the version part of the stream + if char == '.': + version += '.' + # If the version is set and the character is not a period or + # digit, then the remainder of the stream is a suffix like "-beta" + else: + break + + # Remove the periods and pad the numbers if necessary + version = ''.join([section.zfill(2) for section in version.split('.')]) + + if version: + if right_pad: + version += (6 - len(version)) * '0' + # Since the version must be stored as a number, we convert the string back to + # an integer which consequently drops the leading zero if there is one + return int(version) + + def get_buildrequired_base_modules(self): + """ + Find the base modules in the modulemd's xmd section. + + :return: a list of ModuleBuild objects of the base modules that are buildrequired with the + ordering in conf.base_module_names preserved + :rtype: list + :raises RuntimeError: when the xmd section isn't properly filled out by MBS + """ + rv = [] + xmd = self.mmd().get_xmd() + with make_session(conf) as db_session: + for bm in conf.base_module_names: + # xmd is a GLib Variant and doesn't support .get() syntax + try: + bm_dict = xmd['mbs']['buildrequires'].get(bm) + except KeyError: + raise RuntimeError( + 'The module\'s mmd is missing information in the xmd section') + + if not bm_dict: + continue + base_module = self.get_build_from_nsvc( + db_session, bm, bm_dict['stream'], bm_dict['version'], bm_dict['context']) + if not base_module: + log.error('Module #{} buildrequires "{}" but it wasn\'t found in the database' + .format(self.id, repr(bm_dict))) + continue + rv.append(base_module) + + return rv + def __repr__(self): return (("") diff --git a/module_build_service/resolver/MBSResolver.py b/module_build_service/resolver/MBSResolver.py index 9d26862..bbfa663 100644 --- a/module_build_service/resolver/MBSResolver.py +++ b/module_build_service/resolver/MBSResolver.py @@ -29,10 +29,11 @@ import logging import kobo.rpmlib import requests -from module_build_service import db +from module_build_service import db, conf from module_build_service import models from module_build_service.errors import UnprocessableEntity from module_build_service.resolver.base import GenericResolver +from module_build_service.utils.general import import_mmd log = logging.getLogger() @@ -354,5 +355,9 @@ class MBSResolver(GenericResolver): raise RuntimeError( 'The module "{0}" didn\'t contain either a commit hash or a' ' version in MBS'.format(module_name)) + # If the module is a base module, then import it in the database so that entries in + # the module_builds_to_module_buildrequires table can be created later on + if module_name in conf.base_module_names: + import_mmd(db.session, mmd) return new_requires diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py index 46543d0..bfecd7e 100644 --- a/module_build_service/utils/general.py +++ b/module_build_service/utils/general.py @@ -334,6 +334,8 @@ def import_mmd(session, mmd): build.time_submitted = datetime.utcnow() build.time_modified = datetime.utcnow() build.time_completed = datetime.utcnow() + if build.name in conf.base_module_names: + build.stream_version = models.ModuleBuild.get_stream_version(stream) session.add(build) session.commit() msg = "Module {} imported".format(nsvc) diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 28bfce1..f97ba68 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -239,36 +239,14 @@ def get_prefixed_version(mmd): return version # The platform version (e.g. prefix1.2.0 => 010200) - version_prefix = '' - for char in base_module_stream: - try: - # See if the current character is an integer, signifying the version - # has started - int(char) - version_prefix += char - except ValueError: - # If version_prefix isn't set, then a digit hasn't been encountered - if version_prefix: - # If the character is a period and the version_prefix is set, then - # the loop is still processing the version part of the stream - if char == '.': - version_prefix += '.' - # If the version_prefix is set and the character is not a period or - # digit, then the remainder of the stream is a suffix like "-beta" - else: - break - - # Remove the periods and pad the numbers if necessary - version_prefix = ''.join([section.zfill(2) for section in version_prefix.split('.')]) + version_prefix = models.ModuleBuild.get_stream_version(base_module_stream, right_pad=False) if not version_prefix: log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s ' 'version'.format(base_module, base_module_stream)) return version - # Since the version must be stored as a number, we convert the string back to - # an integer which consequently drops the leading zero if there is one - new_version = int(version_prefix + str(version)) + new_version = int(str(version_prefix) + str(version)) if new_version > GLib.MAXUINT64: log.warning('The "{0}" stream "{1}" caused the module\'s version prefix to be ' 'too long'.format(base_module, base_module_stream)) diff --git a/module_build_service/utils/views.py b/module_build_service/utils/views.py index de6acd2..f46eb06 100644 --- a/module_build_service/utils/views.py +++ b/module_build_service/utils/views.py @@ -29,6 +29,7 @@ from datetime import datetime from flask import request, url_for, Response from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean +from sqlalchemy.orm import aliased from module_build_service import models, api_version, conf from module_build_service.errors import ValidationError, NotFound @@ -265,6 +266,55 @@ def filter_module_builds(flask_request): elif context == 'before': query = query.filter(column <= item_datetime) + br_joined = False + module_br_alias = None + for item in ('base_module_br', 'name', 'stream', 'version', 'context', 'stream_version', + 'stream_version_lte', 'stream_version_gte'): + if item == 'base_module_br': + request_arg_name = item + else: + request_arg_name = 'base_module_br_{}'.format(item) + request_arg = flask_request.args.get(request_arg_name) + + if not request_arg: + continue + + if not br_joined: + module_br_alias = aliased(models.ModuleBuild, name='module_br') + # Shorten this table name for clarity in the query below + mb_to_br = models.module_builds_to_module_buildrequires + # The following joins get added: + # JOIN module_builds_to_module_buildrequires + # ON module_builds_to_module_buildrequires.module_id = module_builds.id + # JOIN module_builds AS module_br + # ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id + query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id)\ + .join(module_br_alias, + mb_to_br.c.module_buildrequire_id == module_br_alias.id) + br_joined = True + + if item == 'base_module_br': + try: + name, stream, version, context = flask_request.args['base_module_br'].split(':') + except ValueError: + raise ValidationError( + 'The filter argument for "base_module_br" must be in the format of N:S:V:C') + query = query.filter( + module_br_alias.name == name, + module_br_alias.stream == stream, + module_br_alias.version == version, + module_br_alias.context == context + ) + elif item.endswith('_lte'): + column = getattr(module_br_alias, item[:-4]) + query = query.filter(column <= request_arg) + elif item.endswith('_gte'): + column = getattr(module_br_alias, item[:-4]) + query = query.filter(column >= request_arg) + else: + column = getattr(module_br_alias, item) + query = query.filter(column == request_arg) + query = _add_order_by_clause(flask_request, query, models.ModuleBuild) page = flask_request.args.get('page', 1, type=int) diff --git a/tests/__init__.py b/tests/__init__.py index bf5efa3..4031146 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -293,6 +293,8 @@ def scheduler_init_data(tangerine_state=None): mmd.upgrade() mmd.get_rpm_components()['tangerine'].set_buildorder(0) + platform_br = module_build_service.models.ModuleBuild.query.get(1) + build_one = module_build_service.models.ModuleBuild() build_one.name = 'testmodule' build_one.stream = 'master' @@ -314,6 +316,7 @@ def scheduler_init_data(tangerine_state=None): build_one.rebuild_strategy = 'changed-and-after' build_one.modulemd = mmd.dumps() build_one_component_release = get_rpm_release(build_one) + build_one.buildrequires.append(platform_br) component_one_build_one = module_build_service.models.ComponentBuild() component_one_build_one.package = 'perl-Tangerine' @@ -381,6 +384,7 @@ def scheduler_init_data(tangerine_state=None): component_four_build_one.build_time_only = True with make_session(conf) as session: + session.add(platform_br) session.add(build_one) session.add(component_one_build_one) session.add(component_two_build_one) @@ -398,6 +402,8 @@ def reuse_component_init_data(): mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path) mmd.upgrade() + platform_br = module_build_service.models.ModuleBuild.query.get(1) + build_one = module_build_service.models.ModuleBuild() build_one.name = 'testmodule' build_one.stream = 'master' @@ -422,6 +428,7 @@ def reuse_component_init_data(): xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba' mmd.set_xmd(glib.dict_values(xmd)) build_one.modulemd = mmd.dumps() + build_one.buildrequires.append(platform_br) component_one_build_one = module_build_service.models.ComponentBuild() component_one_build_one.package = 'perl-Tangerine' @@ -506,6 +513,7 @@ def reuse_component_init_data(): xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8' mmd.set_xmd(glib.dict_values(xmd)) build_two.modulemd = mmd.dumps() + build_two.buildrequires.append(platform_br) component_one_build_two = module_build_service.models.ComponentBuild() component_one_build_two.package = 'perl-Tangerine' @@ -550,6 +558,7 @@ def reuse_component_init_data(): component_four_build_two.build_time_only = True with make_session(conf) as session: + session.add(platform_br) session.add(build_one) session.add(component_one_build_one) session.add(component_two_build_one) diff --git a/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml b/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml index 2d19b26..7804863 100644 --- a/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml +++ b/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml @@ -28,7 +28,7 @@ data: testmodule: {ref: 147dca4ca65aa9a1ac51f71b7e687f9178ffa5df, stream: master, version: '20170616125652', context: '321'} requires: - platform: {ref: virtual, stream: f28, version: '3'} + platform: {ref: virtual, stream: f28, version: '3', context: '00000000'} commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a rpms: ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216} diff --git a/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml b/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml index efe96fc..9e884f1 100644 --- a/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml +++ b/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml @@ -25,7 +25,7 @@ data: xmd: mbs: buildrequires: - platform: {ref: virtual, stream: f28, version: '3'} + platform: {ref: virtual, stream: f28, version: '3', context: '00000000'} commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a rpms: ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216} diff --git a/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml b/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml index 81923a4..f725e33 100644 --- a/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml +++ b/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml @@ -25,7 +25,7 @@ data: xmd: mbs: buildrequires: - platform: {ref: virtual, stream: f28, version: '3'} + platform: {ref: virtual, stream: f28, version: '3', context: '00000000'} commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a rpms: ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216} diff --git a/tests/test_builder/test_mock.py b/tests/test_builder/test_mock.py index 9633574..5ca883e 100644 --- a/tests/test_builder/test_mock.py +++ b/tests/test_builder/test_mock.py @@ -50,18 +50,6 @@ class TestMockModuleBuilder: mmd = Modulemd.Module().new_from_file(os.path.join( base_dir, '..', 'staged_data', 'testmodule-with-filters.yaml')) mmd.upgrade() - - module = ModuleBuild.create( - session, - conf, - name="mbs-testmodule", - stream="test", - version="20171027111452", - modulemd=mmd.dumps(), - scmurl="file:///testdir", - username="test", - ) - module.koji_tag = "module-mbs-testmodule-test-20171027111452" mmd.set_xmd(glib.dict_values({ 'mbs': { 'rpms': { @@ -74,13 +62,16 @@ class TestMockModuleBuilder: 'version': '20171024133034', 'filtered_rpms': [], 'stream': 'master', - 'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45' + 'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45', + 'context': '00000000' }, 'platform': { 'version': '20171028112959', 'filtered_rpms': [], 'stream': 'master', - 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'} + 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa', + 'context': '00000000' + } }, 'scmurl': 'file:///testdir', 'commit': '5566bc792ec7a03bb0e28edd1b104a96ba342bd8', @@ -89,11 +80,23 @@ class TestMockModuleBuilder: 'version': '20171028112959', 'filtered_rpms': [], 'stream': 'master', - 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'} + 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa', + 'context': '00000000' + } } } })) - module.modulemd = mmd.dumps() + module = ModuleBuild.create( + session, + conf, + name="mbs-testmodule", + stream="test", + version="20171027111452", + modulemd=mmd.dumps(), + scmurl="file:///testdir", + username="test", + ) + module.koji_tag = "module-mbs-testmodule-test-20171027111452" module.batch = batch session.add(module) @@ -101,6 +104,7 @@ class TestMockModuleBuilder: cb = ComponentBuild(**dict(build, format="rpms", state=state)) session.add(cb) session.commit() + session.commit() return module diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index eb3dd17..51b328d 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -93,6 +93,16 @@ class TestModels: build_one = ModuleBuild.query.get(2) assert build_one.siblings == [3, 4] + def test_get_stream_version(self): + """Test the ModuleBuild.get_stream_version method when right_pad is True.""" + assert ModuleBuild.get_stream_version('f27') == 270000 + assert ModuleBuild.get_stream_version('f27.02.30') == 270230 + + def test_get_stream_version_no_right_pad(self): + """Test the ModuleBuild.get_stream_version method when right_pad is False.""" + assert ModuleBuild.get_stream_version('f27', False) == 27 + assert ModuleBuild.get_stream_version('f27.02.30', False) == 270230 + class TestModelsGetStreamsContexts: diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 8658e1b..7e6c068 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -213,7 +213,7 @@ class TestModuleWait: @patch('module_build_service.resolver.DBResolver') @patch('module_build_service.resolver.GenericResolver') @patch("module_build_service.config.Config.base_module_names", - new_callable=mock.PropertyMock, return_value=set(["base-runtime"])) + new_callable=mock.PropertyMock, return_value=set(["base-runtime", "platform"])) def test_set_cg_build_koji_tag( self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg): """ diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 4fdb9a1..c6cf7c4 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -21,6 +21,7 @@ # Written by Matt Prahl import json +from datetime import datetime import module_build_service.scm @@ -39,6 +40,7 @@ from module_build_service.models import ModuleBuild from module_build_service import db, version, Modulemd import module_build_service.config as mbs_config import module_build_service.scheduler.handlers.modules +from module_build_service.utils import import_mmd, load_mmd user = ('Homer J. Simpson', set(['packager'])) @@ -172,6 +174,7 @@ class TestViews: def test_query_build_with_verbose_mode(self): rv = self.client.get('/module-build-service/1/module-builds/2?verbose=true') data = json.loads(rv.data) + assert data['base_module_buildrequires'] == [] assert data['component_builds'] == [1, 2] assert data['context'] == '00000000' # There is no xmd information on this module, so these values should be None @@ -216,6 +219,21 @@ class TestViews: assert data['rebuild_strategy'] == 'changed-and-after' assert data['siblings'] == [] + def test_query_build_with_br_verbose_mode(self): + reuse_component_init_data() + rv = self.client.get('/module-build-service/1/module-builds/2?verbose=true') + data = json.loads(rv.data) + assert data['base_module_buildrequires'] == [{ + 'context': '00000000', + 'id': 1, + 'name': 'platform', + 'state': 5, + 'state_name': 'ready', + 'stream': 'f28', + 'stream_version': 280000, + 'version': '3' + }] + def test_pagination_metadata(self): rv = self.client.get('/module-build-service/1/module-builds/?per_page=2&page=2') meta_data = json.loads(rv.data)['meta'] @@ -662,6 +680,77 @@ class TestViews: assert data['error'] == 'Bad Request' assert data['message'] == 'An invalid order_by or order_desc_by key was supplied' + def test_query_base_module_br_filters(self): + reuse_component_init_data() + mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True) + mmd.set_stream('f30.1.3') + import_mmd(db.session, mmd) + platform_f300103 = ModuleBuild.query.filter_by(stream='f30.1.3').one() + build = ModuleBuild( + name='testmodule', + stream='master', + version=20170109091357, + state=5, + build_context='dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3', + runtime_context='ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7', + context='7c29193d', + koji_tag='module-testmodule-master-20170109091357-7c29193d', + scmurl='git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79', + batch=3, + owner='Dr. Pepper', + time_submitted=datetime(2018, 11, 15, 16, 8, 18), + time_modified=datetime(2018, 11, 15, 16, 19, 35), + rebuild_strategy='changed-and-after', + modulemd='---' + ) + build.buildrequires.append(platform_f300103) + db.session.add(build) + db.session.commit() + # Query by NSVC + rv = self.client.get( + '/module-build-service/1/module-builds/?base_module_br=platform:f28:3:00000000') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + rv = self.client.get( + '/module-build-service/1/module-builds/?base_module_br=platform:f30.1.3:3:00000000') + data = json.loads(rv.data) + assert data['meta']['total'] == 1 + # Query by non-existent NVC + rv = self.client.get( + '/module-build-service/1/module-builds/?base_module_br=platform:f12:3:00000000') + data = json.loads(rv.data) + assert data['meta']['total'] == 0 + # Query by name and stream + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_name=platform' + '&base_module_br_stream=f28') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + # Query by stream version + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version=' + '280000') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + # Query by lte stream version + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'lte=290000') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + # Query by lte stream version with no results + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'lte=270000') + data = json.loads(rv.data) + assert data['meta']['total'] == 0 + # Query by gte stream version + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'gte=270000') + data = json.loads(rv.data) + assert data['meta']['total'] == 3 + # Query by gte stream version with no results + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'gte=320000') + data = json.loads(rv.data) + assert data['meta']['total'] == 0 + @pytest.mark.parametrize('api_version', [1, 2]) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') @@ -702,6 +791,15 @@ class TestViews: mmd = Modulemd.Module().new_from_string(data['modulemd']) mmd.upgrade() + # Make sure the buildrequires entry was created + module = ModuleBuild.query.get(8) + assert len(module.buildrequires) == 1 + assert module.buildrequires[0].name == 'platform' + assert module.buildrequires[0].stream == 'f28' + assert module.buildrequires[0].version == '3' + assert module.buildrequires[0].context == '00000000' + assert module.buildrequires[0].stream_version == 280000 + @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') @patch('module_build_service.config.Config.rebuild_strategy_allow_override',