From 4f9fffff3b4e5c06370ded9c2ad8eeb904b1ccaf Mon Sep 17 00:00:00 2001 From: mprahl Date: Oct 16 2018 14:26:39 +0000 Subject: [PATCH 1/2] Move some of the logic in `get_prefixed_version` to a ModuleBuild static method The logic used to parse the stream version will need to be reused elsewhere to support the filtering and ordering of a base module stream version. --- diff --git a/module_build_service/models.py b/module_build_service/models.py index 281738c..ae40b4e 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -620,6 +620,42 @@ class ModuleBuild(MBSBase): return ModuleBuildTrace.query.filter_by( module_id=module_id).order_by(ModuleBuildTrace.state_time).all() + @staticmethod + def get_stream_version(stream): + """ + Parse the supplied stream to find its version. + + This will parse a stream such as "f27" and return 270000. Another example would be a stream + of "f27.0.1" and return 270001. + :param str stream: the module stream + :return: a stream version represented as an integer + :rtype: int or None if the stream doesn't have a valid version + """ + # The platform version (e.g. prefix1.2.0 => 010200) + version = '' + for char in stream: + # See if the current character is an integer, signifying the version has started + if char.isdigit(): + version += char + # If version isn't set, then a digit hasn't been encountered + elif version: + # If the character is a period and the version is set, then + # the loop is still processing the version part of the stream + if char == '.': + version += '.' + # If the version is set and the character is not a period or + # digit, then the remainder of the stream is a suffix like "-beta" + else: + break + + # Remove the periods and pad the numbers if necessary + version = ''.join([section.zfill(2) for section in version.split('.')]) + + if version: + # Since the version must be stored as a number, we convert the string back to + # an integer which consequently drops the leading zero if there is one + return int(version) + def __repr__(self): return (("") diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 28bfce1..729758d 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -239,36 +239,14 @@ def get_prefixed_version(mmd): return version # The platform version (e.g. prefix1.2.0 => 010200) - version_prefix = '' - for char in base_module_stream: - try: - # See if the current character is an integer, signifying the version - # has started - int(char) - version_prefix += char - except ValueError: - # If version_prefix isn't set, then a digit hasn't been encountered - if version_prefix: - # If the character is a period and the version_prefix is set, then - # the loop is still processing the version part of the stream - if char == '.': - version_prefix += '.' - # If the version_prefix is set and the character is not a period or - # digit, then the remainder of the stream is a suffix like "-beta" - else: - break - - # Remove the periods and pad the numbers if necessary - version_prefix = ''.join([section.zfill(2) for section in version_prefix.split('.')]) + version_prefix = models.ModuleBuild.get_stream_version(base_module_stream) if not version_prefix: log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s ' 'version'.format(base_module, base_module_stream)) return version - # Since the version must be stored as a number, we convert the string back to - # an integer which consequently drops the leading zero if there is one - new_version = int(version_prefix + str(version)) + new_version = int(str(version_prefix) + str(version)) if new_version > GLib.MAXUINT64: log.warning('The "{0}" stream "{1}" caused the module\'s version prefix to be ' 'too long'.format(base_module, base_module_stream)) diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index eb3dd17..f6f764c 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -93,6 +93,11 @@ class TestModels: build_one = ModuleBuild.query.get(2) assert build_one.siblings == [3, 4] + def test_get_stream_version(self): + """Test the ModuleBuild.get_stream_version method when right_pad is True.""" + assert ModuleBuild.get_stream_version('f27') == 270000 + assert ModuleBuild.get_stream_version('f27.02.30') == 270230 + class TestModelsGetStreamsContexts: From 599c8817142c055e2d5b10ed5e146497e5ffa485 Mon Sep 17 00:00:00 2001 From: mprahl Date: Oct 17 2018 11:47:31 +0000 Subject: [PATCH 2/2] Add the ability to query by the base modules a module build buildrequires Future use cases will require the ability to find compatible module builds to buildrequire based on the base module the module used to build. This commit adds an association table that will contain module build IDs and the base module they buildrequire. Addresses FACTORY-3353 --- diff --git a/README.rst b/README.rst index b885721..757dcf2 100644 --- a/README.rst +++ b/README.rst @@ -369,6 +369,15 @@ parameters:: { "items": [ { + "base_module_buildrequires": [ + { + "context": "00000000", + "name": "platform", + "stream": "f29", + "stream_version": 290000, + "version": "5" + } + ], "component_builds": [ 57047, 57048 @@ -479,6 +488,16 @@ Filtering module builds The module builds can be filtered by a variety of GET parameters. Some of these parameters include: +- ``base_module_br`` - the name:stream:version:context of a base module the module buildrequires +- ``base_module_br_context`` - the context of a base module the module buildrequires +- ``base_module_br_name`` - the name of a base module the module buildrequires +- ``base_module_br_stream`` - the stream of a base module the module buildrequires +- ``base_module_br_stream_version`` - the stream version of a base module the module buildrequires +- ``base_module_br_stream_version_lte`` - less than or equal to the stream version of a base module + the module buildrequires +- ``base_module_br_stream_version_gte`` - greater than or equal to the stream version of a base + module the module buildrequires +- ``base_module_br_version`` - the version of a base module the module buildrequires - ``batch`` - ``cg_build_koji_tag`` - ``completed_after`` - Zulu ISO 8601 format e.g. ``completed_after=2016-08-23T09:40:07Z`` diff --git a/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py b/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py new file mode 100644 index 0000000..b663936 --- /dev/null +++ b/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py @@ -0,0 +1,107 @@ +"""Add an association table for module buildrequires + +Revision ID: 526fb7d445f7 +Revises: 9d5e6938588f +Create Date: 2018-10-11 12:46:36.060460 + +""" + +# revision identifiers, used by Alembic. +revision = '526fb7d445f7' +down_revision = '9d5e6938588f' + +from alembic import op +import sqlalchemy as sa + +# Data migration imports +from module_build_service import Modulemd, conf +from module_build_service.models import ModuleBuild + +# Data migration tables +mb = sa.Table( + 'module_builds', + sa.MetaData(), + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.String()), + sa.Column('stream', sa.String()), + sa.Column('version', sa.String()), + sa.Column('context', sa.String()), + sa.Column('stream_version', sa.Integer()), + sa.Column('modulemd', sa.String()) +) + +mb_to_mbr = sa.Table( + 'module_builds_to_module_buildrequires', + sa.MetaData(), + sa.Column('module_id', sa.Integer(), nullable=False), + sa.Column('module_buildrequire_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']), + sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_builds.id']), + sa.UniqueConstraint('module_id', 'module_buildrequire_id') +) + + +def upgrade(): + with op.batch_alter_table('module_builds', schema=None) as batch_op: + batch_op.add_column(sa.Column('stream_version', sa.Integer())) + + op.create_table( + 'module_builds_to_module_buildrequires', + sa.Column('module_id', sa.Integer(), nullable=False), + sa.Column('module_buildrequire_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_buildrequires.id']), + sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']), + sa.UniqueConstraint('module_id', 'module_buildrequire_id') + ) + + connection = op.get_bind() + # Create all the base module buildrequire entries + for build in connection.execute(mb.select()): + if not build.modulemd: + # If the modulemd is empty, skip this build + continue + + brs = None + try: + mmd = Modulemd.Module().new_from_string(build.modulemd) + mmd.upgrade() + brs = mmd.get_xmd()['mbs']['buildrequires'] + except Exception: + # If the modulemd isn't parseable then skip this build + continue + + for base_module in conf.base_module_names: + base_module_dict = brs.get(base_module) + if not base_module_dict: + # If this base module isn't a buildrequire, continue to see if the next one is + continue + + select = mb.select()\ + .where(mb.c.name == base_module)\ + .where(mb.c.stream == base_module_dict['stream'])\ + .where(mb.c.version == base_module_dict['version'])\ + .where(mb.c.context == base_module_dict['context']) + br = connection.execute(select).fetchone() + if not br: + # If the buildrequire isn't in the datbase, then skip it + continue + + connection.execute(mb_to_mbr.insert().values( + module_id=build.id, + module_buildrequire_id=br.id + )) + + for base_module in conf.base_module_names: + for build in connection.execute(mb.select().where(mb.c.name == base_module)): + stream_version = ModuleBuild.get_stream_version(build.stream) + if not stream_version: + # If a stream version isn't parseable, then skip it + continue + connection.execute(mb.update().where(mb.c.id == build.id).values( + stream_version=stream_version)) + + +def downgrade(): + with op.batch_alter_table('module_builds', schema=None) as batch_op: + batch_op.drop_column(sa.Column('stream_version')) + op.drop_table('module_builds_to_module_buildrequires') diff --git a/module_build_service/models.py b/module_build_service/models.py index ae40b4e..9b8ad9b 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -166,6 +166,15 @@ class MBSBase(db.Model): __abstract__ = True +module_builds_to_module_buildrequires = db.Table( + 'module_builds_to_module_buildrequires', + db.Column('module_id', db.Integer, db.ForeignKey('module_builds.id'), nullable=False), + db.Column('module_buildrequire_id', db.Integer, db.ForeignKey('module_builds.id'), + nullable=False), + db.UniqueConstraint('module_id', 'module_buildrequire_id', name='unique_buildrequires') +) + + class ModuleBuild(MBSBase): __tablename__ = "module_builds" id = db.Column(db.Integer, primary_key=True) @@ -195,6 +204,16 @@ class ModuleBuild(MBSBase): # components. Think like 'mockchain --recurse' batch = db.Column(db.Integer, default=0) + # This is only used for base modules for ordering purposes (f27.0.1 => 270001) + stream_version = db.Column(db.Integer) + buildrequires = db.relationship( + 'ModuleBuild', + secondary=module_builds_to_module_buildrequires, + primaryjoin=module_builds_to_module_buildrequires.c.module_id == id, + secondaryjoin=module_builds_to_module_buildrequires.c.module_buildrequire_id == id, + backref='buildrequire_for' + ) + rebuild_strategies = { 'all': 'All components will be rebuilt', 'changed-and-after': ('All components that have changed and those in subsequent batches ' @@ -429,6 +448,11 @@ class ModuleBuild(MBSBase): # Add a state transition to "init" mbt = ModuleBuildTrace(state_time=now, state=module.state) module.module_builds_trace.append(mbt) + + # Record the base modules this module buildrequires + for base_module in module.get_buildrequired_base_modules(): + module.buildrequires.append(base_module) + session.add(module) session.commit() if publish_msg: @@ -535,8 +559,8 @@ class ModuleBuild(MBSBase): return query.first() - def short_json(self): - return { + def short_json(self, show_stream_version=False): + rv = { 'id': self.id, 'state': self.state, 'state_name': INVERSE_BUILD_STATES[self.state], @@ -545,6 +569,9 @@ class ModuleBuild(MBSBase): 'name': self.name, 'context': self.context, } + if show_stream_version: + rv['stream_version'] = self.stream_version + return rv def json(self, show_tasks=True): json = self.short_json() @@ -576,7 +603,9 @@ class ModuleBuild(MBSBase): state_url = None if show_state_url: state_url = get_url_for('module_build', api_version=api_version, id=self.id) + json.update({ + 'base_module_buildrequires': [br.short_json(True) for br in self.buildrequires], 'build_context': self.build_context, 'modulemd': self.modulemd, 'ref_build_context': self.ref_build_context, @@ -621,13 +650,15 @@ class ModuleBuild(MBSBase): module_id=module_id).order_by(ModuleBuildTrace.state_time).all() @staticmethod - def get_stream_version(stream): + def get_stream_version(stream, right_pad=True): """ Parse the supplied stream to find its version. This will parse a stream such as "f27" and return 270000. Another example would be a stream of "f27.0.1" and return 270001. :param str stream: the module stream + :kwarg bool right_pad: determines if the right side of the stream version should be padded + with zeroes (e.g. `f27` => `27` vs `270000`) :return: a stream version represented as an integer :rtype: int or None if the stream doesn't have a valid version """ @@ -652,10 +683,44 @@ class ModuleBuild(MBSBase): version = ''.join([section.zfill(2) for section in version.split('.')]) if version: + if right_pad: + version += (6 - len(version)) * '0' # Since the version must be stored as a number, we convert the string back to # an integer which consequently drops the leading zero if there is one return int(version) + def get_buildrequired_base_modules(self): + """ + Find the base modules in the modulemd's xmd section. + + :return: a list of ModuleBuild objects of the base modules that are buildrequired with the + ordering in conf.base_module_names preserved + :rtype: list + :raises RuntimeError: when the xmd section isn't properly filled out by MBS + """ + rv = [] + xmd = self.mmd().get_xmd() + with make_session(conf) as db_session: + for bm in conf.base_module_names: + # xmd is a GLib Variant and doesn't support .get() syntax + try: + bm_dict = xmd['mbs']['buildrequires'].get(bm) + except KeyError: + raise RuntimeError( + 'The module\'s mmd is missing information in the xmd section') + + if not bm_dict: + continue + base_module = self.get_build_from_nsvc( + db_session, bm, bm_dict['stream'], bm_dict['version'], bm_dict['context']) + if not base_module: + log.error('Module #{} buildrequires "{}" but it wasn\'t found in the database' + .format(self.id, repr(bm_dict))) + continue + rv.append(base_module) + + return rv + def __repr__(self): return (("") diff --git a/module_build_service/resolver/MBSResolver.py b/module_build_service/resolver/MBSResolver.py index 4b46850..1c57b72 100644 --- a/module_build_service/resolver/MBSResolver.py +++ b/module_build_service/resolver/MBSResolver.py @@ -29,10 +29,11 @@ import logging import kobo.rpmlib import requests -from module_build_service import db +from module_build_service import db, conf from module_build_service import models from module_build_service.errors import UnprocessableEntity from module_build_service.resolver.base import GenericResolver +from module_build_service.utils.general import import_mmd log = logging.getLogger() @@ -363,5 +364,9 @@ class MBSResolver(GenericResolver): raise RuntimeError( 'The module "{0}" didn\'t contain either a commit hash or a' ' version in MBS'.format(module_name)) + # If the module is a base module, then import it in the database so that entries in + # the module_builds_to_module_buildrequires table can be created later on + if module_name in conf.base_module_names: + import_mmd(db.session, mmd) return new_requires diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py index 46543d0..bfecd7e 100644 --- a/module_build_service/utils/general.py +++ b/module_build_service/utils/general.py @@ -334,6 +334,8 @@ def import_mmd(session, mmd): build.time_submitted = datetime.utcnow() build.time_modified = datetime.utcnow() build.time_completed = datetime.utcnow() + if build.name in conf.base_module_names: + build.stream_version = models.ModuleBuild.get_stream_version(stream) session.add(build) session.commit() msg = "Module {} imported".format(nsvc) diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 729758d..f97ba68 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -239,7 +239,7 @@ def get_prefixed_version(mmd): return version # The platform version (e.g. prefix1.2.0 => 010200) - version_prefix = models.ModuleBuild.get_stream_version(base_module_stream) + version_prefix = models.ModuleBuild.get_stream_version(base_module_stream, right_pad=False) if not version_prefix: log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s ' diff --git a/module_build_service/utils/views.py b/module_build_service/utils/views.py index de6acd2..f46eb06 100644 --- a/module_build_service/utils/views.py +++ b/module_build_service/utils/views.py @@ -29,6 +29,7 @@ from datetime import datetime from flask import request, url_for, Response from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean +from sqlalchemy.orm import aliased from module_build_service import models, api_version, conf from module_build_service.errors import ValidationError, NotFound @@ -265,6 +266,55 @@ def filter_module_builds(flask_request): elif context == 'before': query = query.filter(column <= item_datetime) + br_joined = False + module_br_alias = None + for item in ('base_module_br', 'name', 'stream', 'version', 'context', 'stream_version', + 'stream_version_lte', 'stream_version_gte'): + if item == 'base_module_br': + request_arg_name = item + else: + request_arg_name = 'base_module_br_{}'.format(item) + request_arg = flask_request.args.get(request_arg_name) + + if not request_arg: + continue + + if not br_joined: + module_br_alias = aliased(models.ModuleBuild, name='module_br') + # Shorten this table name for clarity in the query below + mb_to_br = models.module_builds_to_module_buildrequires + # The following joins get added: + # JOIN module_builds_to_module_buildrequires + # ON module_builds_to_module_buildrequires.module_id = module_builds.id + # JOIN module_builds AS module_br + # ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id + query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id)\ + .join(module_br_alias, + mb_to_br.c.module_buildrequire_id == module_br_alias.id) + br_joined = True + + if item == 'base_module_br': + try: + name, stream, version, context = flask_request.args['base_module_br'].split(':') + except ValueError: + raise ValidationError( + 'The filter argument for "base_module_br" must be in the format of N:S:V:C') + query = query.filter( + module_br_alias.name == name, + module_br_alias.stream == stream, + module_br_alias.version == version, + module_br_alias.context == context + ) + elif item.endswith('_lte'): + column = getattr(module_br_alias, item[:-4]) + query = query.filter(column <= request_arg) + elif item.endswith('_gte'): + column = getattr(module_br_alias, item[:-4]) + query = query.filter(column >= request_arg) + else: + column = getattr(module_br_alias, item) + query = query.filter(column == request_arg) + query = _add_order_by_clause(flask_request, query, models.ModuleBuild) page = flask_request.args.get('page', 1, type=int) diff --git a/tests/__init__.py b/tests/__init__.py index bf5efa3..4031146 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -293,6 +293,8 @@ def scheduler_init_data(tangerine_state=None): mmd.upgrade() mmd.get_rpm_components()['tangerine'].set_buildorder(0) + platform_br = module_build_service.models.ModuleBuild.query.get(1) + build_one = module_build_service.models.ModuleBuild() build_one.name = 'testmodule' build_one.stream = 'master' @@ -314,6 +316,7 @@ def scheduler_init_data(tangerine_state=None): build_one.rebuild_strategy = 'changed-and-after' build_one.modulemd = mmd.dumps() build_one_component_release = get_rpm_release(build_one) + build_one.buildrequires.append(platform_br) component_one_build_one = module_build_service.models.ComponentBuild() component_one_build_one.package = 'perl-Tangerine' @@ -381,6 +384,7 @@ def scheduler_init_data(tangerine_state=None): component_four_build_one.build_time_only = True with make_session(conf) as session: + session.add(platform_br) session.add(build_one) session.add(component_one_build_one) session.add(component_two_build_one) @@ -398,6 +402,8 @@ def reuse_component_init_data(): mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path) mmd.upgrade() + platform_br = module_build_service.models.ModuleBuild.query.get(1) + build_one = module_build_service.models.ModuleBuild() build_one.name = 'testmodule' build_one.stream = 'master' @@ -422,6 +428,7 @@ def reuse_component_init_data(): xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba' mmd.set_xmd(glib.dict_values(xmd)) build_one.modulemd = mmd.dumps() + build_one.buildrequires.append(platform_br) component_one_build_one = module_build_service.models.ComponentBuild() component_one_build_one.package = 'perl-Tangerine' @@ -506,6 +513,7 @@ def reuse_component_init_data(): xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8' mmd.set_xmd(glib.dict_values(xmd)) build_two.modulemd = mmd.dumps() + build_two.buildrequires.append(platform_br) component_one_build_two = module_build_service.models.ComponentBuild() component_one_build_two.package = 'perl-Tangerine' @@ -550,6 +558,7 @@ def reuse_component_init_data(): component_four_build_two.build_time_only = True with make_session(conf) as session: + session.add(platform_br) session.add(build_one) session.add(component_one_build_one) session.add(component_two_build_one) diff --git a/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml b/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml index 2d19b26..7804863 100644 --- a/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml +++ b/tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml @@ -28,7 +28,7 @@ data: testmodule: {ref: 147dca4ca65aa9a1ac51f71b7e687f9178ffa5df, stream: master, version: '20170616125652', context: '321'} requires: - platform: {ref: virtual, stream: f28, version: '3'} + platform: {ref: virtual, stream: f28, version: '3', context: '00000000'} commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a rpms: ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216} diff --git a/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml b/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml index efe96fc..9e884f1 100644 --- a/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml +++ b/tests/staged_data/local_builds/module-testmodule-master-20170816080815/results/modules.yaml @@ -25,7 +25,7 @@ data: xmd: mbs: buildrequires: - platform: {ref: virtual, stream: f28, version: '3'} + platform: {ref: virtual, stream: f28, version: '3', context: '00000000'} commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a rpms: ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216} diff --git a/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml b/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml index 81923a4..f725e33 100644 --- a/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml +++ b/tests/staged_data/local_builds/module-testmodule-master-20170816080816/results/modules.yaml @@ -25,7 +25,7 @@ data: xmd: mbs: buildrequires: - platform: {ref: virtual, stream: f28, version: '3'} + platform: {ref: virtual, stream: f28, version: '3', context: '00000000'} commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a rpms: ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216} diff --git a/tests/test_builder/test_mock.py b/tests/test_builder/test_mock.py index 9633574..5ca883e 100644 --- a/tests/test_builder/test_mock.py +++ b/tests/test_builder/test_mock.py @@ -50,18 +50,6 @@ class TestMockModuleBuilder: mmd = Modulemd.Module().new_from_file(os.path.join( base_dir, '..', 'staged_data', 'testmodule-with-filters.yaml')) mmd.upgrade() - - module = ModuleBuild.create( - session, - conf, - name="mbs-testmodule", - stream="test", - version="20171027111452", - modulemd=mmd.dumps(), - scmurl="file:///testdir", - username="test", - ) - module.koji_tag = "module-mbs-testmodule-test-20171027111452" mmd.set_xmd(glib.dict_values({ 'mbs': { 'rpms': { @@ -74,13 +62,16 @@ class TestMockModuleBuilder: 'version': '20171024133034', 'filtered_rpms': [], 'stream': 'master', - 'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45' + 'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45', + 'context': '00000000' }, 'platform': { 'version': '20171028112959', 'filtered_rpms': [], 'stream': 'master', - 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'} + 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa', + 'context': '00000000' + } }, 'scmurl': 'file:///testdir', 'commit': '5566bc792ec7a03bb0e28edd1b104a96ba342bd8', @@ -89,11 +80,23 @@ class TestMockModuleBuilder: 'version': '20171028112959', 'filtered_rpms': [], 'stream': 'master', - 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'} + 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa', + 'context': '00000000' + } } } })) - module.modulemd = mmd.dumps() + module = ModuleBuild.create( + session, + conf, + name="mbs-testmodule", + stream="test", + version="20171027111452", + modulemd=mmd.dumps(), + scmurl="file:///testdir", + username="test", + ) + module.koji_tag = "module-mbs-testmodule-test-20171027111452" module.batch = batch session.add(module) @@ -101,6 +104,7 @@ class TestMockModuleBuilder: cb = ComponentBuild(**dict(build, format="rpms", state=state)) session.add(cb) session.commit() + session.commit() return module diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index f6f764c..51b328d 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -98,6 +98,11 @@ class TestModels: assert ModuleBuild.get_stream_version('f27') == 270000 assert ModuleBuild.get_stream_version('f27.02.30') == 270230 + def test_get_stream_version_no_right_pad(self): + """Test the ModuleBuild.get_stream_version method when right_pad is False.""" + assert ModuleBuild.get_stream_version('f27', False) == 27 + assert ModuleBuild.get_stream_version('f27.02.30', False) == 270230 + class TestModelsGetStreamsContexts: diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 8658e1b..7e6c068 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -213,7 +213,7 @@ class TestModuleWait: @patch('module_build_service.resolver.DBResolver') @patch('module_build_service.resolver.GenericResolver') @patch("module_build_service.config.Config.base_module_names", - new_callable=mock.PropertyMock, return_value=set(["base-runtime"])) + new_callable=mock.PropertyMock, return_value=set(["base-runtime", "platform"])) def test_set_cg_build_koji_tag( self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg): """ diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 4fdb9a1..c6cf7c4 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -21,6 +21,7 @@ # Written by Matt Prahl import json +from datetime import datetime import module_build_service.scm @@ -39,6 +40,7 @@ from module_build_service.models import ModuleBuild from module_build_service import db, version, Modulemd import module_build_service.config as mbs_config import module_build_service.scheduler.handlers.modules +from module_build_service.utils import import_mmd, load_mmd user = ('Homer J. Simpson', set(['packager'])) @@ -172,6 +174,7 @@ class TestViews: def test_query_build_with_verbose_mode(self): rv = self.client.get('/module-build-service/1/module-builds/2?verbose=true') data = json.loads(rv.data) + assert data['base_module_buildrequires'] == [] assert data['component_builds'] == [1, 2] assert data['context'] == '00000000' # There is no xmd information on this module, so these values should be None @@ -216,6 +219,21 @@ class TestViews: assert data['rebuild_strategy'] == 'changed-and-after' assert data['siblings'] == [] + def test_query_build_with_br_verbose_mode(self): + reuse_component_init_data() + rv = self.client.get('/module-build-service/1/module-builds/2?verbose=true') + data = json.loads(rv.data) + assert data['base_module_buildrequires'] == [{ + 'context': '00000000', + 'id': 1, + 'name': 'platform', + 'state': 5, + 'state_name': 'ready', + 'stream': 'f28', + 'stream_version': 280000, + 'version': '3' + }] + def test_pagination_metadata(self): rv = self.client.get('/module-build-service/1/module-builds/?per_page=2&page=2') meta_data = json.loads(rv.data)['meta'] @@ -662,6 +680,77 @@ class TestViews: assert data['error'] == 'Bad Request' assert data['message'] == 'An invalid order_by or order_desc_by key was supplied' + def test_query_base_module_br_filters(self): + reuse_component_init_data() + mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True) + mmd.set_stream('f30.1.3') + import_mmd(db.session, mmd) + platform_f300103 = ModuleBuild.query.filter_by(stream='f30.1.3').one() + build = ModuleBuild( + name='testmodule', + stream='master', + version=20170109091357, + state=5, + build_context='dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3', + runtime_context='ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7', + context='7c29193d', + koji_tag='module-testmodule-master-20170109091357-7c29193d', + scmurl='git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79', + batch=3, + owner='Dr. Pepper', + time_submitted=datetime(2018, 11, 15, 16, 8, 18), + time_modified=datetime(2018, 11, 15, 16, 19, 35), + rebuild_strategy='changed-and-after', + modulemd='---' + ) + build.buildrequires.append(platform_f300103) + db.session.add(build) + db.session.commit() + # Query by NSVC + rv = self.client.get( + '/module-build-service/1/module-builds/?base_module_br=platform:f28:3:00000000') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + rv = self.client.get( + '/module-build-service/1/module-builds/?base_module_br=platform:f30.1.3:3:00000000') + data = json.loads(rv.data) + assert data['meta']['total'] == 1 + # Query by non-existent NVC + rv = self.client.get( + '/module-build-service/1/module-builds/?base_module_br=platform:f12:3:00000000') + data = json.loads(rv.data) + assert data['meta']['total'] == 0 + # Query by name and stream + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_name=platform' + '&base_module_br_stream=f28') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + # Query by stream version + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version=' + '280000') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + # Query by lte stream version + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'lte=290000') + data = json.loads(rv.data) + assert data['meta']['total'] == 2 + # Query by lte stream version with no results + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'lte=270000') + data = json.loads(rv.data) + assert data['meta']['total'] == 0 + # Query by gte stream version + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'gte=270000') + data = json.loads(rv.data) + assert data['meta']['total'] == 3 + # Query by gte stream version with no results + rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_' + 'gte=320000') + data = json.loads(rv.data) + assert data['meta']['total'] == 0 + @pytest.mark.parametrize('api_version', [1, 2]) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') @@ -702,6 +791,15 @@ class TestViews: mmd = Modulemd.Module().new_from_string(data['modulemd']) mmd.upgrade() + # Make sure the buildrequires entry was created + module = ModuleBuild.query.get(8) + assert len(module.buildrequires) == 1 + assert module.buildrequires[0].name == 'platform' + assert module.buildrequires[0].stream == 'f28' + assert module.buildrequires[0].version == '3' + assert module.buildrequires[0].context == '00000000' + assert module.buildrequires[0].stream_version == 280000 + @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') @patch('module_build_service.config.Config.rebuild_strategy_allow_override',