#1038 Add the ability to query by the base modules a module build buildrequires
Merged 11 months ago by mprahl. Opened a year ago by mprahl.

file modified
+19

@@ -369,6 +369,15 @@ 

      {

        "items": [

          {

+           "base_module_buildrequires": [

+             {

+               "context": "00000000",

+               "name": "platform",

+               "stream": "f29",

+               "stream_version": 290000,

+               "version": "5"

+             }

+           ],

            "component_builds": [

              57047,

              57048

@@ -479,6 +488,16 @@ 

  The module builds can be filtered by a variety of GET parameters. Some of these

  parameters include:

  

+ - ``base_module_br`` - the name:stream:version:context of a base module the module buildrequires

+ - ``base_module_br_context`` - the context of a base module the module buildrequires

+ - ``base_module_br_name`` - the name of a base module the module buildrequires

+ - ``base_module_br_stream`` - the stream of a base module the module buildrequires

+ - ``base_module_br_stream_version`` - the stream version of a base module the module buildrequires

+ - ``base_module_br_stream_version_lte`` - less than or equal to the stream version of a base module

+   the module buildrequires

+ - ``base_module_br_stream_version_gte`` - greater than or equal to the stream version of a base

+   module the module buildrequires

+ - ``base_module_br_version`` - the version of a base module the module buildrequires

  - ``batch``

  - ``cg_build_koji_tag``

  - ``completed_after`` - Zulu ISO 8601 format e.g. ``completed_after=2016-08-23T09:40:07Z``

@@ -0,0 +1,107 @@ 

+ """Add an association table for module buildrequires

+ 

+ Revision ID: 526fb7d445f7

+ Revises: 9d5e6938588f

+ Create Date: 2018-10-11 12:46:36.060460

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = '526fb7d445f7'

+ down_revision = '9d5e6938588f'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ # Data migration imports

+ from module_build_service import Modulemd, conf

+ from module_build_service.models import ModuleBuild

+ 

+ # Data migration tables

+ mb = sa.Table(

+     'module_builds',

+     sa.MetaData(),

+     sa.Column('id', sa.Integer, primary_key=True),

+     sa.Column('name', sa.String()),

+     sa.Column('stream', sa.String()),

+     sa.Column('version', sa.String()),

+     sa.Column('context', sa.String()),

+     sa.Column('stream_version', sa.Integer()),

+     sa.Column('modulemd', sa.String())

+ )

+ 

+ mb_to_mbr = sa.Table(

+     'module_builds_to_module_buildrequires',

+     sa.MetaData(),

+     sa.Column('module_id', sa.Integer(), nullable=False),

+     sa.Column('module_buildrequire_id', sa.Integer(), nullable=False),

+     sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']),

+     sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_builds.id']),

+     sa.UniqueConstraint('module_id', 'module_buildrequire_id')

+ )

+ 

+ 

+ def upgrade():

+     with op.batch_alter_table('module_builds', schema=None) as batch_op:

+         batch_op.add_column(sa.Column('stream_version', sa.Integer()))

+ 

+     op.create_table(

+         'module_builds_to_module_buildrequires',

+         sa.Column('module_id', sa.Integer(), nullable=False),

+         sa.Column('module_buildrequire_id', sa.Integer(), nullable=False),

+         sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_buildrequires.id']),

+         sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']),

+         sa.UniqueConstraint('module_id', 'module_buildrequire_id')

+     )

+ 

+     connection = op.get_bind()

+     # Create all the base module buildrequire entries

+     for build in connection.execute(mb.select()):

+         if not build.modulemd:

+             # If the modulemd is empty, skip this build

+             continue

+ 

+         brs = None

+         try:

+             mmd = Modulemd.Module().new_from_string(build.modulemd)

+             mmd.upgrade()

+             brs = mmd.get_xmd()['mbs']['buildrequires']

+         except Exception:

+             # If the modulemd isn't parseable then skip this build

+             continue

+ 

+         for base_module in conf.base_module_names:

+             base_module_dict = brs.get(base_module)

+             if not base_module_dict:

+                 # If this base module isn't a buildrequire, continue to see if the next one is

+                 continue

+ 

+             select = mb.select()\

+                 .where(mb.c.name == base_module)\

+                 .where(mb.c.stream == base_module_dict['stream'])\

+                 .where(mb.c.version == base_module_dict['version'])\

+                 .where(mb.c.context == base_module_dict['context'])

+             br = connection.execute(select).fetchone()

+             if not br:

+                 # If the buildrequire isn't in the datbase, then skip it

+                 continue

+ 

+             connection.execute(mb_to_mbr.insert().values(

+                 module_id=build.id,

+                 module_buildrequire_id=br.id

+             ))

+ 

+     for base_module in conf.base_module_names:

+         for build in connection.execute(mb.select().where(mb.c.name == base_module)):

+             stream_version = ModuleBuild.get_stream_version(build.stream)

+             if not stream_version:

+                 # If a stream version isn't parseable, then skip it

+                 continue

+             connection.execute(mb.update().where(mb.c.id == build.id).values(

+                 stream_version=stream_version))

+ 

+ 

+ def downgrade():

+     with op.batch_alter_table('module_builds', schema=None) as batch_op:

+         batch_op.drop_column(sa.Column('stream_version'))

+     op.drop_table('module_builds_to_module_buildrequires')

file modified
+103 -2

@@ -166,6 +166,15 @@ 

      __abstract__ = True

  

  

+ module_builds_to_module_buildrequires = db.Table(

+     'module_builds_to_module_buildrequires',

+     db.Column('module_id', db.Integer, db.ForeignKey('module_builds.id'), nullable=False),

+     db.Column('module_buildrequire_id', db.Integer, db.ForeignKey('module_builds.id'),

+               nullable=False),

+     db.UniqueConstraint('module_id', 'module_buildrequire_id', name='unique_buildrequires')

+ )

+ 

+ 

  class ModuleBuild(MBSBase):

      __tablename__ = "module_builds"

      id = db.Column(db.Integer, primary_key=True)

@@ -195,6 +204,16 @@ 

      # components.  Think like 'mockchain --recurse'

      batch = db.Column(db.Integer, default=0)

  

+     # This is only used for base modules for ordering purposes (f27.0.1 => 270001)

+     stream_version = db.Column(db.Integer)

+     buildrequires = db.relationship(

+         'ModuleBuild',

+         secondary=module_builds_to_module_buildrequires,

+         primaryjoin=module_builds_to_module_buildrequires.c.module_id == id,

+         secondaryjoin=module_builds_to_module_buildrequires.c.module_buildrequire_id == id,

+         backref='buildrequire_for'

+     )

+ 

      rebuild_strategies = {

          'all': 'All components will be rebuilt',

          'changed-and-after': ('All components that have changed and those in subsequent batches '

@@ -429,6 +448,11 @@ 

          # Add a state transition to "init"

          mbt = ModuleBuildTrace(state_time=now, state=module.state)

          module.module_builds_trace.append(mbt)

+ 

+         # Record the base modules this module buildrequires

+         for base_module in module.get_buildrequired_base_modules():

+             module.buildrequires.append(base_module)

+ 

          session.add(module)

          session.commit()

          if publish_msg:

@@ -535,8 +559,8 @@ 

  

          return query.first()

  

-     def short_json(self):

-         return {

+     def short_json(self, show_stream_version=False):

+         rv = {

              'id': self.id,

              'state': self.state,

              'state_name': INVERSE_BUILD_STATES[self.state],

@@ -545,6 +569,9 @@ 

              'name': self.name,

              'context': self.context,

          }

+         if show_stream_version:

+             rv['stream_version'] = self.stream_version

+         return rv

  

      def json(self, show_tasks=True):

          json = self.short_json()

@@ -576,7 +603,9 @@ 

          state_url = None

          if show_state_url:

              state_url = get_url_for('module_build', api_version=api_version, id=self.id)

+ 

          json.update({

+             'base_module_buildrequires': [br.short_json(True) for br in self.buildrequires],

              'build_context': self.build_context,

              'modulemd': self.modulemd,

              'ref_build_context': self.ref_build_context,

@@ -620,6 +649,78 @@ 

          return ModuleBuildTrace.query.filter_by(

              module_id=module_id).order_by(ModuleBuildTrace.state_time).all()

  

+     @staticmethod

+     def get_stream_version(stream, right_pad=True):

+         """

+         Parse the supplied stream to find its version.

+ 

+         This will parse a stream such as "f27" and return 270000. Another example would be a stream

+         of "f27.0.1" and return 270001.

+         :param str stream: the module stream

+         :kwarg bool right_pad: determines if the right side of the stream version should be padded

+             with zeroes (e.g. `f27` => `27` vs `270000`)

+         :return: a stream version represented as an integer

+         :rtype: int or None if the stream doesn't have a valid version

+         """

+         # The platform version (e.g. prefix1.2.0 => 010200)

+         version = ''

+         for char in stream:

+             # See if the current character is an integer, signifying the version has started

+             if char.isdigit():

+                 version += char

+             # If version isn't set, then a digit hasn't been encountered

+             elif version:

+                 # If the character is a period and the version is set, then

+                 # the loop is still processing the version part of the stream

+                 if char == '.':

+                     version += '.'

+                 # If the version is set and the character is not a period or

+                 # digit, then the remainder of the stream is a suffix like "-beta"

+                 else:

+                     break

+ 

+         # Remove the periods and pad the numbers if necessary

+         version = ''.join([section.zfill(2) for section in version.split('.')])

+ 

+         if version:

+             if right_pad:

+                 version += (6 - len(version)) * '0'

+             # Since the version must be stored as a number, we convert the string back to

+             # an integer which consequently drops the leading zero if there is one

+             return int(version)

+ 

+     def get_buildrequired_base_modules(self):

+         """

+         Find the base modules in the modulemd's xmd section.

+ 

+         :return: a list of ModuleBuild objects of the base modules that are buildrequired with the

+             ordering in conf.base_module_names preserved

+         :rtype: list

+         :raises RuntimeError: when the xmd section isn't properly filled out by MBS

+         """

+         rv = []

+         xmd = self.mmd().get_xmd()

+         with make_session(conf) as db_session:

+             for bm in conf.base_module_names:

+                 # xmd is a GLib Variant and doesn't support .get() syntax

+                 try:

+                     bm_dict = xmd['mbs']['buildrequires'].get(bm)

+                 except KeyError:

+                     raise RuntimeError(

+                         'The module\'s mmd is missing information in the xmd section')

+ 

+                 if not bm_dict:

+                     continue

+                 base_module = self.get_build_from_nsvc(

+                     db_session, bm, bm_dict['stream'], bm_dict['version'], bm_dict['context'])

+                 if not base_module:

+                     log.error('Module #{} buildrequires "{}" but it wasn\'t found in the database'

+                               .format(self.id, repr(bm_dict)))

+                     continue

+                 rv.append(base_module)

+ 

+         return rv

+ 

      def __repr__(self):

          return (("<ModuleBuild %s, id=%d, stream=%s, version=%s, state %r,"

                   " batch %r, state_reason %r>")

@@ -29,10 +29,11 @@ 

  import kobo.rpmlib

  import requests

  

- from module_build_service import db

+ from module_build_service import db, conf

  from module_build_service import models

  from module_build_service.errors import UnprocessableEntity

  from module_build_service.resolver.base import GenericResolver

+ from module_build_service.utils.general import import_mmd

  

  log = logging.getLogger()

  

@@ -363,5 +364,9 @@ 

                  raise RuntimeError(

                      'The module "{0}" didn\'t contain either a commit hash or a'

                      ' version in MBS'.format(module_name))

+             # If the module is a base module, then import it in the database so that entries in

+             # the module_builds_to_module_buildrequires table can be created later on

+             if module_name in conf.base_module_names:

+                 import_mmd(db.session, mmd)

  

          return new_requires

@@ -334,6 +334,8 @@ 

      build.time_submitted = datetime.utcnow()

      build.time_modified = datetime.utcnow()

      build.time_completed = datetime.utcnow()

+     if build.name in conf.base_module_names:

+         build.stream_version = models.ModuleBuild.get_stream_version(stream)

      session.add(build)

      session.commit()

      msg = "Module {} imported".format(nsvc)

@@ -239,36 +239,14 @@ 

              return version

  

          # The platform version (e.g. prefix1.2.0 => 010200)

-         version_prefix = ''

-         for char in base_module_stream:

-             try:

-                 # See if the current character is an integer, signifying the version

-                 # has started

-                 int(char)

-                 version_prefix += char

-             except ValueError:

-                 # If version_prefix isn't set, then a digit hasn't been encountered

-                 if version_prefix:

-                     # If the character is a period and the version_prefix is set, then

-                     # the loop is still processing the version part of the stream

-                     if char == '.':

-                         version_prefix += '.'

-                     # If the version_prefix is set and the character is not a period or

-                     # digit, then the remainder of the stream is a suffix like "-beta"

-                     else:

-                         break

- 

-         # Remove the periods and pad the numbers if necessary

-         version_prefix = ''.join([section.zfill(2) for section in version_prefix.split('.')])

+         version_prefix = models.ModuleBuild.get_stream_version(base_module_stream, right_pad=False)

  

          if not version_prefix:

              log.warning('The "{0}" stream "{1}" couldn\'t be used to prefix the module\'s '

                          'version'.format(base_module, base_module_stream))

              return version

  

-         # Since the version must be stored as a number, we convert the string back to

-         # an integer which consequently drops the leading zero if there is one

-         new_version = int(version_prefix + str(version))

+         new_version = int(str(version_prefix) + str(version))

          if new_version > GLib.MAXUINT64:

              log.warning('The "{0}" stream "{1}" caused the module\'s version prefix to be '

                          'too long'.format(base_module, base_module_stream))

@@ -29,6 +29,7 @@ 

  

  from flask import request, url_for, Response

  from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean

+ from sqlalchemy.orm import aliased

  

  from module_build_service import models, api_version, conf

  from module_build_service.errors import ValidationError, NotFound

@@ -265,6 +266,55 @@ 

                  elif context == 'before':

                      query = query.filter(column <= item_datetime)

  

+     br_joined = False

+     module_br_alias = None

+     for item in ('base_module_br', 'name', 'stream', 'version', 'context', 'stream_version',

+                  'stream_version_lte', 'stream_version_gte'):

+         if item == 'base_module_br':

+             request_arg_name = item

+         else:

+             request_arg_name = 'base_module_br_{}'.format(item)

+         request_arg = flask_request.args.get(request_arg_name)

+ 

+         if not request_arg:

+             continue

+ 

+         if not br_joined:

+             module_br_alias = aliased(models.ModuleBuild, name='module_br')

+             # Shorten this table name for clarity in the query below

+             mb_to_br = models.module_builds_to_module_buildrequires

+             # The following joins get added:

+             # JOIN module_builds_to_module_buildrequires

+             #     ON module_builds_to_module_buildrequires.module_id = module_builds.id

+             # JOIN module_builds AS module_br

+             #     ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id

+             query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id)\

+                          .join(module_br_alias,

+                                mb_to_br.c.module_buildrequire_id == module_br_alias.id)

+             br_joined = True

+ 

+         if item == 'base_module_br':

+             try:

+                 name, stream, version, context = flask_request.args['base_module_br'].split(':')

+             except ValueError:

+                 raise ValidationError(

+                     'The filter argument for "base_module_br" must be in the format of N:S:V:C')

+             query = query.filter(

+                 module_br_alias.name == name,

+                 module_br_alias.stream == stream,

+                 module_br_alias.version == version,

+                 module_br_alias.context == context

+             )

+         elif item.endswith('_lte'):

+             column = getattr(module_br_alias, item[:-4])

+             query = query.filter(column <= request_arg)

+         elif item.endswith('_gte'):

+             column = getattr(module_br_alias, item[:-4])

+             query = query.filter(column >= request_arg)

+         else:

+             column = getattr(module_br_alias, item)

+             query = query.filter(column == request_arg)

+ 

      query = _add_order_by_clause(flask_request, query, models.ModuleBuild)

  

      page = flask_request.args.get('page', 1, type=int)

file modified
+9

@@ -293,6 +293,8 @@ 

      mmd.upgrade()

      mmd.get_rpm_components()['tangerine'].set_buildorder(0)

  

+     platform_br = module_build_service.models.ModuleBuild.query.get(1)

+ 

      build_one = module_build_service.models.ModuleBuild()

      build_one.name = 'testmodule'

      build_one.stream = 'master'

@@ -314,6 +316,7 @@ 

      build_one.rebuild_strategy = 'changed-and-after'

      build_one.modulemd = mmd.dumps()

      build_one_component_release = get_rpm_release(build_one)

+     build_one.buildrequires.append(platform_br)

  

      component_one_build_one = module_build_service.models.ComponentBuild()

      component_one_build_one.package = 'perl-Tangerine'

@@ -381,6 +384,7 @@ 

      component_four_build_one.build_time_only = True

  

      with make_session(conf) as session:

+         session.add(platform_br)

          session.add(build_one)

          session.add(component_one_build_one)

          session.add(component_two_build_one)

@@ -398,6 +402,8 @@ 

      mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)

      mmd.upgrade()

  

+     platform_br = module_build_service.models.ModuleBuild.query.get(1)

+ 

      build_one = module_build_service.models.ModuleBuild()

      build_one.name = 'testmodule'

      build_one.stream = 'master'

@@ -422,6 +428,7 @@ 

      xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'

      mmd.set_xmd(glib.dict_values(xmd))

      build_one.modulemd = mmd.dumps()

+     build_one.buildrequires.append(platform_br)

  

      component_one_build_one = module_build_service.models.ComponentBuild()

      component_one_build_one.package = 'perl-Tangerine'

@@ -506,6 +513,7 @@ 

      xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'

      mmd.set_xmd(glib.dict_values(xmd))

      build_two.modulemd = mmd.dumps()

+     build_two.buildrequires.append(platform_br)

  

      component_one_build_two = module_build_service.models.ComponentBuild()

      component_one_build_two.package = 'perl-Tangerine'

@@ -550,6 +558,7 @@ 

      component_four_build_two.build_time_only = True

  

      with make_session(conf) as session:

+         session.add(platform_br)

          session.add(build_one)

          session.add(component_one_build_one)

          session.add(component_two_build_one)

@@ -28,7 +28,7 @@ 

          testmodule: {ref: 147dca4ca65aa9a1ac51f71b7e687f9178ffa5df, stream: master,

            version: '20170616125652', context: '321'}

        requires:

-         platform: {ref: virtual, stream: f28, version: '3'}

+         platform: {ref: virtual, stream: f28, version: '3', context: '00000000'}

        commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a

        rpms:

          ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216}

@@ -25,7 +25,7 @@ 

    xmd:

      mbs:

        buildrequires:

-         platform: {ref: virtual, stream: f28, version: '3'}

+         platform: {ref: virtual, stream: f28, version: '3', context: '00000000'}

        commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a

        rpms:

          ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216}

@@ -25,7 +25,7 @@ 

    xmd:

      mbs:

        buildrequires:

-         platform: {ref: virtual, stream: f28, version: '3'}

+         platform: {ref: virtual, stream: f28, version: '3', context: '00000000'}

        commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a

        rpms:

          ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216}

file modified
+20 -16

@@ -50,18 +50,6 @@ 

          mmd = Modulemd.Module().new_from_file(os.path.join(

              base_dir, '..', 'staged_data', 'testmodule-with-filters.yaml'))

          mmd.upgrade()

- 

-         module = ModuleBuild.create(

-             session,

-             conf,

-             name="mbs-testmodule",

-             stream="test",

-             version="20171027111452",

-             modulemd=mmd.dumps(),

-             scmurl="file:///testdir",

-             username="test",

-         )

-         module.koji_tag = "module-mbs-testmodule-test-20171027111452"

          mmd.set_xmd(glib.dict_values({

              'mbs': {

                  'rpms': {

@@ -74,13 +62,16 @@ 

                          'version': '20171024133034',

                          'filtered_rpms': [],

                          'stream': 'master',

-                         'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45'

+                         'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45',

+                         'context': '00000000'

                      },

                      'platform': {

                          'version': '20171028112959',

                          'filtered_rpms': [],

                          'stream': 'master',

-                         'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'}

+                         'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa',

+                         'context': '00000000'

+                     }

                  },

                  'scmurl': 'file:///testdir',

                  'commit': '5566bc792ec7a03bb0e28edd1b104a96ba342bd8',

@@ -89,11 +80,23 @@ 

                          'version': '20171028112959',

                          'filtered_rpms': [],

                          'stream': 'master',

-                         'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'}

+                         'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa',

+                         'context': '00000000'

+                     }

                  }

              }

          }))

-         module.modulemd = mmd.dumps()

+         module = ModuleBuild.create(

+             session,

+             conf,

+             name="mbs-testmodule",

+             stream="test",

+             version="20171027111452",

+             modulemd=mmd.dumps(),

+             scmurl="file:///testdir",

+             username="test",

+         )

+         module.koji_tag = "module-mbs-testmodule-test-20171027111452"

          module.batch = batch

          session.add(module)

  

@@ -101,6 +104,7 @@ 

              cb = ComponentBuild(**dict(build, format="rpms", state=state))

              session.add(cb)

              session.commit()

+         session.commit()

  

          return module

  

@@ -93,6 +93,16 @@ 

          build_one = ModuleBuild.query.get(2)

          assert build_one.siblings == [3, 4]

  

+     def test_get_stream_version(self):

+         """Test the ModuleBuild.get_stream_version method when right_pad is True."""

+         assert ModuleBuild.get_stream_version('f27') == 270000

+         assert ModuleBuild.get_stream_version('f27.02.30') == 270230

+ 

+     def test_get_stream_version_no_right_pad(self):

+         """Test the ModuleBuild.get_stream_version method when right_pad is False."""

+         assert ModuleBuild.get_stream_version('f27', False) == 27

+         assert ModuleBuild.get_stream_version('f27.02.30', False) == 270230

+ 

  

  class TestModelsGetStreamsContexts:

  

@@ -213,7 +213,7 @@ 

      @patch('module_build_service.resolver.DBResolver')

      @patch('module_build_service.resolver.GenericResolver')

      @patch("module_build_service.config.Config.base_module_names",

-            new_callable=mock.PropertyMock, return_value=set(["base-runtime"]))

+            new_callable=mock.PropertyMock, return_value=set(["base-runtime", "platform"]))

      def test_set_cg_build_koji_tag(

              self, cfg, generic_resolver, resolver, create_builder, koji_get_session, dbg):

          """

@@ -21,6 +21,7 @@ 

  # Written by Matt Prahl <mprahl@redhat.com>

  

  import json

+ from datetime import datetime

  

  import module_build_service.scm

  

@@ -39,6 +40,7 @@ 

  from module_build_service import db, version, Modulemd

  import module_build_service.config as mbs_config

  import module_build_service.scheduler.handlers.modules

+ from module_build_service.utils import import_mmd, load_mmd

  

  

  user = ('Homer J. Simpson', set(['packager']))

@@ -172,6 +174,7 @@ 

      def test_query_build_with_verbose_mode(self):

          rv = self.client.get('/module-build-service/1/module-builds/2?verbose=true')

          data = json.loads(rv.data)

+         assert data['base_module_buildrequires'] == []

          assert data['component_builds'] == [1, 2]

          assert data['context'] == '00000000'

          # There is no xmd information on this module, so these values should be None

@@ -216,6 +219,21 @@ 

          assert data['rebuild_strategy'] == 'changed-and-after'

          assert data['siblings'] == []

  

+     def test_query_build_with_br_verbose_mode(self):

+         reuse_component_init_data()

+         rv = self.client.get('/module-build-service/1/module-builds/2?verbose=true')

+         data = json.loads(rv.data)

+         assert data['base_module_buildrequires'] == [{

+             'context': '00000000',

+             'id': 1,

+             'name': 'platform',

+             'state': 5,

+             'state_name': 'ready',

+             'stream': 'f28',

+             'stream_version': 280000,

+             'version': '3'

+         }]

+ 

      def test_pagination_metadata(self):

          rv = self.client.get('/module-build-service/1/module-builds/?per_page=2&page=2')

          meta_data = json.loads(rv.data)['meta']

@@ -662,6 +680,77 @@ 

          assert data['error'] == 'Bad Request'

          assert data['message'] == 'An invalid order_by or order_desc_by key was supplied'

  

+     def test_query_base_module_br_filters(self):

+         reuse_component_init_data()

+         mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True)

+         mmd.set_stream('f30.1.3')

+         import_mmd(db.session, mmd)

+         platform_f300103 = ModuleBuild.query.filter_by(stream='f30.1.3').one()

+         build = ModuleBuild(

+             name='testmodule',

+             stream='master',

+             version=20170109091357,

+             state=5,

+             build_context='dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3',

+             runtime_context='ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7',

+             context='7c29193d',

+             koji_tag='module-testmodule-master-20170109091357-7c29193d',

+             scmurl='git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79',

+             batch=3,

+             owner='Dr. Pepper',

+             time_submitted=datetime(2018, 11, 15, 16, 8, 18),

+             time_modified=datetime(2018, 11, 15, 16, 19, 35),

+             rebuild_strategy='changed-and-after',

+             modulemd='---'

+         )

+         build.buildrequires.append(platform_f300103)

+         db.session.add(build)

+         db.session.commit()

+         # Query by NSVC

+         rv = self.client.get(

+             '/module-build-service/1/module-builds/?base_module_br=platform:f28:3:00000000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 2

+         rv = self.client.get(

+             '/module-build-service/1/module-builds/?base_module_br=platform:f30.1.3:3:00000000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 1

+         # Query by non-existent NVC

+         rv = self.client.get(

+             '/module-build-service/1/module-builds/?base_module_br=platform:f12:3:00000000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 0

+         # Query by name and stream

+         rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_name=platform'

+                              '&base_module_br_stream=f28')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 2

+         # Query by stream version

+         rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version='

+                              '280000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 2

+         # Query by lte stream version

+         rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_'

+                              'lte=290000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 2

+         # Query by lte stream version with no results

+         rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_'

+                              'lte=270000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 0

+         # Query by gte stream version

+         rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_'

+                              'gte=270000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 3

+         # Query by gte stream version with no results

+         rv = self.client.get('/module-build-service/1/module-builds/?base_module_br_stream_version_'

+                              'gte=320000')

+         data = json.loads(rv.data)

+         assert data['meta']['total'] == 0

+ 

      @pytest.mark.parametrize('api_version', [1, 2])

      @patch('module_build_service.auth.get_user', return_value=user)

      @patch('module_build_service.scm.SCM')

@@ -702,6 +791,15 @@ 

          mmd = Modulemd.Module().new_from_string(data['modulemd'])

          mmd.upgrade()

  

+         # Make sure the buildrequires entry was created

+         module = ModuleBuild.query.get(8)

+         assert len(module.buildrequires) == 1

+         assert module.buildrequires[0].name == 'platform'

+         assert module.buildrequires[0].stream == 'f28'

+         assert module.buildrequires[0].version == '3'

+         assert module.buildrequires[0].context == '00000000'

+         assert module.buildrequires[0].stream_version == 280000

+ 

      @patch('module_build_service.auth.get_user', return_value=user)

      @patch('module_build_service.scm.SCM')

      @patch('module_build_service.config.Config.rebuild_strategy_allow_override',

Future use cases will require the ability to find compatible module builds to buildrequire based on the base module the module used to build. This commit adds a ModuleBuildrequire table that will contain entries of base modules that module builds buildrequire. The linking is done in an association table.

Addresses FACTORY-3353

Probably this error message may contain more specific info about what is missing, that would be helpful to debug what need to be fixed.

FYI, according to the above comment xmd is a GLib Variant and doesn't support .get() syntax, I guess xmd['mbs']['buildrequires'] could raise ValueError if xmd is not filled properly. If yes, maybe following change could be more straightforward about what problem the try-except is trying to solve.

xmd = self.mmd().get_xmd()
try:
    buildrequires = xmd['mbs']['buildrequires']
except KeyError:
    raise RuntimeError('The module\'s mmd is missing information in the xmd section')
for base_module in conf.base_module_names:
    ...

How about use char.isdigit() instead?

Do we really need this new class and table?

I think ModuleBuild can only buildrequire other ModuleBuild. I'm thinking if we can only keep module_builds_to_module_buildrequires relation and let it to map ModuleBuild to buildrequired ModuleBuilds as 1:N. We can then also add "stream_version" to ModuleBuild to allow searching for ModuleBuilds with particular stream_version.

Is there a reason why to limit this feature just to base modules? It sems that with simple change to not compute stream_version for non-base modules, it would just work as expected and you would be able to find out all buildrequired modules, which might be useful for Freshmajer or some external scripts.

After thinking about it, I think all of our use cases are covered by the filters on the API. I'll refactor.

Edit:
There is some consideration around local builds that will be discussed in IRC.

I was trying to limit the scope of the change, but I can enable it for all buildrequires.

rebased onto 4f9ffff

11 months ago

@jkaluza based on your feedback and our IRC discussions, I've rewritten the PR. Could you please take a look?

You can use get_build_from_nsvc, but this is not review blocker. Just wanted to point out we have that method :).

+1, it looks good to me. It surely needs some staging tests to be 100% sure it does not break anything, but I did not find any issue so far.

Thanks for pointing that out. I'll replace this.

2 new commits added

  • Add the ability to query by the base modules a module build buildrequires
  • Move some of the logic in `get_prefixed_version` to a ModuleBuild static method
11 months ago

Commit a33e082 fixes this pull-request

Pull-Request has been merged by mprahl

11 months ago

Pull-Request has been merged by mprahl

11 months ago